#!/usr/bin/env python3
# Erik de Bueger

import os
import sys
import datetime
import logging
import optparse
from random import randint
import re
import json
import boto3
import pickle
import requests

standarddateformat  = '%Y%m%d'

# Daily data
URLS = {'daily': 'https://www.daggegevens.knmi.nl/klimatologie/daggegevens',
            'hourly': 'https://www.daggegevens.knmi.nl/klimatologie/uurgegevens',
            'daily_rain': 'https://www.daggegevens.knmi.nl/klimatologie/monv/reeksen'}

# STN      LON(east)   LAT(north)     ALT(m)  NAME
# 235:         4.785       52.924       0.50  DE KOOY
# 209         4.518       52.465      0.00        IJmond
STATIONREGEX=re.compile(r'^#\s*(?P<NUM>[0-9]+)\s+(?P<LON>[0-9\.]+)\s+(?P<LAT>[0-9\.]+)\s+(?P<ALT>[0-9\.-]+)\s+(?P<Station>.*)$', re.U)
# EV24     = Potential evapotranspiration (Makkink) (in 0.1 mm);
LEGENDREGEX=re.compile(r'^#\s+(?P<SYMBOL>[A-Z0-9]+)\s+:\s*(?P<legend>.*)$', re.U)
## STN,YYYYMMDD,DDVEC,FHVEC,   FG,  FHX, FHXH,  FHN, FHNH,  FXX, FXXH,   TG,   TN,  TNH,   TX,  TXH, T10N,T10NH,   SQ,   SP,    Q,   DR,   RH,  RHX, RHXH,   PG,   PX,  PXH,   PN,  PNH,  VVN, VVNH,  VVX, VVXH,   NG,   UG,   UX,  UXH,   UN,  UNH, EV24
HEADERREGEX=re.compile(r'^#\s+STN,\s*YYYYMMDD,(?P<header>.*)$', re.U)
#  209,20231019,  145,   60,   70,  100,   24,   50,   11,  130,    1,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,     ,
DATAREGEX=re.compile(r'^(\s*[0-9]*,){1,}(\s*[0-9]*){0,1}$', re.U)

# {event type:date}
class Cache(object) :
    def __init__(self, region,logger,starttime):
        self.logger    = logger
        self.starttime = starttime
        self.counter   = 0
        self.cache     =  None
        self.d         = {}
    def Get(self,key) :
        return(self.starttime)
    def Write(self,key, value):
        self.d[key]=value
        self.counter+=1
        if self.counter % 10 == 0:
            self.Flush()
    def Close(self) :
        self.Flush()
    def Flush(self) :
        pass
    def Print(self,fp) :
        for k,v in self.d.items() :
            fp.write('{} -- {}\n'.format(k,v.strftime(standarddateformat)))

class CacheFile(Cache) :
    def __init__(self, region,logger,starttime):
        self.logger = logger
        self.starttime = starttime
        self.counter = 0
        self.file = region
        self.fp = None
        self.d={}
        try:
            self.fp = open(os.path.join(sys.path[0],self.file),'rb')
            # use dict as fast cache
            self.d = pickle.load(self.fp)
            self.fp.close()
        except Exception as e:
            self.logger.warning(str(e))
            self.logger.warning('Filecache could not be opened: {}, will create later'.format(os.path.join(sys.path[0],self.file)))
            self.d={}

    def Get(self,key) :
        try:
            return(self.d[key])
        except KeyError:
            # Not in cache, use earliest date
            return(self.starttime)

    # write dict to cache
    def Flush(self) :
        try:
            self.fp = open(os.path.join(sys.path[0],self.file),'wb')
            pickle.dump(self.d,self.fp, protocol=0)
            self.fp.truncate()
            self.fp.flush()
            self.fp.close()
        except Exception as e:
            self.logger.warning(str(e))
            self.logger.warning('Error writing last_eventid to cache')

class CacheAWSParameter(Cache) :
    def __init__(self,region,logger,starttime):
        self.logger    = logger
        self.starttime = starttime
        self.counter   = 0
        self.cache     = boto3.client('ssm', region_name=region)
        self.d         = {}

    # Check last event tokens or timestamps per key in AWS cache
    # key is event type, value is date: knmi-daily:20211123
    def Get(self,key) :
        # First try local cache
        try:
            return(self.d[key])
        except KeyError:
            pass
        # Retrieve from remote cache
        cachekey = 'knmi-{}'.format(key)
        try:
            cachevalue = self.cache.get_parameter(Name=cachekey)['Parameter']['Value']
            # We have a string back from the cache, should be timestamp
            # Try timestamp
            self.d[key] = datetime.datetime.strptime(cachevalue, standarddateformat)
            return(self.d[key])
        except Exception as e:
            # No cache value found
            self.logger.warning(str(e))
            self.logger.warning('Error: {} not found in cache! Starting from {}. \n'.format(cachekey,self.starttime))
            return(self.starttime)

    # write dict to cache
    def Flush(self) :
        for k,v in self.d.items() :
            cachekey = 'knmi-{}'.format(k)
            try:
                # try date format
                x = self.cache.put_parameter(Name=cachekey, Type='String', Overwrite=True,Value=v.strftime(standarddateformat))
            except Exception as e:
                self.logger.warning(str(e))
                self.logger.warning('Error writing last_eventid to cache')

def MarshalGeneric(d) :
    # Remove emty fields
    if d==None: return(d)
    d=dict((k, v) for k, v in d.items() if v)
    for k,v in d.items() :
        try:
            d[k]=int(d[k])
        except:
            pass
    try:
        d['timestamp'] = datetime.datetime.strptime(str(d['YYYYMMDD']), standarddateformat).strftime('%Y-%m-%d:%H%M%S')
    except :
        pass
    return(d)

# Class for writing events to destination, either Splunk or Elastic.
# Overwrites possible for alternate destinations
class EventWriter(object) :
    def __init__(self, **kwargs):
        self.marshal  = kwargs.get('mfunc',None)
        self.logger   = kwargs.get('logger',None)

    # Send a kv-formatted message to stdout
    # message is json dict
    def send(self, message):
        if self.marshal != None :
            data=self.marshal(message)
        else :
            data=message
        #    data = message.encode('utf-8', 'replace')
        #sys.stdout.write('{}\n\n'.format(json.dumps(data).encode('utf-8', 'replace')))
        sys.stdout.write('{}\n\n'.format(json.dumps(data)))

    def flush(self) :
        pass

class EventWriterCached(EventWriter) :
    def __init__(self, **kwargs):
        self.marshal  = kwargs.get('mfunc',None)
        self.logger   = kwargs.get('logger',None)
        self.batch    = kwargs.get('batch',1)
        self.cache    = []

    def send(self, message):
        if self.marshal != None :
            data=self.marshal(message)
        else :
            data=message
        self.cache.append(data)
        if len(self.cache) >= self.batch :
            s='\n'.join([json.dumps(x) for x in self.cache])
            sys.stdout.write(s)
            sys.stdout.write('\n')
            self.cache=[]

    def flush(self) :
        if len(self.cache) > 0:
            s='\n'.join([json.dumps(x) for x in self.cache])
            sys.stdout.write(s)
            sys.stdout.write('\n')
            self.cache=[]

class EventWriterJSON(EventWriter) :
    def __init__(self, **kwargs):
        self.marshal  = kwargs.get('mfunc',None)
        self.logger   = kwargs.get('logger',None)
        self.url      = kwargs.get('endpoint',None)

    # message is dict
    def send(self, message):
        if self.marshal != None :
            data=self.marshal(message)
        else :
            data=message
        response = requests.post(self.url, data=json.dumps(data))
        if self.logger != None:
            self.logger.info('Endpoint response: {}'.format(response.text))

class EventWriterCachedJSON(EventWriter) :
    def __init__(self, **kwargs):
        self.marshal  = kwargs.get('mfunc',None)
        self.logger   = kwargs.get('logger',None)
        self.url      = kwargs.get('endpoint',None)
        self.batch    = kwargs.get('batch',1)
        self.cache    = []

    # message is dict
    def send(self, message):
        if self.marshal != None :
            data=self.marshal(message)
        else :
            data=message
        self.cache.append(data)
        if len(self.cache) >= self.batch :
            s='\n'.join([x for x in json.dumps(self.cache)])
            response = requests.post(self.url, data=s)
            self.cache=[]
            if self.logger != None:
                self.logger.info('Endpoint response: {}'.format(response.text))

    def flush(self) :
        if len(self.cache) > 0:
            s='\n'.join([x for x in json.dumps(self.cache)])
            response = requests.post(self.url, s)
            self.cache=[]

def GetDailyData(url, startdate,enddate, logger) :
    params = dict(stns='ALL')
    start = startdate.strftime(standarddateformat)
    end = enddate.strftime(standarddateformat)
    params.update(start=start)
    params.update(end=end)
    params.update(vars='ALL')
    r = requests.post(url=url, data=params)
    if r.status_code != 200:
        raise requests.HTTPError(r.status_code, url, params)
        return(None)
    return(r)

# Iterator over KNMI csv format file, returns dicts
def data_digger(raw):
    cols=['STN','YYYYMMDD']
    headerseen=False
    for line in raw.text.splitlines():
        if headerseen:
            m=DATAREGEX.match(line)
            if m:
                lColumns=[x.strip() for x in line.lstrip().rstrip().split(',')]
                dFormat=dict(list(zip(cols,lColumns)))
                yield(dFormat)
        else:
            m=HEADERREGEX.match(line)
            if m:
                headerseen=True
                cols=cols+[x.strip() for x in m.group('header').split(',')]

# Iterator over KNMI csv format file, returns dicts
def location_digger(raw):
    for line in raw.text.splitlines():
        m=HEADERREGEX.match(line)
        if m:
            return
        m =  STATIONREGEX.match(line)
        if m != None:
            d = {'STN':int(m.group('NUM')), 'lat':m.group('LAT'), 'lon':m.group('LON'), 'alt':m.group('ALT')}
            yield (d)

def legend_digger(raw):
    for line in raw.text.splitlines():
        m=HEADERREGEX.match(line)
        if m:
            return
        m=LEGENDREGEX.match(line)
        if m:
            try:
                yield({'symbol':m.group('SYMBOL'),'desc':m.group('legend')})
            except:
                pass

def station_digger(raw):
    for line in raw.text.splitlines():
        m=HEADERREGEX.match(line)
        if m:
            return
        m=STATIONREGEX.match(line)
        if m :
            # we have a station spec
            try:
                yield({'STN':int(m.group('NUM')),'LON':m.group('LON'), 'LAT':m.group('LAT'), 'ALT':m.group('ALT'), 'Station':m.group('Station').strip()})
            except:
                pass

# Helper to iterate over days
def daterange(start_date, end_date):
    for n in range(int((end_date - start_date).days)):
        yield start_date + datetime.timedelta(n)

def planning(latest,history,maxdays) :
    startdate= datetime.datetime.now()-datetime.timedelta(days=opts.history)
    enddate= datetime.datetime.now()-datetime.timedelta(days=opts.latest)

def DoCollect(startdate,enddate,cachedate,opts,eventwriter,cache,logger):
    # Decide which days we will try to collect: startdate=the start date requested, enddate=latest possible end date, cachedate=last date collected, or equal to startdate
    daysback= (datetime.datetime.now()-cachedate).days
    if daysback > opts.maxdays :
        enddate = cachedate + datetime.timedelta(days=opts.maxdays)
    startdate=cachedate + datetime.timedelta(days=1)
    logger.info('Planning to collect from {} to {}'.format(startdate.strftime(standarddateformat), enddate.strftime(standarddateformat)))

    # Write legend + stations inventory only on some random occasions
    writelegend = (randint(1,5)==3)

    # Start collecting on per-day basis
    for thedate in daterange(startdate, enddate):
        thedata = GetDailyData(URLS['daily'], thedate,thedate,logger)
        # Send data
        nEvents=0
        for d in data_digger(thedata):
            if d != None:
                d.update({'type':'daily'})
                eventwriter.send(d)
                nEvents+=1
        if writelegend:
            for d in legend_digger(thedata):
                if d != None:
                    d.update({'type':'legend'})
                    eventwriter.send(d)
            for d in station_digger(thedata):
                if d != None:
                    d.update({'type':'station'})
                    eventwriter.send(d)
            writelegend=False
        # Update cache
        if nEvents > 10:
            cache.Write('daily',thedate)
        logger.info('Send {} events for day {}'.format(nEvents,thedate.strftime(standarddateformat)))

# Main code
def main():
    # Parse command line options
    p = optparse.OptionParser("usage: %prog [options]")
    p.add_option("-u", "--url", dest="url", default=URLS['daily'], \
                     help = 'url for dutch KNMI daily data extraction, defaults to: {}\nAlternate option: {}'.format(URLS['daily'], URLS['hourly']))
    p.add_option("-y", "--hourly", dest="hourly", action='store_true', default=False, \
                     help = 'Extract hourly data as opposed to daily, defaults to: {}'.format(False))
    p.add_option("-t", "--stations", dest="stations", action='store_true', default=False, \
                 help='Extract list of stations and write to stdout, default=False')
    p.add_option("-g", "--legend", dest="legend", action='store_true', default=False, \
                 help='send legend output only, default=False')
    p.add_option("-f", "--format", dest="format", default='stdout', \
                 help='send output to {}, default output is stdout'.format('sumo,splunk,elastic, awselastic,json or stdout'))
    p.add_option("--latest", dest="latest", type='int', default=2, \
                     help = 'Collect this many days in the past as the latest available data, more recent days will not be populated, defaults to 30')
    p.add_option("--history", dest="history", type='int', default=30, \
                     help = 'Collect this many days of historic data, defaults to 10')
    p.add_option("--maxdays", dest="maxdays", type='int', default=5, \
                     help = 'Maximum amount of days to be collected in one run , defaults to 5')
    p.add_option("-b","--batch", dest="batch", type='int', default=10, \
                     help = 'output batch size, defaults to 10')
    p.add_option("--endpoint", dest="endpoint", default=None, \
                  help = 'endpoint (sumo, splunk, elastic url) where data is delivered, defaults to: {}'.format('None'))
    p.add_option("-n", "--stationnumber", dest="stationnumber", type='int', default=-1, \
                  help = 'Collect data for specific station, identified by station number, defaults to -1')
    p.add_option("--nolambda", dest="nolambda", action='store_true', default=False, \
                 help='if set donot run as as lambda function, default=False')
    p.add_option("--region", dest="region", default='us-east-2', \
                     help = 'AWS region, defaults to: {}'.format('us-east-2'))
    p.add_option("-l", "--log", dest="loglevel", default='WARNING', \
                 help = 'loglevel field, defaults to WARNING, options are: CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET')
    p.add_option("--cache", dest="cache", default='lastknmievents', \
                 help = 'name of cache file or endpoint for AWS memcached service, defaults to : lastknmievent')
    opts,args = p.parse_args()

    # For AWS lambda: try to overwrite some options from environment
    for x in ('format','endpoint','logtype','url','loglevel','stationnumber','region','history','latest','maxdays','batch') :
        try:
            exec('opts.{}=os.environ[\'{}\']'.format(x,x))
        except Exception as e:
            pass
    try:
        opts.history = int(opts.history)
    except:
        pass
    try:
        opts.maxdays = int(opts.maxdays)
    except:
        pass
    try:
        opts.latest = int(opts.latest)
    except:
        pass
    try:
        opts.stationnumber = int(opts.stationnumber)
    except:
        pass
    try:
        opts.batch = int(opts.batch)
    except:
        pass

    startdate= datetime.datetime.now()-datetime.timedelta(days=opts.history)
    enddate= datetime.datetime.now()-datetime.timedelta(days=opts.latest)

    # Initiate Logging
    numeric_level = getattr(logging, opts.loglevel.upper(), None)
    if not isinstance(numeric_level, int):
        raise ValueError('Invalid log level: %s' % opts.loglevel)

    formatter = logging.Formatter('%(asctime)s,Level=%(levelname)s,\
                                   LOGGING=%(message)s', '%m/%d/%Y %H:%M:%S')

    sh = logging.StreamHandler(sys.stderr)
    sh.setFormatter(formatter)
    logger        = logging.getLogger('KNMI')
    logger.handlers.clear()
    logger.addHandler(sh)
    logger.level=numeric_level

    # Open up the cache, which remembers per station what the date of the most recent event was
    if opts.nolambda:
        # revert to local file cache
        try:
            cache = CacheFile(opts.cache, logger, startdate)
        except Exception as e:
            # giving up
            logger.warning(str(e)+'\n')
            logger.warning('Could not open file cache, exit')
            return
    else:
        try:
            cache = CacheAWSParameter(opts.region,logger,startdate)
        except Exception as e:
            logger.warning(str(e)+'\n')
            logger.warning('Could not open AWS cache, exit')
            return

    # Retrieve latest date from cache
    cachedate = cache.Get('daily')

    # Create output channel
    if opts.format=='stdout':
        eventwriter=EventWriterCached(logger=logger,mfunc=MarshalGeneric,batch=opts.batch)
        logger.info('Writing events to stdout')
    elif opts.format=='sumo':
        #eventwriter=EventWriterJSON(endpoint=opts.endpoint,mfunc=MarshalGeneric, logger=logger)
        eventwriter=EventWriterCachedJSON(endpoint=opts.endpoint,mfunc=MarshalGeneric, logger=logger,batch=opts.batch)
        logger.info('Writing events to Sumo')
    else :
        logger.err('Unknow destination format provided: {}. Options are: stdout-splunk-elastic-sumo'.format(opts.format))
        sys.exit(0)

    # Process options
    if opts.stations:
        thedata = GetDailyData(URLS['daily'], startdate,startdate,logger)
        for d in station_digger(thedata):
            if d != None:
                d.update({'type':'station'})
                eventwriter.send(d)
        sys.exit()
    if opts.legend:
        thedata = GetDailyData(URLS['daily'], startdate,startdate,logger)
        for d in legend_digger(thedata):
            if d != None:
                d.update({'type':'legend'})
                eventwriter.send(d)
        sys.exit()
    if opts.stationnumber > 0 :
        thedata = GetDailyData(URLS['daily'], startdate,startdate,logger)
        for d in data_digger(thedata):
            if int(d['STN'])==opts.stationnumber:
                eventwriter.send(d)
                sys.exit()
        sys.exit()

    DoCollect(startdate,enddate,cachedate,opts,eventwriter,cache,logger)

    cache.Close()
    eventwriter.flush()

if __name__ == '__main__':
    main()

def lambda_handler(event, context):
    main()
    return {
        'statusCode': 200,
        'body': json.dumps('KNMI Lambda returns\n')
    }


from django.urls import reverse
from django.core import mail
from rest_framework import status
from diary.models import User
from .base_setup import BaseUserTestCase

class UserAccountActivationTestCase(BaseUserTestCase):
    reset_sequences = True

    def test_user_account_activation(self):
        self.client.post(reverse('diary:signup'), data=self.user)
        activation_url = mail.outbox[0].body.split(' ')[-1]
        self.client.get(activation_url)
        active_user = User.objects.get(pk=1)
        self.assertTrue(active_user.is_active)

    def test_user_account_activation_link_is_one_time(self):
        self.client.post(reverse('diary:signup'), data=self.user)
        activation_url = mail.outbox[0].body.split(' ')[-1]
        self.client.get(activation_url)
        response = self.client.get(activation_url)
        self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)

    def test_user_account_activation_with_invalid_token(self):
        response = self.client.get(
            reverse('diary:activate', kwargs={'slug_field': 'jkdjjdddhjjh'}),
            {'token': 'hjhgedgvgdgwegghegvghv'}
            )
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)


class Solution:
    def bs(self, num,  d1, d2, cnt1, cnt2):
        both = num // lcm(d1, d2)
        first = num // d1
        second = num // d2

        both_add = num - (first + second - both)

        cnt1 = max(0, cnt1 - (second - both))
        cnt2 = max(0, cnt2 - (first - both))

        return both_add >= cnt1 + cnt2

    def minimizeSet(self, divisor1: int, divisor2: int, uniqueCnt1: int, uniqueCnt2: int) -> int:
        low, high = 0, 10**11
        while low + 1 < high:
            mid = (low + high) // 2
            if self.bs(mid,  divisor1, divisor2, uniqueCnt1, uniqueCnt2):
              high = mid
            else:
              low = mid
        return high


# Databricks notebook source
# MAGIC %md
# MAGIC Author: milos.colic@databricks.com

# COMMAND ----------

# DBTITLE 1,Lib Installation
# MAGIC %pip install geopandas h3 s2 shapely folium

# COMMAND ----------

# MAGIC %md
# MAGIC ## Point Data Preparation (Trips Data)

# COMMAND ----------

# MAGIC %md
# MAGIC For our point data we will use New York City Taxis data. </br>
# MAGIC This data provides us a volume of 1.6B records and in addition it contains two different Point Features - pick up and drop off. </br>
# MAGIC Our Polygon features dataset will remain the New York City Neighbourhoods.

# COMMAND ----------

# DBTITLE 1,Read NYC Taxis Data
taxi_trips = spark.read.format("delta").load("/databricks-datasets/nyctaxi/tables/nyctaxi_yellow/")

# COMMAND ----------

# DBTITLE 1,Visualise a Sample of Data
display(taxi_trips)

# COMMAND ----------

# DBTITLE 1,Verify Row Count
taxi_trips.count()

# COMMAND ----------

# MAGIC %md
# MAGIC In order to perform Point In Polygon joins we will make sure our point data contains H3 indices at resolution 10. </br>
# MAGIC We will create an User Defined Function that will perform this for us and we will apply it to the data.

# COMMAND ----------

# DBTITLE 1,UDF for H3 encoding
import h3
from pyspark.sql import functions as F

@udf("string")
def lat_lng_2_h3(lat, lng, res):
  import h3
  try:
    result = h3.geo_to_h3(lat, lng, res)
    return result
  except:
    return None # invalid coordinates will result in null index value.

taxi_trips = taxi_trips.withColumn(
  "h3_pickup", lat_lng_2_h3(F.col("pickup_latitude"), F.col("pickup_longitude"), F.lit(10))
).withColumn(
  "h3_dropoff", lat_lng_2_h3(F.col("dropoff_latitude"), F.col("dropoff_longitude"), F.lit(10))
)

# COMMAND ----------

# DBTITLE 1,View that H3 columns
display(taxi_trips)

# COMMAND ----------

# DBTITLE 1,Define location for data storage (silver layer)
username = "milos.colic" #please update with a correct user
silver_data_location = f"Users/{username}/geospatial/workshop/data/silver"

# COMMAND ----------

# DBTITLE 1,Write data out to a delta table
dbutils.fs.mkdirs(f"dbfs:/{silver_data_location}/h3/trips")
taxi_trips.write.format("delta").save(f"/{silver_data_location}/h3/trips/")

# COMMAND ----------

# MAGIC %md
# MAGIC For convenience purpose we will generate a managed database in the metastore. </br>
# MAGIC We will register the tables with our data against this database. </br>
# MAGIC This simplifies the way we are handling the data at consumption time.

# COMMAND ----------

# DBTITLE 1,Create a database for our workshop
# MAGIC %sql
# MAGIC Create DATABASE if Not EXISTS GeospatialWorkshop

# COMMAND ----------

# DBTITLE 1,Use the database
# MAGIC %sql
# MAGIC use GeospatialWorkshop

# COMMAND ----------

# DBTITLE 1,Create the table that will contain trips with h3 indices
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_trips
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/trips' -- please updated the username value in the path

# COMMAND ----------

# MAGIC %md
# MAGIC Delta is able to physically colocate the similar data based on z-order. </br>
# MAGIC Since we know we will be serving PIP joins using h3 columns we will z-order data by both h3 columns we have created.

# COMMAND ----------

# DBTITLE 1,ZOrder Data
# MAGIC %sql
# MAGIC optimize silver_h3_trips ZORDER by (h3_pickup, h3_dropoff)

# COMMAND ----------

# DBTITLE 1,View Trips Data
taxi_trips = spark.read.table("GeospatialWorkshop.silver_h3_trips")
display(taxi_trips)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Polygon Data Preparation - Neighbourhoods Data

# COMMAND ----------

# DBTITLE 1,Fetch Neighbours Data
# MAGIC %sh wget -O /tmp/neighbourhoods.zip https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/nynta2020_21d.zip

# COMMAND ----------

# DBTITLE 1,Unzip the Data
# MAGIC %sh
# MAGIC rm -R /tmp/neighbourhoods/
# MAGIC mkdir /tmp/neighbourhoods/
# MAGIC unzip /tmp/neighbourhoods.zip -d /tmp/neighbourhoods

# COMMAND ----------

# DBTITLE 1,Move the Data to dbfs
username = "milos.colic" #please update with a correct user
raw_data_location = f"Users/{username}/geospatial/workshop/data/raw/"
dbutils.fs.mkdirs(f"dbfs:/{raw_data_location}/")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.prj", f"dbfs:/{raw_data_location}/neighbourhoods.prj")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.shx", f"dbfs:/{raw_data_location}/neighbourhoods.shx")
dbutils.fs.cp("file:/tmp/neighbourhoods/nynta2020_21d/nynta2020.shp", f"dbfs:/{raw_data_location}/neighbourhoods.shp")

# COMMAND ----------

# DBTITLE 1,Load Data to geopandas
import geopandas as gpd

neighborhoods_pd = gpd.read_file(f"/dbfs/{raw_data_location}/neighbourhoods.shp")
display(neighborhoods_pd.head())

# COMMAND ----------

# MAGIC %md
# MAGIC In order to prepare our polygon data we will be using two set representation via h3 decompositon. </br>
# MAGIC The only difference to what we discussed in the notebook 01 is that we will be using WKB representation for the chips. </br>
# MAGIC Binary representation of polygons will result in more optimal runtime handling of chips. </br>
# MAGIC In addition we will be returning a single array of structs instead of separate collections for core and border chips. </br>
# MAGIC This will simplify our join logic.

# COMMAND ----------

# DBTITLE 1,H3 Two Set Representation
import shapely
import math
import h3

@udf("array<struct<index: string, is_border: boolean, chip: binary>>")
def h3_2set_polyfill(geometry, resolution):
  # we need to account for possibility of MultiPolygon shapes
  def to_h3(geometry):
    if 'MultiPolygon' in geometry.geom_type:
      geometry = list(geometry.geoms)
    else:
      geometry = [geometry]

    hex_set = set()
    for p in geometry:
      p_geojson = shapely.geometry.mapping(p)
      hex_set = hex_set.union(h3.polyfill_geojson(p_geojson, resolution))

    return hex_set

  # convenience method for converting an index to a chip of a polygon
  def get_result_struct(hex_i, polygon, dirty_set):
    if hex_i in dirty_set:
      hex_polygon = shapely.geometry.Polygon(h3.h3_to_geo_boundary(hex_i, geo_json=True))
      intersection = polygon.intersection(hex_polygon)
      if intersection.is_empty:
        return None
      elif intersection.equals(hex_polygon):
        return (hex_i, False, None)
      else:
        return (hex_i, True, intersection.wkb)
    else:
      return (hex_i, False, None)

  polygon = shapely.wkb.loads(bytes(geometry))

  # compute the buffer radius
  # we cannot use the hexagon side lenght due to curvatrure
  # we use minimum rotated rectange - we assume that the rectangle is near rectangle in shape
  # the alternative would be to iterate through boundary vertices and take the longest side
  cent = polygon.centroid.xy
  centroid_hex = h3.geo_to_h3(cent[1][0], cent[0][0], resolution)
  centroid_geom = shapely.geometry.Polygon(h3.h3_to_geo_boundary(centroid_hex, geo_json=True))
  radius = math.sqrt(centroid_geom.minimum_rotated_rectangle.area)/2

  # any index that may touch the boundary
  dirty = polygon.boundary.buffer(radius)

  original_set = to_h3(polygon)
  dirty_set = to_h3(dirty)

  result = [get_result_struct(hex_i, polygon, dirty_set) for hex_i in list(original_set.union(dirty_set))]

  return result

# COMMAND ----------

# MAGIC %md
# MAGIC Our methods expect wkb representation. When we have loaded geopandas dataframe into spark, all our geometries were loaded as wkt. </br>
# MAGIC We will create an udf to perform the necessary conversions.

# COMMAND ----------

# DBTITLE 1,Convert WKT to WKB in the Input Data
from pyspark.sql import functions as F

@udf("binary")
def wkt_to_wkb(wkt_poly):
  from shapely import wkt as wkt_loader
  from shapely import wkb as wkb_writer

  geom = wkt_loader.loads(wkt_poly)
  wkb_poly = wkb_writer.dumps(geom, hex=False)

  return wkb_poly

neighborhoods_pd = neighborhoods_pd.to_crs("EPSG:4326")
neighborhoods_pd["wkt_polygon"] = neighborhoods_pd["geometry"].apply(lambda x: str(x.wkt))
neighborhoods_df = spark.createDataFrame(neighborhoods_pd[["wkt_polygon"]])
neighborhoods_df = neighborhoods_df.withColumn("wkb_polygon", wkt_to_wkb(F.col("wkt_polygon")))

# COMMAND ----------

# DBTITLE 1,View Conversions between WKT and WKB
display(neighborhoods_df)

# COMMAND ----------

# DBTITLE 1,Represent Geometries via H3 Two Set Decomposition
neighborhoods_df = neighborhoods_df.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(10)))

# COMMAND ----------

# DBTITLE 1,View Decompositions
display(neighborhoods_df)

# COMMAND ----------

# MAGIC %md
# MAGIC We will perform 2 different write operations with Neighbourhoods data. </br>
# MAGIC One option is to keep the data in compact format with the array data kept at storage but exploded at join time. </br>
# MAGIC The other option is to pre-explode the arrays, store the data in flat way and z order by the index values. </br>

# COMMAND ----------

# DBTITLE 1,Write out Neighbourhoods Data
dbutils.fs.mkdirs(f"dbfs:/{silver_data_location}/h3/neighbourhoods")

neighborhoods_df = neighborhoods_df.select(
  F.monotonically_increasing_id().alias("id"),
  F.col("wkb_polygon"),
  F.col("chips")
).cache()

neighborhoods_df.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/compact/")
neighborhoods_df.select(
  F.col("id"),
  F.col("wkb_polygon"),
  F.explode("chips").alias("chip")
).write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/flat/")

# COMMAND ----------

# DBTITLE 1,Create Compact Table
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_neighbourhoods_compact
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/neighbourhoods/compact' -- please updated the username value in the path

# COMMAND ----------

# DBTITLE 1,Create Flat Table
# MAGIC %sql
# MAGIC CREATE TABLE IF NOT EXISTS silver_h3_neighbourhoods_flat
# MAGIC using delta
# MAGIC LOCATION '/Users/milos.colic/geospatial/workshop/data/silver/h3/neighbourhoods/flat' -- please updated the username value in the path

# COMMAND ----------

# DBTITLE 1,Optimize Flat Table
# MAGIC %sql
# MAGIC OPTIMIZE silver_h3_neighbourhoods_flat ZORDER BY (chip.index)

# COMMAND ----------

# DBTITLE 1,View Compact Data
neighborhoods_compact = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_compact")
display(neighborhoods_compact)

# COMMAND ----------

# DBTITLE 1,View Flat Data
neighborhoods_flat = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_flat")
display(neighborhoods_flat)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Polygon Data Preparation - Ranomized Polygons

# COMMAND ----------

neighborhoods_df = spark.read.table("GeospatialWorkshop.silver_h3_neighbourhoods_compact").select("id", "wkb_polygon")

# COMMAND ----------

# MAGIC %md
# MAGIC We will generate 2 polygon datasets of ~2.5M polygons to demonstrate polygon intersection joins. </br>
# MAGIC We will randomly translate existing polygons many times in order to generate a dataset with many to many replationships.

# COMMAND ----------

from shapely import wkb
from shapely import affinity
from pyspark.sql import functions as F

@udf("binary")
def translate_polygon(in_wkb):
  from shapely import wkb
  from shapely import affinity
  import random

  polygon = wkb.loads(bytes(in_wkb))
  x_d = random.uniform(-1, 1)
  y_d = random.uniform(-1.5, 1.5)
  translated = affinity.translate(polygon, xoff=x_d, yoff=y_d)
  return translated.wkb

@udf("array<int>")
def gerate_simple_array(size):
  return [i for i in range(size)]

# COMMAND ----------

translated = neighborhoods_df.withColumn(
  "id2", F.explode(gerate_simple_array(F.lit(100))) # generate 100 numers that we will use to explode the data
).withColumn(
  "id3", F.explode(gerate_simple_array(F.lit(100))) # generate 100 numers that we will use to explode the data
).repartition(
  320, F.rand() # repartition the data
).withColumn(
  "wkb_polygon",
  translate_polygon(F.col("wkb_polygon")) # randomly translate the data
).withColumn(
  "id", F.concat_ws("-", F.col("id"), F.col("id2"), F.col("id3")) # generate a single ID by combining the original ID with two new IDs
).drop("id2", "id3")

display(translated)

# COMMAND ----------

# MAGIC %md
# MAGIC We will run the "same" run operation twice, both data outputs will be different. </br>
# MAGIC This is due to radom translations and the fact we have not fixed the random seed.

# COMMAND ----------

translated.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1")

# COMMAND ----------

translated.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2")

# COMMAND ----------

# MAGIC %md
# MAGIC We will run the decomposition operation on both of these randomized datasets and store the data.

# COMMAND ----------

from pyspark.sql import functions as F

polygons1 = spark.read.format("delta").load(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1")
polygons1 = polygons1.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(8)))
polygons1.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_1_decomposed")

# COMMAND ----------

from pyspark.sql import functions as F

polygons2 = spark.read.format("delta").load(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2")
polygons2 = polygons2.withColumn("chips", h3_2set_polyfill(F.col("wkb_polygon"), F.lit(8)))
polygons2.write.format("delta").mode("overwrite").option("overwriteSchema", "true").save(f"/{silver_data_location}/h3/neighbourhoods/random/dataset_2_decomposed")

# COMMAND ----------

sessions/2021-12-14 | Geospatial Analytics and AI at Scale/notebooks/mosaic-notebooks/02-DataPreparation.py


import json

from kafka import KafkaConsumer

topic = "foobar"
consumer = KafkaConsumer(topic, bootstrap_servers=['localhost:9092'],
                         value_deserializer=lambda m: m.decode('utf-8'))

for message in consumer:  # boucle infinie
    print(type(message.value))
    print(message.value)


import pandas as pd
import folium #pip install folium

filePath='./Python_practice/university_locations.xlsx'
df_excel=pd.read_excel(filePath,engine='openpyxl',header=None)
df_excel.columns=['학교명','주소','lng','lat']

name_list=df_excel['학교명'].to_list() # 학교명을 name_list에 다 담음
addr_list=df_excel['주소'].to_list()
lng_list=df_excel['lng'].to_list()
lat_list=df_excel['lat'].to_list()

fMap=folium.Map(location=[37.553175,126.989326],zoom_start=10)

for i in range(len(name_list)):
    if lng_list[i] !=0:#위/경도 값이 0이 아니면
        marker=folium.Marker([lat_list[i],lng_list[i]],popup=name_list[i],
                             icon=folium.Icon(color='blue'))
        marker.add_to(fMap)

fMap.save('./Python_practice/Korea_universites.html')


"""gtn_copliot package."""
from importlib import metadata

from gtn_copliot.chain import get_graphql_chain

try:
    __version__ = metadata.version(__package__)
except metadata.PackageNotFoundError:
    # Case where package metadata is not available.
    __version__ = ""

__all__ = [__version__, "get_graphql_chain"]


from pyutils import *

def parse(lines):
    instructions = []
    for line in lines:
        instructions.append(0)
        if line.startswith('addx'):
            instructions.append(int(line.split()[1]))
    return instructions

def register_history(instructions):
    h = [1]
    for n in instructions:
        h.append(h[-1]+n)
    return h

@expect({'test': 13140})
def solve1(instructions):
    history = register_history(instructions)
    return sum((c+1)*history[c] for c in range(19, 220, 40))

def solve2(instructions):
    history = register_history(instructions)

    screen = []
    for c in range(240):
        if c % 40 in range(history[c]-1, history[c]+2):
            screen.append('#')
        else:
            screen.append('.')

    for i in range(0, 201, 40):
        print(''.join(screen[i:i+40]))


#!/usr/bin/env python3
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os.path
import json
from time import sleep

import RPi.GPIO as GPIO

import google.oauth2.credentials
from google.assistant.library import Assistant
from google.assistant.library.event import EventType
from google.assistant.library.file_helpers import existing_file

PIN_BTN = 17
PIN_LED = 22
ASSISTANT = None

def callback(channel):
    global CREDENTIALS
    if not CREDENTIALS:
        return
    print("callback")
    assistant = Assistant(CREDENTIALS)
    ret = assistant.start_conversation()
    print(ret)

GPIO.setmode(GPIO.BCM)
GPIO.setup(PIN_BTN, GPIO.IN, GPIO.PUD_UP)
GPIO.setup(PIN_LED, GPIO.OUT)
GPIO.output(PIN_LED, GPIO.LOW)

def callback_start_conversation(channel):
    global ASSISTANT
    if not ASSISTANT:
        return
    ASSISTANT.start_conversation()

def process_event(event):
    if event.type == EventType.ON_CONVERSATION_TURN_STARTED:
        print('Google Assistant is listening to you now.')
        GPIO.output(PIN_LED, GPIO.HIGH)
    if event.type == EventType.ON_RECOGNIZING_SPEECH_FINISHED:
        print("You:", event.args['text'])
    if event.type == EventType.ON_ALERT_STARTED:
        print('An alert has started sounding.')
    if event.type == EventType.ON_ALERT_FINISHED:
        print('The alert has finished sounding.')
    if (event.type == EventType.ON_CONVERSATION_TURN_FINISHED and
            event.args and event.args['with_follow_on_turn'] == True):
        #print('Google Assistant is listening to your response.')
        pass
    if event.type == EventType.ON_END_OF_UTTERANCE:
        print('Google Assistant may not have understood what you has said')
    if (event.type == EventType.ON_CONVERSATION_TURN_FINISHED and
            event.args and event.args['with_follow_on_turn'] == False):
        print('Conversation finished.')
        GPIO.output(PIN_LED, GPIO.LOW)

def main():
    global ASSISTANT

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('--credentials', type=existing_file,
                        metavar='OAUTH2_CREDENTIALS_FILE',
                        default=os.path.join(
                            os.path.expanduser('~/.config'),
                            'google-oauthlib-tool',
                            'credentials.json'
                        ),
                        help='Path to store and read OAuth2 credentials')
    args = parser.parse_args()
    with open(args.credentials, 'r') as f:
        credentials = google.oauth2.credentials.Credentials(token=None,
                                                            **json.load(f))
    ASSISTANT = Assistant(credentials)
    GPIO.add_event_detect(PIN_BTN, GPIO.FALLING,
                callback=callback_start_conversation, bouncetime=300)
    for event in ASSISTANT.start():
        process_event(event)

    while True:
        pass

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        print("Google Assistant finised.")
        GPIO.cleanup()


# https://codeforces.com/contest/1515/problem/B

import heapq

def main():
    n, m, x = map(int, input().split())
    h = list(map(int, input().split()))
    h = list(zip(h, range(n)))
    h.sort(reverse=True)
    # print(h)
    ans = [0]*n
    q = []
    max_sum = 0
    for i in range(m):
        heapq.heappush(q, (h[i][0], i+1))
        ans[h[i][1]] = i+1
        max_sum = max(max_sum, h[i][0])
    for i in range(m, n):
        sumh, mi = heapq.heappop(q)
        heapq.heappush(q, (sumh + h[i][0], mi))
        ans[h[i][1]] = mi
        max_sum = max(max_sum, sumh + h[i][0])
    # print(ans)
    # print(q)
    # print(max_sum)
    if max_sum - q[0][0] > x:
        print("NO")
    else:
        print("YES")
        print(*ans)
    return

t = int(input())
for _ in range(t):
    main()


'''
Given a non-empty array of integers, every element appears twice except for one. Find that single one.

Note:

Your algorithm should have a linear runtime complexity. Could you implement it without using extra memory?

Example 1:

Input: [2,2,1]
Output: 1
Example 2:

Input: [4,1,2,1,2]
Output: 4
'''

from unittest import TestCase
from typing import List

def singleNumber(nums: List[int]) -> int:
    ret = 0
    for n in nums:
        ret ^= n

    return ret

if __name__ == "__main__":
    test_case = TestCase()
    test_case.assertEqual(23^23^8, 8)
    test_case.assertEqual(singleNumber([3,4,4,4,4]), 3)


from typing import List
from aws_manage.aws_ec2_image_manager import AwsEc2ImageManager
from aws_manage.aws_ec2_instance_manager import AwsEc2InstanceManager
from aws_manage.aws_ec2_security_group_manager import AwsEc2SecurityGroupManager
from aws_util.aws_client import AwsEc2Client
from aws_util.aws_ec2_instance import AwsEc2Instance
from aws_util.aws_ec2_security_group import AwsEc2SecurityGroup
from aws_util.aws_exception import InvalidValueException, NotFoundException

class AwsEc2Manager(object):
    ##########################
    # for Public
    ##########################

    #
    # Add/Del Security Group
    @classmethod
    def verify_for_add_security_group_to_instance(cls, instance_name :str, sg_name :str) -> bool:
        instance = AwsEc2InstanceManager.fetch_by_name(instance_name)
        sg = AwsEc2SecurityGroupManager.fetch_by_name(sg_name)
        if not instance.is_valid():
            raise NotFoundException(instance_name)
        if not sg.is_valid():
            raise NotFoundException(sg_name)
        return True

    @classmethod
    def add_security_group_to_instance(cls, instance_name :str, sg_name :str, dry_run :bool = False)->bool:
        cls.verify_for_add_security_group_to_instance(instance_name, sg_name)
        instance = AwsEc2InstanceManager.fetch_by_name(instance_name)
        sg = AwsEc2SecurityGroupManager.fetch_by_name(sg_name)
        sgs = instance.security_groups()
        sgs.append(sg)
        return cls.modify_network_interface_attribute(dry_run, instance, sgs)

    @classmethod
    def verify_for_delete_security_group_to_instance(cls, instance_name :str, sg_name :str) -> bool:
        instance = AwsEc2InstanceManager.fetch_by_name(instance_name)
        sg = AwsEc2SecurityGroupManager.fetch_by_name(sg_name)
        if not instance.is_valid():
            raise NotFoundException(instance_name)
        if not sg.is_valid():
            raise NotFoundException(sg_name)
        return True

    @classmethod
    def delete_security_group_to_instance(cls, instance_name :str, sg_name :str, dry_run :bool = False)->bool:
        cls.verify_for_delete_security_group_to_instance(instance_name, sg_name)
        instance = AwsEc2InstanceManager.fetch_by_name(instance_name)
        sg = AwsEc2SecurityGroupManager.fetch_by_name(sg_name)
        sgs = list(filter(lambda _sg: _sg.group_id()!=sg.group_id(), instance.security_groups()))
        return cls.modify_network_interface_attribute(dry_run, instance, sgs)

    #
    # Create Image
    @classmethod
    def verify_for_create_image_from_instance(cls, image_name :str, instance_name :str) -> bool:
        base_instance = AwsEc2InstanceManager.fetch_by_name(instance_name)
        image = AwsEc2ImageManager.fetch_by_name(image_name)
        if image.is_valid():
            raise InvalidValueException(image_name)
        if not base_instance.is_valid():
            raise InvalidValueException(instance_name)
        return True

    @classmethod
    def create_image_from_instance(cls, image_name :str, instance_name :str) -> str:
        cls.verify_for_create_image_from_instance(image_name, instance_name)
        base_instance = AwsEc2InstanceManager.fetch_by_name(instance_name)

        client = AwsEc2Client()
        res = client.create_image(False, image_name, base_instance.instance_id())
        if not res.is_http_success():
            raise APICallException( { "cls": cls.__name__, "func": sys._getframe().f_code.co_name })

        return res._data.get('ImageId')

    ##########################
    # for Private
    ##########################
    @classmethod
    def modify_network_interface_attribute(cls, dry_run :bool, instance :AwsEc2Instance, security_groups :List[AwsEc2SecurityGroup])->bool:
        if not instance.is_valid():
            raise InvalidValueException()
        ###############################
        # This validation is hoped, but this security_groups is expected from AwsEc2Instance, that do not have enough param for verify AwsEc2SecurityGroup.
        ###############################
        #for sg in security_groups:
        #    if not sg.is_valid():
        #        raise InvalidValueException()
        if len(security_groups) > AwsEc2SecurityGroup.SECURITY_GROUP_MAX:
                raise InvalidValueException()

        client = AwsEc2Client()
        res = client.modify_network_interface_attribute(dry_run, instance.network_interfaces()[0].id(),
                                                        [sg.group_id() for sg in security_groups])
        return res.is_http_success()


'''
MATLAB .MAT file exporting.
'''

from datetime import datetime
from glob import glob
import os.path
import string
import struct

import logging
logger = logging.getLogger('idelib')
logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s")

# NOTE: 64 bit Scipy is unstable; avoid using it for now (v0.13.2, 12/2014).
# from scipy.io.matlab import mio5_params as MP

class MP(object):
    """ MAT data and matrix types. """
    miINT8 =            0x01
    miUINT8 =           0x02
    miINT16 =           0x03
    miUINT16 =          0x04
    miINT32 =           0x05
    miUINT32 =          0x06
    miSINGLE =          0x07
    miDOUBLE =          0x09
    miINT64 =           0x0c
    miUINT64 =          0x0d
    miMATRIX =          0x0e
    miCOMPRESSED =      0x0f
    miUTF8 =            0x10
    miUTF16 =           0x11
    miUTF32 =           0x12

    mxCELL_CLASS =      0x01
    mxSTRUCT_CLASS =    0x02
    mxOBJECT_CLASS =    0x03
    mxCHAR_CLASS =      0x04
    mxSPARSE_CLASS =    0x05
    mxDOUBLE_CLASS =    0x06
    mxSINGLE_CLASS =    0x07
    mxINT8_CLASS =      0x08
    mxUINT8_CLASS =     0x09
    mxFUNCTION_CLASS =  0x10
    mxINT16_CLASS =     0x0a
    mxUINT16_CLASS =    0x0b
    mxINT32_CLASS =     0x0c
    mxUINT32_CLASS =    0x0d
    mxINT64_CLASS =     0x0e
    mxUINT64_CLASS =    0x0f
    mxOPAQUE_CLASS =    0x11
    mxOBJECT_CLASS_FROM_MATRIX_H = 0x12

#===============================================================================
#
#===============================================================================

def splitNum(name, digits=string.digits):
    """ Split a string that ends with a number between the 'body' of the
        string and its numeric suffix. Both parts are returned as strings.

        :param name: The string to split.
        :keyword digits: The set of numeric characters. Defaults to 0-9.
        :return: A tuple containing the base name and the numeric suffix.
    """
    base = name.rstrip(digits)
    return base, name[len(base):]

def serialFilename(basename, numDigits=2, minNumber=1, inc=1, pad='_'):
    """ Generate a new filename with an incremented numeric suffix.

        :param basename: The name to make unique.
        :keyword numDigits: The minimum number of digits to use when adding a
            new number to a name. Used if there are no existing files.
        :keyword numNumber: The minimum serial number, if no file currently
            exists.
        :keyword inc: The serial number increment.
        :keyword pad: A string to appear between the base name and the number.
        :return: A unique filename.
    """
    base, ext = os.path.splitext(basename)
    existing = glob(base.rstrip(string.digits)+'*'+ext)
    existing = [x for x in existing if os.path.splitext(x)[0][-1].isdigit()]
    if len(existing) > 0:
        existing.sort()
        lastName, lastNum = splitNum(os.path.splitext(existing[-1])[0])
        numDigits = len(existing[-1]) - len(lastName) - len(ext)
        lastNum = int(lastNum)+inc
    else:
        lastNum = minNumber
    return '%s%s%s%s' % (base, pad, str(lastNum).rjust(numDigits, '0'), ext)

# def dump8(data, start=0, end=None):
#     """ Debugging tool. Prints data as 8 columns of hex. """
#     end = len(data) if end is None else end
#     for i in range(start,end,8):
#         print '%s:' % (str(i).rjust(4)),
#         try:
#             for j in range(8):
#                 print "%02x" % (ord(data[i+j])),
#         except IndexError:
#             print "(end)",
#             break
#         print
#
# def hexdump(data):
#     """ Debugging tool. Dumps a string or a collection of numbers as hex. """
#     if isinstance(data, (str, bytes, bytearray)):
#         a = ["%02x" % ord(c) for c in data]
#     else:
#         a = ["%02x" % c for c in data]
#     return ' '.join(a)

# MATLAB reserved words.
# TODO: Add standard functions as well
RESERVED_WORDS = ('break', 'case', 'catch','continue', 'else', 'elseif', 'end',
                  'for', 'function', 'global', 'if', 'otherwise', 'persistent',
                  'return', 'switch', 'try', 'while')

def sanitizeName(s, validChars=string.ascii_letters+string.digits+'_',
                 prefix="v", reservedWords=RESERVED_WORDS):
    """ Convert an arbitrary string into a valid MATLAB variable name.

        :keyword validChars: A string of all valid characters
    """
    s = s.strip().encode('ascii', 'replace')
    result = [c if c in validChars else '_' for c in (chr(x) for x in s.strip())]
    result = ''.join(result).strip('_ ')
    if result[0].isdigit() or result in reservedWords:
        result = prefix + result.title()
    return result.replace('__', '_').replace('__', '_')

#===============================================================================
#
#===============================================================================

class MatStream(object):
    """
    """
    MAX_LENGTH = (2**31)-9 # Accounts for data being rounded to the next x8.
    MAX_SIZE = int(MAX_LENGTH * .95) # scale back by 5%, just to be certain
    DEFAULT_HEADER = "MATLAB 5.0 MAT-file MIDE IDE to MAT"

    # Map MATLAB types to the struct formatting character
    typeFormatChars = {
        MP.miINT8:   'b',
        MP.miUINT8:  'B',
        MP.miINT16:  'h',
        MP.miUINT16: 'H',
        MP.miINT32:  'i',
        MP.miUINT32: 'I',
        MP.miINT64:  'q',
        MP.miUINT64: 'Q',
        MP.miUTF8:   'c',
        MP.miSINGLE: 'f',
        MP.miDOUBLE: 'd',
    }

    # Map data types to matrix types
    classTypes = {
        MP.miINT8:   MP.mxINT8_CLASS,
        MP.miUINT8:  MP.mxUINT8_CLASS,
        MP.miINT16:  MP.mxINT16_CLASS,
        MP.miUINT16: MP.mxUINT16_CLASS,
        MP.miINT32:  MP.mxINT32_CLASS,
        MP.miUINT32: MP.mxUINT32_CLASS,
        MP.miINT64:  MP.mxINT64_CLASS,
        MP.miUINT64: MP.mxUINT64_CLASS,
        MP.miUTF8:   MP.mxCHAR_CLASS,
        MP.miSINGLE: MP.mxSINGLE_CLASS,
        MP.miDOUBLE: MP.mxDOUBLE_CLASS,
    }

    intPack = struct.Struct('I')

    def __init__(self, filename, doc=None, msg=None, serialize=True,
                 maxFileSize=MAX_SIZE, timeScalar=1, writeCal=False,
                 calChannels=None, writeStart=False, writeInfo=False):
        """ Constructor. Create a new .MAT file.

            :param filename: The name of the new file, or `None`.
            :keyword doc: The `idelib.dataset.Dataset` from which to export.
            :keyword msg: The message string to appear at the start of the
                MAT file. The first 4 bytes must be non-zero.
            :keyword serialize: If `True`, start the first file with a
                number. Successive files are always numbered.
            :keyword maxFileSize: The maximum size of each exported file.
                Must not exceed the maximum size allowed for MATs.
            :keyword timeScalar: Timestamp scaling factor.
            :keyword writeCal: If `True`, write calibration data to each
                exported file. If `"channel"`, write calibration by channel
                name, ignoring any transforms not being used.
            :keyword writeStart: If `True`, write the doc's start time.
            :keyword writeInfo: If `True`, write doc/recorder info.
        """
        if msg is None:
            msg = self.DEFAULT_HEADER if doc is None else self.makeHeader(doc)
        self.basename = filename
        self.filename = filename
        self.msg = msg.encode('utf8') or self.DEFAULT_HEADER
        self.stream = None
        self.timeScalar = timeScalar
        self.doc = doc
        if doc is not None:
            self._writeCal = writeCal and len(doc.transforms) > 0
            self._calPerChannel = "channel" in str(writeCal).lower()
            self._writeInfo = writeInfo
            self.startTime = writeStart and doc.sessions[0].utcStartTime
        else:
            self._writeCal = self._writeInfo = self._writeStart = False
        self.calChannels = calChannels

        # MATLAB identifies the file as level 4 if the first byte is 0.
        if b'\x00' in self.msg[:4]:
            self.msg = self.DEFAULT_HEADER

        self.maxFileSize = min(self.MAX_SIZE, self.next8(maxFileSize))

        # Default array parameters. Typically set by startArray().
        self._inArray = False
        self.arrayName = None
        self.arrayMType = None
        self.arrayDType = MP.miDOUBLE
        self.arrayFlags = 0
        self.arrayNoTimes = False
        self.arrayHasTimes = False
        self.arrayColNames = None

        self.exportedFiles = []

        if filename is not None:
            self.newFile(serialize)
        else:
            self.stream = None

    def newFile(self, serialize=True, offset=None):
        """ Used internally. Wraps file creation.
        """
        if self.stream is not None:
            self.close()

        if serialize:
            self.filename = serialFilename(self.basename)
        else:
            self.filename = self.basename
        self.stream = open(self.filename, 'wb')
        self._write = self.stream.write
        self._seek = self.stream.seek

        self._write(struct.pack(b'116s II H 2s', self.msg, 0, 0, 0x0100, b'IM'))

        if self._writeInfo:
            self.writeRecorderInfo(self.doc.recorderInfo)
        if self._writeCal:
            if self._calPerChannel:
                self.writeCalPerChannel(self.doc)
            else:
                self.writeCalibration(self.doc.transforms)
        if self.startTime:
            self.writeValue('start_time_utc', self.startTime, MP.miINT64)

        self._inArray = False
        self.arrayName = None
        self.arrayColNames = None
        self.arrayStartTime = None

        self.exportedFiles.append(self.filename)

    def checkFileSize(self, size, pad=0):
        """ Check if adding data of the given size will exceed the maximum
            file size; start a new file if it will. Also creates a variable
            for the current array's start time (if provided).
        """
        if self.next8(self.stream.tell() + size + pad) >= self.maxFileSize:
#             print "checkFileSize"
            inArray = self._inArray
            if inArray:
                colNames = self.arrayColNames
                baseName = self.arrayBaseName
                dtype = self.arrayDType
                mtype = self.arrayMType
                noTimes = self.arrayNoTimes
                hasTimes = self.arrayHasTimes
                num = self.arrayNumber + 1
                cols = self.numCols if self.arrayNoTimes else self.numCols - 1
            self.newFile()
            if inArray:
                self.startArray(baseName, cols, arrayNumber=num,
                                colNames=colNames, mtype=mtype, dtype=dtype,
                                noTimes=noTimes, hasTimes=hasTimes)

    @property
    def closed(self):
        return True if self.stream is None else self.stream.closed

    @classmethod
    def next8(cls, num):
        """ Return the next multiple of 8; elements within a MAT file must be
            aligned to 64 bits.
        """
        if num % 8 == 0:
            return num
        return 8*(int(num/8)+1)

    def pack(self, fmt, args, dtype=MP.miUINT32):
        """ Write data to the file, proceeded by the type and size (aligned to
            64 bits). Used internally.

            :param fmt: A struct formatting string for the data.
            :param args: Arguments for the struct formatting string (i.e. data).
            :keyword dtype: The type for the data size element.
        """
        dataSize = struct.calcsize(fmt)
        result = struct.pack('II'+fmt, dtype, dataSize, *args)
        self._write(result.ljust(self.next8(len(result)), b'\0'))
        return self.next8(dataSize + 8)

    def packStr(self, s, maxlen=31):
        """ Write a string to the file, proceeded by type and size info, aligned
            to 64 bits. Used internally.
        """
        s = s[:maxlen].encode('utf8')
        n = self.next8(len(s))
        fmt = 'II %ds' % n
        self._write(struct.pack(fmt, MP.miINT8, len(s), s))

    def endArray(self):
        """ End an array, updating all the sizes.
        """
        if not self._inArray:
            return False
        self._inArray = False

        dataEndPos = self.stream.tell()
        arrayEndPos = self.next8(dataEndPos)

        # Move back and rewrite the actual total size
        self._seek(self.dataStartPos-4)
        self._write(self.intPack.pack(self.next8(dataEndPos-self.dataStartPos)))

        # Move back and rewrite the actual number of rows (columns, actually)
        self._seek(self.rowsPos)
        self._write(self.intPack.pack(self.numRows))

        # Move back and write the actual payload size (the 'real' portion only)
        self._seek(self.prSize)
        self._write(self.intPack.pack(self.numRows * self.rowFormatter.size))

        # Go back to the end.
        self._seek(dataEndPos)
        if dataEndPos < arrayEndPos:
            self._write('\0' * (arrayEndPos-dataEndPos))

        if self.arrayHasTimes and self.arrayNoTimes and self.arrayStartTime is not None:
            self.writeValue('%s_start' % self.arrayBaseName, self.arrayStartTime, MP.miUINT64)

        self.arrayMType = None
        self.arrayColNames = None
        self.arrayStartTime = None

        return True

    def startArray(self, name, cols, rows=1, arrayNumber=0,
                   mtype=None, dtype=MP.miDOUBLE, flags=0, noTimes=True,
                   hasTimes=True, colNames=None):
        """ Begin a 2D array for storing the recorded data.

            :param name: The name of the matrix (array).
            :param cols: The number of columns in the data (excluding time).
            :keyword rows: The number of rows in the data (if known).
            :keyword dtype: The Matlab data type in the matrix.
            :keyword mtype: The Matlab matrix type. Defaults to match `dtype`.
            :keyword flags: A set of bit flags for the matrix.
        """
        if self._inArray:
            self.endArray()

        self.arrayBaseName = sanitizeName(name)

        # Calculate size of data, plus any applicable 'header' data that needs
        # to be in the same file as the array (column names, start time offset,
        # etc.)
        headerSize = 0
        if colNames is not None:
            headerSize += self.getNamesSize(colNames, '%s_names' % name, noTimes)
        if noTimes and hasTimes:
            headerSize += self.getValueSize('%s_start' % self.arrayBaseName, self.arrayStartTime, MP.miUINT64)

        newSize = 80 + self.next8(len(name)) + headerSize
        self.checkFileSize(newSize)

        self._inArray = True
        self.arrayNumber = arrayNumber
        self.numRows = 0
        self.expectedRows = rows
        self.arrayMType = mtype
        self.arrayDType = dtype
        self.arrayFlags = flags
        self.arrayNoTimes = noTimes
        self.arrayHasTimes = hasTimes
        self.arrayColNames = colNames
        self.arrayStartTime = None

        if arrayNumber > 0:
            name = "%s%d" % (self.arrayBaseName, arrayNumber)

        if self.arrayColNames:
            self.writeNames(self.arrayColNames, '%s_names' % self.arrayBaseName, self.arrayNoTimes)

        if self.arrayMType is None:
            self.arrayMType = self.classTypes[self.arrayDType]

        if self.arrayNoTimes:
            self.numCols = cols
        else:
            self.numCols = cols+1
        fchar = self.typeFormatChars.get(self.arrayDType, self.typeFormatChars[MP.miDOUBLE]) * self.numCols
        self.rowFormatter = struct.Struct(fchar)

        # Start of matrix element, initial size of 0 (rewritten at end)
        self._write(struct.pack("II", MP.miMATRIX, 0)) # Start
        self.dataStartPos = self.stream.tell()

        # Write flags and matrix type
        self.pack('BBBBBBBB', (self.arrayMType, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00))

        # Write matrix dimensions. Because the file stores data column first,re
        # the recording data is stored 'sideways': lots of columns. The
        # second dimension is rewritten at the end.
        self.pack('ii', (self.numCols, rows), dtype=MP.miINT32)
        self.rowsPos = self.stream.tell() - 4

        # Write the matrix name
        self.packStr(sanitizeName(name))

        # Write the start of the 'PR' element; the size will be filled in later.
        size = min(self.rowFormatter.size * rows, 2 ** 32 - 1)
        self._write(struct.pack('II', self.arrayDType, size))
        self.prSize = self.stream.tell() - 4

    def getStringArraySize(self, title, strings):
        """ Get the size of an array of strings before writing it.
        """
        textSize = max([len(s) for s in strings])
        return 40 + self.next8(textSize*len(strings)) + 8 + self.next8(len(title))

    def writeStringArray(self, title, strings):
        """ Write a set of strings as a MATLAB character array.
        """
        textSize = max([len(s) for s in strings])
        strings = [n.ljust(textSize) for n in strings]
        payload = ''.join([''.join(x) for x in zip(*strings)])

        totalSize = self.getStringArraySize(title, strings)

        # Ensure that this won't exceed the max file size before writing
        self.checkFileSize(totalSize)

        self._write(struct.pack("II", MP.miMATRIX, totalSize)) # Start
        self.pack('BBBBBBBB', (0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00))
        self.pack('II', (len(strings), textSize), dtype=MP.miINT32)
        self.packStr(sanitizeName(title))
        self._write(struct.pack("II", MP.miUTF8, textSize*len(strings)))
        self._write(payload.ljust(self.next8(len(payload)), '\0').encode('utf8'))

    def getNamesSize(self, names, title="channel_names", noTimes=False):
        if noTimes:
            return self.getStringArraySize(title, names)
        return self.getStringArraySize(title, ['Time']+names)

    def writeNames(self, names, title="channel_names", noTimes=False):
        """ Write IDE column names to the MAT file, for easy identification
            of rows in MATLAB (IDE data is written in columns in order to
            stream).
        """
        if not noTimes:
            names = names[:]
            names.insert(0, 'Time')
        self.writeStringArray(title, names)

    def writeRow(self, event):
        """ Write a sample to the array.

            :param event: The sample, in the format `(time, (v1, v2, ...))`
        """
        if self.arrayStartTime is None:
            self.arrayStartTime = event[0]
        self.checkFileSize(self.rowFormatter.size, pad=96)

        try:
            if self.arrayNoTimes:
                data = self.rowFormatter.pack(*event[:1])
            else:
                data = self.rowFormatter.pack(event[0]*self.timeScalar, *event[1:])
        except struct.error as err:
            logger.exception("ERROR: %s, formatter=%r, event=%r" % (self.arrayBaseName, self.rowFormatter.format, event))
            raise

        self._write(data)
        self.numRows += 1

    def getValueSize(self, name, val, dtype=MP.miDOUBLE):
        """ Get the total size of a value as written.
        """
        name = sanitizeName(name)
        fchar = self.typeFormatChars.get(dtype,'d')
        dsize = struct.calcsize(fchar)
        return 40 + self.next8(len(name)) + 8 + self.next8(dsize)

    def writeValue(self, name, val, dtype=MP.miDOUBLE):
        """ Write a single numeric value to the MAT file. Don't use for strings.
        """
        name = sanitizeName(name)
        mtype = self.classTypes.get(dtype, MP.mxDOUBLE_CLASS)
        fchar = self.typeFormatChars.get(dtype,'d')
        totalSize = self.getValueSize(name, val, dtype)

        self.checkFileSize(totalSize)

        self._write(struct.pack("II", MP.miMATRIX, totalSize)) # Start
        self.pack('BBBBBBBB', (mtype, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00)) #flags
        self.pack('II', (1, 1), dtype=MP.miINT32) # dimensions
        self.packStr(name)
        self.pack(fchar, (val,), dtype=dtype)

    def writeRecorderInfo(self, info):
        """ Write an IDE file's 'RecorderInfo' data to the MAT as a set of
            strings.
        """
        if info:
            s = ['%s:\t%s' % i for i in info.items()]
            self.writeStringArray('recorder_info', s)

    def _writeCalData(self, name, cal, doc=None):
        if isinstance(cal, int):
            cal = doc.transforms.get(cal, None)
        if cal is None:
            return
        vals = cal.references + cal.coefficients
        self.startArray(name, len(vals), dtype=MP.miDOUBLE, noTimes=True, hasTimes=False)
        self.writeRow((0,vals))
        self.endArray()

    def writeCalibration(self, cals):
        """ Write a dictionary of Transform objects (keyed by ID) to the file.
        """
#         print "writeCalibration %s" % self.stream.name
        for calId, cal in cals.items():
            self._writeCalData("calibration%d" % calId, cal)

    def writeCalPerChannel(self, doc):
        """ Write the calibration used by a document's Channels/Subchannels.
        """
        channels = [c for c in doc.channels.values() if self.calChannels is None or c.id in self.calChannels]
        for c in channels:
            self._writeCalData("%s_calibration" % c.displayName, c.transform, doc)
            for subc in c.subchannels:
                self._writeCalData("%s_calibration" % subc.displayName, subc.transform, doc)

    def close(self):
        """ Close the file.
        """
        if self._inArray:
            self.endArray()
        return self.stream.close()

#===============================================================================
#
#===============================================================================

    @classmethod
    def makeHeader(cls, doc, session=-1, prefix="MATLAB 5.0 MAT-file"):
        """ Generate MAT file header text from a `Dataset` document.
        """
        if not isinstance(prefix, (str, bytes, bytearray)):
            prefix = ''
        elif not prefix.endswith(' '):
            prefix += ' '

        msg = "%sGenerated from %s" % (prefix, os.path.basename(doc.filename))

        s = doc.sessions[session]
        if s.utcStartTime:
            createTime = s.utcStartTime
        else:
            try:
                createTime = os.path.getctime(doc.filename)
            except IOError:
                createTime = None

        if createTime is not None:
            msg = "%s recorded %s UTC" % (msg, datetime.utcfromtimestamp(createTime))

        return msg

#===============================================================================
#
#===============================================================================

def exportMat(events, filename, start=0, stop=-1, step=1, subchannels=True,
              callback=None, callbackInterval=0.01, timeScalar=1,
              raiseExceptions=False, useUtcTime=False, headers=True,
              removeMean=None, meanSpan=None, display=False, matArgs={},
              noBivariates=False, **kwargs):
    """ Export a `dataset.EventList` as a Matlab .MAT file. Works in a manner
        similar to the standard `EventList.exportCsv()` method.

        :param events: an `EventList` from which to export.
        :param filename: The path/name of the .MAT file to write.
        :keyword start: The first event index to export (defaults to first).
        :keyword stop: The last event index to export (defaults to last).
        :keyword step: The number of events between exported lines.
        :keyword subchannels: A sequence of individual subchannel numbers
            to export. Only applicable to objects with subchannels.
            `True` (default) exports them all.
        :keyword callback: A function (or function-like object) to notify
            as work is done. It should take four keyword arguments:
            `count` (the current line number), `total` (the total number
            of lines), `error` (an exception, if raised during the
            export), and `done` (will be `True` when the export is
            complete). If the callback object has a `cancelled`
            attribute that is `True`, the MAT export will be aborted.
            The default callback is `None` (nothing will be notified).
        :keyword callbackInterval: The frequency of update, as a
            normalized percent of the total lines to export.
        :keyword timeScalar: A scaling factor for the even times.
            The default is 1 (microseconds).
        :keyword raiseExceptions: If `False`, all exceptions will be handled
            quietly, passed along to the callback.
        :keyword useUtcTime: If `True`, times are written as the UTC
            timestamp. If `False`, times are relative to the recording.
        :keyword removeMean: If `True`, remove the mean from the output.
        :keyword meanSpan: The span over which the mean is calculated. -1
            for the total mean.
        :keyword display: If `True`, export using the EventList's 'display'
            transform (e.g. unit conversion).
        :keyword matArgs: A dictionary of keyword arguments supplied to the
            `MatStream` constructor.
        :return: Tuple: The number of rows exported and the elapsed time.
    """
    noCallback = callback is None
    events = events.copy()

    events.noBivariates = noBivariates
    if removeMean is not None:
        events.removeMean = removeMean
    if meanSpan is not None:
        events.rollingMeanSpan = meanSpan

    start = (1 + start + len(events)) if start < 0 else start
    stop = (1 + stop + len(events)) if stop < 0 else stop
    totalLines = int((stop - start) / (step + 0.0))
    updateInt = int(totalLines * callbackInterval)

    # Catch all or no exceptions
    ex = None if raiseExceptions or noCallback else Exception

    t0 = datetime.now()

    createTime = 0
    if useUtcTime:
        if events.session.utcStartTime:
            createTime = events.session.utcStartTime
        else:
            try:
                createTime = os.path.getctime(events.dataset.filename)
            except IOError:
                pass

        # Scale to increments used in the source.
        createTime /= timeScalar

    # If specific subchannels are specified, export them in order.
    if events.hasSubchannels:
        if subchannels is True:
            numCols = len(events.parent.subchannels)
            formatter = None
            names = [x.name for x in events.parent.subchannels]
        else:
            numCols = len(subchannels)
            # Create a function instead of chewing the subchannels every time
            formatter = eval("lambda x: (%s,)" % \
                       ",".join([("x[%d]" % c) for c in subchannels]))
            names = [events.parent.subchannels[x].name for x in subchannels]
    else:
        numCols = 1
        formatter = lambda x: (x,)
        names = [events.parent.name]

    totalSamples = totalLines * numCols
    if headers is False:
        names = None

    comments = MatStream.makeHeader(events.dataset, events.session.sessionId)
    matfile = MatStream(filename, events.dataset, comments,
                        timeScalar=timeScalar, **matArgs)

    matfile.startArray(events.parent.name, numCols, rows=totalLines,
                       colNames=names, noTimes=False)

    try:
        for num, evt in enumerate(events.iterSlice(start, stop, step, display)):
            t, v = evt[0], tuple(evt[1:])
            if formatter is not None:
                v = formatter(v)

            matfile.writeRow((createTime + t,)+v)

            if callback is not None:
                if getattr(callback, 'cancelled', False):
                    callback(done=True)
                    break
                if updateInt == 0 or num % updateInt == 0:
                    callback(num*numCols, total=totalSamples,
                             filename=matfile.filename)

        if callback:
            callback(done=True)

    except Exception as e:
        if raiseExceptions or noCallback:
            raise
        callback(error=e)

    matfile.close()

    return num + 1, datetime.now() - t0

#===============================================================================
#
#===============================================================================


"""Write a function that check whether a string is a palindrome or not.

Usage of any reversing functions is prohibited. To check your implementation you can use
strings from [here](https://en.wikipedia.org/wiki/Palindrome#Famous_palindromes).
"""

def is_palindrome(input_str: str) -> bool:
    input_str = input_str.lower()
    for index, _char in enumerate(input_str):
        if input_str[index] != input_str[-index - 1]:
            return False
    return True
    # return input_str.lower() == input_str.lower()[::-1]

if __name__ == '__main__':
    s_1 = 'Repaper'
    s_2 = 'Repazer'
    s_3 = 'qwq'
    print("s_1 is palindrome:", is_palindrome(s_1))
    print("s_2 is palindrome:", is_palindrome(s_2))
    print("s_3 is palindrome:", is_palindrome(s_3))


# -*- coding: utf-8 -*-
"""
TRAINEE BURGING GOOSE AERO DESEING UFPR.

Gráfico tração em função da velocidade hélice 20x8E
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

file = 'dados.txt'  # importando arquivo txt com os dados convertidos
columnsnames = ['x1', 'y1']  # Nomeando as colunas

df = pd.read_csv(file, skiprows=1, names=columnsnames, sep='\s+')

lista_x = df["x1"].tolist()  # Transformando coluna x em lista
lista_y = df["y1"].tolist()  # Transformando coluna y em lista

x = np.array(lista_x)  # Inserindo pontos da velocidade em x

y = np.array(lista_y)  # Inserindo pontos da tração em y

poly = np.polyfit(x, y, 2)  # Ajustando polinomio

a, b, c = poly

print("Coeficientes:")
print(a, b, c)  # Print dos coeficientes

p = 1.206  # Densidade local calculada
p0 = 1.225  # Densidade nível do mar

#plt.scatter(x, y, label='tração')
plt.plot(x, a*x**2 + b*x + c, color='k', label='h nível do mar')
plt.plot(x, (a*x**2 + b*x + c)*(p/p0), color='darkcyan', label='h calculado')
plt.title('Scorpion 4020KV, APC20X8E a 5000RPM')
plt.legend()
plt.xlabel('V (m/s)')
plt.ylabel('T (N)')
plt.ylim(0, 50)
plt.xlim(0, 26)
plt.yticks(np.arange(0, 55, 5))
plt.xticks(np.arange(0, 28, 2))


import requests  #Importamos la librería requests
#especie = input()
especie = "human"
print("SOLICITANDO INFORMACION DE INTERNET")
print("espere....")
URL = "https://rickandmortyapi.com/api/character/?species="+especie #configuramos la url
#solicitamos la información y guardamos la respuesta en data.
dataPage = requests.get(URL)
dataPage = dataPage.json() #convertimos la respuesta en dict
paginas = dataPage['info']['pages']
cantidad = dataPage['info']['count']
#for element in data: #iteramos sobre data
    #print(element) #Accedemos a sus valores
#print("Total de "+str(especie)+" "+str(data['info']['count']))

j = 0
for i in range(1,paginas+1):
    URL = "https://rickandmortyapi.com/api/character/?page="+str(i)+"&species="+especie
    data = requests.get(URL)
    data = data.json() #convertimos la respuesta en dict
    for j in data['results']:
        print(j['name']+" "+['species'])


from __future__ import absolute_import
import numpy as np

from scipy.optimize import linear_sum_assignment as linear_assignment
from . import kalman_filter
import time
import copy
import logging
import numba as nb
from . import tracker

INFTY_COST = 1e18

def get_remain_pair_matrix(cost_matrix, row_indices):

    track_ids = np.arange(cost_matrix.shape[0])
    row_indices = np.array(row_indices)
    if row_indices.shape[0] > 0:
        remain_track_ids = np.delete(track_ids, row_indices)
        cost_matrix = cost_matrix[remain_track_ids, :]

    return cost_matrix

def better_np_unique(arr):

    sort_indexes = np.argsort(arr)
    arr = np.asarray(arr)[sort_indexes]
    vals, first_indexes, _, counts = np.unique(arr,
                                               return_index=True, return_inverse=True, return_counts=True)
    indexes = np.split(sort_indexes, first_indexes[1:])
    for x in indexes:
        x.sort()
    return vals, indexes, counts

def split_conflict(pre_matches):

    vals, indexes, counts = better_np_unique(pre_matches[:, 0])

    single_idxs = np.where(counts == 1)[0]
    single_det_indices = [int(indexes[i]) for i in single_idxs]
    matches = pre_matches[single_det_indices]

    conflict_matches = []
    multi_idxs = np.where(counts > 1)[0]
    for i in multi_idxs:
        conflict_matches.append(((pre_matches[indexes[i], 0]), vals[i]))

    return list(map(tuple, matches)), conflict_matches

def split_TD_conflict(pre_matches):

    T_vals, T_indexes, T_counts = better_np_unique(pre_matches[:, 0])
    D_vals, D_indexes, D_counts = better_np_unique(pre_matches[:, 1])

    matches, matches_ind = [], []

    single_idxs = np.where(T_counts == 1)[0]
    for i in single_idxs:
        T_ind = int(T_indexes[i])
        dv = pre_matches[T_ind, 1]
        c = D_counts[np.argwhere(D_vals == dv)]
        if c == 1:
            matches.append(pre_matches[T_ind, :2])
            matches_ind.append(T_ind)

    two_idxs = np.where(T_counts == 2)[0]
    for i in two_idxs:
        T_ind0 = int(T_indexes[i][0])
        T_ind1 = int(T_indexes[i][1])
        dv0 = pre_matches[T_ind0, 1]
        dv1 = pre_matches[T_ind1, 1]
        if dv0 == dv1:
            matches.append(pre_matches[T_ind0, :2])
            matches_ind.append(T_ind0)
            matches_ind.append(T_ind1)
    matches = np.array(matches, dtype=np.int)

    conflicts_ind = list(
        set(np.arange(pre_matches.shape[0])) - set(matches_ind))
    conflict_matches = pre_matches[conflicts_ind, :]

    return list(map(tuple, matches)), conflict_matches

def _assign(level, cost_matrix, track_info, track_indices, detection_indices, gate):
    cost_matrix = filter_pairs(cost_matrix, gate)
    row_indices, col_indices = linear_assignment(cost_matrix)

    pre_matches = []
    for row, col in zip(row_indices, col_indices):
        track_idx = track_indices[row]
        detection_idx = detection_indices[col]
        if cost_matrix[row, col] <= gate:
            pre_matches.append(
                (track_idx, detection_idx, track_info[track_idx]))
    pre_matches = np.array(pre_matches)

    if pre_matches.shape[0]:
        unmatched_tracks = list(set(track_indices) -
                                set(list(pre_matches[:, 0])))
        unmatched_detections = list(set(detection_indices) -
                                    set(list(pre_matches[:, 1])))
        if level == 0:
            pre_matches = np.hstack(
                (pre_matches, np.zeros(pre_matches.shape[0]).reshape(-1, 1)))
        else:
            pre_matches = np.hstack(
                (pre_matches, np.ones(pre_matches.shape[0]).reshape(-1, 1)))
    else:
        unmatched_tracks = track_indices
        unmatched_detections = detection_indices

    return pre_matches, unmatched_tracks, unmatched_detections

def dist_cost_matrix(track_previous_ct, det_previous_ct, tracks, dets, motion_gate=1.1e18, overlap_thresh=0.05):
    cost_matrix = (((det_previous_ct.reshape(1, -1, 2) -
                   track_previous_ct.reshape(-1, 1, 2)) ** 2).sum(axis=2))

    tracks_wh = np.array([track.tlwh[2:4] for track in tracks])
    track_size = tracks_wh[:, 0] * tracks_wh[:, 1]
    det_wh = np.array([det.tlwh[2:4] for det in dets])
    item_size = det_wh[:, 0] * det_wh[:, 1]

    track_boxes = np.array([[track.tlwh[0],
                           track.tlwh[1],
                           track.tlwh[0]+track.tlwh[2],
                           track.tlwh[1]+track.tlwh[3]]
                            for track in tracks], np.float32)
    det_boxes = np.array([[det.tlwh[0],
                         det.tlwh[1],
                         det.tlwh[0]+det.tlwh[2],
                         det.tlwh[1]+det.tlwh[3]]
                          for det in dets], np.float32)
    box_ious = bbox_overlaps_py(track_boxes, det_boxes)

    invalid = ((cost_matrix > track_size.reshape(-1, 1))
               + (cost_matrix > item_size.reshape(1, -1))
               + (box_ious < overlap_thresh)
               ) > 0
    cost_matrix = cost_matrix + invalid * motion_gate

    return cost_matrix

def bbox_overlaps_py(boxes, query_boxes):

    n_ = boxes.shape[0]
    k_ = query_boxes.shape[0]
    overlaps = np.zeros((n_, k_), dtype=np.float)
    for k in range(k_):
        query_box_area = (
            query_boxes[k, 2] - query_boxes[k, 0] + 1) * (query_boxes[k, 3] - query_boxes[k, 1] + 1)
        for n in range(n_):
            iw = min(boxes[n, 2], query_boxes[k, 2]) - \
                max(boxes[n, 0], query_boxes[k, 0]) + 1
            if iw > 0:
                ih = min(boxes[n, 3], query_boxes[k, 3]) - \
                    max(boxes[n, 1], query_boxes[k, 1]) + 1
                if ih > 0:
                    box_area = (boxes[n, 2] - boxes[n, 0] + 1) * \
                        (boxes[n, 3] - boxes[n, 1] + 1)
                    all_area = float(box_area + query_box_area - iw * ih)
                    overlaps[n, k] = iw * ih / all_area
    return overlaps

def cost_metric(metric, sigma, level, tracks, dets, track_indices, detection_indices):
    features = np.array([dets[i].feature for i in detection_indices])
    targets = np.array([tracks[i].track_id for i in track_indices])
    tracks = [tracks[i] for i in track_indices]
    dets_use = [dets[i] for i in detection_indices]

    det_previous_ct = np.array([d.previous_ct for d in dets_use])
    track_previous_ct = np.array([t.ct for t in tracks])

    delta_xy = np.array([t.mean[4:6] for t in tracks])
    alpha = tracker.metric_gaussian_motion(delta_xy, sigma)
    if level == 0:
        start_time = time.time()

        motion_cost = dist_cost_matrix(
            track_previous_ct, det_previous_ct, tracks, dets_use)
        motion_time = time.time()
        motion_t = motion_time - start_time

        cost_matrix = motion_cost
    else:

        start_time = time.time()
        appearance_cost = metric.distance(features, targets)
        appear_time = time.time()
        appear_t = appear_time - start_time

        cost_matrix = appearance_cost

    tracks_info = {}
    for i, idx in enumerate(track_indices):
        tracks_info[idx] = alpha[i]

    return cost_matrix, tracks_info

def metric_assign(metric, sigma, level, tracks, detections, track_indices, detection_indices, metric_gate):

    start_time = time.time()

    cost_matrix, info = cost_metric(metric,
                                    sigma, level, tracks, detections, track_indices, detection_indices)

    dis_time = time.time()
    dis_t = dis_time - start_time

    matches, unmatched_tracks, unmatched_detections = _assign(
        level, cost_matrix, info, track_indices, detection_indices, metric_gate)
    assign_time = time.time()
    assign_t = assign_time - dis_time

    return (matches, unmatched_tracks, unmatched_detections)

def test(a):
    time.sleep(0.002)
    a = np.array([(1, 1, 1, 1)] * 15)
    return a, a, a

def pre_assignment(opt, level, tracks, detections, track_indices, detection_indices, metric, motion_distance, appearance_distance):

    if level == 0:

        start_time = time.time()
        results = []
        for i, gate in enumerate([motion_distance, appearance_distance]):
            results.append(
                metric_assign(metric, opt.sigma, i, tracks, detections, track_indices, detection_indices, gate))

        results = [r for r in results]

        single_time = time.time()
        single_t = single_time - start_time

        motion_results = results[0]
        appearance_results = results[1]

        pre_matches = np.vstack((motion_results[0], appearance_results[0])
                                ) if appearance_results[0].shape[0] else motion_results[0]
        unmatched_tracks = list(set(motion_results[1]).intersection(
            set(appearance_results[1])))
        unmatched_detections = list(set(motion_results[2]).intersection(
            set(appearance_results[2])))

        return pre_matches, unmatched_tracks, unmatched_detections
    else:

        appearance_matches, appearance_unmatched_tracks, appearance_unmatched_detections = metric_assign(
            metric, opt.sigma, 1, tracks, detections, track_indices, detection_indices, appearance_distance)

        return appearance_matches, list(appearance_unmatched_tracks), list(appearance_unmatched_detections)

def pre_assignmentv1(cost_matrix, tracks, detections, track_indices, detection_indices, gate, max_distance, distance_metric):

    unmatched_track_indices = copy.deepcopy(track_indices)
    pre_matches, unmatched_tracks, unmatched_detections, = [], [], []

    while len(unmatched_track_indices):
        row_indices, col_indices = linear_assignment(cost_matrix)

        cur_assignment_idx = []
        for row, col in zip(row_indices, col_indices):
            track_idx = unmatched_track_indices[row]
            detection_idx = detection_indices[col]

            if cost_matrix[row, col] > gate:
                unmatched_tracks.append(track_idx)
                unmatched_detections.append(detection_idx)
            else:
                pre_matches.append((track_idx, detection_idx))

            cur_assignment_idx.append(track_idx)

        for idx in cur_assignment_idx:
            unmatched_track_indices.remove(idx)

        cost_matrix = get_remain_pair_matrix(cost_matrix, row_indices)

        if len(unmatched_track_indices):
            gate = max_distance

            cost_matrix, _ = distance_metric(
                1, tracks, detections, unmatched_tracks, unmatched_detections)
            cost_matrix = filter_pairs(cost_matrix, gate=gate)

    pre_matches = np.array(sorted(pre_matches, key=lambda p: p[0]))

    return pre_matches, unmatched_tracks

def extract_conflict_tracks_info(tracklet_idx, det_id_index, tracks_info, alpha_thredhold):
    fisrt_tracks, second_tracks = [], []
    for idx in tracklet_idx:
        appearance_cost = tracks_info[idx]["appearance_cost"][det_id_index]

        miss_nums = tracks_info[idx]["miss_nums"]
        if tracks_info[idx]['alpha'] >= alpha_thredhold:
            fisrt_tracks.append((idx, appearance_cost,  miss_nums))

        else:
            second_tracks.append(
                (idx, appearance_cost, miss_nums))

    fisrt_tracks = np.array(fisrt_tracks)
    second_tracks = np.array(second_tracks)

    return fisrt_tracks, second_tracks

def first_round_association(fisrt_tracks, second_tracks, det_id, tolerate_threshold):

    if len(fisrt_tracks) == 0:
        return [], []

    if second_tracks.shape[0] > 0:
        min_second_appearance = np.min(second_tracks[:, 1])
        accepted_appearance = np.where((fisrt_tracks[:, 1] - min_second_appearance) /
                                       min_second_appearance <= tolerate_threshold, True, False)
    else:
        accepted_appearance = np.array(fisrt_tracks.shape[0] * [True])

    matches, unmatched_tracks = [], []

    matches_tracks_ids = fisrt_tracks[accepted_appearance, 0]
    matches = [(int(i), det_id) for i in matches_tracks_ids]

    unmatched_tracks = list(fisrt_tracks[~accepted_appearance, 0])

    if len(matches):

        return [matches[0]], unmatched_tracks
    else:
        return matches, unmatched_tracks

def second_round_association(second_tracks, det_id, max_discount_num, det_used):

    matches, unmatched_tracks, kf_ids = [], [], []
    while second_tracks.shape[0]:
        if not det_used:

            min_id = np.argmin(second_tracks[:, 1])
            track_id = second_tracks[min_id, 0]
            matches.append((int(track_id), det_id))

            remain_ids = [True] * second_tracks.shape[0]
            remain_ids[min_id] = False
            second_tracks = second_tracks[remain_ids, :]
        else:
            discount = np.where(
                second_tracks[:, -1] <= max_discount_num, True, False)

            kf_ids = list(second_tracks[discount, 0].astype(np.int32))
            unmatched_tracks = list(second_tracks[~discount, 0])

            break

    return matches, unmatched_tracks, kf_ids

def two_round_match_v1(conflict_matches, tracks_info, detection_indices, alpha_thredhold=0.5, tolerate_threshold=0.1, max_discount_num=10):

    matches, unmatched_tracks, kf_ids = [], [], []
    for pair in conflict_matches:
        tracklet_idx, det_id = pair[0], pair[1]

        det_id_index = detection_indices.index(det_id)
        fisrt_tracks, second_tracks = extract_conflict_tracks_info(
            tracklet_idx, det_id_index, tracks_info, alpha_thredhold)

        matches_first, unmatched_tracks_first = first_round_association(
            fisrt_tracks, second_tracks, det_id, tolerate_threshold)

        det_used = False if len(matches_first) == 0 else True
        matches_second, unmatched_tracks_second, kf_ids = second_round_association(
            second_tracks, det_id, max_discount_num, det_used)

        matches = matches_first + matches_second
        unmatched_tracks = unmatched_tracks_first + unmatched_tracks_second

    return matches, unmatched_tracks, kf_ids

def filter_pairs(cost_matrix, gate):
    cost_matrix[cost_matrix > gate] = gate + INFTY_COST

    return cost_matrix

def iou_cost_matching(
    distance_metric, max_distance, tracks, detections, track_indices=None,
        detection_indices=None):

    if track_indices is None:
        track_indices = np.arange(len(tracks))
    if detection_indices is None:
        detection_indices = np.arange(len(detections))

    if len(detection_indices) == 0 or len(track_indices) == 0:
        return [], track_indices, detection_indices

    cost_matrix = distance_metric(
        tracks, detections, track_indices, detection_indices)

    cost_matrix[cost_matrix > max_distance] = max_distance + INFTY_COST

    row_indices, col_indices = linear_assignment(cost_matrix)

    matches, unmatched_tracks, unmatched_detections = [], [], []
    for col, detection_idx in enumerate(detection_indices):
        if col not in col_indices:
            unmatched_detections.append(detection_idx)
    for row, track_idx in enumerate(track_indices):
        if row not in row_indices:
            unmatched_tracks.append(track_idx)
    for row, col in zip(row_indices, col_indices):
        track_idx = track_indices[row]
        detection_idx = detection_indices[col]
        if cost_matrix[row, col] > max_distance:
            unmatched_tracks.append(track_idx)
            unmatched_detections.append(detection_idx)
        else:
            matches.append((track_idx, detection_idx))

    return matches, unmatched_tracks, unmatched_detections

def select_pairs(matches, col_id, pairs):
    for t in pairs:
        ind = np.argwhere(matches[:, col_id] == t)
        if len(ind):
            matches[ind[0], -1] = 0
    matches = matches[matches[:, -1] == 1, :]

    return matches

def two_round_match(conflicts, alpha_gate=0.9):
    if conflicts.shape[0] == 0:
        return [], [], []

    first_round = conflicts[conflicts[:, -2] >= alpha_gate, :]
    matches_a = first_round[first_round[:, -1] == 1, :2]

    second_round = conflicts[conflicts[:, -2] < alpha_gate, :]
    second_round = second_round[second_round[:, -1] == 0, :]
    mask = np.ones(second_round.shape[0]).reshape(-1, 1)
    second_round = np.hstack((second_round, mask))

    second_round = select_pairs(second_round, 0, matches_a[:, 0])

    second_round = select_pairs(second_round, 1, matches_a[:, 1])
    matches_b = second_round[:, :2]

    matches_ = np.vstack((matches_a, matches_b))
    matches_ = matches_.astype(np.int)
    unmatched_tracks_ = list(set(conflicts[:, 0]) - set(matches_[:, 0]))
    unmatched_tracks_ = [int(i) for i in unmatched_tracks_]
    unmatched_detections_ = list(set(conflicts[:, 1]) - set(matches_[:, 1]))
    unmatched_detections_ = [int(i) for i in unmatched_detections_]

    return list(map(tuple, matches_)), unmatched_tracks_, unmatched_detections_

def min_cost_matching(
        opt, level, metric, appearance_distance, tracks, detections, track_indices=None,
        detection_indices=None, motion_distance=1e+18):

    if track_indices is None:
        track_indices = np.arange(len(tracks))
    if detection_indices is None:
        detection_indices = np.arange(len(detections))

    if len(detection_indices) == 0 or len(track_indices) == 0:
        return [], track_indices, detection_indices, []

    start_time = time.time()
    pre_matches, unmatched_tracks, unmatched_detections = pre_assignment(opt, level, tracks, detections,
                                                                         track_indices, detection_indices,
                                                                         metric, motion_distance, appearance_distance)
    preassign_time = time.time()
    preassign_t = preassign_time - start_time

    if pre_matches.shape[0]:
        matches, conflicts = split_TD_conflict(pre_matches)
        conflict_time = time.time()
        conflict_t = conflict_time - preassign_time

        matches_, unmatched_tracks_, unmatched_detections_ = two_round_match(
            conflicts, opt.alpha_gate)
        tworound_time = time.time()
        tworound_t = tworound_time - conflict_time
        tot_t = time.time() - start_time
        matches += matches_
        unmatched_tracks += unmatched_tracks_
        unmatched_detections += unmatched_detections_
        kf_ids = []

    else:
        matches = []
        kf_ids = []

    matches = sorted(matches, key=lambda m: m[0])
    return matches, unmatched_tracks, unmatched_detections, kf_ids

def matching_cascade(opt,
                     metric, max_distance, cascade_depth, tracks, detections,
                     track_indices=None, detection_indices=None):

    if track_indices is None:
        track_indices = list(range(len(tracks)))
    if detection_indices is None:
        detection_indices = list(range(len(detections)))

    unmatched_detections = detection_indices
    matches, kf_ids = [], []

    for level in range(cascade_depth):
        if len(unmatched_detections) == 0:
            break

        track_indices_l = [
            k for k in track_indices if tracks[k].time_since_update == 1 + level]
        if len(track_indices_l) == 0:
            continue

        matches_l, _, unmatched_detections, kf_ids_l = min_cost_matching(opt,
                                                                         level, metric, max_distance, tracks, detections,
                                                                         track_indices_l, unmatched_detections)

        matches += matches_l
        kf_ids += kf_ids_l

    unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches))

    return matches, unmatched_tracks, unmatched_detections, kf_ids

def gate_cost_matrix(
        kf, cost_matrix, tracks, detections, track_indices, detection_indices,
        gated_cost=INFTY_COST, only_position=False):

    gating_dim = 2 if only_position else 4
    gating_threshold = kalman_filter.chi2inv95[gating_dim]
    measurements = np.asarray(
        [detections[i].to_xyah() for i in detection_indices])
    for row, track_idx in enumerate(track_indices):
        track = tracks[track_idx]
        gating_distance = kf.gating_distance(
            track.mean, track.covariance, measurements, only_position)
        cost_matrix[row, gating_distance > gating_threshold] = gated_cost
    return cost_matrix

def motion_cost_matrix(
        kf, tracks, detections, track_indices, detection_indices,
        gated_cost=INFTY_COST, only_position=False):

    def normalization(data):

        if data.shape[0] == 1:
            data[0, 0] = 1.0
            return data
        else:
            _range = np.max(data) - np.min(data)
            return (data - np.min(data)) / _range

    measurements = np.asarray(
        [detections[i].to_xyah() for i in detection_indices])

    cost_matrix = np.zeros((len(track_indices), len(detection_indices)))
    for row, track_idx in enumerate(track_indices):
        track = tracks[row]
        gating_distance = kf.gating_distance(
            track.mean, track.covariance, measurements, only_position)

        cost_matrix[row, :] = gating_distance

    return normalization(cost_matrix)