# **************************************************************************
# *
# * Authors:     Grigory Sharov (gsharov@mrc-lmb.cam.ac.uk)
# *
# * MRC Laboratory of Molecular Biology (MRC-LMB)
# *
# * This program is free software; you can redistribute it and/or modify
# * it under the terms of the GNU General Public License as published by
# * the Free Software Foundation; either version 3 of the License, or
# * (at your option) any later version.
# *
# * This program is distributed in the hope that it will be useful,
# * but WITHOUT ANY WARRANTY; without even the implied warranty of
# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# * GNU General Public License for more details.
# *
# * You should have received a copy of the GNU General Public License
# * along with this program; if not, write to the Free Software
# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
# * 02111-1307  USA
# *
# *  All comments concerning this program package may be sent to the
# *  e-mail address 'scipion@cnb.csic.es'
# *
# **************************************************************************

import math
from enum import Enum

import pyworkflow.protocol.params as params
from pyworkflow.constants import PROD
from pwem.protocols import ProtCreateMask3D
from pwem.objects import VolumeMask

import relion.convert as convert
from ..constants import MASK_AND

class outputs(Enum):
    outputMask = VolumeMask

class ProtRelionCreateMask3D(ProtCreateMask3D):
    """ This protocols creates a 3D mask using Relion.
    The mask is created from a 3d volume or by comparing two input volumes.
    """
    _label = 'create 3d mask'
    _devStatus = PROD
    _possibleOutputs = outputs

    # --------------------------- DEFINE param functions ----------------------
    def _defineParams(self, form):
        form.addSection(label='Mask generation')
        form.addParam('inputVolume', params.PointerParam,
                      pointerClass="Volume",
                      label="Input volume",
                      help="Select the volume that will be used to create the mask")
        form.addParam('initialLowPassFilterA', params.FloatParam,
                      default=15.,
                      label='Lowpass filter map by (A)',
                      help='Lowpass filter that will be applied to the input map, '
                           'prior to binarization. To calculate solvent masks, a '
                           'lowpass filter of 15-20A may work well.')
        # TODO: add wizard
        form.addParam('threshold', params.FloatParam, default=0.02,
                      label='Initial binarisation threshold',
                      help="This threshold is used to make an initial binary "
                           "mask from the average of the two unfiltered "
                           "half-reconstructions. If you don't know what "
                           "value to use, display one of the unfiltered "
                           "half-maps in a 3D surface rendering viewer and "
                           "find the lowest threshold that gives no noise "
                           "peaks outside the reconstruction.")

        form.addParam('doCompare', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Compare with another volume to produce a mask?',
                      help='Logical comparison of two input volumes to produce a mask')

        form.addParam('inputVolume2', params.PointerParam,
                      pointerClass="Volume",
                      condition='doCompare',
                      expertLevel=params.LEVEL_ADVANCED,
                      label="Input volume (second)",
                      help="Select the volume that will be compared to the first one")

        form.addParam('operation', params.EnumParam, default=MASK_AND,
                      condition='doCompare',
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Operation',
                      choices=['AND', 'OR', 'AND_NOT', 'OR_NOT'],
                      help='*AND*: Pixels in the initial mask will be one if the input '
                           'AND the second volume are above the threshold value.\n'
                           '*OR*: Pixels in the initial mask will be one if the input '
                           'OR the second volume are above the threshold value.\n'
                           '*AND_NOT*: pixels in the initial mask will be one if '
                           'the input is above the threshold AND the second volume '
                           'is below it.\n*OR_NOT*: pixels in the initial mask will be '
                           'one if the input is above the threshold OR the second '
                           'volume is below it.')

        form.addParam('extend', params.IntParam, default=3,
                      label='Extend binary mask by (px)',
                      help='The initial binary mask is extended this number of '
                           'pixels in all directions.')

        form.addParam('edge', params.IntParam, default=3,
                      label='Add a soft-edge (px)',
                      help='The extended binary mask is further extended with '
                           'a raised-cosine soft edge of the specified width.')

        form.addParam('doInvert', params.BooleanParam, default=False,
                      expertLevel=params.LEVEL_ADVANCED,
                      label='Invert final mask',
                      help='Invert the final mask')

        form.addParallelSection(threads=4, mpi=0)

    # --------------------------- INSERT steps functions ----------------------
    def _insertAllSteps(self):
        self.maskFile = self._getExtraPath('mask.mrc')
        self._insertFunctionStep('convertInputStep', self.inputVolume.get().getObjId())
        self._insertFunctionStep('createMaskStep')
        self._insertFunctionStep('createOutputStep')

    # --------------------------- STEPS functions -----------------------------
    def convertInputStep(self, volId):
        self.inputVolFn = convert.convertBinaryVol(
            self.inputVolume.get(), self._getTmpPath())

        if self.doCompare:
            self.inputVol2Fn = convert.convertBinaryVol(
                self.inputVolume2.get(), self._getTmpPath())

    def createMaskStep(self):
        argsDict = {'--i ': self.inputVolFn,
                    '--ini_threshold ': self.threshold.get(),
                    '--extend_inimask ': self.extend.get(),
                    '--width_soft_edge ': self.edge.get(),
                    '--angpix ': self.inputVolume.get().getSamplingRate()
                    }
        if self.initialLowPassFilterA.get() != -1:
            argsDict['--lowpass '] = self.initialLowPassFilterA.get()

        args = ' --o %s ' % self.maskFile
        args += ' '.join(['%s %s' % (k, v) for k, v in argsDict.items()])

        if self.doCompare:
            op = self.operation.get()
            opStr = ['--and', '--or', '--and_not', '--or_not'][op]
            args += ' %s %s' % (opStr, self.inputVol2Fn)

        if self.doInvert:
            args += ' --invert'

        if self.numberOfThreads > 1:
            args += ' --j %d' % self.numberOfThreads

        self.runJob("relion_mask_create", args)

        return [self.maskFile]

    def createOutputStep(self):
        volMask = VolumeMask()
        volMask.setFileName(self.maskFile)
        volMask.setSamplingRate(self.inputVolume.get().getSamplingRate())

        self._defineOutputs(**{outputs.outputMask.name: volMask})
        self._defineSourceRelation(self.inputVolume, self.outputMask)

    # --------------------------- INFO functions ------------------------------
    def _validate(self):
        errors = []

        if self.doCompare:
            pixVol1 = self.inputVolume.get().getSamplingRate()
            pixVol2 = self.inputVolume2.get().getSamplingRate()

            if not math.isclose(pixVol1, pixVol2, abs_tol=0.001):
                errors.append('Pixel size should be the same for both input volumes.')

        return errors

    def _summary(self):
        messages = [
            "Created a mask from input volume using threshold *%0.5f*"
            % self.threshold.get(),
            "*Mask processing*",
            "   Extend by *%d* pixels" % self.extend,
            "   Apply soft edge of *%d* pixels" % self.edge]
        if self.doInvert:
            messages.append("   Inverted")

        return messages

    def _citations(self):
        return []

    def _methods(self):
        messages = [
            "*Mask creation*",
            "We processed the volume %s." % self.getObjectTag('inputVolume'),
            "We binarized it at threshold of %0.3f. " % self.threshold,
            "We extended binary mask by %d voxels." % self.extend,
            "And, we smoothed it by applying a soft edge of %d voxels."
            % self.edge.get()
        ]
        if self.doInvert:
            messages.append("We inverted the mask. ")
            messages.append("And, we smoothed it by applying a soft edge of "
                            "%d voxels." % self.edge)
        if self.hasAttribute('outputMask'):
            messages.append('We refer to the output mask as %s.'
                            % self.getObjectTag('outputMask'))

        return messages


from dataclasses import dataclass
import datetime
from typing import List, NamedTuple, Tuple
import json
from copy import deepcopy
from event import Event, Location, Timeframe

from mapsy import timeDistance

@dataclass
class Task:
    id: int
    begin: datetime.datetime
    end: datetime.datetime
    dur: datetime.timedelta
    @property
    def loc_id(self):
        return self.id

class Interval(NamedTuple):
    begin: datetime.datetime
    end: datetime.datetime
    loc_id_b: Tuple
    loc_id_e: Tuple

class Planner:
    def __init__(self, events: List[Event], home_location: Location):
        self.events = events
        self.home_location = home_location
        self.rigid_tasks, self.flexible_tasks, s_date, e_date = self.parse()
        home_loc_id = -1

        if datetime.datetime.now().date() != s_date.date():
            days = [s_date + datetime.timedelta(days=i)
                    for i in range((e_date - s_date).days + 1)]
            self.freetime = [
                Interval(d + datetime.timedelta(hours=7),
                         d + datetime.timedelta(hours=23),
                         home_loc_id,
                         home_loc_id) for d in days
            ]

        else:
            today_int = Interval(
                datetime.datetime.now() + datetime.timedelta(minutes=1),
                datetime.datetime.now().replace(hour=23, minute=0),
                home_loc_id,
                home_loc_id
            )

            days = [s_date + datetime.timedelta(days=i)
                    for i in range(1, (e_date - s_date).days + 1)]

            self.freetime = [today_int] + [
                Interval(d + datetime.timedelta(hours=7),
                         d + datetime.timedelta(hours=23),
                         home_loc_id,
                         home_loc_id) for d in days
            ]

        self.placed_tasks = dict()
        self.solution_found = False
        self.distances = dict()

    def __repr__(self):
        return str(self.placed_tasks)

    def parse(self):
        dt_fmt = "%d.%m.%Y %H:%M"
        rigid_tasks = []
        flexible_tasks = []

        for i, e in enumerate(self.events):
            b_st = e.timeframe.start_date + " " + e.timeframe.start_time
            b_t = datetime.datetime.strptime(b_st, dt_fmt)

            e_st = e.timeframe.end_date + " " + e.timeframe.end_time
            e_t = datetime.datetime.strptime(e_st, dt_fmt)

            if e.timeframe.duration is not None:
                h, m = [int(e) for e in e.timeframe.duration.split(":")]
                duration = datetime.timedelta(hours=h, minutes=m)
            else:
                duration = e_t - b_t
            task = Task(i, b_t, e_t, duration)
            if e.isFixed():
                rigid_tasks.append(task)
            else:
                flexible_tasks.append(task)

        s_time = min(rigid_tasks+flexible_tasks,
                     key=lambda t: t.begin).begin.replace(hour=0, minute=0)
        e_time = max(rigid_tasks+flexible_tasks,
                     key=lambda t: t.end).end.replace(hour=0, minute=0)

        return rigid_tasks, flexible_tasks, s_time, e_time

    def dist(self, loc_id1, loc_id2, time):
        if loc_id1 == -1:  # home destination
            l1 = self.home_location
        else:
            l1 = self.events[loc_id1].location

        if loc_id2 == -1:
            l2 = self.home_location
        else:
            l2 = self.events[loc_id2].location

        if (l1.latlngtup(), l2.latlngtup(), time) in self.distances:
            return self.distances[(l1.latlngtup(), l2.latlngtup(), time)]
        else:

            #dist = datetime.timedelta(0)
            dist = timeDistance(l1, l2, time)

            self.distances[(l1.latlngtup(), l2.latlngtup(), time)] = dist
            return dist

    def place(self, t: Task, invls):
        for ivl in invls:
            s_time = max(ivl.begin, t.begin)
            e_time = min(ivl.end,   t.end)
            aval_time = e_time - s_time
            if aval_time > datetime.timedelta(0):
                tot_dur = self.dist(ivl.loc_id_b, t.loc_id, s_time) + \
                    t.dur + \
                    self.dist(t.loc_id, ivl.loc_id_e, s_time + t.dur)
                if tot_dur <= aval_time:
                    # print("Posssible", t, ivl)
                    break
        else:
            # print("Cannot allocate task")
            return False
        if ivl.begin != s_time:
            invls.append(
                Interval(ivl.begin,
                         s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time),
                         ivl.loc_id_b,
                         t.loc_id)
            )

        if s_time + tot_dur < ivl.end:
            invls.append(
                Interval(s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time) + t.dur,
                         ivl.end,
                         t.loc_id,
                         ivl.loc_id_e)
            )
        invls.remove(ivl)
        invls.sort()

        self.placed_tasks[t.id] = (
            s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time),
            s_time + self.dist(ivl.loc_id_b, t.loc_id, s_time) + t.dur
        )

        return True

    def allocate(self, task_l, invls):
        if self.solution_found:
            return

        if len(task_l) == 0:
            self.solution_found = True
            return

        for t in task_l:
            cinvls = deepcopy(invls)
            allocated = self.place(t, cinvls)

            if not allocated:
                return

            t_tmp = deepcopy(task_l)
            t_tmp.remove(t)
            self.allocate(t_tmp, cinvls)
            if self.solution_found:
                return

    def plan(self):
        invls = self.freetime
        if len(self.rigid_tasks) > 0:
            for t in self.rigid_tasks:
                self.place(t, invls)
        if len(self.flexible_tasks) > 0:
            self.flexible_tasks.sort(key=lambda task: -task.dur)
            self.allocate(self.flexible_tasks, invls)

        if not self.solution_found:
            n_missing = len(self.placed_tasks) - \
                len(set(self.placed_tasks.values()))
            print(
                f"No solution found, {n_missing} tasks remains without place")

        for i, t in self.placed_tasks.items():  # task_time_start, task_time_end, task
            st = t[0].timetuple()
            et = t[1].timetuple()
            diff = t[1] - t[0]
            start_date = f"{st[2]:02d}.{st[1]:02d}.{st[0]}"
            start_time = f"{st[3]:02d}:{st[4]:02d}"
            end_date = f"{et[2]:02d}.{et[1]:02d}.{et[0]}"
            end_time = f"{et[3]:02d}:{et[4]:02d}"
            ntf = Timeframe(start_date, start_time, end_date, end_time)
            self.events[i].reschedule(ntf)

def test(n):
    HOME_LOCATION = Location(52.22977, 21.01178)  # PKIN

    print(f"------Test with n = {n}------")
    events = []
    for i in range(n):
        t = Timeframe("01.04.2020", "13:00", "02.04.2020", "13:21", "3:00")
        e = Event(f"a{i}", "nothing", "lol",
                  Location(52.237082385, 21.025083233), False, t)
        events.append(e)

    print("Before:")
    for e in events:
        print(e.timeframe.start_date, e.timeframe.start_time)
    planner = Planner(events, HOME_LOCATION)
    planner.plan()
    print("After:")
    for e in events:
        print(e.scheluded_datetime.start_date, e.scheluded_datetime.start_time)

if __name__ == "__main__":
    test(3)
    test(7)


'''
Created on 09/11/2009
@author: Nahuel
'''

from matrix.NDimMatrix import NDimMatrix
from matrix.MatrixException import MatrixException

class Matrix(NDimMatrix):
    '''Two dimension Matrix.'''
    def __init__(self, size):
        '''
        Constructor of Matrix.
        @note: checks if size has a dimension = 2.
        @raise MatrixException: if the size has more or less dimension than 2.
        '''
        if not len(size) == 2:
            raise MatrixException("Matrix Error #3 : Dimension isn't 2.")
        else:
            NDimMatrix.__init__(self, size)

    def rows(self):
        '''Returns the number of rows in the matrix.'''
        return self.get_size()[0]

    def cols(self):
        '''Returns the number of columns in the matrix.'''
        return self.get_size()[1]

    def substract(self, another_matrix):
        '''
        Subtract element by element the receiver with a matrix received.

        @param another_matrix: is 'y' in (x - y). x is the receiver.
        @note: Don't alter the receiver. Returns a new Matrix.
        @precondition: The two matrix has the same size.
        @precondition: The two matrix are complete (don't have None's).
        '''
        res = Matrix(self.get_size())
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                res.at_put((i, j), self.at((i, j)) - another_matrix.at((i, j)))
        return res

    def add(self, another_matrix):
        '''
        Add element by element the receiver with a matrix received.

        @note: Alter the receiver. Don't create a new matrix.
        @precondition: The two matrix has the same size.
        @precondition: The two matrix are complete (don't have None's).
        '''
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                self.at_put((i, j), self.at((i, j)) + another_matrix.at((i, j)))

    def fill_with(self, obj):
        '''
        Fills the entire matrix with the received element.

        @param obj: the element that fills the matrix.
        '''
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                self.at_put((i, j), obj)

    def get_max(self):
        '''
        Returns the max element in the matrix.

        @note: if all the elements are None's, returns 0.
        '''
        max = 0
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                curr_elem = self.at((i, j))
                if curr_elem is not None and curr_elem > max:
                    max = curr_elem
        return max

    def fill_row(self, row, values):
        '''
        Fills a row (index received) with values received.

        @param row: the row number.
        @param values: tuple with the new values.
        '''
        for i in range(1, self.cols() + 1):
            self.at_put((row, i), values[i-1])

    def have_some_none(self):
        '''Checks if there is some None in the matrix.'''
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                if self.at((i, j)) is None:
                    return True
        return False

    def row_to_tuple(self, row_num):
        '''
        Make a tuple with a row specified.

        @param row_num: the row number.
        '''
        res = []
        for j in range(1, self.cols() + 1):
            res.append(self.at((row_num, j)))
        return tuple(res)

    def sum_row(self, row_num, tup):
        '''
        Add element by element a row specified
        with values received as argument.

        @note: Alter the receiver.
        '''
        for j in range(1, self.cols() + 1):
            self.at_put(((row_num, j)), self.at((row_num, j)) + tup[j - 1])

    def substract_row(self, row_num, tup):
        '''
        Add element by element a row specified
        with values received as argument.

        @note: Alter the receiver.
        '''
        for j in range(1, self.cols() + 1):
            self.at_put(((row_num, j)), self.at((row_num, j)) - tup[j - 1])

    def __repr__(self):
        '''Pretty print for two dimension Matrix.'''
        spaces = len(str(self.get_max())) + 1
        res = ''
        if self.have_some_none() and spaces < 4:
            spaces = 5                            #necessary space for "None"
        for i in range(1, self.rows() + 1):
            res += '| '
            for j in range(1, self.cols() + 1):
                str_elem = str(self.at((i, j)))
                res += str_elem
                res += ' ' * (spaces - len(str_elem))
            res += ' |\n'
        return res

    def has_all(self, obj):
        '''Checks if all the elements in the matrix are 'obj'.'''
        for i in range(1, self.rows() + 1):
            for j in range(1, self.cols() + 1):
                if not self.at((i, j)) == obj:
                    return False
        return True

    def copy(self):
        '''Returns a new matrix, copy of the receiver.'''
        new_matrix = Matrix(self.get_size())
        new_matrix.set_data(self.get_data().copy())
        return new_matrix

    def new_row(self, values):
        '''
        Add a new row in the matrix (at the end)
        and fills it with 'values'.
        '''
        self.set_size((self.rows() + 1, self.cols()))
        last_row = self.rows()
        for j in range(1, self.cols() + 1):
            self.at_put((last_row, j), values[j - 1])


import FWCore.ParameterSet.Config as cms

from Configuration.StandardSequences.Eras import eras
from cp3_llbb.Framework import Framework
from cp3_llbb.Framework.CmdLine import CmdLine

options = CmdLine(defaults=dict(runOnData=1, era="25ns", globalTag='80X_dataRun2_2016SeptRepro_v7', process='RECO'))

framework = Framework.Framework(options)

framework.addAnalyzer('dilepton', cms.PSet(
        type = cms.string('dilepton_analyzer'),
        prefix = cms.string('dilepton_'),
        enable = cms.bool(True),
        categories_parameters = cms.PSet(
            mll_cut = cms.untracked.double(20)
            ),
        parameters = cms.PSet(
            standalone = cms.untracked.bool(True),
            muons_wp = cms.untracked.string('loose'),
            electrons_wp = cms.untracked.string('loose')
            )
        ))

framework.addAnalyzer('test', cms.PSet(
        type = cms.string('test_analyzer'),
        prefix = cms.string('test_'),
        enable = cms.bool(True)
        ))

framework.applyElectronRegression()
framework.applyElectronSmearing()

from cp3_llbb.Framework.JetsProducer import discriminators_deepFlavour
framework.redoJEC(addBtagDiscriminators=discriminators_deepFlavour)
framework.doSystematics(['jec', 'jer'])

process = framework.create()

process.source.fileNames = cms.untracked.vstring(
        '/store/data/Run2016G/DoubleMuon/MINIAOD/03Feb2017-v1/100000/007796A5-78EB-E611-8EFA-A4BF01011FD0.root'
        )

process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1000))


# Imports
import random
import time

# LINEAR SEARCH
# Linear Search method that will search list one element at a time to see if it matches "a"
def linear_search(numeric_list, a):
    for j in range(len(numeric_list)):
        if numeric_list[j] == a:
            return j
    # If match can't be found, return -1
    return -1

# BINARY SEARCH
# Binary search which will requires sorted data. It will look at midpoint and throw away half the data at a time until,
# it finds data it is looking for, or runs out of data
def binary_search(numeric_list, b):
    low = 0
    high = len(numeric_list) - 1
    while low <= high:
        # Find middle using low and high
        middle = (low + high) // 2
        # If b is greater than value at middle index, ignore left side including the middle index
        if b > numeric_list[middle]:
            low = middle + 1
        # if b is less than value at middle index, ignore right side including the middle index
        if b < numeric_list[middle]:
            high = middle - 1
        else:
            return middle
    return -1

# BUBBLE SORT
# Python program for implementation of Bubble Sort. Obtained from
# "https://www.geeksforgeeks.org/python-program-for-bubble-sort/"
def bubblesort(arr):
    n = len(arr)
    # optimize code, so if the array is already sorted, it doesn't need
    # to go through the entire process
    swapped = False
    # Traverse through all array elements
    for i in range(n - 1):
        # range(n) also work but outer loop will
        # repeat one time more than needed.
        # Last i elements are already in place
        for j in range(0, n - i - 1):

            # traverse the array from 0 to n-i-1
            # Swap if the element found is greater
            # than the next element
            if arr[j] > arr[j + 1]:
                swapped = True
                arr[j], arr[j + 1] = arr[j + 1], arr[j]

        if not swapped:
            # if we haven't needed to make a single swap, we
            # can just exit the main loop.
            return

# SELECTION SORT
# Selection sort in Python
# Obtained and adapted from https://www.geeksforgeeks.org/python-program-for-selection-sort/
# time complexity O(n*n)
# sorting by finding min_index
def selectionsort(array):
    for ind in range(len(array)):
        min_index = ind

        for j in range(ind + 1, len(array)):
            # select the minimum element in every iteration
            if array[j] < array[min_index]:
                min_index = j
        # swapping the elements to sort the array
        (array[ind], array[min_index]) = (array[min_index], array[ind])

# INSERTION SORT
# Obtained from https://www.geeksforgeeks.org/python-program-for-insertion-sort/
def insertionsort(arr):
    n = len(arr)  # Get the length of the array

    if n <= 1:
        return  # If the array has 0 or 1 element, it is already sorted, so return

    for i in range(1, n):  # Iterate over the array starting from the second element
        key = arr[i]  # Store the current element as the key to be inserted in the right position
        j = i - 1
        while j >= 0 and key < arr[j]:  # Move elements greater than key one position ahead
            arr[j + 1] = arr[j]  # Shift elements to the right
            j -= 1
        arr[j + 1] = key  # Insert the key in the correct position

if __name__ == '__main__':

    # Bubble sort execution
    # First randomly generated array of 10000 values
    ten_k_array = [random.randint(0, 1000) for _ in range(10000)]

    # Perform linear search for constant value "500"
    VALUE = 500
    ls_index = linear_search(ten_k_array, VALUE)
    if ls_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(ls_index))

    """print("\nRaw array is:")
    for j in range(len(ten_k_array)):
        print("% d" % ten_k_array[j], end=" ")"""

    start = time.process_time()
    bubblesort(ten_k_array)
    end = time.process_time()
    print("\nTime to bubble sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(ten_k_array)):
        print("% d" % ten_k_array[k], end=" ")"""

    # Randomly generated array of x values - to approximate 30 seconds
    thirty_sec_array = [random.randint(0, 1000) for _ in range(31000)]

    """print("\nRaw array is below. Size of array is " + str(len(thirty_sec_array)))
    for j in range(len(thirty_sec_array)):
        print("% d" % thirty_sec_array[j], end=" ")"""

    start = time.process_time()
    bubblesort(thirty_sec_array)
    end = time.process_time()
    print("\nTime to bubble sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(thirty_sec_array)):
        print("% d" % thirty_sec_array[k], end=" ")"""

    # Perform binary search for constant value 500
    bs_index = binary_search(ten_k_array, VALUE)
    if bs_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(bs_index))

    # Second randomly generated array of 10000 values
    ten_k_array_second = [random.randint(0, 1000) for _ in range(10000)]

    # Perform linear search for constant value "500"
    VALUE = 500
    ls_index = linear_search(ten_k_array_second, VALUE)
    if ls_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(ls_index))

    """print("\nRaw array is:")
    for j in range(len(ten_k_array_second)):
        print("% d" % ten_k_array_second[j], end=" ")"""

    start = time.process_time()
    selectionsort(ten_k_array_second)
    end = time.process_time()
    print("\nTime to selection sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(ten_k_array_second)):
        print("% d" % ten_k_array_second[k], end=" ")"""

    # Randomly generated array of x values - to approximate 30 seconds
    thirty_sec_array_second = [random.randint(0, 1000) for _ in range(30000)]

    """print("\nRaw array is below. Size of array is " + str(len(thirty_sec_array_second)))
    for j in range(len(thirty_sec_array_second)):
        print("% d" % thirty_sec_array_second[j], end=" ")"""

    start = time.process_time()
    bubblesort(thirty_sec_array_second)
    end = time.process_time()
    print("\nTime to selection sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(thirty_sec_array_second)):
        print("% d" % thirty_sec_array_second[k], end=" ")"""

    # Perform binary search for constant value 500
    bs_index = binary_search(ten_k_array_second, VALUE)
    if bs_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(bs_index))

    # Third randomly generated array of 10000 values
    ten_k_array_third = [random.randint(0, 1000) for _ in range(10000)]

    # Perform linear search for constant value "500"
    VALUE = 500
    ls_index = linear_search(ten_k_array_third, VALUE)
    if ls_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(ls_index))

    """print("\nRaw array is:")
    for j in range(len(ten_k_array_third)):
        print("% d" % ten_k_array_third[j], end=" ")"""

    start = time.process_time()
    insertionsort(ten_k_array_third)
    end = time.process_time()
    print("\nTime to insertion sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(ten_k_array_third)):
        print("% d" % ten_k_array_third[k], end=" ")"""

    # Randomly generated array of x values - to approximate 30 seconds
    thirty_sec_array_third = [random.randint(0, 1000) for _ in range(32000)]

    """print("\nRaw array is below. Size of array is " + str(len(thirty_sec_array_second)))
    for j in range(len(thirty_sec_array_second)):
        print("% d" % thirty_sec_array_second[j], end=" ")"""

    start = time.process_time()
    bubblesort(thirty_sec_array_third)
    end = time.process_time()
    print("\nTime to insertion sort was " + str(end - start) + " seconds")

    """print("\nSorted array is:")
    for k in range(len(thirty_sec_array_second)):
        print("% d" % thirty_sec_array_second[k], end=" ")"""

    # Perform binary search for constant value 500
    bs_index = binary_search(ten_k_array_third, VALUE)
    if bs_index == -1:
        print("\nValue 500 not found in array")
    else:
        print("\nValue 500 found in index " + str(bs_index))

##### INSERTION SORT

##### SUMMARY AND CONCLUSION


import uuid
from datetime import datetime

from django.urls import reverse
from rest_framework import status
from rest_framework.test import APITestCase

from core.models import Reading, Customer, Device

class TestReadingView(APITestCase):
    def setUp(self):
        self.url = reverse('reading')

    def test_send_request_success(self):
        """
        In this method we test add new reading successfully.
        """

        body = {
            "timestamp": "2017-01-15T12:18:49.541124+00:00",
            "device_id": str(uuid.uuid4()),
            "customer_id": str(uuid.uuid4()),
            "value": 12.6
        }
        response = self.client.post(self.url, body)

        readings_count = Reading.objects.count()
        self.assertEquals(response.status_code, status.HTTP_201_CREATED)
        self.assertEqual(readings_count, 1)

class TestReadingAggregateView(APITestCase):
    def setUp(self):
        self.url = reverse('reading-aggregator')
        self.customer = Customer.objects.create()
        self.device = Device.objects.create(customer=self.customer)
        timestamp = datetime.now()
        values = [540, 600, 350]

        for value in values:
            Reading.objects.create(timestamp=timestamp, value=value, customer=self.customer, device=self.device)

        self.avg_value = round(sum(values) / len(values), 2)

    def test_send_request_success(self):
        """
        In this method we test retrieve aggregated reading successfully.
        """
        response = self.client.get(self.url)
        data = response.json()
        reading_aggregated_object = data[0]

        self.assertEquals(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(data), 1)
        self.assertEqual(reading_aggregated_object.get('device_id'), str(self.device.id))
        self.assertEqual(reading_aggregated_object.get('customer_id'), str(self.customer.id))
        self.assertEqual(reading_aggregated_object.get('aggregated_values')[0].get('value'), self.avg_value)


import numpy as np
from PIL import Image, ImageDraw, ImageFilter
import streamlit as st

# 何個作るか聞く
num_trials = st.slider('何個試作しますか？',0,10,0)
button = st.button('上記の設定で試作')

# 画像を後で入れる空のリスト
imgs = []

# URLを下になる画像から順に入れる
urls = ['base.png', 'field.jpg', 'yattazeyossya.png', 'Splatoon2logo.png',
        'decorators-12_2x.png', 'decorators-12_2x.png']
genres = ['base', 'back', 'person', 'logo', 'deco', 'deco']
for i, url in enumerate(urls):
    imgs.append(Image.open(url))

if button:
    for trials in range(num_trials):
        # 背景に色付け
        draw = ImageDraw.Draw(imgs[0])
        full_size = np.array(imgs[0].size)
        base_color = np.random.randint(0, 255, 3)
        draw.rectangle((0, 0, tuple(full_size)), fill=(
            base_color[0], base_color[1], base_color[2]))

        # 背景と混ぜた枠をセット※mode'RGBA'と'RGB'とかを揃える必要あり
        imgs[1] = imgs[1].resize(tuple(full_size))
        imgs[0] = Image.blend(imgs[0], imgs[1].convert('RGBA'), 0.5)

        # 背景画像をセット
        imgs[1] = imgs[1].resize(tuple(full_size-[20, 20]))
        imgs[0].paste(imgs[1], (15, 10))

        # 背景画像(通常1280,720)内に画像を散りばめる乱数を生成
        number = len(imgs)
        widths = []
        heights = []
        for k in range(number-2):
            img_ratio = (imgs[k+2].width / imgs[k+2].height) ** 0.5
            if genres[k+2] == 'logo':
                size_base = 400
            elif genres[k+2] == 'deco':
                size_base = 100
            elif genres[k+2] == 'person':
                size_base = 700
            new_size = tuple([round(size_base*img_ratio), round(size_base/img_ratio)])
            imgs[k+2] = imgs[k+2].resize(new_size)
            widths.append(np.random.randint(0, full_size[0]-new_size[0]))
            heights.append(np.random.randint(0, full_size[1]-new_size[1]))

        # 画像を配置
        for j, x, y in zip(range(number-2), widths, heights):
            mask = imgs[j+2].copy().convert('RGBA')
            imgs[0].paste(imgs[j+2], (x, y), mask)

        st.title('AutoThumbnail')
        st.image(imgs[0], caption='squid', use_column_width=True)


from aiohttp import web

from asyncpg.exceptions import UniqueViolationError

from . import permissions
from .dataaccess import environmentda
from . import auditing

async def get_envs(request):
    env_list = await environmentda.get_envs()
    envs = {'envs': [{'name': e} for e in env_list]}
    return web.json_response(envs)

async def add_env(request):
    body = await request.json()
    env_name = body.get('name', None)
    user = request.get('user')

    if not env_name.isidentifier():
        return web.json_response({'Message': "Not a valid name"},
                                 status=400)

    await permissions.check_permissions(user, permissions.Action.create_env)
    try:
        response = await environmentda.add_env(env_name)
        await auditing.audit_event(
            'environment.add', user, {'env_name': env_name})
        return web.json_response(response, status=201)

    except UniqueViolationError as e:
        return web.json_response(
            {'Message': "The environment name '{}' already exists"
                .format(env_name)},
            status=409)

async def delete_env(request):
    env = request.match_info['name']
    user = request.get('user')

    await permissions.check_permissions(user, permissions.Action.delete_env)

    await environmentda.delete_env(env)
    await auditing.audit_event(
        'environment.remove', user, {'env_name': env})
    return web.HTTPNoContent()


"""
Module to train the decision tree model.
"""
import logging
import typing
import pickle

import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

logger = logging.getLogger(__name__)

def train_test_split_data(data:pd.DataFrame, target_col:str, test_size: float = 0.4,
                          random_state: int = 42) -> typing.Union[pd.DataFrame, pd.Series]:
    """
    Split the data into a training and testing set.

    Args:
        data (pd.DataFrame): The data to be split.
        target_col (str): The target column.
        test_size (float): The proportion of the data to use for testing.
        random_state (int): The seed for the random number generator.

    Returns:
        A tuple containing the training and testing data.
    """

    if not isinstance(data, pd.DataFrame):
        logger.error("Invalid data")
        raise TypeError("data must be a pandas DataFrame")
    if not isinstance(target_col, str):
        logger.error("Invalid target column")
        raise TypeError("target_col must be a string")

    # Return whole data if test_size is 0
    if test_size == 0:
        logger.info("Test size is 0, returning whole data")
        return data

    if test_size < 0 or test_size >= 1:
        logger.error("Invalid test size")
        raise ValueError("test_size must be between 0 and 1")

    try:
        features = data.drop(target_col, axis=1)
        target = data[target_col]
        # Split the data into a training and testing set
        logger.info("Splitting data into training and testing set")
        x_train, x_test, y_train, y_test = train_test_split(features, target, test_size=test_size,
                                                            random_state=random_state)

    # Catch exceptions for train_test_split
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err
    except Exception as err:
        logger.error("Error: %s", err)
        raise err

    # Return the training and testing data
    return x_train, x_test, y_train, y_test

def train_dt_model(x_train: pd.DataFrame, y_train: pd.Series,  # pylint: disable=too-many-arguments
                   initial_features: typing.List[str],
                   random_state: int = 42) -> DecisionTreeClassifier:
    """
    Train the decision tree model.

    Args:
        x_train (pd.DataFrame): The training features.
        y_train (pd.Series): The training labels.
        initial_features (list): The initial features.
        random_state (int): The seed for the random number generator.

    Returns:
        dt_model (DecisionTreeClassifier): The decision tree model.
    """

    # Checking xtrain and ytrain
    if not isinstance(x_train, pd.DataFrame):
        logger.error("Invalid training features")
        raise ValueError("x_train must be a pandas DataFrame")

    if not isinstance(y_train, pd.Series):
        logger.error("Invalid training labels")
        raise ValueError("y_train must be a pandas Series")

    try:
        # Train the decision tree model
        logger.info("Training the decision tree model")
        dt_model = DecisionTreeClassifier(random_state=random_state)
        logger.info("Fitting the decision tree model")
        dt_model.fit(x_train[initial_features], y_train.values.ravel())

    # Catch exceptions for training the decision tree model
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    # Return the trained model
    return dt_model

def train(input_path: str,x_train_path: str, y_train_path: str, x_test_path: str, y_test_path: str,
            model_path: str, target_column: str,initial_features: typing.List[str], test_size: float,
            random_state: int):
    """
    Train the decision tree model.

    Args:
        input_path (str): The path to the input data.
        x_train_path (str): The path to save the training features.
        y_train_path (str): The path to save the training labels.
        x_test_path (str): The path to save the testing features.
        y_test_path (str): The path to  savethe testing labels.
        model_path (str): The path to save the model.
        target_column (str): The target column.
        initial_features (list): The initial features used to train the model.
        test_size (float): The proportion of the data to use for testing.
        random_state (int): The seed for the random number generator.

    Returns:
        None
    """

    # Load the data
    logger.info("Loading the data")
    try:
        data = pd.read_csv(input_path)
    except FileNotFoundError as err:
        logger.error("Error: %s", err)
        raise err

    # Split the data into a training and testing set
    logger.info("Splitting data into training and testing set")
    try:
        x_train, x_test, y_train, y_test = train_test_split_data(data, target_column, test_size, random_state)
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    # Train the decision tree model
    try:
        logger.info("Training the decision tree model")
        dt_model = train_dt_model(x_train, y_train, initial_features, random_state)
    except ValueError as err:
        logger.error("Error: %s", err)
        raise err
    except KeyError as err:
        logger.error("Error: %s", err)
        raise err

    try:
        # Save the model
        logger.info("Saving the model")
        with open(model_path, "wb") as file:
            pickle.dump(dt_model, file)
    except FileNotFoundError as err:
        logger.error("Error: %s", err)
        raise err

    # Save the training and testing data
    logger.info("Saving the training and testing data")
    try:
        x_train.to_csv(x_train_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        y_train.to_csv(y_train_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        x_test.to_csv(x_test_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err
    try:
        y_test.to_csv(y_test_path, index=False)
    except PermissionError as err:
        logger.error("Error: %s", err)
        raise err


#!/usr/bin/env python3

CLOUD = False
try:
    from polyinterface import Controller,LOGGER
except ImportError:
    from pgc_interface import Controller,LOGGER

try:
    import polyinterface
except ImportError:
    import pgc_interface as polyinterface
    CLOUD = True

import sys
import json
import time
import http.client
import urllib.parse
from datetime import datetime
import os
import os.path
import re
import logging
from copy import deepcopy

from pgSession import pgSession
from nodes import Thermostat
from node_funcs import *

LOGGER = polyinterface.LOGGER

ECOBEE_API_URL = 'api.ecobee.com'

class Controller(Controller):
    def __init__(self, polyglot):
        super().__init__(polyglot)
        self.name = 'Ecobee Controller'
        self.tokenData = {}
        self.msgi = {}
        self.in_discover = False
        self.discover_st = False
        self.refreshingTokens = False
        self.pinRun = False
        self._last_dtns = False
        self.hb = 0
        self.ready = False
        self.waiting_on_tokens = False
        self._cloud = CLOUD

    def start(self):
        LOGGER.info('Started Ecobee v2 NodeServer')
        self.removeNoticesAll()
        self.heartbeat()
        self.serverdata = get_server_data(LOGGER)
        LOGGER.info('Ecobee NodeServer Version {}'.format(self.serverdata['version']))
        nsv = 'nodeserver_version'
        ud  = False
        cust_data = self.polyConfig['customData']
        if not nsv in cust_data:
            LOGGER.info("Adding {}={} to customData".format(nsv,self.serverdata['version']))
            cust_data[nsv] = self.serverdata['version']
            ud = True
        elif cust_data[nsv] != self.serverdata['version']:
            LOGGER.info("Update {} from {} to {} in customData".format(nsv,cust_data[nsv],self.serverdata['version']))
            cust_data[nsv] = self.serverdata['version']
            ud = True
        # Delete the saved token data we were doing for a while
        # For PG3 we can start doing it again, but not in PG2 since it costs $$
        # Can't iterate over keys, since the keys get removed, causes python crash
        keys = list(cust_data.keys())
        for key in keys:
            if re.match('tokenData2020.*',key):
                LOGGER.info("Deleting old customData key {}".format(key))
                del cust_data[key]
        if ud:
            self.saveCustomDataWait(cust_data)
        #LOGGER.debug("init=\n"+json.dumps(self.poly.init,sort_keys=True,indent=2))
        LOGGER.debug("customData=\n"+json.dumps(cust_data,sort_keys=True,indent=2))
        self.set_debug_mode()
        self.get_session()
        # Anything special to do in pgtest development mode?
        #if self.poly.init['development']:
        #
        # Cloud uses OAuth, local users PIN
        #
        self.pg_test = False
        if self._cloud:
            self.grant_type = 'authorization_code'
            self.api_key    = self.serverdata['api_key']
            # TODO: Need a better way to tell if we are on pgtest!
            #       "logBucket": "pgc-test-logbucket-19y0vctj4zlk5",
            if self.poly.stage == 'test':
                self.pg_test = True
                LOGGER.warning("Looks like we are running on to pgtest")
                self.redirect_url = 'https://pgtest.isy.io/api/oauth/callback'
            else:
                LOGGER.warning("Looks like we are running on to pgc")
                self.redirect_url = 'https://polyglot.isy.io/api/oauth/callback'
        else:
            self.grant_type = 'ecobeePin'
            self.api_key    = self.serverdata['api_key_pin']
            self.redirect_url = None
        # Force to false, and successful communication will fix it
        #self.set_ecobee_st(False) Causes it to always stay false.
        # Make sure they are using the latest API
        if self.check_api():
            if 'tokenData' in self.polyConfig['customData']:
                self.tokenData = self.polyConfig['customData']['tokenData']
                if self._checkTokens():
                    LOGGER.info("start: Calling discover...")
                    self.discover()
            else:
                LOGGER.info('No tokenData, will need to authorize...')
                self.authorize()
                self.reportDrivers()
        else:
            LOGGER.info('No tokenData, will need to authorize...')
            self._reAuth("Using old api key.")
        self.ready = True
        LOGGER.debug('done')

    def shortPoll(self):
        if not self.ready:
            LOGGER.debug("{}:shortPoll: not run, not ready...".format(self.address))
            return False
        if self.in_discover:
            LOGGER.debug("{}:shortPoll: Skipping since discover is still running".format(self.address))
            return
        if self.waiting_on_tokens is False:
            return
        elif self.waiting_on_tokens == "OAuth":
            LOGGER.debug("{}:shortPoll: Waiting for user to authorize...".format(self.address))
        else:
            # Must be waiting on our PIN Authorization
            LOGGER.debug("{}:shortPoll: Try to get tokens...".format(self.address))
            if self._getTokens(self.waiting_on_tokens):
                self.removeNoticesAll()
                LOGGER.info("shortPoll: Calling discover now that we have authorization...")
                self.discover()

    def longPoll(self):
        # Call discovery if it failed on startup
        LOGGER.debug("{}:longPoll".format(self.address))
        self.check_api() # In case the message went away...
        self.heartbeat()
        if not self.ready:
            LOGGER.debug("{}:longPoll: not run, not ready...".format(self.address))
            return False
        if self.waiting_on_tokens is not False:
            LOGGER.debug("{}:longPoll: not run, waiting for user to authorize...".format(self.address))
            return False
        if self.in_discover:
            LOGGER.debug("{}:longPoll: Skipping since discover is still running".format(self.address))
            return
        if self.discover_st is False:
            LOGGER.info("longPoll: Calling discover...")
            self.discover()
        self.updateThermostats()

    def heartbeat(self):
        LOGGER.debug('heartbeat hb={}'.format(self.hb))
        if self.hb == 0:
            self.reportCmd("DON",2)
            self.hb = 1
        else:
            self.reportCmd("DOF",2)
            self.hb = 0

    # sends a stop command for the nodeserver to Polyglot
    def exit(self):
        LOGGER.info('Asking Polyglot to stop me.')
        self.poly.send({"stop": {}})    # sends a stop command for the nodeserver to Polyglot

    def delete(self):
        LOGGER.warning("Nodeserver is being deleted...")
        # Ecobee delete tokens not working, need info from Ecobee
        #if self.ecobeeDelete():
        #    self.tokenData = {}

    def get_session(self):
        self.session = pgSession(self,self.name,LOGGER,ECOBEE_API_URL,debug_level=self.debug_level)

    def check_api(self):
        """
        Check if api being used is old and user needs to re-auth
        """
        msg = None
        if 'api_key' in self.polyConfig['customData']:
            if self.polyConfig['customData']['api_key'] != self.api_key:
                msg = 'You are using old api key "{}".'.format(self.polyConfig['customData']['api_key'])
                return False
        elif 'tokenData' in self.polyConfig['customData']:
            msg = 'No api key found, if you have previously authorized, '
        if msg is not None:
            msg += ' Please Signin to your Ecobee account. Click on Profile > My Apps > Remove App of the old Nodeserver, and resart this NodeServer'
            LOGGER.warning(msg)
            self.addNotice({'check_api': msg})
            return False
        return True

    def authorize(self):
        if self._cloud is True:
            self._getOAuth()
        else:
            self._getPin()

    def _reAuth(self, reason):
        # Need to re-auth!
        LOGGER.error('_reAuth because: {}'.format(reason))
        self.addNotice({'reauth': "Must Re-Authorize because {}".format(reason)})
        cdata = deepcopy(self.polyConfig['customData'])
        if not 'tokenData' in cdata:
            LOGGER.error('No tokenData in customData: {}'.format(cdata))
        self.saveCustomDataWait(cdata)
        self.authorize()

    def _getPin(self):
        # Ask Ecobee for our Pin and present it to the user in a notice
        res = self.session_get('authorize',
                              {
                                  'response_type':  'ecobeePin',
                                  'client_id':      self.api_key,
                                  'scope':          'smartWrite'
                              })
        if res is False:
            self.refreshingTokens = False
            return False
        res_data = res['data']
        res_code = res['code']
        if 'ecobeePin' in res_data:
            msg = 'Please Signin to your Ecobee account. Click on Profile > My Apps > Add Application and enter PIN: {} You have 10 minutes to complete this. The NodeServer will check every 60 seconds.'.format(res_data['ecobeePin'])
            LOGGER.info('_getPin: {}'.format(msg))
            self.addNotice({'getPin': msg})
            # cust_data = deepcopy(self.polyConfig['customData'])
            # cust_data['pinData'] = data
            # self.saveCustomDataWait(cust_data)
            # This will tell shortPoll to check for PIN
            self.waiting_on_tokens = res_data
        else:
            msg = 'ecobeePin Failed code={}: {}'.format(res_code,res_data)
            self.addNotice({'getPin': msg})

    def _getOAuthInit(self):
        """
        See if we have the oauth data stored already
        """
        sdata = {}
        if self._cloud:
            error = False
            if 'clientId' in self.poly.init['oauth']:
                sdata['api_client'] =  self.poly.init['oauth']['clientId']
            else:
                LOGGER.warning('Unable to find Client ID in the init oauth data: {}'.format(self.poly.init['oauth']))
                error = True
            if 'clientSecret' in self.poly.init['oauth']:
                sdata['api_key'] =  self.poly.init['oauth']['clientSecret']
            else:
                LOGGER.warning('Unable to find Client Secret in the init oauth data: {}'.format(self.poly.init['oauth']))
                error = True
            if error:
                return False
        return sdata

    def _getOAuth(self):
        # Do we have it?
        sdata = self._getOAuthInit()
        LOGGER.debug("_getOAuth: sdata={}".format(sdata))
        if sdata is not False:
            LOGGER.debug('Init={}'.format(sdata))
            self.serverdata['api_key'] = sdata['api_key']
            self.serverdata['api_client'] = sdata['api_client']
        else:
            #LOGGER.error(self.poly.init)
            url = 'https://{}/authorize?response_type=code&client_id={}&redirect_uri={}&state={}'.format(ECOBEE_API_URL,self.api_key,self.redirect_url,self.poly.init['worker'])
            msg = 'No existing Authorization found, Please Authorize access to your Ecobee Account'.format(url)
            self.addNotice({'oauth': msg})
            LOGGER.warning(msg)
            self.waiting_on_tokens = "OAuth"

    def oauth(self, oauth):
        LOGGER.info('OAUTH Received: {}'.format(oauth))
        if 'code' in oauth:
            if self._getTokens(oauth):
                self.removeNoticesAll()
                self.polyConfig['customData']['api_key'] = self.api_key
                self.discover()

    def _expire_delta(self):
        if not 'expires' in self.tokenData:
            return False
        ts_exp = datetime.strptime(self.tokenData['expires'], '%Y-%m-%dT%H:%M:%S')
        return ts_exp - datetime.now()

    def _checkTokens(self):
        if self.refreshingTokens:
            LOGGER.error('Waiting for token refresh to complete...')
            while self.refreshingTokens:
                time.sleep(.1)
        if 'access_token' in self.tokenData:
            exp_d = self._expire_delta()
            if exp_d is not False:
                # We allow for 10 long polls to refresh the token...
                if exp_d.total_seconds() < int(self.polyConfig['longPoll']) * 10:
                    self.l_info('_checkTokens','Tokens {} expires {} will expire in {} seconds, so refreshing now...'.format(self.tokenData['refresh_token'],self.tokenData['expires'],exp_d.total_seconds()))
                    return self._getRefresh()
                else:
                    # Only print this ones, then once a minute at most...
                    sd = True
                    if 'ctdt' in self.msgi:
                        md = datetime.now() - self.msgi['ctdt']
                        if md.total_seconds() < 60:
                            sd = False
                    if sd:
                        self.l_debug('_checkTokens',0,'Tokens valid until: {} ({} seconds, longPoll={})'.format(self.tokenData['expires'],exp_d.seconds,int(self.polyConfig['longPoll'])))
                    self.msgi['ctdt'] = datetime.now()
                    self.set_auth_st(True)
                    return True
            else:
                self.l_error('_checkTokens', 'No expires in tokenData:{}'.format(self.tokenData))
        else:
            self.set_auth_st(False)
            self.l_error('_checkTokens','tokenData or access_token not available')
            # self.saveCustomDataWait({})
            # this._getPin()
            return False

    def _startRefresh(self,test=False):
        # See if someone else already refreshed it?  Very small chance of this happening on PGC, but it could.
        if 'tokenData' in self.polyConfig['customData']:
            if 'refresh_token' in self.polyConfig['customData']['tokenData']:
                LOGGER.debug("{}= {}".format(self._data_tag,self.polyConfig['customData'][self._data_tag]))
                LOGGER.debug("_last_dtns={}".format(self._last_dtns))
                if self._last_dtns is not False:
                    # Check that it was ours
                    if not self._last_dtns == self.polyConfig['customData'][self._data_tag]:
                        LOGGER.error("Someone changed the db?")
                        LOGGER.error("{}= {}".format(self._data_tag,self.polyConfig['customData'][self._data_tag]))
                        LOGGER.error("_last_dtns={}".format(self._last_dtns))
                        try:
                            l_dt = datetime.fromtimestamp(self._last_dtns).strftime(self._lock_fmt)
                            c_dt = datetime.fromtimestamp(self.polyConfig['customData'][self._data_tag]).strftime(self._lock_fmt)
                            if c_dt < l_dt:
                                LOGGER.error("But it is older than what we wrote, so will ignore it...")
                            else:
                                LOGGER.error("And it's newer than what we wrote so will use it?")
                                LOGGER.error(" Mine:    {}".format(self.tokenData))
                                LOGGER.error(" Current: {}".format(self.polyConfig['customData']['tokenData']))
                                LOGGER.error("We will use the new tokens...")
                                self.tokenData = deepcopy(self.polyConfig['customData']['tokenData'])
                                return False
                        except:
                            LOGGER.error("Failed determing age of last db write by someone else, will ignore it.",exc_info=True)
        rval = self.lockCustomData()
        if (rval):
            self.refreshingTokens = True
        return rval

    # This is only called when refresh fails, when it works saveTokens clears
    # it, otherwise we get_ a race on who's customData is saved...
    def _endRefresh(self,refresh_data=False,test=False):
        cdata = deepcopy(self.polyConfig['customData'])
        if refresh_data is not False:
            if 'expires_in' in refresh_data:
                ts = time.time() + refresh_data['expires_in']
                refresh_data['expires'] = datetime.fromtimestamp(ts).strftime("%Y-%m-%dT%H:%M:%S")
            # Save new token data in customData
            cdata['tokenData'] = refresh_data
            # And save in our variable for checking, unless in test mode...
            if not test:
                self.tokenData = deepcopy(refresh_data)
            self.set_auth_st(True)
            self.removeNoticesAll()
        self.saveCustomDataWait(cdata)
        LOGGER.info('cleared lock')
        self.refreshingTokens = False

    # test option is passed in to force a refresh and save to db, but not our
    # locally saved self.tokenData.  This makes it look like someone else
    # refreshed the token, so we just grab from the db.
    def _getRefresh(self,test=False):
        if 'refresh_token' in self.tokenData:
            if not self._startRefresh(test=test):
                return False
            LOGGER.info('Attempting to refresh tokens...')
            res = self.session.post('token',
                params = {
                    'grant_type':    'refresh_token',
                    'client_id':     self.api_key,
                    'refresh_token': self.tokenData['refresh_token']
                })
            if res is False:
                self.set_ecobee_st(False)
                self._endRefresh(test=test)
                return False
            self.set_ecobee_st(True)
            res_data = res['data']
            res_code = res['code']
            if res_data is False:
                LOGGER.error('No data returned.')
            else:
                # https://www.ecobee.com/home/developer/api/documentation/v1/auth/auth-req-resp.shtml
                if 'error' in res_data:
                    self.set_ecobee_st(False)
                    self.addNotice({'grant_error': "{}: {} ".format(res_data['error'], res_data['error_description'])})
                    #self.addNotice({'grant_info': "For access_token={} refresh_token={} expires={}".format(self.tokenData['access_token'],self.tokenData['refresh_token'],self.tokenData['expires'])})
                    LOGGER.error('Requesting Auth: {} :: {}'.format(res_data['error'], res_data['error_description']))
                    LOGGER.error('For access_token={} refresh_token={} expires={}'.format(self.tokenData['access_token'],self.tokenData['refresh_token'],self.tokenData['expires']))
                    # Set auth to false for now, so user sees the error, even if we correct it later...
                    # JimBo: This can only happen if our refresh_token is bad, so we need to force a re-auth
                    if res_data['error'] == 'invalid_grant':
                        exp_d = self._expire_delta()
                        if exp_d is False:
                            self.addNotice({'grant_info': "No token expire data available, so will have to re-auth...".format(exp_d.total_seconds())})
                            LOGGER.error("No token expire data available, so will have to re-auth...".format(exp_d.total_seconds()))
                            self._reAuth('{} No token expire data available'.format(res_data['error']))
                        else:
                            # Only do this check for new authorization which tracks api_key in customData
                            if 'api_key' in self.polyConfig['customData']:
                                if exp_d.total_seconds() > 0:
                                    self.addNotice({'grant_info_2': "But token still has {} seconds to expire, so assuming this is an Ecobee server issue and will try to refresh on next poll...".format(exp_d.total_seconds())})
                                    LOGGER.error("But token still has {} seconds to expire, so assuming this is an Ecobee server issue and will try to refresh on next poll...".format(exp_d.total_seconds()))
                                else:
                                    self.addNotice({'grant_info_2': "Token expired {} seconds ago, so will have to re-auth...".format(exp_d.total_seconds())})
                                    LOGGER.error("Token expired {} seconds ago, so will have to re-auth...".format(exp_d.total_seconds()))
                                    # May need to remove the re-auth requirement because we get these and they don't seem to be real?
                                    self._reAuth('{} and Token expired'.format(res_data['error']))
                    elif res_data['error'] == 'invalid_client':
                        # We will Ignore it because it may correct itself on the next poll?
                        LOGGER.error('Ignoring invalid_client error, will try again later for now, but may need to mark it invalid if we see more than once?  See: https://github.com/Einstein42/udi-ecobee-poly/issues/60')
                    #elif res_data['error'] == 'authorization_expired':
                    #    self._reAuth('{}'.format(res_data['error']))
                    else:
                        # Should all other errors require re-auth?
                        #self._reAuth('{}'.format(res_data['error']))
                        LOGGER.error('Unknown error, not sure what to do here.  Please Generate Log Package and Notify Author with a github issue: https://github.com/Einstein42/udi-ecobee-poly/issues')
                    self._endRefresh(test=test)
                    return False
                elif 'access_token' in res_data:
                    self._endRefresh(res_data,test=test)
                    return True
        else:
            self._reAuth(' refresh_token not Found in tokenData={}'.format(self.tokenData))
        self._endRefresh(test=test)
        return False

    def _getTokens(self, pinData):
        LOGGER.debug('Attempting to get tokens for {}'.format(pinData))
        res = self.session.post('token',
            params = {
                        'grant_type':  self.grant_type,
                        'client_id':   self.api_key,
                        'code':        pinData['code'],
                        'redirect_uri': self.redirect_url
                    })
        if res is False:
            self.set_ecobee_st(False)
            self.set_auth_st(False)
            return False
        res_data = res['data']
        res_code = res['code']
        if res_data is False:
            LOGGER.error('_getTokens: No data returned.')
            self.set_auth_st(False)
            return False
        if 'error' in res_data:
            LOGGER.error('_getTokens: {} :: {}'.format(res_data['error'], res_data['error_description']))
            self.set_auth_st(False)
            if res_data['error'] == 'authorization_expired' or res_data['error'] == 'invalid_grant':
                msg = 'Nodeserver exiting because {}, please restart when you are ready to authorize.'.format(res_data['error'])
                LOGGER.error('_getTokens: {}'.format(msg))
                self.waiting_on_tokens = False
                self.removeNoticesAll()
                self.addNotice({'getTokens': msg})
                self.exit()
            return False
        if 'access_token' in res_data:
            self.waiting_on_tokens = False
            LOGGER.debug('Got tokens sucessfully.')
            self.removeNoticesAll()
            self.addNotice({'getTokens': 'Tokens obtained!'})
            self.polyConfig['customData']['api_key'] = self.api_key
            self.polyConfig['customData']['api_code'] = pinData['code']
            self._endRefresh(res_data)
            return True
        self.set_auth_st(False)

    def updateThermostats(self,force=False):
        LOGGER.debug("{}:updateThermostats: start".format(self.address))
        thermostats = self.getThermostats()
        if not isinstance(thermostats, dict):
            LOGGER.error('Thermostats instance wasn\'t dictionary. Skipping...')
            return
        for thermostatId, thermostat in thermostats.items():
            LOGGER.debug("{}:updateThermostats: {}".format(self.address,thermostatId))
            if self.checkRev(thermostat):
                address = self.thermostatIdToAddress(thermostatId)
                if address in self.nodes:
                    LOGGER.debug('Update detected in thermostat {}({}) doing full update.'.format(thermostat['name'], address))
                    fullData = self.getThermostatFull(thermostatId)
                    if fullData is not False:
                        self.nodes[address].update(thermostat, fullData)
                    else:
                        LOGGER.error('Failed to get updated data for thermostat: {}({})'.format(thermostat['name'], thermostatId))
                else:
                    LOGGER.error("Thermostat id '{}' address '{}' is not in our node list. thermostat: {}".format(thermostatId,address,thermostat))
            else:
                LOGGER.info("No {} '{}' update detected".format(thermostatId,thermostat['name']))
        LOGGER.debug("{}:updateThermostats: done".format(self.address))

    def checkRev(self, tstat):
        if tstat['thermostatId'] in self.revData:
            curData = self.revData[tstat['thermostatId']]
            if (tstat['thermostatRev'] != curData['thermostatRev']
                    or tstat['alertsRev'] != curData['alertsRev']
                    or tstat['runtimeRev'] != curData['runtimeRev']
                    or tstat['intervalRev'] != curData['intervalRev']):
                return True
        return False

    def query(self):
        self.reportDrivers()
        for node in self.nodes:
            self.nodes[node].reportDrivers()

    def stop(self):
        LOGGER.debug('NodeServer stoping...')
        self.set_ecobee_st(False)

    def thermostatIdToAddress(self,tid):
        return 't{}'.format(tid)

    def discover(self, *args, **kwargs):
        # True means we are in dsocvery
        if self.in_discover:
            LOGGER.info('Discovering Ecobee Thermostats already running?')
            return True
        self.in_discover = True
        self.discover_st = False
        try:
            self.discover_st = self._discover()
        except Exception as e:
            self.l_error('discover','failed: {}'.format(e),True)
            self.discover_st = False
        self.in_discover = False
        return self.discover_st

    def _discover(self, *args, **kwargs):
        LOGGER.info('Discovering Ecobee Thermostats')
        if not 'access_token' in self.tokenData:
            return False
        self.revData = {} # Intialize in case we fail
        thermostats = self.getThermostats()
        if thermostats is False:
            LOGGER.error("Discover Failed, No thermostats returned!  Will try again on next long poll")
            return False
        self.revData = deepcopy(thermostats)
        #
        # Build or update the profile first.
        #
        self.check_profile(thermostats)
        #
        # Now add our thermostats
        #
        for thermostatId, thermostat in thermostats.items():
            address = self.thermostatIdToAddress(thermostatId)
            if not address in self.nodes:
                fullData = self.getThermostatFull(thermostatId)
                if fullData is not False:
                    tstat = fullData['thermostatList'][0]
                    useCelsius = True if tstat['settings']['useCelsius'] else False
                    self.addNode(Thermostat(self, address, address, thermostatId,
                                            'Ecobee - {}'.format(get_valid_node_name(thermostat['name'])),
                                            thermostat, fullData, useCelsius))
        return True

    def check_profile(self,thermostats):
        self.profile_info = get_profile_info(LOGGER)
        #
        # First get all the climate programs so we can build the profile if necessary
        #
        climates = dict()
        for thermostatId, thermostat in thermostats.items():
            # Only get program data if we have the node.
            fullData = self.getThermostatSelection(thermostatId,includeProgram=True)
            if fullData is not False:
                programs = fullData['thermostatList'][0]['program']
                climates[thermostatId] = list()
                for climate in programs['climates']:
                    climates[thermostatId].append({'name': climate['name'], 'ref':climate['climateRef']})
        LOGGER.debug("check_profile: climates={}".format(climates))
        #
        # Set Default profile version if not Found
        #
        cdata = deepcopy(self.polyConfig['customData'])
        LOGGER.info('check_profile: profile_info={}'.format(self.profile_info))
        LOGGER.info('check_profile:   customData={}'.format(cdata))
        if not 'profile_info' in cdata:
            update_profile = True
        elif self.profile_info['version'] == cdata['profile_info']['version']:
            # Check if the climates are different
            update_profile = False
            LOGGER.info('check_profile: update_profile={} checking climates.'.format(update_profile))
            if 'climates' in cdata:
                current = cdata['climates']
                if not update_profile:
                    # Check if the climates have changed.
                    for id in climates:
                        if id in current:
                            if len(climates[id]) == len(current[id]):
                                for i in range(len(climates[id])):
                                    if climates[id][i] != current[id][i]:
                                        update_profile = True
                            else:
                                update_profile = True
                        else:
                            update_profile = True
            else:
                update_profile = True
        else:
            update_profile = True
        LOGGER.warning('check_profile: update_profile={}'.format(update_profile))
        if update_profile:
            self.write_profile(climates)
            self.poly.installprofile()
            cdata['profile_info'] = self.profile_info
            cdata['climates'] = climates
            self.saveCustomDataWait(cdata)

    def write_profile(self,climates):
      pfx = '{}:write_profile:'.format(self.address)
      #
      # Start the nls with the template data.
      #
      en_us_txt = "profile/nls/en_us.txt"
      make_file_dir(en_us_txt)
      LOGGER.info("{0} Writing {1}".format(pfx,en_us_txt))
      nls_tmpl = open("template/en_us.txt", "r")
      nls      = open(en_us_txt,  "w")
      for line in nls_tmpl:
        nls.write(line)
      nls_tmpl.close()
      # Open the nodedef custom for writing
      nodedef_f = 'profile/nodedef/custom.xml'
      LOGGER.info("{0} Writing {1}".format(pfx,nodedef_f))
      nodedef_h = open(nodedef_f, "w")
      nodedef_h.write('<nodedefs>\n')
      # Open the editor custom for writing
      editor_f = 'profile/editor/custom.xml'
      LOGGER.info("{0} Writing {1}".format(pfx,editor_f))
      editor_h = open(editor_f, "w")
      editor_h.write('<editors>\n')
      for id in climates:
        # Read thermostat template to write the custom version.
        in_h  = open('template/thermostat.xml','r')
        for line in in_h:
            nodedef_h.write(re.sub(r'tstatid',r'{0}'.format(id),line))
        in_h.close()
        # Read the editor template to write the custom version
        in_h  = open('template/editors.xml','r')
        for line in in_h:
            line = re.sub(r'tstatid',r'{0}'.format(id),line)
            line = re.sub(r'tstatcnta',r'{0}'.format(len(climateList)-1),line)
            # This is minus 3 because we don't allow selecting vacation or smartAway, ...
            # But not currently using this because we don't have different list for
            # status and programs?
            line = re.sub(r'tstatcnt',r'{0}'.format(len(climateList)-5),line)
            editor_h.write(line)
        in_h.close()
        # Then the NLS lines.
        nls.write("\n")
        nls.write('ND-EcobeeC_{0}-NAME = Ecobee Thermostat {0} (C)\n'.format(id))
        nls.write('ND-EcobeeC_{0}-ICON = Thermostat\n'.format(id))
        nls.write('ND-EcobeeF_{0}-NAME = Ecobee Thermostat {0} (F)\n'.format(id))
        nls.write('ND-EcobeeF_{0}-ICON = Thermostat\n'.format(id))
        # ucfirst them all
        customList = list()
        for i in range(len(climateList)):
            customList.append(climateList[i][0].upper() + climateList[i][1:])
        # Now see if there are custom names
        for i in range(len(climateList)):
            name = climateList[i]
            # Find this name in the map and replace with our name.
            for cli in climates[id]:
                if cli['ref'] == name:
                    customList[i] = cli['name']
        LOGGER.debug("{} customList={}".format(pfx,customList))
        for i in range(len(customList)):
            nls.write("CT_{}-{} = {}\n".format(id,i,customList[i]))
      nodedef_h.write('</nodedefs>\n')
      nodedef_h.close()
      editor_h.write('</editors>\n')
      editor_h.close()
      nls.close()
      LOGGER.info("{} done".format(pfx))

    # Calls session.get and converts params to weird ecobee formatting.
    def session_get (self,path,data):
        if path == 'authorize':
            # All calls before with have auth token, don't reformat with json
            return self.session.get(path,data)
        else:
            res = self.session.get(path,{ 'json': json.dumps(data) },
                                    auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token'])
                                    )
            if res is False:
                return res
            if res['data'] is False:
                return False
            self.l_debug('session_get', 0, 'res={}'.format(res))
            if not 'status' in res['data']:
                return res
            res_st_code = int(res['data']['status']['code'])
            if res_st_code == 0:
                return res
            LOGGER.error('Checking Bad Status Code {} for {}'.format(res_st_code,res))
            if res_st_code == 14:
                self.l_error('session_get', 'Token has expired, will refresh')
                # TODO: Should this be a loop instead ?
                if self._getRefresh() is True:
                    return self.session.get(path,{ 'json': json.dumps(data) },
                                     auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token']))
            elif res_st_code == 16:
                self._reAuth("session_get: Token deauthorized by user: {}".format(res))
            return False

    def getThermostats(self):
        if not self._checkTokens():
            LOGGER.debug('getThermostat failed. Couldn\'t get tokens.')
            return False
        LOGGER.debug('getThermostats: Getting Summary...')
        res = self.session_get('1/thermostatSummary',
                               {
                                    'selection': {
                                        'selectionType': 'registered',
                                        'selectionMatch': '',
                                        'includesEquipmentStatus': True
                                    },
                                })
        if res is False:
            self.set_ecobee_st(False)
            return False
        self.set_ecobee_st(True)
        thermostats = {}
        res_data = res['data']
        res_code = res['code']
        if res_data is False:
            self.l_error('getThermostats','Ecobee returned code {} but no data? ({})'.format(res_code,res_data))
            return thermostats
        if 'revisionList' in res_data:
            if res_data['revisionList'] is False:
                self.l_error('getThermostats','Ecobee returned code {} but no revisionList? ({})'.format(res_code,res_data['revisionList']))
            for thermostat in res_data['revisionList']:
                revisionArray = thermostat.split(':')
                thermostats['{}'.format(revisionArray[0])] = {
                    'name': revisionArray[1],
                    'thermostatId': revisionArray[0],
                    'connected': revisionArray[2],
                    'thermostatRev': revisionArray[3],
                    'alertsRev': revisionArray[4],
                    'runtimeRev': revisionArray[5],
                    'intervalRev': revisionArray[6]
                }
        return thermostats

    def getThermostatFull(self, id):
        return self.getThermostatSelection(id,True,True,True,True,True,True,True,True,True,True,True,True)

    def getThermostatSelection(self,id,
                               includeEvents=False,
                               includeProgram=False,
                               includeSettings=False,
                               includeRuntime=False,
                               includeExtendedRuntime=False,
                               includeLocation=False,
                               includeEquipmentStatus=False,
                               includeVersion=False,
                               includeUtility=False,
                               includeAlerts=False,
                               includeWeather=False,
                               includeSensors=False
                               ):
        if not self._checkTokens():
            LOGGER.error('getThermostat failed. Couldn\'t get tokens.')
            return False
        LOGGER.info('Getting Thermostat Data for {}'.format(id))
        res = self.session_get('1/thermostat',
                               {
                                   'selection': {
                                       'selectionType': 'thermostats',
                                       'selectionMatch': id,
                                       'includeEvents': includeEvents,
                                       'includeProgram': includeProgram,
                                       'includeSettings': includeSettings,
                                       'includeRuntime': includeRuntime,
                                       'includeExtendedRuntime': includeExtendedRuntime,
                                       'includeLocation': includeLocation,
                                       'includeEquipmentStatus': includeEquipmentStatus,
                                       'includeVersion': includeVersion,
                                       'includeUtility': includeUtility,
                                       'includeAlerts': includeAlerts,
                                       'includeWeather': includeWeather,
                                       'includeSensors': includeSensors
                                       }
                               }
                           )
        self.l_debug('getThermostatSelection',0,'done'.format(id))
        self.l_debug('getThermostatSelection',1,'data={}'.format(res))
        if res is False or res is None:
            return False
        return res['data']

    def ecobeePost(self, thermostatId, postData = {}):
        if not self._checkTokens():
            LOGGER.error('ecobeePost failed. Tokens not available.')
            return False
        LOGGER.info('Posting Update Data for Thermostat {}'.format(thermostatId))
        postData['selection'] = {
            'selectionType': 'thermostats',
            'selectionMatch': thermostatId
        }
        res = self.session.post('1/thermostat',params={'json': 'true'},payload=postData,
            auth='{} {}'.format(self.tokenData['token_type'], self.tokenData['access_token']),dump=True)
        if res is False:
            self.refreshingTokens = False
            self.set_ecobee_st(False)
            return False
        self.set_ecobee_st(True)
        if 'error' in res:
            LOGGER.error('ecobeePost: error="{}" {}'.format(res['error'], res['error_description']))
            return False
        res_data = res['data']
        res_code = res['code']
        if 'status' in res_data:
            if 'code' in res_data['status']:
                if res_data['status']['code'] == 0:
                    return True
                else:
                    LOGGER.error('Bad return code {}:{}'.format(res_data['status']['code'],res_data['status']['message']))
        return False

    def ecobeeDelete(self):
        if 'access_token' in self.tokenData:
            res = self.session.delete("/oauth2/acess_tokens/"+self.tokenData['access_token'])
            if res is False:
                return False
            if 'error' in res:
                LOGGER.error('ecobeePost: error="{}" {}'.format(res['error'], res['error_description']))
                return False
            res_data = res['data']
            res_code = res['code']
            if 'status' in res_data:
                if 'code' in res_data['status']:
                    if res_data['status']['code'] == 204:
                        LOGGER.info("Revoke successful")
                        return True
                    else:
                        LOGGER.error('Bad return code {}:{}'.format(res_data['status']['code'],res_data['status']['message']))
        else:
            LOGGER.warning("No access_token to revoke...")
        return False

    #
    # This method "locks" the DB by adding setting _data_lock to true.
    # There is no unlock method, currently it's done in saveCustomDataWait
    # because don't want to do multipe writes to save the data, then unlock it.
    #
    # TODO: Add a "wait" function with "timeout"?
    #
    # This holds the lock.
    _data_lock = '_data_lock'
    _lock_fmt  = '%Y-%m-%dT%H:%M:%S.%f'
    def lockCustomData(self):
        cdata = deepcopy(self.polyConfig['customData'])
        # Now see if someone is trying to refresh it.
        lock = cdata.get(self._data_lock)
        rval = False
        if lock is None or lock is False:
            rval = True
        else:
            LOGGER.error("Someone is already refreshing at {} will wait a while to see if it clears or take it...".format(lock))
            # See if it has expired
            ts_now = datetime.now()
            try:
                ts_start = datetime.strptime(lock,self._lock_fmt)
            except Exception as e:
                LOGGER.error('convert time {} failed {}, will grab the lock'.format(lock,e))
                rval = True
            else:
                ts_diff  = ts_now - ts_start
                if ts_diff.total_seconds() > 120:
                    LOGGER.error("But their attempt was {} seconds ago, so we will grab the lock...".format(ts_diff.total_seconds()))
                    rval = True
        if (rval):
            rval = self.saveCustomDataWait(cdata,lock=True)
        return rval

    # This holds the last date time we did saveCustomData
    _data_tag  = '_data_dtm'
    def saveCustomDataWait(self,ndata,lock=False,timeout=10):
        dtns = datetime.now().strftime(self._lock_fmt)
        ndata[self._data_tag] = dtns
        LOGGER.info("saveCustomData: {}".format(dtns))
        # Old stuff
        if 'pinData' in ndata:
            del ndata['pinData']
        if 'refresh_status' in ndata:
            del ndata['refresh_status']
        # lock=None means do nothing to the lock.
        if lock is not None:
            # If True then set to curent date/time
            # Otherwise set to what they desired, typically False
            ndata[self._data_lock] = dtns if lock else lock
        # Save it...
        done = False
        tcnt = 0
        while (not done):
            tcnt += 1
            LOGGER.info("Sending try={}\n{}={}\n{}={}\ntokenData=".format(tcnt,self._data_tag,ndata[self._data_tag],self._data_lock,ndata[self._data_lock]))
            if 'tokenData' in ndata:
                LOGGER.info("tokenData="+json.dumps(ndata['tokenData'],sort_keys=True,indent=2))
            self.saveCustomData(ndata)
            cnt = 0
            while (not done and cnt < 900):
                cd = deepcopy(self.polyConfig['customData'])
                if cd.get(self._data_tag) == dtns:
                    done = True
                else:
                    LOGGER.info("Waiting for custom data save to happen {}={} expecting {}".format(self._data_tag,cd.get(self._data_tag),dtns))
                    time.sleep(1)
                    cnt += 1
            if (done):
                self._last_dtns = dtns
                # IF we set auth false then set it back to true
                if (tcnt > 1):
                    self.set_auth_st(True)
            else:
                LOGGER.error("This may cause problems... timeout waiting for custom data save to happen {}={} expecting {}".format(self._data_tag,cd.get(self._data_tag),dtns))
                # No idea what to do in this case?  For now set auto false so can trigger a failure...
                self.set_auth_st(False)
        LOGGER.info("Done\n{}={}\n{}={}".format(self._data_tag,cd[self._data_tag],self._data_lock,cd[self._data_lock]))
        if 'tokenData' in cd:
            LOGGER.info("tokenData="+json.dumps(cd['tokenData'],sort_keys=True,indent=2))
        return True

    def cmd_poll(self,  *args, **kwargs):
        LOGGER.debug("{}:cmd_poll".format(self.address))
        self.updateThermostats(force=True)
        self.query()

    def cmd_query(self, *args, **kwargs):
        LOGGER.debug("{}:cmd_query".format(self.address))
        self.query()

    def cmd_upload_profile(self, *args, **kwargs):
        LOGGER.debug("{}:cmd_upload_profile".format(self.address))
        self.poly.installprofile()

    def cmd_debug_mode(self,command):
        val = int(command.get('value'))
        self.l_info("cmd_debug_mode",val)
        self.set_debug_mode(val)

    # This does a refresh, but doesn't record it to the local data structure so
    # the next time the NS tries to update it's runtime token will be invalid so
    # it will try to refresh, then see the DB has a new one and users it.
    # This is to manually test Issue #57
    def cmd_test_refresh(self,  *args, **kwargs):
        LOGGER.debug("{}".format(self.address))
        self._getRefresh(test=True)

    # This locks the DB, to test the auto-unlock timout feature for an old lock
    def cmd_test_lock(self,  *args, **kwargs):
        LOGGER.debug("{}".format(self.address))
        self.lockCustomData()

    def set_all_logs(self,level):
        self.l_info("set_all_logs",level)
        LOGGER.setLevel(level)
        #logging.getLogger('requests').setLevel(level)
        #logging.getLogger('urllib3').setLevel(level)

    def set_debug_mode(self,level=None):
        self.l_info("set_debug_mode",level)
        if level is None:
            try:
                level = self.getDriver('GV2')
            except:
                self.l_error('set_debug_mode','getDriver(GV2) failed',False)
            if level is None:
                level = 20
        level = int(level)
        self.debug_mode = level
        try:
            self.setDriver('GV2', level)
        except:
            self.l_error('set_debug_mode','setDriver(GV2) failed',True)
        self.debug_level = 0
        if level < 20:
            self.set_all_logs(logging.DEBUG)
            # 9 & 8 incrase pgsession debug level
            if level == 9:
                self.debug_level = 1
            elif level == 8:
                self.debug_level = 2
        elif level <= 20:
            self.set_all_logs(logging.INFO)
        elif level <= 30:
            self.set_all_logs(logging.WARNING)
        elif level <= 40:
            self.set_all_logs(logging.ERROR)
        elif level <= 50:
            self.set_all_logs(logging.CRITICAL)
        else:
            self.l_error("set_debug_mode","Unknown level {0}".format(level))
        self.l_info("set_debug_mode"," session debug_level={}".format(self.debug_level))

    def set_ecobee_st(self,val):
      ival = 1 if val else 0
      LOGGER.debug("{}:set_ecobee_st: {}={}".format(self.address,val,ival))
      self.setDriver('GV1',ival)

    def set_auth_st(self,val):
      ival = 1 if val else 0
      LOGGER.debug("{}:set_auth_st: {}={}".format(self.address,val,ival))
      self.setDriver('GV3',ival)

    def l_info(self, name, string):
        LOGGER.info("%s:%s:%s: %s" %  (self.id,self.name,name,string))

    def l_error(self, name, string, exc_info=False):
        LOGGER.error("%s:%s:%s: %s" % (self.id,self.name,name,string), exc_info=exc_info)

    def l_warning(self, name, string):
        LOGGER.warning("%s:%s:%s: %s" % (self.id,self.name,name,string))

    def l_debug(self, name, level, string, exc_info=False):
        if level <= self.debug_level:
            LOGGER.debug("%s:%s:%s: %s" % (self.id,self.name,name,string), exc_info=exc_info)

    id = 'ECO_CTR'
    commands = {
        'DISCOVER': discover,
        'QUERY': cmd_query,
        'POLL': cmd_poll,
        'DEBUG': cmd_debug_mode,
        'UPLOAD_PROFILE': cmd_upload_profile,
        'TESTREFRESH': cmd_test_refresh,
        'TESTLOCK': cmd_test_lock
    }
    drivers = [
        {'driver': 'ST', 'value': 1, 'uom': 2},
        {'driver': 'GV1', 'value': 0, 'uom': 2},
        {'driver': 'GV2', 'value': 30, 'uom': 25},
        {'driver': 'GV3', 'value': 0, 'uom': 2}
    ]


#!/usr/bin/env python3

from decimal import Decimal
import argparse
import json

from robinhood.RobinhoodCachedClient import RobinhoodCachedClient, FORCE_LIVE
from robinhood.util import ORDER_TYPES, ORDER_SIDES

# Set up the client
client = RobinhoodCachedClient()
client.login()

def place_order(order_type, order_side, symbol, quantity, price):
  quote = client.get_crypto_quote(symbol)
  currency_pair_id = quote['id']
  currency_pair = client.get_crypto_currency_pair(currency_pair_id)
  name = currency_pair['name']

  is_tradable = currency_pair['tradability'] == 'tradable'
  if not is_tradable:
    print("Sorry, {} ({}) isn't tradable".format(symbol, name))
    exit()

  min_order_size = Decimal(currency_pair['min_order_size'])
  max_order_size = Decimal(currency_pair['max_order_size'])
  crypto_increment = Decimal(currency_pair['asset_currency']['increment'])
  price_increment = Decimal(currency_pair['quote_currency']['increment'])

  if price % price_increment != 0:
    print("Sorry, price must be a multiple of {}".format(price_increment))
    exit()
  if quantity < min_order_size or quantity > max_order_size:
    print("Sorry, order quantity must be between {} and {}".format(min_order_size, max_order_size))
    exit()
  if quantity % crypto_increment != 0:
    print("Sorry, quantity must be a multiple of {}".format(crypto_increment))
    exit()

  print('')
  print('!!!!!!!!!!!!!!!!!! CAUTION !!!!!!!!!!!!!!!!!!')
  confirm = input('Are you sure that you want to {} {} {} {} ({}) for ${:.2f}? [N/y]? '.format(
    order_type,
    order_side,
    quantity,
    symbol,
    name,
    price)).lower()
  if confirm not in ['y', 'yes']:
    print('Bailed out!')
    exit()

  order = client.order_crypto(
      currency_pair_id, order_type, order_side, quantity, price)
  print(json.dumps(order, indent=4))

if __name__ == '__main__':
  parser = argparse.ArgumentParser(description='Place an order')
  parser.add_argument('order_type', choices=ORDER_TYPES)
  parser.add_argument('order_side', choices=ORDER_SIDES)
  parser.add_argument('symbol', type=str.upper, help='The cryptocurrency + currency ticker')
  parser.add_argument('quantity', type=Decimal)
  parser.add_argument('price', type=Decimal)
  args = parser.parse_args()
  place_order(
      args.order_type,
      args.order_side,
      args.symbol,
      args.quantity,
      args.price
  )


'''
https://www.codewars.com/kata/51c8e37cee245da6b40000bd/python
'''
def solution(string:str,markers):
    s_lst=string.split('\n')
    content = []
    for line in s_lst:
        find = False
        for mark in markers:
            if mark in line:
                find = True
                line = line.split(mark)[0]
                if [mark2 for mark2 in markers if mark2 in line]: continue
                content.append(line.strip())
        if find: continue
        content.append(line.strip())
    return '\n'.join(content)

print(solution("apples, pears # and bananas\ngrapes\nbananas !#apples", ["#", "!"]))