# coding: utf-8

from systemtools.basics import *
from datatools.jsonreader import *
from datatools.url import *
from machinelearning.bandit import *
from datastructuretools.setqueue import *
from machinelearning.function import *
from datatools.csvreader import *
from systemtools.basics import *
from systemtools import duration
from systemtools.duration import *
from systemtools.file import *
from systemtools.logger import *
from systemtools.location import *
from systemtools.system import *
import selenium
from selenium import webdriver
import sh
import random
import html2text
import re
from networktools import ipgetter
from threading import Thread, Lock, Semaphore, active_count, BoundedSemaphore
import math
import numpy
from hjwebbrowser.utils import *
from hjwebbrowser.browser import *
from hjwebbrowser.httpbrowser import *
from hjwebbrowser.tor import *
from queue import *
import gc

class Crawler:
    """
        The ajaxSleep param for all browser is important : you need to choose
        a long sleep (5 seconds for instance) to allow a full ajax completion
        BUT the Bandit will automatically increase the parallelRequests parameter
        to crawl more urls while some browser are waiting for Ajax. So we don't
        have to care about if there are ajax requests or not because tuhe crawling
        speed will automticaly fit the ajaxSleep param.
    """
    def __init__\
    (
        self,
        startUrls,
        paramsDomain=None,
        alpha=[0.99],
        beta=[NormalizedLawBeta.LOG],
        parallelRequests=[2, 3],
        browserCount=[5, 8],
        banditRoundDuration=120, # in seconds
        verbose=True,
        logger=None,
        alreadyCrawledFunct=None, # For example a mongo funct
        alreadyFailedFunct=None, # For example a mongo funct
        proxiesPath=None,
        checkProxy=False, # This feature doesn't work
        useProxies=True,
        proxies=None,
        maxRetryFailed=20, # 20 is a good choice
        ajaxSleep=5.0, # > 3.0 for production crawl task, you need to set a lot of browsers
        pageLoadTimeout=25,
        browserDurationHistoryCount=60, # The better choice is 60
        queueMinSize=1000, # A good choice is 1000
        terminatedCallback=None, # To get failed urls when the crawler stop
        crawlingCallback=None,
        browsersVerbose=True,
        browsersDriverType=None,
        browsersHeadless=None,
        browserMaxDuplicatePerDomain=3,
        banditExploreRate=0.2,
        banditVerbose=True,
        banditParamHistoryCount=60, # Depends on the number of possible params combinason
        stopCrawlerAfterSeconds=1000, # best is 1000
        beforeAjaxSleepCallback=None,
        afterAjaxSleepCallback=None,
        failedCallback=None,
        noPipedBrowsers=True, # Piped browser are experimentals so this bool ensure it will not use
        # This callback will, for instance, fill a form and send it,
        # DO NOT catch timeout exception because the browser (which call pipCallback)
        # DO NOT edit the given pipedMessage
        # Will automatically handle this case.
        pipCallback=None,
        notUniqueUrlsGetter=None,
        mainThreadTimeInterval=0.5,
        loadImages=True,
        phantomjsKillPattern=".*phantomjs.*octopeek.*",
        chromeKillPattern=".*chrome.*",
        sameBrowsersParallelRequestsCount=False,
        beforeGetCallback=None,
        browserUseFastError404Detection=True,
        firstRequestSleepMin=0.0,
        firstRequestSleepMax=5.0,
        allRequestsSleepMin=0.0,
        allRequestsSleepMax=0.0,
        useHTTPBrowser=False,
        allowRestartTor=False,
        deleteCrawlingElement=True,
        httpBrowserParams={},
        browserParams={},
        isInvalidFunct=None,
    ):
        """
            startUrls can be a list, an iterator or a generator
            pipCallback replace the get() function of selenium,
            so you can for example click on an element
            When you click, Selenium wait until the page is loaded !
            You don't need to do this : https://blog.codeship.com/get-selenium-to-wait-for-page-load/
            sameBrowsersParallelRequestsCount: to force the browser count to be the same as the parallel request count
        """
        # We check bandit params:
        if not isinstance(alpha, list):
            alpha = [alpha]
        if not isinstance(beta, list):
            beta = [beta]
        if not isinstance(parallelRequests, list):
            parallelRequests = [parallelRequests]
        if not isinstance(browserCount, list):
            browserCount = [browserCount]

        if sameBrowsersParallelRequestsCount:
            browserCount = parallelRequests

        # We have to remove elements from parallelRequests > max(browserCount)
        maxBrowserCount = max(browserCount)
        newParallelRequests = []
        for current in parallelRequests:
            if current <= maxBrowserCount:
                newParallelRequests.append(current)
        parallelRequests = newParallelRequests
        if len(parallelRequests) == 0:
            parallelRequests = [maxBrowserCount]

        # Domain params:
        if paramsDomain is None:
            paramsDomain = \
            {
                "proxyInstanciationRate":
                {
                    # 0.99 will choose all proxies uniformly, it's important to keep this param because
                    # proxies can fall down in the performance classement (mistake) and we will not use
                    # it again...
                    "alpha": alpha, # [0.0, 0.05, 0.2, 0.5, 0.5, 0.95, 0.99]
                    "beta": beta, # NormalizedLawBeta.EXP
                },
                "browserCount": browserCount,
                "parallelRequests": parallelRequests,
            }

        # Callback and functions:
        self.beforeAjaxSleepCallback = beforeAjaxSleepCallback
        self.afterAjaxSleepCallback = afterAjaxSleepCallback
        self.crawlingCallback = crawlingCallback
        self.terminatedCallback = terminatedCallback
        self.alreadyCrawledFunct = alreadyCrawledFunct
        self.alreadyFailedFunct = alreadyFailedFunct
        self.pipCallback = pipCallback
        self.notUniqueUrlsGetter = notUniqueUrlsGetter
        self.failedCallback = failedCallback
        self.beforeGetCallback = beforeGetCallback
        self.isInvalidFunct = isInvalidFunct

        # Experimental piped browsers:
        self.noPipedBrowsers = noPipedBrowsers

        # Memory:
        self.processing = []
        self.browsers = OrderedSetQueue()
        self.threads = []
        self.alreadyCrawled = set()
        self.failedUrls = dict()
        self.pipedBrowsers = [] # structure [(browser, dict), ...]

        # To start with a delay:
        self.startedBrowsers = SerializableDict(limit=1000)
        self.firstRequestSleepMin = firstRequestSleepMin
        self.firstRequestSleepMax = firstRequestSleepMax
        self.allRequestsSleepMin = allRequestsSleepMin
        self.allRequestsSleepMax = allRequestsSleepMax

        # Proxies:
        self.allowRestartTor = allowRestartTor
        self.useProxies = useProxies
        self.proxies = None
        if self.useProxies:
            if proxies is not None:
                self.proxies = proxies
            elif proxiesPath is not None:
                try:
                    self.proxies = getProxies(proxiesPath)
                except: pass
        if self.useProxies and (self.proxies is None or len(self.proxies) == 0):
            self.useProxies = False
            self.proxies = None

        # For the browser:
        self.browserParams = browserParams
        self.browserParams["isInvalidFunct"] = self.isInvalidFunct
        self.browsersHeadless = browsersHeadless
        self.browsersDriverType = browsersDriverType
        if self.browsersDriverType is None:
            self.browsersDriverType = DRIVER_TYPE.chrome
        self.loadImages = loadImages
        self.checkProxy = checkProxy
        self.browsersVerbose = browsersVerbose
        self.pageLoadTimeout = pageLoadTimeout
        self.ajaxSleep = ajaxSleep
        self.browserUseFastError404Detection = browserUseFastError404Detection

        # If we work with an httpBrowser:
        self.useHTTPBrowser = useHTTPBrowser
        self.httpBrowserParams = httpBrowserParams
        self.httpBrowserParams["isInvalidFunct"] = self.isInvalidFunct

        # Others:
        self.deleteCrawlingElement = deleteCrawlingElement
        self.phantomjsKillPattern = phantomjsKillPattern
        self.chromeKillPattern = chromeKillPattern
        self.mainThreadTimeInterval = mainThreadTimeInterval
        self.browserMaxDuplicatePerDomain = browserMaxDuplicatePerDomain
        self.browserDurationHistoryCount = browserDurationHistoryCount
        self.banditRoundDuration = banditRoundDuration
        self.paramsDomain = paramsDomain
        self.logger = logger
        self.maxRetryFailed = maxRetryFailed
        self.verbose = verbose
        self.previousParams = None
        self.stopCrawlerAfterSeconds = stopCrawlerAfterSeconds
        self.score = 0
        self.banditVerbose = banditVerbose
        self.banditExploreRate = banditExploreRate
        self.paused = False
        self.timer = None
        self.initTimer()
        self.initProxiesScores()

        # For the bandit:
        self.sameBrowsersParallelRequestsCount = sameBrowsersParallelRequestsCount
        self.banditParamHistoryCount = banditParamHistoryCount
        self.bandit = Bandit(paramsDomain=self.paramsDomain,
                             verbose=self.banditVerbose,
                             exploreRate=self.banditExploreRate,
                             logger=self.logger,
                             paramHistoryCount=self.banditParamHistoryCount,
                             paramsChecker=self.banditParamsChecker)

        # For this class:
        self.tt = None
        self.semaphoreCount = paramsDomain["parallelRequests"][0]

        # Locks:
        # print("d" * 1000)
        self.cacheLock = Lock() # with self.cacheLock:
        self.queueLock = Lock() # with self.queueLock:
        self.parallelSemaphore = None

        # For the queue:
        # iter convert a list, a generator or an iterator to an iterator so you can call next(self.startUrls)
        self.startUrls = iter(startUrls)
        # queueMinSize will define an event when the queue be filled:
        self.queueMinSize = queueMinSize
        # And at each event, we fill the queue with:
        self.queueFillCount = self.queueMinSize * 2
        # Finally the queue max size is:
        self.queueMaxSize = self.queueMinSize * 100
        self.queue = OrderedSetQueue(self.queueMaxSize)

        # The main thread:
        self.mainThread = None

        # We init params:
        self.nextBanditRound()

    def banditParamsChecker(self, params):
        if self.sameBrowsersParallelRequestsCount and params["browserCount"] != params["parallelRequests"]:
            return False
        elif params["browserCount"] < params["parallelRequests"]:
            return False
        return True

    def initTimer(self):
#         print("initTimer")
        if self.banditRoundDuration > 0:
            if self.timer is not None:
                self.timer.stop()
            self.timer = duration.Timer(self.nextBanditRound, self.banditRoundDuration, sleepFirst=True)

    def start(self):
#         print("start")
        # This call will start the crawling, so we set stopped as False
        self.paused = False
        # Now we can start:
        self.mainThread = Thread(target=self.run)
        self.mainThread.start()

    def run(self):
#         print("run")
        # Now for each url:
        while True:
            # We add elements to the queue if necessary:
            self.fillQueue()
            # We stop if requested by the user of this class:
            if self.paused:
                break
            else:
                # We get the next url:
                crawlingElement = None
                try:
                    # But if there are no url more, we timeout and retry fillQueue:
                    with self.cacheLock:
                        with self.queueLock:
#                             log("aa", self)
                            # First we try to get an existing pipedBrowser, it's the priority:
                            if self.hasPipedBrowser():
                                pipedMessage = self.pipedBrowsers[0][1]
                                crawlingElement = CrawlingElement(pipedMessage, type=CrawlingElement.TYPE.pipedMessage)
                            else:
                                # But a piped browser can be in the processing, so we don't get other crawlingElement
                                # which can produce other browsers clones because it can be too much:
                                if len(self.processing) < 1.5 * self.previousParams["browserCount"]: # self.noPipedBrowsers or
#                                     log("bb", self)
#                                 if self.pipedMessageInProcessingCount() < self.previousParams["browserCount"]:
#                                 if self.pipedMessageInProcessingCount() < self.previousParams["browserCount"] / 2:
#                                 if len(self.processing) < self.previousParams["browserCount"] / 2:
                                    # Now if all is ok we can get a not unique url (user function):
                                    if self.notUniqueUrlsGetter is not None:
                                        crawlingElement = self.notUniqueUrlsGetter()
                                        if isinstance(crawlingElement, str):
                                            crawlingElement = CrawlingElement(crawlingElement, type=CrawlingElement.TYPE.notUniqueUrl)
                                    # Or we get url from the unique url queue:
                                    if crawlingElement is None:
#                                         log("cc", self)
                                        crawlingElement = self.queue.get(True, self.mainThreadTimeInterval)
#                                         log("dd", self)
                                        crawlingElement = tryUrlToCrawlingElement(crawlingElement)
#                                         log("ee", self)
                                # If we have too much processing, we just sleep to prevent clonage of too much browsers:
                                else:
                                    time.sleep(self.mainThreadTimeInterval)
#                                     log("ff", self)
#                                     pass
                            # Here if we got a crawlingElement, we throw it to the thread crawl
                            # But here we just add the element to processing, we throw it below because we need
                            # to unlock:
                            if crawlingElement is not None:
                                # We add the url in processing:
                                self.processing.append(crawlingElement)
#                                 log("gg", self)
                                # We None the tt because we got an url:
                                self.tt = None
                except Exception as e:
#                     log("hh", self)
                    if isinstance(e, Empty):
                        pass
                    else:
                        logError("Exception location: Crawler.run():", self)
                        logError(str(e), self)
                    with self.cacheLock:
                        # Here if the user want to stop when we have no url for stopCrawlerAfterSecond
                        # We also have to check if there are url currently in a crawling process:
                        if self.stopCrawlerAfterSeconds is not None \
                            and self.stopCrawlerAfterSeconds > 0 \
                            and len(self.processing) == 0:
#                             log("ii", self)
                            if self.tt is None:
                                # We init the TicToc to watch if the queue is empty for stopCrawlerAfterSeconds
                                self.tt = TicToc()
                                self.tt.tic(display=False)
                            # We check if the total duration since the first fail is > stopCrawlerAfterSecond and we stop:
                            elif self.tt.toc(display=False) > self.stopCrawlerAfterSeconds:
#                                 log("jj", self)
                                self.tt = None
                                stopperThread = self.stopInThread()
                # We crawl only if we got an url, otherwise we retry to fill the queue:
                if crawlingElement is not None:
#                     log("kk", self)
                    self.crawl(crawlingElement)

    def pipedMessageInProcessingCount(self):
        count = 0
        for current in self.processing:
            if current.type == CrawlingElement.TYPE.pipedMessage:
                count += 1
        return count

    def stopInThread(self):
#         print("stopInThread")
        stopperThread = Thread(target=self.stop)
        stopperThread.start()
        return stopperThread

    def pauseInThread(self):
#         print("pauseInThread")
        pauserThread = Thread(target=self.pause)
        pauserThread.start()
        return pauserThread

    def stop(self):
        """
            public function
            The second list returned in terminatedCallback is urls which failed maxRetryFailed times
            The first one failed < maxRetryFailed : in most case this list is empty. You can start the
            crawler again if you want to
        """
#         print("stop")
        self.pause()
        # Now we purge and save duplicates:
#         Browser.duplicates.purgeAndSave() # Deprecated
        # Here we can display all url which failed:
#         if len(self.failedUrls) > 0:
#             logWarning("These urls failed:\n" + listToStr(self.failedUrls), self)
        # And we can display url count:
        logInfo(str(len(self.alreadyCrawled)) + " unique urls were crawled during this session", self)
        # We can add all failed urls in mongodb (an other collection):
        if self.terminatedCallback is not None:
            urlsFailedEnough = []
            urlsFailedNotEnough = []
            for key, value in self.failedUrls.items():
                if value == self.maxRetryFailed:
                    urlsFailedEnough.append(key)
                else:
                    urlsFailedNotEnough.append(key)
            self.terminatedCallback(urlsFailedNotEnough, urlsFailedEnough)

    def pause(self, display=True):
        """
            public function
        """
#         print("pause")
#         with self.cacheLock: # cacheLock en conflict avec cacheLock de start et fillQueue
        # We stop the start method:
        self.paused = True
        # And stop the timer:
        if self.timer is not None:
            self.timer.stop()
        self.initTimer()
        # Finally we join all threads:
        for thread in self.threads:
            thread.join()
        self.threads = []
        # Finally we join the main thread:
        self.mainThread.join()

#     def putCrawlingElement(self, crawlingElement):
#         """
#             public function
#             crawlingElement can be an url or a crawlingElement
#             {"data": "<url>", "type": <CRAWLING_ELEMENT_TYPE>}
#         """
#         # We convert the url:
#         crawlingElement = urlToCrawlingElement(crawlingElement)
#         with self.cacheLock:
#             if self.paused:
#                 return False
#             self.put(crawlingElement["data"])
#         return True

    def put(self, crawlingElement):
        """
            You can use this function if you want to put new url when receiving html in the callback funct
        """
        # We convert the url:
        crawlingElement = tryUrlToCrawlingElement(crawlingElement)
#         print("put")
        # We check if the url is to crawl (not in mongo, arlready crawled or already failed...):
        with self.queueLock:
#             print("3>>>>>>>>>>>>>>>>>>>>>> paused: " + str(self.paused))
            if self.isCrawlingElementToCrawl(crawlingElement):
                self.queue.put(crawlingElement)

    def putFail(self, crawlingElement):
        # We convert the url:
        crawlingElement = tryUrlToCrawlingElement(crawlingElement)
        log("We add this url to failedUrls and remove it from alreadyCrawled: " + str(crawlingElement), self)
        with self.queueLock:
            if crawlingElement not in self.failedUrls:
                self.failedUrls[crawlingElement] = 0
            self.failedUrls[crawlingElement] += 1
            # Now we add the url in alreadyCrawled to not crawl it again in this session:
            if crawlingElement in self.alreadyCrawled:
                self.alreadyCrawled.remove(crawlingElement)

    def fillQueue(self):
#         log("fillQueue", self)
#         log("queueSize: " + str(self.queue.size()), self)
        with self.cacheLock:
            with self.queueLock:
                queueSize = self.queue.size()
#                 log("queueSize: " + str(queueSize), self)
            # If the queue is too small:
            if queueSize < self.queueMinSize:
#                 log("a", self)
                # We first add failed urls to the queue:
                # We retry all url which failed:
                for key, value in self.failedUrls.items():
                    # We apply a max failed count and we check if the url is not actually "in retrying":
#                     log("b", self)
                    if value < self.maxRetryFailed:
#                         log("put", self)
                        # So we add this url to to tryingFailed:
                        self.put(key)
                # Then we add urls from self.startUrls:
                for i in range(self.queueFillCount):
#                     log("c", self)
                    try:
                        crawlingElement = next(self.startUrls)
                        crawlingElement = tryUrlToCrawlingElement(crawlingElement)
#                         log("d", self)
                        self.put(crawlingElement)
                    except Exception as e:
                        if isinstance(e, StopIteration):
#                             log("e", self)
                            pass
                            break
                        else:
                            logException(e, self, location="Crawler.fillQueue()")
#                             logError("Exception location: :", self)
#                             logError(str(e), self)
#                             print(str(type(e)))

#     def normalizeParamsDomain(self):
#         """
#             This method will set paramsDomain['browserCount'] to value > max(paramsDomain['parallelRequests'])
#         """
#         maxRIP = max(self.paramsDomain['parallelRequests'])
#         newBrowserCount = []
#         for current in self.paramsDomain['browserCount']:
#             if current >= maxRIP:
#                 newBrowserCount.append(current)
#         self.paramsDomain['browserCount'] = newBrowserCount

    def initProxiesScores(self):
        # We init allScores to all ip with the timeout duration:
        allScores = {}
        if self.proxies is not None and self.useProxies:
            for current in self.proxies:
                # 0.0 request duration to take this proxy at least one time:
                allScores[current["ip"]] = [0.0]
        self.proxiesScores = allScores

    def getScores(self):
        """
=            This function takes all scores for each proxy ip (2 browsers can have the same ip)
            and average the scre on all durations.
        """
#         random.seed(0)
#         theDict = {}
#         for i in range(100):
#             theDict[getRandomInt(1, 10000)] = getRandomFloat(1.5, 3.0)
#         return theDict

        # We return a fake score if we don't use proxies:
        if not self.useProxies:
            return {"0.0.0.0": 0.0}

        # First we reset all scores of browser we just used in this round
        # All others proxies not used at this round will keep previous scores:
        browserList = queueToList(self.browsers)
        for b in browserList:
            if b.durationHistory is not None:
                if len(removeNone(list(b.durationHistory))) > 0:
                    self.proxiesScores[b.getProxy()["ip"]] = [0.0]
        # Then we update score according to already used browsers (several browser per proxy):
        for b in browserList:
            # We check if the history of this brwoser is None:
            if b.durationHistory is not None:
                # Then we check if there are values in:
                durationHistoryWithoutNone = removeNone(list(b.durationHistory))
                if durationHistoryWithoutNone is not None and len(durationHistoryWithoutNone) > 0:
                    # Then we can update self.proxiesScores:
                    self.proxiesScores[b.getProxy()["ip"]] += durationHistoryWithoutNone
        allScores = {}
        # Finally we compute means of all proxies:
        for ip, scoreList in self.proxiesScores.items():
            if scoreList is not None and len(scoreList) > 0:
                allScores[ip] = sum(scoreList) / len(scoreList)
            else:
                logError("This ip has no score list!", self)
#                 exit()

        return allScores

    def getProxyInstanciation(self, params, correctRound=True, display=True):
        """
            The instanciation is a list of ip with a number of browser according to his score
            (the mean time duration of requests)
        """
        def instanciationSameOrder(scores, instanciation):
            newInstanciation = []
            if isinstance(instanciation, list):
                instanciation = tuplesToDict(instanciation)
            for ip, score in scores:
                newInstanciation.append((ip, instanciation[ip]))
            return newInstanciation

        if self.proxies is None or not self.useProxies:
            return {}

        # We get params:
        proxyInstanciationRate = params["proxyInstanciationRate"]
        alpha = float(proxyInstanciationRate["alpha"])
        beta = proxyInstanciationRate["beta"]
        browserCount = params["browserCount"]
        # We get scores, min, max...
        scores = self.getScores()
        scores = sortBy(scores, desc=False)
        # We generate the instanciation according to normalizedLaw, normalized numbers
        # for example 0.6, 0.1, 0.1, 0.1, 0.1 which mean the best proxy will have 60% of all
        # browser etc:
        instanciationList = [normalizedLaw(x / len(scores), alpha=alpha, beta=beta) for x in range(len(scores))]
        instanciationList = normalize(instanciationList)
        instanciation = list(scores)
        instanciation = sortBy(instanciation, desc=False)
        noChanceMemory = [] # We use this to retain float number association to correct the browserCount...
        i = 0
        newInstanciation = []
        for ip, score in instanciation:
            rate = instanciationList[i] * browserCount
            noChanceMemory.append((ip, rate))
            newInstanciation.append((ip, round(rate)))
            i += 1
        instanciation = newInstanciation
        # Here we have:
        #  * instanciation which contains ip and a number of browser
        #  * noChanceMemory which contains ip with a number of browser (float value)
        #  * scores which contains ip with each score (lower the score is, better the proxy is)
        # And if we have not the correct sum of browser, we find which element
        # was unlucky : for exemple if an ip have 5.45 browsers, he got 5 browser -> unlucky
        # The same but inverse if we have too much browsers. If
        if correctRound:
            theSum = sum(nbBrowser for ip, nbBrowser in instanciation)
            def getNoChance(noChanceMemory):
                noChance = []
                for ip, instanciationFloat in noChanceMemory:
                    currentNoChance = instanciationFloat - math.floor(instanciationFloat)
                    if currentNoChance < 0.5:
                        currentNoChance += 1.0
                    noChance.append((ip, currentNoChance))
                return noChance
            if theSum == browserCount:
                pass
            elif theSum < browserCount:
                diff = browserCount - theSum
                noChance = getNoChance(noChanceMemory)
                noChance = sortBy(noChance, index=1, desc=True)
                instanciation = tuplesToDict(instanciation)
                for i in range(diff):
                    ip = noChance[i][0]
                    instanciation[ip] += 1
                instanciation = dictToTuples(instanciation)
                instanciation = instanciationSameOrder(scores, instanciation)
            elif theSum > browserCount:
                diff = theSum - browserCount
                noChance = getNoChance(noChanceMemory)
                # The first is the one who had the most luck:
                noChance = sortBy(noChance, index=1, desc=False)
                instanciation = tuplesToDict(instanciation)
                for i in range(diff):
                    ip = noChance[i][0]
                    instanciation[ip] -= 1
                instanciation = dictToTuples(instanciation)
                instanciation = instanciationSameOrder(scores, instanciation)

        if display:
            u = 0
            noChanceMemoryDict = tuplesToDict(noChanceMemory)
            currentScores = self.getScores()
            strLarge = "Proxies instanciation (score (mean duration in seconds), ip, instanciation):\n"
            for key, value in instanciation:
                strLarge += str(int(currentScores[key])) + "\t"
                strLarge += str(key) + "\t"
                for i in range(value):
                    strLarge += "-"
                strLarge += "\n"
                u += 1
            logInfo(strLarge, self)

#             somme = 0
#
#             for key, value in instanciation:
#                 somme += value
#             if somme < browserCount:
#                 print(alpha)
#                 print(somme)
#             logInfo("proxyInstanciationRate=" + str(proxyInstanciationRate), self)
#             print("browserCount=" + str(browserCount))
#             print("somme=" + str(somme))
#             print("\n\n")

        instanciation = tuplesToDict(instanciation)
        return instanciation

    def isCrawlingElementToCrawl(self, crawlingElement):
        """
            The caller have to cacheLock
        """
        if crawlingElement.type != CrawlingElement.TYPE.uniqueUrl:
            return True
        # If the url is in mongo db, we don't crawl it again:
        if self.alreadyCrawledFunct is not None:
            if self.alreadyCrawledFunct(crawlingElement):
                return False
        if self.alreadyFailedFunct is not None:
            if self.alreadyFailedFunct(crawlingElement):
                return False
        # Or if the url is currently in processing, we don't add it:
        if crawlingElement in self.processing:
            return False
        # Or if this url is already crawled in the current session, we don't crawl it agani:
        if crawlingElement in self.alreadyCrawled:
            return False
        return True

    def addBrowser(self, proxyIp):
#         print("addBrowser")
        # We find the right proxy:
        theProxy = None
        if self.useProxies:
            if isinstance(proxyIp, dict):
                theProxy = proxyIp
            else:
                for current in self.proxies:
                    if current["ip"] == proxyIp:
                        theProxy = current
                        break
            if theProxy is None:
                logError("The proxy " + str(proxyIp) + " doesn't exist!", self)
                exit()
        if self.useHTTPBrowser:
            b = HTTPBrowser \
            (
                verbose=self.browsersVerbose,
                logger=self.logger,
                proxy=theProxy,
                pageLoadTimeout=self.pageLoadTimeout,
                htmlCallback=self.htmlCallback,
                maxDuplicatePerDomain=self.browserMaxDuplicatePerDomain,
                durationHistoryCount=self.browserDurationHistoryCount,
                **self.httpBrowserParams,
            )
        else:
            # Then we create the browser:
            b = Browser \
            (
                proxy=theProxy,
                headless=self.browsersHeadless,
                driverType=self.browsersDriverType,
                pageLoadTimeout=self.pageLoadTimeout,
                ajaxSleep=self.ajaxSleep,
                durationHistoryCount=self.browserDurationHistoryCount,
                verbose=self.browsersVerbose,
                logger=self.logger,
                htmlCallback=self.htmlCallback,
                beforeAjaxSleepCallback=self.beforeAjaxSleepCallback,
                afterAjaxSleepCallback=self.afterAjaxSleepCallback,
                maxDuplicatePerDomain=self.browserMaxDuplicatePerDomain,
                loadImages=self.loadImages,
                useFastError404Detection=self.browserUseFastError404Detection,
                **self.browserParams,
            )
            if self.checkProxy:
                if not b.checkProxy():
                    return False
        if b.driver is not None or self.useHTTPBrowser:
            self.browsers.put(b)
        return True

    def closeBrowser(self, b):
        return b.quit()

    def close(self):
        self.resetBrowsers()

    def resetBrowsers(self):
#         print("resetBrowsers")
        allClosed = True
        if self.browsers is not None:
            browsersList = queueToList(self.browsers)
            if browsersList is not None:
                for b in browsersList:
                    currentClosed = self.closeBrowser(b)
                    if not currentClosed:
                        allClosed = False
            self.browsers = OrderedSetQueue()
#         if not allClosed:
        if self.pipCallback is None:
            try:
                if not self.useHTTPBrowser:
                    # Because sometimes a phantomjs can't be stopped, so memory overflow...:
                    sh.pkill("-f", self.phantomjsKillPattern)
                    sh.pkill("-f", self.chromeKillPattern)
            except Exception as e:
                pass
#                 logException(e, self, location="crawler.resetBrowsers()")

    def nextBanditRound(self):
#         print("nextBanditRound")
#         # First we skip this bandit round if a browser is piped:
#         # we don't need to do this because piped browsers are protected in self.pipedBrowsers
#         with self.cacheLock:
#             if self.hasPipedBrowser():
#                 self.score = 0
#                 return
        # We have to pause the thread because some threads can remains and cacheLock
        # We do this only if we are runnning, i.e.:
        alreadyPaused = False
        if self.mainThread is not None:
            if self.paused:
                alreadyPaused = True
            pauserThread = self.pauseInThread()
            pauserThread.join()
#             print("2>>>>>>>>>>>>>>>>>>>>>> paused: " + str(self.paused))
        with self.cacheLock:
            # First we have the overall score
            # Warning, the score of a browser is the mean of all requests duration in the history
            # The score of the current param is the number of succeeded requests
            # Now we get next params given by the bandit, we give the current so he can retains
            # that the previous params succed to this score:
            if self.previousParams is None:
                newParams = self.bandit.nextParams()
#                 while True:
#                     newParams = self.bandit.nextParams()
#                     logInfo("These params are OK:", self)
#                     logInfo(listToStr(newParams), self)
#                     time.sleep(0.5)
            else:
                newParams = self.bandit.nextParams(self.score)
            # We restart the tor service:
            try:
                if self.allowRestartTor:
                    if torSingletonExists():
                        getTorSingleton(logger=self.logger, verbose=self.verbose).restart()
                    else:
                        getTorSingleton(logger=self.logger, verbose=self.verbose)
            except Exception as e:
                logException(e, self, location="nextBanditRound")
            # We reset the score for this new round:
            self.score = 0
            # If we have same params, we just continue with same browsers:
            if newParams == self.previousParams:
                # WARNING : here we do not re-instanciate proxies
                # i.e. we do not re-calculate scores of each proxy and
                # do not choose a new instanciation (a new partitionning) of these
                # proxies for each browser...
                pass
            # If this is other params (explore), we reload all:
            else:
                # And we wait all threads finish utl requests:
#                 for currentThread in self.threads:
#                     currentThread.join()
                # We save newParams as previousParams:
                self.previousParams = newParams
                # We first init the parallel semaphore:
                self.parallelSemaphore = BoundedSemaphore(newParams["parallelRequests"])
                # Now we instanciate all browsers:
                proxiesInstanciation = self.getProxyInstanciation(newParams)
                self.resetBrowsers()
#                 if self.mainThread is not None:
#                     time.sleep(1000) # TODO delete
                instanciationThreads = []
                # We define a function which take a list of thread and launch it, the wait all:
                def launchAllThread(currentInstanciationThreads):
                    for current in currentInstanciationThreads:
                        current.start()
                    for current in currentInstanciationThreads:
                        current.join()
                if self.useProxies:
                    # While we still have browsers :
                    parallelBrowserInit = newParams["parallelRequests"]
                    if not self.checkProxy:
                        parallelBrowserInit = newParams["browserCount"]
                    while len(proxiesInstanciation) > 0:
                        # We get the first browser we found in proxiesInstanciation:
                        currentInstan = dictRandomElement(proxiesInstanciation)
                        currentBrowserIp = currentInstan[0]
                        currentBrowserCount = currentInstan[1]
                        # And if the current instanciation row is 0, we pop it and continue the while loop:
                        if currentBrowserCount == 0:
                            proxiesInstanciation.pop(currentBrowserIp)
                        else:
                            # We decrement the browser amount for this ip:
                            proxiesInstanciation[currentBrowserIp] -= 1
                            # Here we create one thread for this browser:
                            theThread = Thread(target=self.addBrowser, args=(currentBrowserIp,))
                            instanciationThreads.append(theThread)
                            # And if we have a number of threads equals to the number of requete in parallel:
                            if len(instanciationThreads) == parallelBrowserInit:
                                # We lanch all and wait:
                                launchAllThread(instanciationThreads)
                                # Then we have to reset the thread list:
                                instanciationThreads = []
                    # Now maybe some threads not started remain, so we have to launch all of them:
                    launchAllThread(instanciationThreads)
                else:
                    for i in range(newParams["browserCount"]):
                        theThread = Thread(target=self.addBrowser, args=(None,))
                        instanciationThreads.append(theThread)
                    instanciationThreadsChunks = chunks(instanciationThreads, cpuCount() * 4)
                    for currentThreads in instanciationThreadsChunks:
                        launchAllThread(currentThreads)

            # Here the we are in the case we first execute nextBanditRound to get
            # first params, so we will start after, we just need to start the timer
            # of the bandit:
            if not alreadyPaused and self.mainThread is None:
                # We start the timer which will call the nextBanditRound method again:
                self.startTimer()
            # Here it's the timer which call nextBanditRound, and we paused before,
            # So we need to restart the crawling and start the timer again,
            # But if the crawler was already paused by a queue timeout, we don't restart it
            elif not alreadyPaused:
                # We start the timer which will call the nextBanditRound method again:
                self.startTimer()
                # Finally we restart the crawler:
                self.start()

    def startTimer(self):
        if self.banditRoundDuration > 0:
            self.timer.start()

    def acquire(self):
#         self.semaphoreCount -= 1
#         print(">>>>>>> Trying to acquire " + str(self.semaphoreCount))
        self.parallelSemaphore.acquire()
#         print(">>>>>>> Acquired ")

    def release(self):
        try:
#         self.semaphoreCount += 1
#         print(">>>>>>> Trying to release " + str(self.semaphoreCount))
            self.parallelSemaphore.release()
#         print(">>>>>>> Released")
        except Exception as e:
            logError("Exception location: crawler.release()\n" + str(e), self)

# Attention, les règles à respecter pour les locks : ne jamais imbriquer lock1 dans lock2 alors que c'est l'inverse autre part dans le code.
# Toujours vérifier que le code qui doit etre parallélisé ne lock rien (typiquement le download)
# Sinon ça ne produit rien de parralèle au final
# toujours regarder la fin d'un with: on doit sortir sans avoir executer de code long.
# Si on a du code long il faut lancer des thread à la fin du with pour que le lock soit libéré

    def crawl(self, crawlingElement):
        """
            Private method
        """
#         print("crawl")
        # We take authorization to launch a new thread:
#         print("tentative d'acquire dans le crawl")
        self.acquire()
        browser = None
        with self.cacheLock:
            if crawlingElement.type == CrawlingElement.TYPE.pipedMessage:
                browser = self.popPipedBrowser(crawlingElement.data)
        # Block here until a brwser is added
        if browser is None:
            browser = self.browsers.get()
#             logInfo("\t==> " + browser.name + " taken! <==", self)

        # Then we take a random browser:
        with self.cacheLock:
            if browser is None:
                logError("The start method can't take a random browsers in self.browsers list!", self)
                exit()
            # Now we start a new thread which have to call html
            # The callback have to release the semaphore and add a point to score if the request suceeded
            # Then this callback also have to store the result in mongodb (with mongodbLock)
            # add the url in self.alreadyCrawled...
            theThread = Thread(target=self.threadedCrawl, args=(crawlingElement, browser,))
            # Now we add the url directly in the run method:
#             # We add the url in processing:
#             self.processing.add(url)
            # We add the thread:
            theThread.start()
            self.threads.append(theThread)

#     def browserCount(self):
#         browserCount = self.browsers.size()
#         for current in self.pipedBrowsers:
#             browserCount += 1

    def threadedCrawl(self, crawlingElement, browser):
        """
            Private method
        """
        # If it is the first time we use this browser,
        # we sleep to not send a lot a request at the same time:
        if browser.name not in self.startedBrowsers or not self.startedBrowsers[browser.name]:
            sleepDuration = randomSleep(self.firstRequestSleepMin, self.firstRequestSleepMax)
            # log(browser.name + " just slept " + str(sleepDuration) + " seconds because it's the first request for this browser.", self)
            self.startedBrowsers[browser.name] = True
        # Then we sleep for each request:
        if self.allRequestsSleepMax > 0.0:
            sleepDuration = randomSleep(self.allRequestsSleepMin, self.allRequestsSleepMax)
            # log(browser.name + " just slept " + str(sleepDuration) + " seconds before requesting " + crawlingElement.toString(), self)

        # First we check if this is a piped browser:
        pipCallback = None
        if crawlingElement.type == CrawlingElement.TYPE.pipedMessage:
            # The we get the pip callback, the user of the class will do some things
            # on the browser instead of a simple get of an url:
            pipCallback = self.pipCallback
        # Here this will be executed in parallel, so we don't cacheLock anything
        if self.beforeGetCallback is not None:
            self.beforeGetCallback(browser)
        ok = browser.get(crawlingElement, pipCallback=pipCallback)
        # Finally we release semaphore:
        self.release()

#     def getPipedBrowser(self, currentCrawlingElement):
#         """
#             The caller have to cacheLock cacheLock
#         """
#         for pipedBrowser, crawlingElement in self.pipedBrowsers:
#             if currentCrawlingElement is crawlingElement:
#                 return pipedBrowser
#         return None

    def hasPipedBrowser(self):
        """
            The caller have to cacheLock cacheLock
        """
        return len(self.pipedBrowsers) > 0

#     def isPipedBrowser(self, browser):
#         """
#             The caller have to cacheLock cacheLock
#         """
#         for pipedBrowser, crawlingElement in self.pipedBrowsers.items():
#             if pipedBrowser is browser:
#                 return True
#         return False

#     def getPipedMessage(self, browser):
#         """
#             The caller have to cacheLock cacheLock
#         """
#         for pipedBrowser, pipedMessage in self.pipedBrowsers:
#             if pipedBrowser is browser:
#                 return pipedMessage
#         return None

#     def popPipedMessage(self, browser):
#         """
#             The caller have to cacheLock cacheLock
#         """
#         newPipedBrowsers = []
#         pipedMessageToReturn = None
#         for pipedBrowser, pipedMessage in self.pipedBrowsers:
#             if pipedBrowser is browser:
#                 pipedMessageToReturn = pipedMessage
#             else:
#                 newPipedBrowsers.append((pipedBrowser, pipedMessage))
#         self.pipedBrowsers = newPipedBrowsers
#         return pipedMessageToReturn

    def popPipedBrowser(self, currentPipedMessage):
        """
            The caller have to cacheLock cacheLock
        """
        newPipedBrowsers = []
        desiredPipedBrowser = None
        for pipedBrowser, pipedMessage in self.pipedBrowsers:
            if pipedMessage is currentPipedMessage:
                desiredPipedBrowser = pipedBrowser
            else:
                newPipedBrowsers.append((pipedBrowser, pipedMessage))
        self.pipedBrowsers = newPipedBrowsers
        return desiredPipedBrowser

    def addPipedBrowser(self, browser, pipedMessage):
        """
            The caller have to cacheLock cacheLock
        """
        if pipedMessage is not None:
            self.pipedBrowsers.append((browser, pipedMessage))

    def removeOneFromProcessing(self, crawlingElement):
        if crawlingElement is not None:
            alreadyPassed = False
            newProcessing = []
            for current in self.processing:
                if not alreadyPassed and crawlingElement == current:
                    alreadyPassed = True
                else:
                    newProcessing.append(current)
#             if len(self.processing) == len(newProcessing) or not alreadyPassed:
#                 logError("We didnt' find the crawling element to delete...", self)
#                 logError(str(crawlingElement), self)
            self.processing = newProcessing
#         else:
#             logError("The crawling element is None...", self)

    def htmlCallback(self, data, browser):
        """
            Private method
        """
        browserToClone = False
        browserToPut = True
        # Now we got the data, we cacheLock browsers:
        with self.cacheLock:
            # Get the url:
            crawlingElement = data["crawlingElement"]
            status = data["status"]
#             data["crawler"] = browser.getCrawlerName()
            # First we remove the crawled url from processing:
            self.removeOneFromProcessing(crawlingElement)
            # Then we process data:
            if data is not None:
                # And if we got html:
                if status.name in "success error404 timeoutWithContent".split(" "):
                    if crawlingElement.type == CrawlingElement.TYPE.uniqueUrl:
                        # If this was a previous failed url, we remove it
                        self.failedUrls.pop(crawlingElement, None)
                        # Now we add the url in alreadyCrawled to not crawl it again in this session:
                        self.alreadyCrawled.add(crawlingElement)
                    # And finally we store this data in mongodb (for example):
                    try:
                        if self.deleteCrawlingElement and "crawlingElement" in data:
                            del data["crawlingElement"]
                        pipedMessage = self.crawlingCallback(data, browser=browser)
                    except Exception as e:
                        logException(e, self, location="Crawler.htmlCallback")
                        pipedMessage = None

                    # 4 possibilités
                    # 1. On a pas de piped message et ce n'est pas une reprise de pipedMessage
                    #     * On ajoute juste le browser dans la queue
                    # 2. On a un pipedMessage et on en avait un aussi avant
                    #     * On ne l'ajoute pas dans la queue
                    #     * On l'ajoute dans les piped browsers
                    # 3. On a un pipedMessage et on en avait pas avant:
                    #     * On doit cloner le browser et ajouter le clone dans la queue
                    #     * On doit ajouter le browser dans les piped browsers
                    # 4. On a pas de piped message mais on en avait un avant
                    #     * On doit close le browser puisqu'on ne l'utilisera plus
                    #     * On ne l'ajoute pas dans la queue
                    if pipedMessage is None: # 1 and 4
                        if crawlingElement.type != CrawlingElement.TYPE.pipedMessage: # 1
                            browserToClone = False
                            browserToPut = True
                        else: # 4
                            self.closeBrowser(browser)
                            browserToClone = False
                            browserToPut = False
                    else: # 2 and 3
                        if crawlingElement.type != CrawlingElement.TYPE.pipedMessage: # 3
                            browserToClone = True
                            browserToPut = True
                            self.addPipedBrowser(browser, pipedMessage)
                        else: # 2
                            browserToClone = False
                            browserToPut = False
                            self.addPipedBrowser(browser, pipedMessage)
                    # The request succeded so we add a point:
                    if status == REQUEST_STATUS.timeoutWithContent:
                        self.score += 0.8
                    else:
                        self.score += 1
                else:
#                     printLTS(data)
                    if self.failedCallback is not None:
                        if self.deleteCrawlingElement and "crawlingElement" in data:
                            del data["crawlingElement"]
                        self.failedCallback(data)
                    if crawlingElement.type == CrawlingElement.TYPE.uniqueUrl:
                        if self.maxRetryFailed > 0:
                            # Here the crawling failed:
                            # So if the url failed the first time, we add it in failedUrls:
                            if crawlingElement not in self.failedUrls:
                                self.failedUrls[crawlingElement] = 1
                            # Else we inc the counter:
                            else:
                                self.failedUrls[crawlingElement] += 1
                    self.score = self.score - 0.1
#                     if self.score < 0.0:
#                         self.score = 0.0
        # Now all is finished with this browser, so we can re-add it in the list
        # Or it can be a new clone:
        if browserToClone:
            browser = browser.clone()
        if browserToPut:
#             logInfo("\t==> " + browser.name + " released! <==", self)
            self.browsers.put(browser)

def funct1(data):
    pass
def funct2(urlsFailedNotEnough, urlsFailedEnough):
    print("urlsFailedNotEnough:")
    printLTS(urlsFailedNotEnough)
    print("urlsFailedEnough:")
    printLTS(urlsFailedEnough)
    tt.toc()

def printCountThread():
#     for i in range(50):
    print(">>>>>>>>>>>>>>> " + str(active_count()))

if __name__ == '__main__':

    countGlobal = 0

    tt = TicToc()
    tt.tic()

    mb = Crawler \
    (
        generateAjaxRandomURLs(100),
        paramsDomain = \
        {
            "proxyInstanciationRate":
            {
                "alpha": [0.99],
                "beta": [NormalizedLawBeta.EXP]
            },
            "browserCount": [10],
            "parallelRequests": [10], # [2, 3, 4, 5, 6, 50]
        },
        pageLoadTimeout=5.0, # 2.1
        crawlingCallback=funct1,
        terminatedCallback=funct2,
        browsersVerbose=True,
        banditRoundDuration=8,
        stopCrawlerAfterSeconds=5,
        queueMinSize=30,
        maxRetryFailed=2,
        ajaxSleep=2,
        browserMaxDuplicatePerDomain=2,
        useProxies=False,
    )

#     timerCount = Timer(printCountThread, 0.1)
#     timerCount.start()

    mb.start()

    def test():
        print("-------" + str(active_count()))
    timer = Timer(test, 1)
#     timer.start()

    mb.mainThread.join()

#     tt.toc()

# Warning: if a proxy doesn't work, it will doesn't work even on localhost with selenium...

#     time.sleep(40)
#
#     mb.pause()
#
#     time.sleep(3)
#
#     mb.fillQueue()
#
#     mb.start()

#     time.sleep(20)
#     printLTS(mb.browsers[0].getUserAgent())
#     printLTS(mb.browsers[0].getScrapedHeader())

#     allProxies = getProxies()
#     random.shuffle(allProxies)
#     allProxies = allProxies[0:3]
#     for current in allProxies:
#         print(current["ip"])
#         print(mb.addBrowser(current["ip"]))
#     printLTS(mb.browsers)


from grid_control import ConfigOverlay, Proxy
from webservice_api import readJSON

class CMSOverlay(ConfigOverlay):
	def __init__(self, config):
		ConfigOverlay.__init__(self, config)
		config.set('jobs', 'monitor', 'dashboard', override = False)
		config.set('grid', 'sites', '-samtest -cmsprodhi', append = True)

		# Rewrite cms storage urls:
		proxy = Proxy.open('VomsProxy', config)
		hnName = readJSON('https://cmsweb.cern.ch/sitedb/json/index/dnUserName', {'dn': proxy.getFQUsername()})
		if not hnName:
			raise ConfigError('Unable to map grid certificate to hypernews name!')

		def rewriteSEurl(url):
			if url.startswith('cms://'): # cms://T2_DE_DESY/project/subdir
				site, url = (url.rstrip('/') + '/').replace('cms://', '').lstrip('/').split('/', 1)
				return readJSON('https://cmsweb.cern.ch/phedex/datasvc/json/prod/lfn2pfn',
					{'node': site, 'protocol': 'srm', 'lfn': '/store/user/%s/%s' % (hnName, url.rstrip('/'))})['phedex']['mapping']
			return url
		#self.rewriteList(lambda (s, i): i.startswith('se') and i.endswith('path'), rewriteSEurl)


#! /usr/bin/env python
#coding: utf-8

import rospy
import tf2_ros
from tf2_geometry_msgs import tf2_geometry_msgs
from geometry_msgs.msg import TransformStamped,Twist
import math

if __name__ == "__main__":

    # 2.初始化节点;
    rospy.init_node("tfs_p")
    # 3.创建订阅对象;
    #3-1 创建缓存对象
    buffer = tf2_ros.Buffer()
    #3-2 创建订阅对象(将缓存传入)
    sub = tf2_ros.TransformListener(buffer)

    # 创建速度消息发布对象
    pub = rospy.Publisher("/turtle2/cmd_vel",Twist,queue_size=100)

    # 5.转换逻辑实现,调用tf封装的算法;
    rate = rospy.Rate(10)
    while not rospy.is_shutdown():
        try:
            #----计算 son1 相对于 son2 的相对关系
            """
                参数1.目标坐标系
                参数2:源坐标系
                参数3:rospy.Time(0) ---取时间间隔最近的两个坐标系帧(son1 相对world 与 son2 相对world)计算结果
                返回值: son1 与 son2 的坐标系关系
            """
            ts = buffer.lookup_transform("turtle2","turtle1",rospy.Time(0))
            rospy.loginfo("父级坐标系: %s,子级坐标系: %s,偏移量(%.2f,%.2f,%.2f)",
                        ts.header.frame_id,
                        ts.child_frame_id,
                        ts.transform.translation.x,
                        ts.transform.translation.y,
                        ts.transform.translation.z
                        )

            # 组织 Twist 消息
            twist = Twist()
            # 线速度 = 系数 * 坐标系原点的间距 = 系数 *（x^2 + y^2）再开方
            # 角速度 = 系数 * 夹角           = 系数 * atan2(y,x)
            twist.linear.x = 0.5 * math.sqrt(math.pow(ts.transform.translation.x,2) + math.pow(ts.transform.translation.y,2))
            twist.angular.z = 4  * math.atan2(ts.transform.translation.y,ts.transform.translation.x)

            # 发布消息
            pub.publish(twist)
        except Exception as e:
            rospy.logwarn("错误提示:%s",e)

        rate.sleep()


def prob30():
	summ = 0
	for i in range(2,1000000):
		print(i)
		currsumm = 0
		for ind in range(len(str(i))):
			currsumm += int(str(i)[ind]) ** 5
		if currsumm == i:
			summ += i
	print(summ)

prob30()


import json

from django.shortcuts import render_to_response
from django.template.context import RequestContext
from datawinners.common.authorization import is_data_sender
from mangrove.form_model.form_model import FormModel

from datawinners.accountmanagement.decorators import valid_web_user
from datawinners.entity.helper import get_organization_telephone_number
from datawinners.entity.views import get_example_sms
from datawinners.main.database import get_database_manager
from datawinners.project.helper import  get_preview_for_field
from datawinners.project.submission_form import SurveyResponseForm
from datawinners.project.wizard_view import create_questionnaire

def get_questions(form_model):
    fields = form_model.fields
    return [get_preview_for_field(field) for field in fields]

def get_questionnaire_form_model(manager, project_info, post):
    return create_questionnaire(post, manager, name=unicode(project_info.get('name')),
                                language=unicode(project_info.get('language')), reporter_id=None)

def get_sms_preview_context(manager, post, project_info):
    form_model = project_info if isinstance(project_info, FormModel) else get_questionnaire_form_model(manager, project_info, post)
    example_sms = "%s" % form_model.form_code
    example_sms += get_example_sms(form_model.fields)
    return {"questionnaire_code": post["questionnaire-code"],
            "questions": get_questions(form_model),
            "project_name": form_model.name,
            "example_sms": example_sms}

@valid_web_user
def sms_preview(request):
    manager = get_database_manager(request.user)
    context = {'org_number': get_organization_telephone_number(request)}
    project_info = json.loads(request.POST['profile_form'])
    context.update(get_sms_preview_context(manager, request.POST, project_info))

    return render_to_response("project/sms_instruction_preview.html", context, context_instance=RequestContext(request))

def get_web_preview_context_from_project_data(manager, request, project_info):
    is_datasender = is_data_sender(request)
    form_model = get_questionnaire_form_model(manager, project_info, request.POST)
    questionnaire_form = SurveyResponseForm(form_model)
    return {'project': project_info,
            'is_datasender': is_datasender,
            'questionnaire_form': questionnaire_form}

@valid_web_user
def web_preview(request):
    project_info = json.loads(request.POST['profile_form'])
    manager = get_database_manager(request.user)

    return render_to_response("project/web_instruction_preview.html",
                              get_web_preview_context_from_project_data(manager, request, project_info),
                              context_instance=RequestContext(request))

@valid_web_user
def smart_phone_preview(request):
    language_code = request.LANGUAGE_CODE
    instruction_template = "alldata/smart_phone_instruction_" + language_code + ".html"

    return render_to_response("project/smart_phone_instruction_preview.html",
            {"instruction_template": instruction_template},
                              context_instance=RequestContext(request))


from pytube import YouTube, Playlist
import csv

playlist_link = "https://www.youtube.com/playlist?list=PLRe9ARNnYSY41I4NXMtfHQ2HN2wap_YtX"

playlist_item = Playlist(playlist_link)

print(f"There are { str(playlist_item.length) } items in the playlist")

video_links = playlist_item.video_urls

file_name = "Youtube_Playlist_videos.csv"
file_headers = ['Name', "Link"]
final_file = []

counter = 1

for link in video_links:
    print(f"Reading { str(counter) }/{ str(playlist_item.length) }")

    # This could take awhile as it reads one by one
    final_file.append([YouTube(link).title, link])

    counter = counter + 1

with open(file_name, 'w', encoding="utf-8", newline='') as csv_file:
    csv_writer = csv.writer(csv_file)
    csv_writer.writerow(file_headers)
    csv_writer.writerows(final_file)


class Node:
    def __init__(Node, data):
        Node.data = data
        Node.next = None

def detectCycle(head):
    fast=head
    slow=head
    loopexist=0
    while(fast!=None and slow!=None and fast.next!=None):
        fast=fast.next.next
        slow=slow.next
        if(fast==slow):
            loopexist=1
            break

    if(loopexist):
        slow=head
        while(slow!=fast):
            slow=slow.next
            fast=fast.next
        return slow
    return None

head = Node('a')
nodeB = Node('b')
nodeC = Node('c')
nodeD = Node('d')
nodeE = Node('e')
nodeF = Node('f')
nodeG = Node('g')
nodeH = Node('h')
nodeI = Node('i')
nodeJ = Node('j')

head.next = nodeB
nodeB.next = nodeC
nodeC.next = nodeD
nodeD.next = nodeE
nodeE.next = nodeF
nodeF.next = nodeG
nodeG.next = nodeH
nodeH.next = nodeI
nodeI.next = nodeJ
nodeJ.next = nodeD

head=detectCycle(head)
print(head.data)


from app.models import db, Review

def seed_reviews():
    review1 = Review(
        user_id=3,
        business_id=1,
        rating=5,
        content="The niku udon is the item to get. The honeycomb tripe had an amazing texture that contrasted well with the tender beef and the chewy udon noodle, and the broth had lot of depth. It came with chili oil on the side that I would've preferred to be spicier, but it didn't take away from anything as the broth has enough flavour to hold its own."
    )
    review2 = Review(
        user_id=4,
        business_id=1,
        rating=4,
        content="I love udon and Raku does it well! Their hot udon noodles are soft and chewy with the perfect amount of flavor in every bite. I typically order the sansai and add on a poached egg and mochi. (mochi lovers out there this is a must!)While their wait time during busy times is typically an hour, its worth it!"
    )
    review3 = Review(
        user_id=5,
        business_id=1,
        rating=3,
        content="Popular place so I had to try it. Udon itself is rly good. Chewy and handmade - just the way I like it. I wanted to try one cold udon and one hot."
    )
    review4 = Review(
        user_id=6,
        business_id=1,
        rating=4,
        content="First of all, the noodles here are soooo chewy and good! My friend and I both got udon and they were pretty good. I got the gyunan and the broth was pretty light and filling. My friend got the tantan, but the broth tasted a LOT like sesame oil.. There were some peanut pieces at the bottom of the broth so make sure to stir through the broth while eating!"
    )
    review5 = Review(
        user_id=6,
        business_id=1,
        rating=5,
        content="Did it! Ventured to Raku. Sat outside and it was cool enough to the point that when we bit into the chicken and sipped the broth for the first time, the warmth was pronounced. Spicy Niku is not too spicy and even those with low spice tolerance would be able to enjoy. "
    )
    review6 = Review(
        user_id=7,
        business_id=1,
        rating=5,
        content="Friendly service. Waiter was knowledgeable on explaining the popular menu items in detail, helped us easily make a decision. I ordered the Ebiten udon with a side of poached egg and shiitake mushroom."
    )
    review7 = Review(
        user_id=1,
        business_id=1,
        rating=5,
        content="I'm not usually an udon person but Raku definitely changed my mind! The Niku Udon might be my favorite bowl of udon ever. The noodles are incredible--the shape is flatter than normal udon noodles, they have the perfect amount of chewiness, and the texture and feel of them is just entirely different from any other noodles I've had."
    )

    review8 = Review(
        user_id=3,
        business_id=2,
        rating=5,
        content="Was pleasantly surprised by how amazing this restaurant was. I had the best chicken parmesan and tiramisu I think I've ever had. Their fried calamari and bread dish was amazing as well."
    )
    review9 = Review(
        user_id=1,
        business_id=2,
        rating=4,
        content="Door is always open. Food is great. Great spot fir you and the kids. Amazing onion soup."
    )
    review10 = Review(
        user_id=4,
        business_id=2,
        rating=4,
        content="We had an excellent time eating here. My lamb burger was delicious, with the fries being the absolute perfect mixture of crispy and thick. The inside was great,  and nothing was wrong with the service."
    )
    review11 = Review(
        user_id=5,
        business_id=2,
        rating=5,
        content="We've past Barn Door a hundred times and saw the parking lot busy. A telltale sign of good food. So we called an made a reservation. Jen was great she found a table last minute. We were served by Bruno who was a happy waiter giving great service. The food was just plain Terrific. "
    )
    review12 = Review(
        user_id=6,
        business_id=2,
        rating=5,
        content="Bored with restaurants close to home so we ventured a bit far and decided to try Barn Door. I ignored the not so good review and gave it a shot.  Glad I did.  I like the rustic feel when I entered the restaurant...it's pretty .  The fireplace is romantic so perfect for date as well."
    )
    review13 = Review(
        user_id=7,
        business_id=2,
        rating=5,
        content="I have always enjoyed eating at Barn Door, but I've apparently never written a review. Well, better late than never! I find the location very cozy to dine in, either in the dining room or at the bar. "
    )

    review14 = Review(
        user_id=2,
        business_id=3,
        rating=5,
        content="Amazing Thai food experience here. One waan thai is a nice atmosphere,great quality and flavorful food. Definitely recommend this place!!!(the best spot in Glen park)"
    )
    review15 = Review(
        user_id=3,
        business_id=3,
        rating=4,
        content="Visiting in SF and a friend suggested this place, so we went to check it out!"
    )
    review16 = Review(
        user_id=4,
        business_id=3,
        rating=4,
        content="I've ordered from One Waan a few times now, it's near my house and orders are usually delivered within the hour. I typically order the same thing of every Thai restaurant and its no different here, I know what I like."
    )
    review17 = Review(
        user_id=1,
        business_id=4,
        rating=5,
        content="This Spanish restaurant is not to be missed. The paella is to die for. "
    )

    review18 = Review(
        user_id=2,
        business_id=4,
        rating=5,
        content="The third in my trio of three favorite restaurants in San Francisco! The Owner/host was incredibly friendly and helpful navigating the menu, and the food was amazing! They have the best Paella I have ever had."
    )

    review19 = Review(
        user_id=5,
        business_id=4,
        rating=4,
        content="More like 3 stars but bumped up for the friendly staff, good service, and ambiance."
    )

    review20 = Review(
        user_id=1,
        business_id=5,
        rating=3.5,
        content="Good pating work"
    )

    review21 = Review(
        user_id=2,
        business_id=5,
        rating=4,
        content="Good pating work"
    )

    review22 = Review(
        user_id=4,
        business_id=5,
        rating=5,
        content="Good pating work"
    )

    review23 = Review(
        user_id=6,
        business_id=5,
        rating=5,
        content="Good pating work"
    )

    review24 = Review(
        user_id=1,
        business_id=5,
        rating=3,
        content="Good pating work"
    )

    review25 = Review(
        user_id=1,
        business_id=6,
        rating=4,
        content="Good job"
    )

    review26 = Review(
        user_id=4,
        business_id=6,
        rating=5,
        content="Good job"
    )

    review27 = Review(
        user_id=6,
        business_id=6,
        rating=5,
        content="Good job"
    )

    review28 = Review(
        user_id=2,
        business_id=7,
        rating=4,
        content="Great service"
    )
    review29 = Review(
        user_id=3,
        business_id=7,
        rating=5,
        content="Great service"
    )
    review30 = Review(
        user_id=6,
        business_id=7,
        rating=4,
        content="Great service"
    )
    review31 = Review(
        user_id=1,
        business_id=8,
        rating=4,
        content="Great service"
    )
    review32 = Review(
        user_id=2,
        business_id=8,
        rating=4,
        content="Great service"
    )
    review33 = Review(
        user_id=1,
        business_id=8,
        rating=5,
        content="Great service"
    )
    review34 = Review(
        user_id=1,
        business_id=8,
        rating=3,
        content="Great service"
    )
    review35 = Review(
        user_id=1,
        business_id=8,
        rating=4,
        content="Great service"
    )
    review36 = Review(
        user_id=1,
        business_id=8,
        rating=5,
        content="Great service"
    )

    review5 = Review(
        user_id=3,
        business_id=5,
        rating=5,
        content="Great room ! Very spacious . I picked this hotel because it was almost the middle of all the Florida Keys, so I was able to have access to all of the MajorKeys and not have to travel too far. 45 miles from Key west, about 50 to 60 from Key Largo. The service was great and the property was by far the nicest Marriott property that I've seen thus far.",
        media_1="https://s3-media0.fl.yelpcdn.com/bphoto/02W5UyMMx0xv058OVMjHOg/o.jpg"
    )
    review6 = Review(
        user_id=1,
        business_id=2,
        rating=5,
        content="Come here with my family, everybody was happy with food here"
    )

    db.session.add(review1)
    db.session.add(review2)
    db.session.add(review3)
    db.session.add(review4)
    db.session.add(review5)
    db.session.add(review6)
    db.session.commit()

def undo_reviews():
    db.session.execute('TRUNCATE reviews RESTART IDENTITY CASCADE;')
    db.session.commit()


from typing import List

class Solution:
    def fullJustify(self, words: List[str], maxWidth: int) -> List[str]:
        temp_length = -1
        temp_line, res = [], []
        for word in words:
            temp_line.append(word)
            temp_length += (len(word) + 1)
            if temp_length > maxWidth:
                temp_line.pop()
                char_length = temp_length - len(word) - len(temp_line)
                if len(temp_line) > 1:
                    space_length = maxWidth - char_length
                    short_space_width = space_length // (len(temp_line) - 1)
                    long_space_num = space_length % (len(temp_line) - 1)
                    short_space = " " * short_space_width
                    long_space = short_space + " "
                    new_line = short_space.join(temp_line)
                    new_line = new_line.replace(short_space, long_space, long_space_num)
                else:
                    new_line = temp_line[0] + " " * (maxWidth - len(temp_line[0]))
                res.append(new_line)
                temp_line = [word]
                temp_length = len(word)
        else:
            new_line = " ".join(temp_line) + " " * (maxWidth - temp_length)
            res.append(new_line)
        return res


#!/usr/bin/env python

import HTSeq
import sys, itertools, csv, os

def interleave_fastq(fastq1, fastq2, interleaved):
    '''Interleave paired fastq files'''

    outfile = open(interleaved, "w")
    for read1, read2 in itertools.izip(HTSeq.FastqReader(fastq1), HTSeq.FastqReader(fastq2)):
        temp1 = read1.name.split()
        temp2 = read2.name.split()
        read1.name = "%s/1 %s" % (temp1[0], temp1[1])
        read2.name = "%s/2 %s" % (temp2[0], temp2[1])

        read1.write_to_fastq_file(outfile)
        read2.write_to_fastq_file(outfile)
    outfile.close()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-f1", "--fastq1", help="Pair1 FastQ")
    parser.add_argument("-f2", "--fastq2", help="Pair2 FastQ")
    parser.add_argument("-i", "--interleaved", help="Name of interleaved fastq file", default="interleaved.fastq")
    parser.add_argument("-e", "--executable", help="MITObim executable", default="MITObim.pl")
    parser.add_argument("-m", "--manifest", help="Manifest containing species in the pools and their reference sequence file")
    args = parser.parse_args()

    if(args.fastq1 and args.fastq2):
        sys.stdout.write("Found paired fastq files. Interleaving\n")
        sys.stdout.write("Pair1: %s, Pair2: %s, Output: %s\n" % (args.pair1, args.pair2, args.interleaved))

        interleave_fastq(args.fastq1, args.fastq2, args.interleaved)

    with open(args.manifest, 'rU') as manifest:
        reader = csv.reader(manifest, delimiter='\t')
        for row in reader:
            sys.stdout.write("Setting up directory %s for species: %s with bait sequence file %s\n" % (row[3], row[0], row[1]))
            os.system("mkdir %s" % row[3])
            os.system("cp %s %s" % (args.interleaved, row[3]))
            os.system("cp %s %s" % (row[1], row[3]))
            os.chdir(row[3])
            sys.stdout.write("Executing: %s -sample testpool -ref bait -readpool %s --quick %s -end 200 --clean &> log" % (args.executable, args.interleaved, row[1]))
            os.system("%s -sample testpool -ref bait -readpool %s --quick %s -end 200 --clean &> log" % (args.executable, args.interleaved, row[1]))

if __name__ == '__main__':
  main()


#!/usr/bin/env python
#
# Redirect data from a TCP/IP connection to a serial port and vice versa.
#
# (C) 2002-2016 Chris Liechti <cliechti@gmx.net>
#
# SPDX-License-Identifier:    BSD-3-Clause

import sys

import time
import asyncio
import asyncio.base_events
import serial_asyncio
import serial
import serial.tools.list_ports

class Server(asyncio.base_events.Server):
    """
    Override base asyncio Server with max_connections implementation from
    https://github.com/python/asyncio/pull/448
    max_connections default updated to 1
    """

    def __init__(self, loop, sockets, protocol_factory, ssl, backlog, max_connections=1):
        self._loop = loop
        self.sockets = sockets
        self._protocol_factory = protocol_factory
        self._ssl = ssl
        self._backlog = backlog
        self._max_connections = max_connections
        self._paused = False
        self._active_count = 0
        self._waiters = []

    def _attach(self):
        assert self.sockets is not None
        self._active_count += 1
        if self._max_connections is not None and \
           not self._paused and \
           self._active_count >= self._max_connections:
            self.pause()

    def _detach(self):
        assert self._active_count > 0
        self._active_count -= 1
        if self._active_count == 0 and self.sockets is None:
            self._wakeup()
        elif self._paused and self._max_connections is not None and \
             self._active_count < self._max_connections:
            self.resume()

    def pause(self):
        """Pause future calls to accept()."""
        assert not self._paused
        self._paused = True
        for sock in self.sockets:
            self._loop.remove_reader(sock.fileno())

    def resume(self):
        """Resume use of accept() on listening socket(s)."""
        assert self._paused
        self._paused = False
        for sock in self.sockets:
            self._loop._start_serving(self._protocol_factory, sock, self._ssl, self)

    def close(self):
        self._protocol_factory = None
        sockets = self.sockets
        if sockets is None:
            return
        self.sockets = None
        for sock in sockets:
            self._loop._stop_serving(sock)
        if self._active_count == 0:
            self._wakeup()

class Pipe(asyncio.Protocol):
    def __init__(self):
        self.other = None  # type: Pipe
        self.transport = None  # type: asyncio.Transport

    def register_other(self, other):
        self.other = other

    def connection_made(self, transport):
        self.transport = transport
        # print('port opened', transport)
        # transport.serial.rts = False  # You can manipulate Serial object via transport
        # transport.write(b'Hello, World!\n')  # Write serial data via transport

    def data_received(self, data):
        if self.other and self.other.transport:
            if self.other.__class__ is Serial and self.other.transport.is_closing():
                asyncio.get_event_loop().stop()

            self.other.transport.write(data)

            print(self.__class__.__name__, repr(data))

    def connection_lost(self, exc):
        print('%s port closed' % self.__class__.__name__)

class Serial(Pipe):
    def connection_lost(self, exc):
        print('%s port closed' % self.__class__.__name__)
        asyncio.get_event_loop().stop()

class TCP(Pipe):
    pass

if __name__ == '__main__':  # noqa
    import argparse

    parser = argparse.ArgumentParser(
        description='Simple Serial to Network (TCP/IP) redirector.',
        epilog="""\
NOTE: no security measures are implemented. Anyone can remotely connect
to this service over the network.
Only one connection at once is supported. When the connection is terminated
it waits for the next connect.
""")

    parser.add_argument(
        'SERIALPORT',
        help="serial port name")

    parser.add_argument(
        '-q', '--quiet',
        action='store_true',
        help='suppress non error messages',
        default=False)

    parser.add_argument(
        '--develop',
        action='store_true',
        help='Development mode, prints Python internals on errors',
        default=False)

    # group = parser.add_argument_group('serial port')

    group = parser.add_argument_group('network settings')

    exclusive_group = group.add_mutually_exclusive_group()

    exclusive_group.add_argument(
        '-P', '--localport',
        type=int,
        help='local TCP port',
        default=7777)

    exclusive_group.add_argument(
        '-c', '--client',
        metavar='HOST:PORT',
        help='make the connection as a client, instead of running a server',
        default=False)

    args = parser.parse_args()

    try:
        with serial.Serial(args.SERIALPORT) as _:
            serial_port = args.SERIALPORT
    except Exception as ex:
        ports = list(serial.tools.list_ports.grep(args.SERIALPORT))
        if not ports:
            raise
        serial_port = ports[0].device

    if not args.quiet:
        sys.stderr.write(
            '--- TCP/IP to Serial redirect on {serial_port} <-> 127.0.0.1:{args.localport} ---\n'
            '--- type Ctrl-C / BREAK to quit\n'.format(args=args, serial_port=serial_port))

    while True:
        ser_pipe = Serial()
        tcp_pipe = TCP()

        ser_pipe.register_other(tcp_pipe)
        tcp_pipe.register_other(ser_pipe)

        loop = asyncio.get_event_loop()
        if loop.is_closed():
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
        ser_coro = serial_asyncio.create_serial_connection(loop, lambda: ser_pipe, serial_port, baudrate=400000)

        asyncio.base_events.Server = lambda loop, sockets: Server(loop, sockets, lambda: tcp_pipe, ssl=None, backlog=100)
        tcp_coro = loop.create_server(lambda: tcp_pipe, '127.0.0.1', port=args.localport)
        try:
            loop.run_until_complete(asyncio.gather(ser_coro, tcp_coro))
            loop.run_forever()
        except Exception as ex:
            print(ex)
        loop.close()
        print("Restarting")
        time.sleep(1)

    sys.stderr.write('\n--- exit ---\n')


'''
Created on Feb 20, 2017

@author: Juan Manuel Acevedo Valle

Script used to compare the Diva's python implementation against the original implementation
in Matlab
'''
import numpy as np
import sys
from matplotlib import pyplot as plt
from divapy import Diva as PyDiva
from divaml import Diva as MlDiva

def get_random_motor_set(system, n_samples):
    n_motor = system.n_motor
    raw_rnd_data = np.random.random((n_samples, n_motor))
    min_values = system.min_motor_values
    max_values = system.max_motor_values

    min_values = np.array(n_samples * [np.array(min_values)])
    max_values = np.array(n_samples * [np.array(max_values)])

    motor_commands = min_values + raw_rnd_data * (max_values - min_values)

    return motor_commands

pydiva_synth = PyDiva()
mldiva_synth = PyDiva()

foo = MlDiva()

n_static_test = 1
n_static_sound_test = 1
n_random_sound_test = 200

static_test = False
static_sound_test = True
random_sound_test = True

if __name__ == '__main__':
    static_commands = get_random_motor_set(foo,
                                           n_static_sound_test)

    print("Validating with static sound samples...")
    for i in range(n_static_sound_test):
        m_tmp = static_commands[i, :]
        arts = np.tile(m_tmp, (80, 1))

        sound_wave1 = mldiva_synth.get_sound(arts)
        sound_wave2 = pydiva_synth.get_sound_opt(arts)
        # try:
        #
        # except ValueError:
        #     print(m_tmp)
        #     sys.exit()
        err_sw = np.linalg.norm(sound_wave1 - sound_wave2)

        if err_sw > 1e-3:
            print("Validation failed in static sound test.")
            print(err_sw)
            plt.figure()
            plt.plot(sound_wave1, 'b')
            plt.hold(True)
            plt.plot(sound_wave2, 'r')
            plt.legend(['Matlab', 'Python'])
            plt.show()
            static_sound_test = False

    if static_sound_test:
        print("Validation in static sound test OK.")

    print("Validating with random sound samples...")
    for i in range(n_random_sound_test):
        arts = get_random_motor_set(foo, 80)

        try:
            sound_wave1 = pydiva_synth.get_sound(arts)
            sound_wave2 = mldiva_synth.get_sound_opt(arts)
        except ValueError:
            print(arts)
            sys.exit()
        err_sw = np.linalg.norm(sound_wave2 - sound_wave1)

        if err_sw > 1e-3:
            print("Validation failed in random sound test.")
            print(err_sw)
            print(arts)
            plt.figure()
            plt.plot(sound_wave2, 'b')
            plt.hold(True)
            plt.plot(sound_wave1, 'r')
            plt.legend(['Matlab', 'Python'])
            plt.show(block=True)

            static_sound_test = False
            x  = raw_input('Press [key]+[Enter] to cancel or [Enter] to continue...')

            if not len(x)==0:
                break

    if static_sound_test:
        print("Validation in random sound test OK.")


#! /usr/bin/env python3

import os, sys, gzip
from optparse import OptionParser
sys.path.insert(1, os.path.join(os.path.dirname(__file__), '../'))
from scanner_core import preprocessing, scanner_io

def getTailna(na_seq, options):
	return na_seq[-options.read_len:len(na_seq)].translate(str.maketrans({'A': 'T', 'T': 'A', 'C': 'G', 'G': 'C'}))[::-1]

def getTailqu(qu_seq, options):
	return qu_seq[-options.read_len:len(qu_seq)][::-1]

#===get params===
parser = OptionParser()

parser.add_option("-i",      dest = "fq_f_name", nargs = 1, default = None,
                  help = "File or folder name of reads. ")
parser.add_option("-d",      dest = "o_dir",     nargs = 1, default = "scNanoGPS_res",
                  help = "Output directory name. "
                         "Default: scNanoGPS_res")
parser.add_option("-l",      dest = "read_len",  nargs = 1, default = 100, type = "int",
                  help = "Length of the extracted first and last read nucleotides. "
                         "Default: 100")
parser.add_option("--o1",    dest = "o1",        nargs = 1, default = "first_tail.fastq.gz",
                  help = "First given length of reads. "
                         "Default: first_tail.fastq.gz")
parser.add_option("--o2",    dest = "o2",        nargs = 1, default = "last_tail.fastq.gz",
                  help = "Last given length of reads. "
                         "Default: last_tail.fastq.gz")

options, arguments = parser.parse_args()

#===pre-check===
termination = False
if not os.path.isdir(options.o_dir):
	os.system("mkdir " + options.o_dir)
if not options.fq_f_name:
	print("\nPlease give valid file or folder name: " + str(options.fq_f_name) + "\n")
	termination = True
if termination:
	parser.print_help()
	sys.exit(1)

#===precheck===
options.isFile   = os.path.isfile(options.fq_f_name)
options.isDir    = os.path.isdir(options.fq_f_name)
options.cwd      = os.getcwd()
options.batch_no = 4000
if options.isDir:
	options.f_no = len(preprocessing.get_valid_input_file_list(options.cwd, options.fq_f_name))
	if options.f_no == 0:
		print("\nCannot find valid input file under directory: " + str(os.path.join(options.cwd, options.fq_f_name)) + "\n")
		parser.print_help()
		sys.exit(1)

#===count read length===
if options.isFile:
	f_list = list([options.fq_f_name])
	f_idx  = 0
	print("Parsing fastQ file: " + str(options.fq_f_name) + " ...")
	reader = scanner_io.open_file(options.fq_f_name, "rt")

if options.isDir:
	f_list = preprocessing.get_valid_input_file_list(options.cwd, options.fq_f_name)
	f_idx  = 0
	if not f_list[f_idx].endswith(".fast5"):
		print("Processing file name: " + str(f_list[f_idx]))
	reader = scanner_io.open_file(os.path.join(options.cwd, options.fq_f_name, f_list[f_idx]), "rt")

rid = 0
while True:
	batch_data = []
	batch_data, eof, rid, reader, f_idx = scanner_io.batch_reading(reader, f_list, f_idx, int(options.batch_no), rid)
	if eof == 1:
		break

	o1h = gzip.open(os.path.join(options.o_dir, options.o1), "at")
	o2h = gzip.open(os.path.join(options.o_dir, options.o2), "at")
	for read in batch_data:
		o1h.write("@" + read['def_line'] + "\n")
		o1h.write(read['na_seq'][0:options.read_len] + "\n")
		o1h.write("+" + read['def_line'] + "\n")
		o1h.write(read['qu_seq'][0:options.read_len] + "\n")

		o2h.write("@" + read['def_line'] + "\n")
		o2h.write(getTailna(read['na_seq'], options) + "\n")
		o2h.write("+" + read['def_line'] + "\n")
		o2h.write(getTailqu(read['qu_seq'], options) + "\n")

	batch_data = []
	o1h.close()
	o2h.close()
print("\nDone.")


from flask import Flask, render_template,send_from_directory, jsonify
import db_output
import json
from datetime import date, datetime

app = Flask(__name__)

def json_serial(obj):

	if isinstance(obj, (datetime,date)):
		return obj.isoformat();

@app.route('/')
def dashboard():
	d = db_output.db.query(db_output.latest_event)
	d[0]["longitude"] = str(d[0]["longitude"])
	d[0]["latitude"] = str(d[0]["latitude"])
	d[0]["soundlevel"] = str(d[0]["soundlevel"])
	res = json.dumps(d[0], default=json_serial)
	return render_template('dashboard.html', summary=res)

@app.route('/templates/<path:path>')
def send_static_stuff(path):
	return send_from_directory('templates',path)

if __name__=='__main__':
	app.run(debug=True,  host='0.0.0.0', port=5000 )


from itertools import accumulate

class Solution:
    def maxSatisfied_1(self, customers, grumpy, X) -> int:
        # find largest sum in window. here windowlength in X

        window_start, satisfied_customers = 0, 0,
        max_window_indices_range = (window_start, X - 1)
        max_grumpy_window_sum = grumpy_window_sum = sum(
            value for index, value in enumerate(customers[:X]) if grumpy[index])

        for index, window_end_char in enumerate(customers[X:]):
            window_end = index + X

            if grumpy[window_end]:
                grumpy_window_sum += window_end_char

            if grumpy[window_start]:
                grumpy_window_sum -= customers[window_start]

            window_start += 1

            if max_grumpy_window_sum < grumpy_window_sum:
                max_grumpy_window_sum = grumpy_window_sum
                max_window_indices_range = (window_start, window_end)

        for index in range(0, len(customers)):
            if not grumpy[index]:
                satisfied_customers += customers[index]
            elif max_window_indices_range[0] <= index <= max_window_indices_range[1]:
                satisfied_customers += customers[index]
            else:
                pass

        return satisfied_customers

    # Approach 2: Here add the already satisfied customer count and make them 0, then run sliding window.
    def maxSatisfied_2(self, customers, grumpy, X):
        satisfied_customers = 0
        for index, customers_count in enumerate(customers):
            if grumpy[index] == 0:
                satisfied_customers += customers_count
                customers[index] = 0

        # now sliding window the remaining customers when grumpy. Find the max customers when he is grumpy.
        max_window_sum, window_start = sum(customers[:X]), 0
        window_sum = max_window_sum

        for window_end_value in customers[X:]:
            window_sum += window_end_value - customers[window_start]
            window_start += 1
            if window_sum > max_window_sum:
                max_window_sum = window_sum

        satisfied_customers += max_window_sum

        return satisfied_customers

    # Approach 2 A. Merging 2 for loops into one.
    def maxSatisfied_2_a(self, customers, grumpy, X):
        window_start, satisfied_customers, max_window_sum, window_sum = 0, 0, 0, 0

        for window_end, customer_count in enumerate(customers):
            if grumpy[window_end] == 0:
                satisfied_customers += customer_count
                customers[window_end] = 0
            else:
                window_sum += customer_count

            if window_end >= X:  # fixed window_size reached
                window_sum -= customers[window_start]
                window_start += 1

            if window_sum > max_window_sum:
                max_window_sum = window_sum

        return satisfied_customers + max_window_sum

    # not so good approach, works nonetheless
    def maxSatisfied_3(self, customers, grumpy, X):
        window_start, satisfied_customers, max_window_sum, window_sum = 0, 0, 0, 0

        for window_end, (customer_count, is_grumpy) in enumerate(zip(customers, grumpy)):
            satisfied_customers += customer_count * (not is_grumpy)
            customers[window_end] = customers[window_end] * is_grumpy
            window_sum += customer_count * is_grumpy

            if window_end >= X:  # fixed window_size reached
                window_sum -= customers[window_start]
                window_start += 1

            if window_sum > max_window_sum:
                max_window_sum = window_sum

        return satisfied_customers + max_window_sum

    # prefix sum approach
    def maxSatisfied(self, customers, grumpy, X):
        satisfied_customers = sum(value for index, value in enumerate(customers) if not grumpy[index])
        dis_satisfied_customers = [value if grumpy[index] else 0 for index, value in enumerate(customers)]

        # prefix sum calculation
        prefix_sum = [0] + list(accumulate(dis_satisfied_customers))
        window_start, max_window_sum = 0, 0
        for index, value in enumerate(prefix_sum[X:]):
            window_end = index + X
            max_window_sum = max(max_window_sum, value - prefix_sum[window_start])
            window_start += 1

        return satisfied_customers + max_window_sum

sol = Solution()

# print(sol.maxSatisfied([2, 6, 6, 9], [0, 0, 1, 1], 1))
print(sol.maxSatisfied([1, 0, 1, 2, 1, 1, 7, 5], [0, 1, 0, 1, 0, 1, 0, 1], 3))
# print(sol.maxSatisfied([9, 10, 4, 5], [1, 0, 1, 1], 1))

sliding_window/1052. Grumpy Bookstore Owner.py


"""
The king wants to share his wealth with a certain number of people N in his
kingdom, so he has provided a certain amount of gold coins A. Knowing that
each person who receives gold coins wants twice as much as the person before
him (and that the first person receives only one gold coin), has the king
provided enough gold coins to give to the N people in his kingdom.

ex :

    there are 3 people
    the king has 15 gold coins

    person 1 : 1 gold coin
    person 2 : 2 gold coins
    person 3 : 4 gold coins

    --> 1+2+4 = 7 <= 15, the king has enough gold coins
"""

def king_n_coins(subjects: int, coins: int) -> bool:
    """
    Solves the problem
    """
    total = 1
    nxt = 1

    for _ in range(subjects - 1):
        nxt *= 2
        total += nxt

    return total <= coins


import numpy as np
from ringBuffer import *
import sidekit

#Function which send tabs into the pipeline
@coroutine
def audio_reader(next_routine, audio_file):
    input_filename = audio_file
    sampling_rate = 16000
    if input_filename.endswith('.sph') or input_filename.endswith('.pcm')\
            or input_filename.endswith('.wav') or input_filename.endswith('.raw'):
        x, rate = sidekit.frontend.io.read_audio(input_filename, sampling_rate)

    # add random noise to avoid any issue due to zeros
    np.random.seed(0)
    if x.ndim == 1:
        x += 0.0001 * np.random.randn(x.shape[0])
    elif x.ndim == 2:
        x[:, 0] += 0.0001 * np.random.randn(x.shape[0])
        if x.shape[1] == 2:
            x[:, 1] += 0.0001 * np.random.randn(x.shape[0])

    (yield None)
    idx = 0
    next_routine.send(x)
    next_routine.close()

@coroutine
def  source(suiv):
    x = [1,2,3,4,5,6,7,8,9,10]
    y = ['a','b','c']
    while (True):
        (yield None)
        suiv.send(x)
        suiv.send(y)
    suiv.close()

#Send every other value
@coroutine
def everyOtherValue(suiv):
    try :
        while True:
            input=yield
            print("Pipe :", input)
            lettre = 1
            while (lettre < len(input)):
                suiv.send(input[lettre])
                lettre = lettre+2;
    except GeneratorExit:
        suiv.close()

#Buffer
@coroutine
def buffer(suiv):
    try :
        buf1 = [None] * 2
        while(True):
            i = 0
            while(i<2):
                input = yield
                print("Buffer :", input)
                buf1[i] = input
                i+= 1
            suiv.send(buf1)
    except GeneratorExit:
        suiv.close()

#Output of the pipeline, print the result of the treatment
@coroutine
def sink():
    try:
        while True:
            input=yield
            print("Sink :", input)
    except GeneratorExit:
        print("ici")

output = sink()
buf = ring_buffer(output, 4, 2 )
inp = audio_reader(buf, "/home/rosalie/Bureau/RichMeeting/SidePipe/RingBuffer/test.raw")
try :
    next(inp)
except StopIteration :
    print("That's all folks")