from io import BytesIO
import re
import sys
import requests
import img2pdf
import PyPDF4
from rich.console import Console
from rich.progress import track

console = Console()

INPUT_SESSION = console.input("🍪  Your '_reader_session' key: ")
ENDPOINT_SCUOLABOOK = "https://webapp.scuolabook.it/books"
HEADER = {'X-Requested-With': 'XMLHttpRequest',
          'Accept': 'application/json, text/javascript, */*; q=0.01',
          'Cookie': f'_reader_session={INPUT_SESSION}'}

class Book:
    """
    A Book interface
    """

    def __init__(self):
        self.book_id: int = 0
        self.title: bytes = ''
        self.author: bytes = ''
        self.publisher: bytes = ''
        self.isbn: bytes = ''
        self.npages: int = 0

def get_books_list_data():
    """
    This function return a dictionary that contains all books present in your library
    """
    data = requests.get(ENDPOINT_SCUOLABOOK,
                        headers=HEADER).text
    books = {}
    for i in range(len(re.findall('"id":(.*?),', data))):
        books[re.findall('"id":(.*?),', data)[i]] = re.findall(
            '"ws_title":"(.*?)"', data)[i].replace("\u0026", "&")
    return books

def get_books_list(books_list_data: dict):
    """
    This function takes in input the `books_list_data` dictionary data that contains all books
    check if the `_reader_session` is valid and if don't exit the program.
    Format them in a readable user manner and output them
    so the user can choose what book wanna download
    """
    if books_list_data == {}:
        console.print(
            "❌  Your library is empty or your '_reader_session' key is expired")
        exit(255)
    console.print("\n📚  [b][Your library][/b]\n")
    for bookid in books_list_data:
        console.print(f"📗  ID:({bookid})  [b]{books_list_data[bookid]}[/b]")

def get_selected_book(books_list_data: dict):
    """
    This function takes in input the `books_list_data` dictionary data that contains all books
    check if the `input_book_id` is valid and  present in the library if don't exit the program.
    Output to the user all the information about the book selected and return a book object
    """
    try:
        input_book_id = int(console.input(
            "\n🔢  Insert ID of the book to download them: "))
        for bookid in books_list_data:
            if not input_book_id != bookid:
                raise ValueError
    except ValueError:
        console.print("❌  Invalid Book ID")
        exit(255)
    book = Book()
    data = requests.get(
        f"{ENDPOINT_SCUOLABOOK}/{input_book_id}", headers=HEADER).text
    book.book_id = input_book_id
    book.title = re.search('"ws_title":"(.*?)"', data).group(1).encode('utf-8')
    book.author = re.search('"ws_author":"(.*?)"',
                            data).group(1).encode('utf-8')
    book.publisher = re.search(
        '"ws_publisher":"(.*?)"', data).group(1).encode('utf-8')
    book.isbn = re.search('"ws_isbn":"(.*?)"', data)
    if book.isbn:
        book.isbn = book.isbn.group(1).encode('utf-8')
    book.npages = int(re.search('"ws_num_pages":"(.*?)"',
                      data).group(1).encode('utf-8'))
    console.print("\n[Selected book]\n")
    console.print(f"📖  Title: {book.title.decode()}")
    console.print(f"👤  Author: {book.author.decode()}")
    console.print(f"🏛  Publisher: {book.publisher.decode()}")
    console.print(f"📜  ISBN: {book.isbn.decode()}")
    console.print(f"📄  Pages: {book.npages}")
    return book

def get_all_pages(book: Book):
    """
    This function take in input a `Book` object  and return a list of page link to download
    """
    data  = ""
    payload = ""
    for i in track(range(1, book.npages + 2)):
        if i % 100 == 0 or i == book.npages + 1:
            data += requests.get(
                f"{ENDPOINT_SCUOLABOOK}/{book.book_id}/pages?{payload[1:]}", headers=HEADER).text
            payload = ""
        payload += f"&pages[]={i}"
    matches = re.findall('":"(.*?)"', data)
    return matches

def dowload_convert_all_pages(book: Book, matches: list):
    """
    This function takes as input a `Book` object and a list of links of pages
    and return a `list` that contains each PDF page in bytes format
    """
    pdfs_bytes = []
    for match in matches:
        console.print(
            f"\r📥  Downloading & Converting page {matches.index(match) + 1}/{book.npages}...")
        sys.stdout.flush()
        link = match.replace("\\u0026", "&").encode('utf-8')
        pdf_bytes = img2pdf.convert(requests.get(link).content)
        pdfs_bytes.append(pdf_bytes)
    return pdfs_bytes

def merging_pdf(pdfs_bytes: list, book: Book):
    """
    This function takes as input a `list` of PDF page of book in bytes format and `Book` object
    each element of the `list` is read and merged with the previous one and at the end of the `list`
    the final merged Book (PDF) is created
    """
    merger = PyPDF4.PdfFileMerger(strict=False)
    output_file = book.title.decode().replace('\u007C', '') + ".pdf"
    console.print("\n☯  Merging everything...")
    for pdf_bytes in pdfs_bytes:
        with BytesIO(pdf_bytes) as pdf_file:
            pdf_file_stream = PyPDF4.PdfFileReader(
                stream=pdf_file, strict=False)
            merger.append(pdf_file_stream)
    console.print(f"⌛  Creating merged book ''{output_file}''...")
    merger.write(output_file)
    console.print(
        f":thumbs_up:  The book: ''{output_file}'' was created succesfully!")

def main():
    """
    This function execute all program
    """
    books_list_data = get_books_list_data()
    get_books_list(books_list_data=books_list_data)
    book = get_selected_book(books_list_data=books_list_data)
    matches = get_all_pages(book=book)
    pdfs_bytes = dowload_convert_all_pages(book=book, matches=matches)
    merging_pdf(pdfs_bytes=pdfs_bytes, book=book)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""{{ cookiecutter.repo_name }} URL Configuration

https://docs.djangoproject.com/en/1.8/topics/http/urls/

"""
from django.conf import settings
from django.conf.urls import include, url
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.staticfiles.storage import staticfiles_storage
from django.http import HttpResponse
from django.views.defaults import (
    server_error
)
from django.views.generic.base import RedirectView

from wagtail.contrib.wagtailapi import urls as wagtailapi_urls
from wagtail.wagtailcore import urls as wagtail_urls
from wagtail.wagtailadmin import urls as wagtailadmin_urls
from wagtail.wagtaildocs import urls as wagtaildocs_urls
from wagtail.wagtailsearch import urls as wagtailsearch_urls

urlpatterns = [

    # www.example.com/500.html
    url(
        regex=r'^500/$',
        view=server_error),
    # www.example.com/django-admin
    url(
        regex=r'^favicon.ico$',
        view=RedirectView.as_view(
            url=staticfiles_storage.url('favicon.ico'),
            permanent=False),
        name="favicon"
    ),
    url(
        regex=r'^django-admin/',
        view=include(admin.site.urls)),
    # www.example.com/admin
    url(
        regex=r'^admin/',
        view=include(wagtailadmin_urls)),
    # www.example.com/search
    url(
        regex=r'^search/',
        view=include(wagtailsearch_urls)),
    # www.example.com/documents
    url(
        regex=r'^documents/',
        view=include(wagtaildocs_urls)),
    # www.example.com/api/vi/{pages, images, documents}
    url(
        regex=r'^api/',
        view=include(wagtailapi_urls)),
    # www.example.com
    url(
        regex=r'',
        view=include(wagtail_urls)),
    # www.example.com/robots.txt
    url(
        regex=r'^robots.txt$',
        view=lambda r: HttpResponse("User-agent: *\nDisallow: /",
                                    content_type="text/plain")),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

# url pattern for django debug toolbar
if settings.DEBUG:
    import debug_toolbar
    urlpatterns += [
        url(r'^__debug__/', include(debug_toolbar.urls)),
    ]


import zipfile
import os

import torchvision.transforms as transforms

# once the images are loaded, how do we pre-process them before being passed into the network
# by default, we resize the images to 64 x 64 in size
# and normalize them to mean = 0 and standard-deviation = 1 based on statistics collected from
# the training set

mean_nums = [0.485, 0.456, 0.406]
std_nums = [0.229, 0.224, 0.225]

resnet_train_transforms = transforms.Compose([
    transforms.Resize(size=224),
    transforms.CenterCrop(size=224),
    transforms.RandomRotation(degrees=10),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor(),
    transforms.Normalize(mean_nums, std_nums)
])

resnet_test_transforms = transforms.Compose([
    transforms.Resize(224),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(mean_nums, std_nums)
])

inceptionv3_train_transforms = transforms.Compose([
    transforms.RandomRotation(degrees=10),
    transforms.RandomHorizontalFlip(),
    transforms.Resize(299),
    transforms.CenterCrop(299),
    transforms.ToTensor(),
    transforms.Normalize(mean_nums, std_nums)
])

inceptionv3_test_transforms = transforms.Compose([
    transforms.Resize(299),
    transforms.CenterCrop(299),
    transforms.ToTensor(),
    transforms.Normalize(mean_nums, std_nums)
])

inceptionv3_params = {
    'batch_size': 32,
    'momentum': 0.9,
    'lr': 0.045,
    'misc': 'RMSProp(0.9, eps=1.0), exp lr decay 0.94 every 2 epochs, grad clipping for stability',
}


from django.contrib.auth.models import User
from django.db import models

class Location(models.Model):
    name = models.CharField(max_length=255)
    world = models.CharField(max_length=255)
    description = models.TextField()
    image = models.ImageField(upload_to='photos/')
    author = models.ForeignKey(User, on_delete=models.CASCADE, blank=True, null=True)

    def __str__(self):
        return self.name

class Character(models.Model):
    name = models.CharField(max_length=255)
    birth_location = models.ForeignKey(Location, on_delete=models.CASCADE, related_name='birth_location')
    location = models.ForeignKey(Location, on_delete=models.CASCADE, related_name='current_location')
    description = models.TextField()
    image = models.ImageField(upload_to='photos/')
    GENDER = (
        ('male', 'Мужской'),
        ('female', 'Женский'),
        ('unknown', 'Неизвестно')
    )
    gender = models.CharField(max_length=10, choices=GENDER)
    race = models.CharField(max_length=55)
    DEAD = (
        ('dead', 'Мертвый'),
        ('alive', 'Живой')
    )
    dead = models.CharField(max_length=10, choices=DEAD)

    def __str__(self):
        return self.name

class Episode(models.Model):
    name = models.CharField(max_length=255)
    number = models.IntegerField()
    number_season = models.IntegerField()
    image = models.ImageField(upload_to='photos/')
    description = models.TextField()
    premiere = models.DateField()
    character = models.ManyToManyField(Character, related_name='char_to_ep')

    def __str__(self):
        return f"{self.number_season} + {self.number}"


import aiohttp
import asyncio
import json

from . import errors

async def json_or_text(response):
    text = await response.text(encoding="utf-8")
    try:
        if "application/json" in response.headers["Content-Type"]:
            return json.loads(text)
    except KeyError:
        # Thanks Cloudflare
        pass

    return text

class Route:
    BASE = "https://g.tenor.com/v1"

    def __init__(self, path):
        self.url = f"{self.BASE}{path}"

class HTTPClient:
    def __init__(self, api_key):
        self.api_key = api_key
        self.session = None

    async def request(self, route, parameters=None):
        if parameters is None:
            parameters = {}
        parameters["key"] = self.api_key

        if self.session is None:
            self.session = await self._create_session()

        params = {}
        for k, v in parameters.items():
            if v is not None:
                params[k] = v

        for tries in range(5):
            async with self.session.get(route.url, params=params) as resp:
                data = await json_or_text(resp)

                if "error" in data and resp.status == 200:
                    raise errors.TenorException(data["error"])

                if 200 <= resp.status < 300:
                    return data

                if resp.status == 404:
                    raise errors.NotFound(data)

                if resp.status == 429:
                    # we are rate limited, wait for 30s
                    await asyncio.sleep(30)
                    continue

                return data

        if resp.status >= 500:
            raise errors.TenorServerError(data)

    async def _create_session(self):
        return aiohttp.ClientSession()

    async def close(self):
        if self.session:
            await self.session.close()

    async def search(self, **parameters):
        return await self.request(Route("/search"),
                                  parameters=parameters)

    async def trending(self, **parameters):
        return await self.request(Route("/trending"),
                                  parameters=parameters)

    async def categories(self, **parameters):
        return await self.request(Route("/categories"),
                                  parameters=parameters)

    async def search_suggestions(self, **parameters):
        return await self.request(Route("/search_suggestions"),
                                  parameters=parameters)

    async def autocomplete(self, **parameters):
        return await self.request(Route("/autocomplete"),
                                  parameters=parameters)

    async def trending_terms(self, **parameters):
        return await self.request(Route("/trending_terms"),
                                  parameters=parameters)

    async def registershare(self, **parameters):
        return await self.request(Route("/registershare"),
                                  parameters=parameters)

    async def gifs(self, **parameters):
        return await self.request(Route("/gifs"),
                                  parameters=parameters)

    async def random(self, **parameters):
        return await self.request(Route("/random"),
                                  parameters=parameters)

    async def anonid(self, **parameters):
        return await self.request(Route("/anonid"),
                                  parameters=parameters)


'''Functions used in river_tracker1.py

Author: guangzhi XU (xugzhi1987@gmail.com; guangzhi.xu@outlook.com)
Update time: 2019-05-10 11:03:36.
'''

from __future__ import print_function
import numpy as np
import pandas as pd
import networkx as nx
from skimage import measure
from skimage import morphology
from scipy import ndimage
import matplotlib.pyplot as plt

import cdms2 as cdms
import MV2 as MV
from genutil import statistics as stats
import cdutil

from utils import rdp
from utils import funcs
from utils import peak_prominence2d as pp2d

NX_VERSION=nx.__version__[0]

def plotGraph(graph,ax=None,show=True):
    '''Helper func to plot the graph of an AR coordinates
    '''

    if ax is None:
        fig=plt.figure()
        ax=fig.add_subplot(111)

    pos=[(ii[1],ii[0]) for ii in graph.nodes()]  # x,y
    pos_dict=dict(zip(graph.nodes(),pos))
    nx.draw(graph,ax=ax,pos=pos_dict,node_size=15,node_color='darkgray',
            edge_color='dimgray')
    if show:
        plt.show(block=False)
    return

def areaFilt(mask,area,min_area=None,max_area=None):
    '''Filter AR binary masks by region areas

    Args:
        mask (ndarray): 2D binary mask with detected objects shown as 1s.
        area (ndarray): 2D map showing grid cell areas in km^2.
        min_area (float or None): if not None, minimum area to filter objects
                                  in <mask>.
        max_area (float or None): if not None, maximum area to filter objects
                                  in <mask>.

    Returns:
        result (ndarray): 2D binary mask with objects area-filtered.
    '''

    if min_area is None and max_area is None:
        return mask

    labels=measure.label(mask,connectivity=1)
    n=labels.max()+1
    areas=ndimage.sum(area,labels,np.arange(n))
    sel=np.ones(n,bool)
    if min_area is not None:
        sel=np.where(areas<min_area,0,sel)
    if max_area is not None:
        sel=np.where(areas>max_area,0,sel)
    # remove background area
    sel[0]=0
    result=sel[labels]

    return result

def spherical2Cart(lat,lon):
    clat=(90-lat)*np.pi/180.
    lon=lon*np.pi/180.
    x=np.cos(lon)*np.sin(clat)
    y=np.sin(lon)*np.sin(clat)
    z=np.cos(clat)

    return np.array([x,y,z])

def cart2Spherical(x,y,z, shift_lon):

    r=np.sqrt(x**2+y**2+z**2)
    clat=np.arccos(z/r)/np.pi*180
    lat=90.-clat
    lon=np.arctan2(y,x)/np.pi*180
    lon=(lon+360)%360

    lon=np.where((lon>=0) & (lon<shift_lon),lon+360,lon)

    return np.array([lat,lon,np.ones(lat.shape)])

def computeTheta(p1,p2):
    '''Tangent line to the arc |p1-p2|
    <p1>,<p2>: (lat,lon) coordinates
    '''
    p1=spherical2Cart(p1[0],p1[1])
    p2=spherical2Cart(p2[0],p2[1])
    theta=p2-np.dot(p1,p2)*p1
    norm=np.linalg.norm(theta)
    if norm>0:
        theta=theta/norm
    return theta

def wind2Cart(u,v,lats,lons):
    '''Convert u,v winds to Cartesian, consistent with spherical2Cart.
    '''

    latsr=lats*np.pi/180
    lonsr=lons*np.pi/180
    vh=v*np.sin(latsr)

    ux=-u*np.sin(lonsr) - vh*np.cos(lonsr)
    uy=u*np.cos(lonsr) - vh*np.sin(lonsr)
    uz=v*np.cos(latsr)
    vs=np.array([ux,uy,uz])

    return vs

def cart2Wind(vs,lats,lons):
    '''Convert winds in Cartesian to u,v, inverse to wind2Cart.
    '''

    latsr=lats*np.pi/180
    lonsr=lons*np.pi/180

    #ux=vs[0]
    uy=vs[1]
    uz=vs[2]

    u=uy/np.cos(lonsr) + uz*np.tan(latsr)*np.tan(lonsr)
    v=uz/np.cos(latsr)

    return u,v

def maskToGraph(mask, quslab, qvslab, costhetas, sinthetas, edge_eps,
        connectivity=2):
    '''Create graph from AR mask

    Args:
        mask (ndarray): 2D binary map showing the location of an AR with 1s.
        quslab (cdms.TransientVariable): 2D map of u-flux.
        qvslab (cdms.TransientVariable): 2D map of v-flux.
        costhetas (cdms.TransientVariable): (n * m) 2D slab of grid cell shape:
                                          cos=dx/sqrt(dx^2+dy^2).
        sinthetas (cdms.TransientVariable): (n * m) 2D slab of grid cell shape:
                                          sin=dy/sqrt(dx^2+dy^2).
        edge_eps (float): float in (0,1), minimal proportion of flux component
                          in a direction to total flux to allow edge building
                          in that direction. Defined in Global preamble.
        connectivity (int): 1 or 2. 4- or 8- connectivity in defining neighbor-
                            hood relationship in a 2D square grid.
    Returns:
        g (networkx.DiGraph): directed planar graph constructed from AR mask
                              and flows.
    '''

    quslab=np.array(quslab)
    qvslab=np.array(qvslab)
    wsslab=np.sqrt(quslab**2+qvslab**2)

    g=nx.DiGraph()

    # 1 connectivity edges
    # the list approach
    '''
    y,x=np.where(mask)
    zipcor=zip(y,x)
    right=[(yi,xi) for yi,xi in zipcor if (yi,xi+1) in zipcor]
    left=[(yi,xi) for yi,xi in zipcor if (yi,xi-1) in zipcor]
    up=[(yi,xi) for yi,xi in zipcor if (yi+1,xi) in zipcor]
    down=[(yi,xi) for yi,xi in zipcor if (yi-1,xi) in zipcor]

    # nodes to the right/left/up/down
    right0=[(yi,xi+1) for yi,xi in right]
    left0=[(yi,xi-1) for yi,xi in left]
    up0=[(yi+1,xi) for yi,xi in up]
    down0=[(yi-1,xi) for yi,xi in down]
    '''

    # the shifting approach
    right=np.roll(mask, -1, axis=1)*mask
    left=np.roll(mask, 1, axis=1)*mask
    up=np.roll(mask, -1, axis=0)*mask
    down=np.roll(mask, 1, axis=0)*mask

    def addWeightedEdges2(nodes1,speedslab,d):
        '''Add directed edges to graph. For shifting approach
        '''

        ratio=np.where(wsslab==0., 0, speedslab/wsslab)
        idx=np.where(ratio>=edge_eps, 1, 0)*nodes1
        idx=zip(*np.where(idx>0))

        for ii, (yii,xii) in enumerate(idx):
            # nii: start, nii2: end
            yii=int(yii)
            xii=int(xii)
            nii=(yii,xii)
            if d=='r':
                nii2=(yii,xii+1)
            elif d=='l':
                nii2=(yii,xii-1)
            elif d=='u':
                nii2=(yii+1,xii)
            elif d=='d':
                nii2=(yii-1,xii)
            elif d=='tr':
                nii2=(yii+1,xii+1)
            elif d=='br':
                nii2=(yii-1,xii+1)
            elif d=='tl':
                nii2=(yii+1,xii-1)
            elif d=='bl':
                nii2=(yii-1,xii-1)

            meanivt=speedslab[yii,xii]

            g.add_edge(nii,nii2,
                    weight=np.exp(-meanivt/1e2),
                    ivt=meanivt)

    def addWeightedEdges(nodes1,nodes2,speedslab):
        '''Add directed edges to graph. For the list approach
        '''

        # nii: start, nii2: end
        for nii,nii2 in zip(nodes1,nodes2):
            if speedslab[nii]/wsslab[nii]>=edge_eps:
                meanivt=speedslab[nii]

                g.add_edge(nii,nii2,
                        weight=np.exp(-meanivt/1e2),
                        ivt=meanivt)

    # add 1 connectivity edges
    #addWeightedEdges(right,right0,quslab)
    #addWeightedEdges(left,left0,-quslab)
    #addWeightedEdges(up,up0,qvslab)
    #addWeightedEdges(down,down0,-qvslab)
    addWeightedEdges2(right,quslab,'r')
    addWeightedEdges2(left,-quslab,'l')
    addWeightedEdges2(up,qvslab,'u')
    addWeightedEdges2(down,-qvslab,'d')

    # 2 connectivity edges
    if connectivity==2:
        # the list approach
        '''
        tr=[(yi,xi) for yi,xi in zipcor if (yi+1,xi+1) in zipcor]
        br=[(yi,xi) for yi,xi in zipcor if (yi-1,xi+1) in zipcor]
        tl=[(yi,xi) for yi,xi in zipcor if (yi+1,xi-1) in zipcor]
        bl=[(yi,xi) for yi,xi in zipcor if (yi-1,xi-1) in zipcor]

        tr0=[(yi+1,xi+1) for yi,xi in tr]
        br0=[(yi-1,xi+1) for yi,xi in br]
        tl0=[(yi+1,xi-1) for yi,xi in tl]
        bl0=[(yi-1,xi-1) for yi,xi in bl]
        '''

        # the shifting approach
        tr=np.roll(np.roll(mask, -1, axis=0), -1, axis=1)*mask
        br=np.roll(np.roll(mask, 1, axis=0), -1, axis=1)*mask
        tl=np.roll(np.roll(mask, -1, axis=0), 1, axis=1)*mask
        bl=np.roll(np.roll(mask, 1, axis=0), 1, axis=1)*mask

        # add 2 connectivity edges
        #addWeightedEdges(tr,tr0,quslab*costhetas+qvslab*sinthetas)
        #addWeightedEdges(br,br0,quslab*costhetas-qvslab*sinthetas)
        #addWeightedEdges(tl,tl0,-quslab*costhetas+qvslab*sinthetas)
        #addWeightedEdges(bl,bl0,-quslab*costhetas-qvslab*sinthetas)
        addWeightedEdges2(tr,quslab*costhetas+qvslab*sinthetas,'tr')
        addWeightedEdges2(br,quslab*costhetas-qvslab*sinthetas,'br')
        addWeightedEdges2(tl,-quslab*costhetas+qvslab*sinthetas,'tl')
        addWeightedEdges2(bl,-quslab*costhetas-qvslab*sinthetas,'bl')

    return g

def getARAxis(g, quslab, qvslab, mask):
    '''Find AR axis from AR region mask

    Args:
        g (networkx.DiGraph): directed planar graph constructed from AR mask
                              and flows. See maskToGraph().
        quslab (cdms.TransientVariable): 2D map of u-flux.
        qvslab (cdms.TransientVariable): 2D map of v-flux.
        mask (ndarray): 2D binary map showing the location of an AR with 1s.

    Returns:
        path (ndarray): Nx2 array storing the AR axis coordinate indices in
                        (y, x) format.
        axismask (ndarray): 2D binary map with same shape as <mask>, with
                            grid cells corresponding to coordinates in <path>
                            set to 1s.
    '''

    nodes=list(g.nodes())

    #---------------Find boundary nodes---------------
    edge=mask-morphology.binary_erosion(mask)

    gy,gx=np.gradient(np.array(mask))
    inedge=(gx*quslab+gy*qvslab)*edge
    inedgecoor=np.where(inedge>0)
    inedgecoor=zip(inedgecoor[0],inedgecoor[1])
    inedgecoor=list(set(inedgecoor).intersection(nodes))

    outedgecoor=np.where(inedge<0)
    outedgecoor=zip(outedgecoor[0],outedgecoor[1])
    outedgecoor=list(set(outedgecoor).intersection(nodes))

    n1=len(inedgecoor)
    n2=len(outedgecoor)

    # when mask is at edge of the map. Rarely happens.
    if n1==0:
        inedgecoor=nodes
        n1=len(inedgecoor)
    if n2==0:
        outedgecoor=nodes
        n2=len(outedgecoor)

    dists=np.zeros((n1,n2))

    def sumDists(path,attr,g):
        '''Sum edge distances along a path'''
        s=0
        for ii in range(len(path)-1):
            if NX_VERSION=='2':
                sii=g[path[ii]][path[ii+1]][attr]
            else:
                sii=g.edge[path[ii]][path[ii+1]][attr]

            # penalize sharp turns. Doesn't make big difference but notably
            # slower
            '''
            if ii+2<len(path):
                pii1=(lats[path[ii][0]], lons[path[ii][1]])
                pii2=(lats[path[ii+1][0]], lons[path[ii+1][1]])
                pii3=(lats[path[ii+2][0]], lons[path[ii+2][1]])
                theta1=computeTheta(pii1,pii2)
                theta2=computeTheta(pii2,pii3)
                dtheta=theta1.dot(theta2)
                dtheta=abs(dtheta)**1

                #if ii==0:
                    #dtheta_old=1.

                #dtheta=np.mean([dtheta,dtheta_old])
                #sii=sii*dtheta
                #dtheta_old=dtheta
            '''
            s+=sii

        return s

    #---------------Find "longest" path---------------
    for ii in range(n1):
        eii=inedgecoor[ii]
        pathsii=nx.single_source_dijkstra_path(g,eii,weight='weight')
        pathsii=dict([(kk,vv) for kk,vv in pathsii.items() if kk in outedgecoor])
        if len(pathsii)>0:
            distdict=dict([(kk, sumDists(vv,'ivt',g)) for kk,vv in pathsii.items()])
            nodeii=sorted(distdict,key=distdict.get)[-1]
            distii=distdict[nodeii]
            dists[ii,outedgecoor.index(nodeii)]=distii

    if np.max(dists)==0:
        # this may happen when a mask is touching the map edges, and inedgecoor
        # outedgecoor can't be linked by a path. Very rarely happen, but damn
        # annoying. A fallback solution is to use an undirected graph linking
        # the most inward and most outward pixels.
        mostin=np.unravel_index(np.argmax(inedge), mask.shape)
        mostout=np.unravel_index(np.argmin(inedge), mask.shape)
        g_und=g.to_undirected()
        try:
            path=nx.dijkstra_path(g_und,mostin,mostout,weight='weight')
        except:
            # if it still can't find a path, make a full connected network
            g_full=maskToGraph(mask, quslab, qvslab, np.ones(mask.shape),
                    np.ones(mask.shape), -np.inf)
            path=nx.dijkstra_path(g_full,mostin,mostout,weight='weight')
    else:
        maxidx=np.argmax(dists)
        yidx,xidx=np.unravel_index(maxidx,(n1,n2))
        path=nx.dijkstra_path(g,inedgecoor[yidx],outedgecoor[xidx],weight='weight')

    # get a mask for axis
    axismask=np.zeros(mask.shape)
    for (y,x) in path:
        axismask[y,x]=1

    path=np.array(path)

    return path, axismask

def cropMask(mask, edge=4):
    '''Cut out a bounding box around mask==1 areas

    Args:
        mask (ndarray): 2D binary map showing the location of an AR with 1s.
        edge (int): number of pixels as edge at 4 sides.

    Returns:
        mask[y1:y2, x1:x2] (ndarray): a sub region cut from <mask> surrouding
                                      regions with value=1.
        (yy,xx): y-, x- indices of the box of the cut region. Can later by
                 used in applyCropIdx(new_slab, (yy,xx)) to crop out the same
                 region from a new array <new_slab>.
    '''

    yidx,xidx=np.where(mask==1)
    if len(yidx)==0:
        raise Exception("mask empty")

    y1=np.min(yidx)
    y2=np.max(yidx)
    x1=np.min(xidx)
    x2=np.max(xidx)

    y1=max(0,y1-edge)
    y2=min(mask.shape[0],y2+edge)
    x1=max(0,x1-edge)
    x2=min(mask.shape[1],x2+edge)
    xx=np.arange(x1,x2)
    yy=np.arange(y1,y2)

    return mask[y1:y2,x1:x2], (yy,xx)

def applyCropIdx(slab, cropidx):
    '''Cut out a bounding box from given 2d slab given corner indices

    Args:
        slab (ndarray): 2D array to cut a box from.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask().

    Returns:
        cropslab (ndarray): 2D sub array cut from <slab> using <cropidx> as
                            boundary indices.
    '''

    cropslab=np.array(slab)[np.ix_(*cropidx)]
    try:
        croplat=slab.getLatitude()[:][cropidx[0]]
        croplon=slab.getLongitude()[:][cropidx[1]]

        croplat=cdms.createAxis(croplat)
        croplat.designateLatitude()
        croplat.id='y'
        croplat.units='degree'
        croplat.name='latitude'

        croplon=cdms.createAxis(croplon)
        croplon.designateLongitude()
        croplon.id='x'
        croplon.units='degree'
        croplon.name='longitude'

        cropslab=MV.array(cropslab)
        cropslab.setAxis(0,croplat)
        cropslab.setAxis(1,croplon)
    except:
        pass

    return cropslab

def insertCropSlab(shape, cropslab, cropidx, axislist=None):
    '''Insert the cropped sub-array back to a larger empty slab

    Args:
        shape (tuple): (n, m) size of the larger slab.
        cropslab (ndarray): 2D array to insert.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask(),
                         defines where <cropslab> will be inserted into.
    Kwargs:
        axislist (list or None): if list, a list of cdms.TransientAxis objs.

    Returns:
        result (ndarray): 2D slab with shape (n, m), an empty array with a
                          box at <cropidx> replaced with data from <cropslab>.
                          Optionally, axes information is added if <axistlist>
                          is not None, making it an TransientVariable.
    '''

    result=np.zeros(shape)
    result[np.ix_(*cropidx)]=cropslab

    if axislist is not None:
        result=MV.array(result)
        result.setAxisList(axislist)

    return result

def getMaskEdge(mask):
    '''Get the ordered boundary cell indices around non-zeros values in a
    binary mask

    Args:
        mask (ndarray): 2D binary mask.

    Returns:
        edge (ndarray): Nx2 array storing (x, y) coordinate indices of the
                        grid cells in <mask> that form the boundary of
                        objects defined as non-zero values.
    '''
    edge=mask-morphology.binary_erosion(mask)
    edge=np.where(edge>0)
    edge=zip(edge[1],edge[0])
    edge=np.array(edge)
    edge=funcs.getLineFromPoints(edge)

    return edge

def partPeaks(cropmask, cropidx, orislab, max_ph_ratio):
    '''Separate local maxima by topographical prominence

    Args:
        cropmask (ndarray): 2D binary array, defines regions of local maxima.
        cropidx (tuple): (y, x) coordinate indices, output from cropMask().
        orislab (ndarray): 2D array, giving magnitude/height/intensity values
                           defining the topography.
        max_ph_ratio (float): maximum peak/height ratio. Local peaks with
                              a peak/height ratio larger than this value is
                              treated as an independent peak.

    Returns:
        result (ndarray): 2D binary array, similar as the input <cropmask>
                          but with connected peaks (if any) separated so that
                          each connected region (with 1s) denotes an
                          independent local maximum.
    '''

    cropslab=applyCropIdx(orislab,cropidx)
    if 0 in cropidx[0] or 0 in cropidx[1] or orislab.shape[0]-1 in\
            cropidx[0] or orislab.shape[1]-1 in cropidx[1]:
        include_edge=True
    else:
        include_edge=False

    # compute prominences
    peaks,peakid,peakpro,peakparents=pp2d.getProminence(cropslab*cropmask,
            10.,include_edge=include_edge,centroid_num_to_center=1,verbose=False)

    peakheights=(peakpro>0)*cropslab*cropmask
    ratios=cropmask*peakpro/peakheights

    # take maxima whose prominence/height ratio> max_ph_ratio
    localpeaks=np.where(ratios>max_ph_ratio)
    localpeaks=zip(localpeaks[0],localpeaks[1])

    mask1=np.zeros(cropmask.shape)    # modified mask
    # residual mask, the union of complimentary masks. A complimentary mask
    # is the sea level mask that separates a peak from its parent. Note that
    # peaks' sea levels are not necessarily at the same height.
    resmask=np.zeros(cropmask.shape)

    def breakPeaks(yidx,xidx,localpeaks,col):
        '''Separate the contour of a peak from its parent by iteratively
        rising the sea level
        '''
        labels=morphology.label(cropslab*cropmask>col)
        dropthis=False
        while True:
            #plabels=[labels[yjj,xjj] for yjj,xjj in localpeaks]
            plabels=[labels[yjj,xjj] for yjj,xjj in localpeaks if labels[yjj, xjj]==labels[yidx, xidx]]
            #if len(set(plabels))==len(localpeaks):
            if len(plabels)==1:
                break
            col+=5.
            if col>cropslab[yidx,xidx]:
                dropthis=True
                col-=5.
                break
            labels=morphology.label(cropslab*cropmask>col)

        tmpmask=np.zeros(cropmask.shape)
        if not dropthis:
            tmpmask[yidx,xidx]=1
            tmpmask=morphology.reconstruction(tmpmask,cropslab>col,'dilation')

        return tmpmask,col

    if len(localpeaks)==1:
        mask1=cropmask
    else:
        # sort by prominence/height ratios
        ratios=[ratios[int(yjj), int(xjj)] for yjj,xjj in localpeaks]
        heights=[peakheights[int(yjj), int(xjj)] for yjj, xjj in localpeaks]
        sortidx=np.argsort(ratios)
        ratios.sort()
        localpeaks=[localpeaks[idjj] for idjj in sortidx]
        '''
        localpeakids=[peakid[yjj,xjj] for yjj,xjj in localpeaks]
        localpeakcols=[peaks[idjj]['col_level'] for idjj in localpeakids]
        c_col=np.min(localpeakcols)-10

        labs=morphology.label(cropslab*cropmask>c_col)
        while True:
            if c_col>np.max(cropslab*cropmask):
                break
            plabels=[labs[yjj,xjj] for yjj,xjj in localpeaks]
            if len(set(plabels))==len(localpeaks):
                break
            c_col+=5.
            labs=morphology.label(cropslab*cropmask>c_col)
        mask1=morphology.reconstruction(localpeaks_map, cropslab>c_col, 'dilation')
        '''

        for yjj,xjj in localpeaks:
            yjj=int(yjj)
            xjj=int(xjj)
            idjj=peakid[yjj,xjj]
            coljj=peaks[idjj]['col_level']
            if peaks[idjj]['parent']==0 and peakheights[yjj,xjj]==np.max(heights):
                # the heighest peak
                tmpmask=np.zeros(cropslab.shape)
                tmpmask[yjj,xjj]=1
                tmpmask=morphology.reconstruction(tmpmask,
                        (cropmask-resmask)>0,'dilation')
                mask1=mask1+tmpmask
            else:
                # separate local peaks
                tmpmask,coljj2=breakPeaks(yjj,xjj,localpeaks,coljj)
                # if childrens overlap, may not need this anymore
                if (tmpmask+mask1).max()>1:
                    tmpmask2=np.zeros(cropmask.shape)
                    tmpmask2[yjj,xjj]=1
                    tmpmask=morphology.reconstruction(tmpmask2,tmpmask-tmpmask*resmask,'dilation')
                mask1=mask1+tmpmask
                resmask=np.where((resmask==1) | (cropslab<coljj2),1,0)

    result=insertCropSlab(orislab.shape,mask1,cropidx)

    return result

def getARData(slab, quslab, qvslab, anoslab, quano, qvano, areas,
        mask_list, axis_list, timestr, param_dict, shift_lon, isplot,
        outputdir):
    '''Fetch AR related data

    Args:
        slab (cdms.TransientVariable): (n * m) 2D array of IVT, in kg/m/s.
        quslab (cdms.TransientVariable): (n * m) 2D array of u-flux, in kg/m/s.
        qvslab (cdms.TransientVariable): (n * m) 2D array of v-flux, in kg/m/s.
        anoslab (cdms.TransientVariable): (n * m) 2D array of IVT anomalies,
                                        in kg/m/s.
        quano (cdms.TransientVariable): (n * m) 2D array of u-flux anomalies,
                                        in kg/m/s.
        qvano (cdms.TransientVariable): (n * m) 2D array of v-flux anomalies,
                                        in kg/m/s.
        areas (cdms.TransientVariable): (n * m) 2D grid cell area slab, in km^2.
        mask_list (list): list of 2D binary masks, each with the same shape as
                      <anoslab> etc., and with 1s denoting the location of a
                      found AR.
        axis_list (list): list of AR axis coordinates. Each coordinate is defined
                     as a Nx2 ndarray storing (y, x) indices of the axis
                     (indices defined in the matrix of corresponding mask
                     in <masks>.)
        timestr (str): string of time snap.
        param_dict (dict): parameter dict defined in Global preamble.
        shift_lon (float): starting longitude of data domain, defined in Global
                           preamble.
        isplot (bool): if True, create plot of AR axis, flux orientation and
                       cross-sectional flux for each AR.
        outputdir (str): folder to save plots. If None, don't save. If <isplot>
                         is False, not relevant.

    Returns:
        labels (cdms.TransientVariable): (n * m) 2D int map showing all ARs
                                       at current time. Each AR is labeled by
                                       an int label, starting from 1. Background
                                       is filled with 0s.
        angles (cdms.TransientVariable): (n * m) 2D map showing orientation
                                       differences between AR axes and fluxes,
                                       for all ARs. In degrees.
        crossfluxes (cdms.TransientVariable): (n * m) 2D map showing cross-
                                           sectional fluxes in all ARs.
                                           In kg/m/s.
        anocrossflux (cdms.TransientVariable): similar as <crossfluxes> but for
                                             anomalous fluxes (corresponding
                                             to <anoslab>).
        df (pandas.DataFrame): AR record table. Each row is an AR, see code
                               below for columns.
    '''

    max_isoq=param_dict['max_isoq']
    min_length=param_dict['min_length']
    min_length_hard=param_dict['min_length_hard']
    rdp_thres=param_dict['rdp_thres']
    min_area=param_dict['min_area']

    lonax=slab.getLongitude()   # NOTE: max > 360
    latax=slab.getLatitude()

    # prepare outputs
    labels=MV.zeros(slab.shape)
    angles=MV.zeros(slab.shape)
    crossfluxes=MV.zeros(slab.shape)
    results={}

    #-----------------Loop through ARs-----------------
    for ii in range(len(mask_list)):

        maskii=mask_list[ii]

        # region properties, in pixel units
        rpii=measure.regionprops(maskii, intensity_image=np.array(slab))[0]

        # get centroid
        centroidy,centroidx=rpii.weighted_centroid
        centroidy=latax[int(centroidy)]
        centroidx=lonax[int(centroidx)]

        # get axis coordinate array
        skelii=axis_list[ii]
        latsii=latax[skelii[:,0]]
        lonsii=lonax[skelii[:,1]]
        axisii=np.c_[latsii,lonsii]

        # segment axis using rdp
        axis_rdpii=np.array(rdp.rdpGC(axisii.tolist(),rdp_thres))  # lat,lon

        # area
        areaii=(maskii*areas).sum()  # km^2

        # compute length
        lens=funcs.greatCircle(axis_rdpii[:-1,0], axis_rdpii[:-1,1],
                axis_rdpii[1:,0], axis_rdpii[1:,1])/1e3
        lenii=lens.sum() #km

        # skip if too small and too short
        if areaii<min_area or lenii<min_length_hard:
            continue

        # mean width
        widthii=areaii/lenii # km

        # mask contour
        contii=funcs.getBinContour(maskii,lonax,latax)

        # isoperimetric quotient
        isoquoii=4*np.pi*rpii.area/rpii.perimeter**2

        # length/width ratio
        ratioii=lenii/widthii

        # mean strength
        slabii=MV.masked_where(maskii==0,slab)
        strengthii=cdutil.averager(slabii,axis='xy',
                weights=['generate','generate'])

        # strength std
        strengthstdii=float(stats.std(slabii,axis='xy'))

        # anomaly strength
        anoslabii=MV.masked_where(maskii==0,anoslab)
        anostrengthii=cdutil.averager(anoslabii,axis='xy',
                weights=['generate','generate'])

        # max strength
        max_strengthii=float(MV.max(slabii))

        # compute angles and cross-section flux of total flux
        cropmask,cropidx=cropMask(maskii)
        cropskelii=skelii-np.array([cropidx[0].min(), cropidx[1].min()])

        cropu=applyCropIdx(quslab,cropidx)
        cropv=applyCropIdx(qvslab,cropidx)

        anglesii,anglesmeanii,crossfluxii,seg_thetasii=crossSectionFlux(
                cropmask, cropu, cropv, axis_rdpii)

        # create plots
        if isplot:
            pass
            #plotARCrosssectionFlux(cropmask, cropu, cropv, cropskelii, axis_rdpii,
                    #'%s AR-%d' %(timestr, ii+1), shift_lon, anglesii, anglesmeanii,
                    #crossfluxii, seg_thetasii, outputdir)

        # insert crop back to the big map
        anglesii=insertCropSlab(maskii.shape,anglesii,cropidx,
                slab.getAxisList())
        anglesii=MV.where(maskii==1,anglesii,0)

        crossfluxii=insertCropSlab(maskii.shape,crossfluxii,cropidx,
                slab.getAxisList())
        crossfluxii=MV.where(maskii==1,crossfluxii,0)

        # mean meridional flux
        cropv=applyCropIdx(qvslab,cropidx)
        cropv=MV.masked_where(cropmask==0,cropv)
        qvmeanii=cdutil.averager(cropv,axis='xy',weights=['generate',\
                'generate'])

        # is candidate a strict AR
        is_relaxedii=False
        if isoquoii>max_isoq or ratioii<2:
            is_relaxedii=True
        if lenii<min_length:
            is_relaxedii=True
        if qvmeanii<=0:
            is_relaxedii=True

        labels=labels+maskii*(ii+1)
        angles=angles+anglesii
        crossfluxes=crossfluxes+crossfluxii

        results[ii+1]={
                'id': ii+1,
                'time':timestr,
                'contour_y': contii.vertices[:,1],
                'contour_x': contii.vertices[:,0],
                'centroid_y': centroidy,
                'centroid_x': centroidx,
                'axis_y':axisii[:,0],
                'axis_x':axisii[:,1],
                'axis_rdp_y':axis_rdpii[:,0],
                'axis_rdp_x':axis_rdpii[:,1],
                'area': areaii,
                'length': lenii,
                'width': widthii,
                'iso_quotient':isoquoii,
                'LW_ratio':ratioii,
                'strength':strengthii,
                'strength_ano':anostrengthii,
                'strength_std':strengthstdii,
                'max_strength':max_strengthii,
                'mean_angle': float(anglesmeanii),
                'is_relaxed':is_relaxedii,
                'qv_mean':qvmeanii
                }

    labels.setAxisList(slab.getAxisList())
    angles.setAxisList(slab.getAxisList())
    crossfluxes.setAxisList(slab.getAxisList())

    labels.id='labels'
    labels.long_name='AR labels'
    labels.standard_name=labels.long_name
    labels.title=labels.long_name
    labels.units=''

    angles.id='angles'
    angles.long_name='AR moisture flux orientation difference'
    angles.standard_name=angles.long_name
    angles.title=angles.long_name
    angles.units='degree'

    crossfluxes.id='ivt_cross'
    crossfluxes.long_name='AR total cross sectional moisture flux'
    crossfluxes.standard_name=crossfluxes.long_name
    crossfluxes.title=crossfluxes.long_name
    crossfluxes.units=getattr(slab, 'units', '')

    keys=['id', 'time', 'contour_y', 'contour_x', 'centroid_y', 'centroid_x',
            'axis_y', 'axis_x', 'axis_rdp_y', 'axis_rdp_x',
            'area', 'length', 'width', 'iso_quotient', 'LW_ratio',
            'strength', 'strength_ano', 'strength_std', 'max_strength',
            'mean_angle', 'is_relaxed', 'qv_mean']

    df=pd.DataFrame(results).T
    if len(df)>0:
        df=df[keys]

    return labels,angles,crossfluxes,df

def uvDecomp(u0, v0, i1, i2):
    '''Decompose background-transient components of u-, v- fluxes

    Args:
        u0 (cdms.TransientVariable): nd array of total u-flux.
        v0 (cdms.TransientVariable): nd array of total v-flux.
        i1 (cdms.TransientVariable): nd array of the reconstruction component
                                   of IVT.
        i2 (cdms.TransientVariable): nd array of the anomalous component
                                   of IVT (i2 = IVT - i1).
    Returns:
        u1 (cdms.TransientVariable): nd array of the u-flux component
                                   corresponding to <i1>, i.e. the background
                                   component.
        v1 (cdms.TransientVariable): nd array of the v-flux component
                                   corresponding to <i1>, i.e. the background
                                   component.
        u2 (cdms.TransientVariable): nd array of the u-flux component
                                   corresponding to <i2>, i.e. the transient
                                   component.
        v2 (cdms.TransientVariable): nd array of the v-flux component
                                   corresponding to <i2>, i.e. the transient
                                   component.
    '''

    i0=i1+i2
    v1=v0*i1/i0
    v2=v0*i2/i0
    u1=u0*i1/i0
    u2=u0*i2/i0

    return u1,u2,v1,v2

def save2DF(result_dict):
    '''Save AR records to a pandas DataFrame

    Args:
        result_dict (dict): key: time str in 'yyyy-mm-dd hh:00'
                            value: pandas dataframe. See getARData().

    Returns:
        result_df (pandas.DataFrame): AR record table containing records
                                      from multiple time steps sorted by
                                      time.
    '''

    for ii,kk in enumerate(result_dict.keys()):
        vv=result_dict[kk]
        if ii==0:
            result_df=vv
        else:
            result_df=pd.concat([result_df,vv],axis=0,ignore_index=True)

    result_df['time']=pd.to_datetime(result_df.time)
    result_df=result_df.sort_values(by='time')

    return result_df

def plotAR(ardf, ax, bmap):
    '''Helper function to plot the regions and axes of ARs

    Args:
        ardf (pandas.DataFrame): table containing AR records.
        ax (matplotlib axis): axis to plot onto.
        bmap (Basemap obj): defining the geo map.
    '''

    for ii in range(len(ardf)):

        vv=ardf.iloc[ii]
        isrelaxkk=vv['is_relaxed']

        # plot contour
        px=vv['contour_x']
        py=vv['contour_y']

        px,py=bmap(px,py)
        linewidth=1 if isrelaxkk else 1
        linestyle=':' if isrelaxkk else '-'
        ax.plot(px,py,color='k',linestyle=linestyle,linewidth=linewidth)

        # plot axis
        px=vv['axis_x']
        py=vv['axis_y']

        px,py=bmap(px,py)
        ax.plot(px,py,'g:',linewidth=1.5)

        # plot cross flux text
        '''
        lenkk=vv['length']
        areakk=vv['area']
        widthkk=vv['width']
        cx=float(vv['centroid_x'])%360
        cy=float(vv['centroid_y'])
        cx,cy=bmap(cx,cy)

        strkk=r'ID=%d, $R=%.0f$' %(ii+1,np.sqrt(areakk/3.14)) + '\n'+\
                r'$L = %d km$' %lenkk +'\n'+\
                r'$W = %d km$' %widthkk

        ax.annotate(strkk,xy=(cx,cy),
                horizontalalignment='center',
                verticalalignment='center',
                fontsize=8,
                bbox=dict(facecolor='white',alpha=0.5))
        '''

    return

def getNormalVectors(point_list, idx):
    '''Get the normal vector and the tagent vector to the plane dividing
    2 sections along the AR axis.

    Args:
        point_list (list): list of (lat, lon) coordinates.
        idx (int): index of the point in <point_list>, denoting the point
                   in question.

    Returns:
        normi (tuple): the (x, y, z) Cartesian coordinate of the unit
                       normal vector, at the point denoted by <idx>,
                       on the Earth surface. This is the normal vector to
                       the plane spanned by the vector Theta and P.
                       Where P is the vector pointing to the point in
                       question (point_list[idx]), and Theta is the
                       tangent vector evenly dividing the angle formed by
                       <P,P1>, and <P,P2>. Where P1, P2 are 2 points on
                       both side of P.
        thetai (tuple): the (x, y, z) Cartesian coordinate of the tangent
                        vector Theta above.
    '''
    pi=point_list[idx]
    pic=spherical2Cart(*pi)
    theta1=computeTheta(pi,point_list[idx-1])
    theta2=computeTheta(pi,point_list[idx+1])
    thetai=(theta1+theta2)/2.

    normi=np.cross(pic,thetai)
    normi=normi/np.linalg.norm(normi)
    thetai=thetai/np.linalg.norm(thetai)

    return normi,thetai

def crossSectionFlux(mask, quslab, qvslab, axis_rdp):
    '''Compute setion-wise orientation differences and cross-section fluxes
    in an AR

    Args:
        mask (ndarray): CROPPED (see cropMask and applyCropIdx) 2D binary map
                        showing the location of an AR with 1s.
        quslab (cdms.TransientVariable): CROPPED (n * m) 2D array of u-flux,
                                       in kg/m/s.
        qvslab (cdms.TransientVariable): CROPPED (n * m) 2D array of v-flux,
                                       in kg/m/s.
        axis_rdp (ndarray): Nx2 array storing the (lat, lon) coordinates of
                           rdp-simplified AR axis.

    Returns:
        angles (TransientVariable): 2D map with the same shape as <mask>,
                                    showing section-wise orientation
                                    differences between horizontal flux (as
                                    in <quslab>, <qvslab>) and the AR axis of
                                    that section. In degrees. Regions outside
                                    of AR (0s in <mask>) are masked.
        anglesmean (float): area-weighted averaged of <angles> inside <mask>.
        crossflux (TransientVariable): 2D map with the same shape as <mask>,
                                       the section-wise cross-section fluxes
                                       in the AR, defined as the projection
                                       of fluxes onto the AR axis, i.e. flux
                                       multiplied by the cos of <angles>.
        seg_thetas (list): list of (x, y, z) Cartesian coordinates of the
                           tangent vectors along section boundaries.
    '''
    # get coordinates
    axislist=quslab.getAxisList()
    lats=quslab.getLatitude()[:]
    lons=quslab.getLongitude()[:]
    lonss,latss=np.meshgrid(lons,lats)

    # convert to cartesian coordinates
    carts=spherical2Cart(latss,lonss)
    vs=wind2Cart(quslab,qvslab,latss,lonss)
    vsnorm=np.linalg.norm(vs,axis=0)
    vsnorm=vs/vsnorm[None,:,:]

    # loop through segments to get orientation differences
    nsegs=len(axis_rdp)-1
    seg_thetas=[]
    angles=np.zeros(mask.shape)

    for ii in range(nsegs):

        pic=spherical2Cart(*axis_rdp[ii])
        pi1c=spherical2Cart(*axis_rdp[ii+1])

        if ii==0:
            setL=1.
            thetai=0 # dummy place holder
        else:
            # get evenly dividing angle theta and normal vector to theta
            normi,thetai=getNormalVectors(axis_rdp, ii)
            # dot products between normal vector and grid coordinates
            dotsi=(normi[:,None,None]*carts).sum(axis=0)
            setL=np.where(dotsi*(normi.dot(pi1c))>=0,1,0)

        if ii==nsegs-1:
            setR=1.
            thetai=0 # dummy place holder
        else:
            normi1,thetai=getNormalVectors(axis_rdp, ii+1)
            dotsi1=(normi1[:,None,None]*carts).sum(axis=0)
            setR=np.where(dotsi1*(normi1.dot(pic))>0,1,0)

        segii=setL*setR*mask
        seg_thetas.append(thetai)

        # sel the correct region if shape too curvy
        segregii=measure.label(segii)
        if segregii.max()>1:
            piidx=[funcs.findIndex(axis_rdp[ii][0],lats),\
                    funcs.findIndex(axis_rdp[ii][1],lons)]
            for jj in range(segregii.max()):
                segjj=np.where(segregii==jj+1,1,0)
                if segjj[piidx[0],piidx[1]]==1:
                    segii=segjj
                    break

        # mean orientation of AR axis segment
        meanori=np.cross(pic,pi1c)
        meanori=meanori/np.linalg.norm(meanori)

        # orientation of flux vectors
        '''
        fluxori=np.cross(carts,vs,axisa=0,axisb=0) # output: ny,nx,3
        norms=np.linalg.norm(fluxori,axis=2)
        fluxori=fluxori/(1e-6+norms[:,:,None])

        # get angles as arccos of the dot product of meanori and fluxori
        anglesii=(meanori[None,None,:]*fluxori).sum(axis=-1)
        anglesii=anglesii*segii
        '''

        # get sin(angle of flux vector and axis segment plane)
        # sign of sin() is: >0: flux vector aligns with meanori, and it
        # is pointing towards the cold side (according to thermal wind).
        # sin() <0: flux vector aligns against meanori, and it is pointing
        # towards the warm side.
        anglesii=(meanori[:,None,None]*vsnorm).sum(axis=0)
        #anglesii=np.sqrt(1-cos_alphaii**2)*np.where(cos_alphaii<0,-1,1)
        anglesii=anglesii*segii

        angles=angles+anglesii

    # compute cross section flux
    angles=np.array(angles)
    cos_angles=np.sqrt(1-angles**2)
    crossflux_c=cos_angles[None,:,:]*vs
    # convert to local tangent winds
    crossflux_u,crossflux_v=cart2Wind(crossflux_c,latss,lonss)
    crossflux=np.sqrt(crossflux_u**2+crossflux_v**2)
    crossflux=MV.masked_where(mask==0,crossflux)
    crossflux.setAxisList(axislist)

    # convert cos to angle in degrees
    angles=np.arcsin(angles)/np.pi*180
    angles=MV.masked_where(mask==0,angles)
    angles.setAxisList(axislist)

    anglesmean=cdutil.averager(angles,axis='xy',weights=['generate','generate'])

    return angles,anglesmean,crossflux,seg_thetas


import os
import tensorflow as tf
import time
import numpy as np
import data_loader
import matplotlib.pyplot as plt
from tqdm import trange

class DNN:
    """DNN模型训练"""

    def __init__(self, path=None, sheet_name="Sheet2",
                 save_model_path="./model/model-1/", batch_size=32,
                 learning_rate=0.001, num_steps=10000, model_dim=(101, 64, 40),
                 steps=10, threshold_value=0.9, dropout=0, save_model_threshold_value=0.8):
        """
        创建模型训练实例
        :param path: 输入xlsx文件路径
        :param save_model_path: 模型保存路径
        :param batch_size: 训练batch大小
        :param num_steps: 迭代步数
        :param steps: 打印评估间隔步数
        :param threshold_value: 输出阈值
        :param dropout: 丢弃率
        :param save_model_threshold_value: 模型保存阈值
        """
        self.path = path
        self.save_model_path = save_model_path
        self.batch_size = batch_size
        self.num_steps = num_steps
        self.steps = steps
        self.all_counts = int(num_steps / steps) - 1
        self.count = 0  # 评估计数
        self.fig_loss = np.zeros([self.all_counts])
        self.fig_acc = np.zeros([self.all_counts])
        self.save_model_threshold_value = save_model_threshold_value  # 保存模型阈值
        self.threshold_value = threshold_value  # 阈值
        self.sheet_name = sheet_name
        self.keep_rate = 1 - dropout  # dropout使神经网络层中随机丢弃神经单元的值的保留率
        self.learning_rate = learning_rate  # 学习率
        self.hidden_dim = model_dim[1:-1]   # 隐藏层维度
        self.input_dim = model_dim[0]   #输入层维度
        self.output_dim = model_dim[-1] #输出层维度

        self.graph = tf.Graph()  # 建立图
        with self.graph.as_default():  # 设为默认的graph
            self.keep_prob = tf.placeholder(tf.float32, None, name="keep_prob")
            self.train_inputs = tf.placeholder(tf.float32, shape=[None, None], name="train_inputs")  # 占位符，样本输入
            self.train_labels = tf.placeholder(tf.float32, shape=[None, None], name="train_labels")  # 占位符，样本标签
            # 使用GPU进行张量计算
            with tf.device('/cpu:0'):
                # 隐藏层1
                self.hidden_layer = self.layer(output_dim=self.hidden_dim[0], input_dim=self.input_dim,
                                               inputs=self.train_inputs,
                                               keep_prob=self.keep_prob,
                                               activation=None)
                # 隐藏层(>1)
                for num in range(len(self.hidden_dim) - 1):
                    self.hidden_layer = self.layer(output_dim=self.hidden_dim[num + 1], input_dim=self.hidden_dim[num],
                                                   inputs=self.hidden_layer,
                                                   keep_prob=self.keep_prob,
                                                   activation=None)
                # 输出层
                self.output_layer = self.layer(output_dim=self.output_dim, input_dim=self.hidden_dim[-1],
                                               inputs=self.hidden_layer,
                                               keep_prob=None,
                                               activation=None)

                # 最终使用sigmod函数输出预测结果
                self.prediction = tf.nn.sigmoid(self.output_layer)
                tf.identity(self.prediction, "output")

            # 使用sigmoid_cross_entropy作为损失函数
            self.loss = tf.reduce_mean(
                tf.nn.sigmoid_cross_entropy_with_logits(logits=self.output_layer, labels=self.train_labels))

            # 定义优化损失函数，使用Adam以及预设学习率训练参数
            self.optimizer = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss)
            self.init = tf.global_variables_initializer()

    def layer(self, output_dim, input_dim, inputs, keep_prob=None, activation=None):
        """
        神经网络层
        :param output_dim: 输出维度
        :param input_dim: 输入维度
        :param inputs: 输入数据
        :param activation: 采用激活函数
        :param keep_prob: drop_out保持率
        :return: 下一层输入
        """
        # 权重
        W = tf.Variable(tf.random_normal([input_dim, output_dim]))
        # 偏置
        b = tf.Variable(tf.random_normal([1, output_dim]))
        XWb = tf.matmul(inputs, W) + b
        if keep_prob is not None:
            XWb = tf.nn.dropout(XWb, keep_prob=keep_prob)
        if activation is None:
            outputs = XWb
        else:
            outputs = activation(XWb)
        return outputs

    def train(self, data_inputs, data_labels, keep_prob):
        """
        模型训练
        :param data_inputs: 样本输入
        :param data_labels: 样本标签
        :param keep_prob: dropout保留神经元概率
        """
        with tf.Session(graph=self.graph) as session:
            self.init.run()
            if not os.path.exists(self.save_model_path):
                os.mkdir(self.save_model_path)
            # 检查是否已存在模型，有则恢复训练，重新训练新模型需先删除模型
            if os.path.exists(self.save_model_path + "model.ckpt.meta"):
                ckpt = tf.train.latest_checkpoint(self.save_model_path)
                tf.train.Saver().restore(session, ckpt)
            total_loss = 0
            total_acc = 0
            max_acc = 0

            # 模型保存管理
            saver = tf.train.Saver(max_to_keep=2)
            start_time = time.time()
            # 使用tqdm.trange管理进度条输出
            with trange(self.num_steps) as t:
                for step in t:
                    t.set_description("Training")
                    batch_inputs, batch_labels, one_hot_label = next(
                        data_loader.generate_batch(data_inputs, data_labels, batch_size=self.batch_size))
                    feed_dict = {self.train_inputs: batch_inputs, self.train_labels: batch_labels,
                                 self.keep_prob: keep_prob}
                    _, loss, output = session.run([self.optimizer, self.loss, self.output_layer], feed_dict)
                    # 阈值判断输出
                    output = [[1 if i > self.threshold_value else 0 for i in j] for j in output]
                    batch_labels = list(batch_labels)
                    for num, value in enumerate(output):
                        if value == list(batch_labels[num]):
                            total_acc = total_acc + (1. / self.batch_size)
                    # 统计损失和准确率
                    total_loss = total_loss + loss

                    # 每steps次进行一次评估输出
                    if step % self.steps == 0 and step != 0:
                        # 计算平均损失
                        average_loss = total_loss / self.steps
                        average_acc = total_acc / self.steps
                        spend_time = time.time() - start_time
                        t.set_postfix(Loss="{:.9f}".format(average_loss), Accuracy="{:.9f}".format(average_acc))
                        self.fig_loss[self.count] = average_loss
                        self.fig_acc[self.count] = average_acc
                        if average_acc > self.save_model_threshold_value:
                            # 如果评估准确率历史最优，保存模型
                            self.save_model_threshold_value = average_acc
                            saver.save(sess=session, save_path=self.save_model_path + "model.ckpt")
                        # 每一次评估后损失与准确率置零
                        total_loss = 0
                        total_acc = 0
                        # 计数+1
                        self.count += 1
                        # 重置时间
                        start_time = time.time()

    def plot(self):
        # 绘制损失与准确率图
        fig, ax1 = plt.subplots()
        ax2 = ax1.twinx()
        lns1 = ax1.plot(np.arange(self.all_counts), self.fig_loss, label="Loss")
        lns2 = ax2.plot(np.arange(self.all_counts), self.fig_acc, 'r', label="Accuracy")
        ax1.set_xlabel('iteration')
        ax1.set_ylabel('training loss')
        ax2.set_ylabel('training accuracy')
        # 合并图例
        lns = lns1 + lns2
        labels = ["Loss", "Accuracy"]
        plt.legend(lns, labels, loc=7)
        plt.show()

    def main(self):
        """模型训练"""
        # 读取样本
        data_inputs, data_labels = data_loader.read_data(path=self.path, sheet_name=self.sheet_name, input_num=self.input_dim, label_num=self.output_dim)
        # 训练样本
        self.train(data_inputs, data_labels, self.keep_rate)
        # 绘制loss与acc图
        self.plot()

if __name__ == '__main__':
    # 创建类的实例
    ###########################################参数说明############################################
    # path: 数据输入路径，sheet_name：数据所在表单名， save_model_path: 模型保存路径
    # batch_size: 训练批次大小，learning_rate: 学习率, num_steps: 总迭代步数
    # model_dim为一个元祖，存放整个模型各层的维度[tuple:（输入层维度, 隐藏层1维度, 隐藏层2维度..., 输出层维度)]
    # steps: 每多少步进行评估，threshold_value: 输出结果阈值，dropout: 防过拟合，随机神经单元丢弃率
    # save_model_threshold_value: 模型保存阈值（评估准确率大于阈值保存模型）
    dnn = DNN(path="./data/样本.xlsx", sheet_name="Sheet2", save_model_path="./models/model-1/",
              batch_size=64, learning_rate=0.001, num_steps=20000, model_dim=(101, 64, 51, 40), steps=100,
              threshold_value=0.9, dropout=0.1, save_model_threshold_value=0.8)
    # 程序入口，开始进行训练
    dnn.main()


from typing import Dict, List, Union, Tuple
from urllib.parse import urljoin

import requests
from PIL import Image
from bs4 import BeautifulSoup
from email_validator import validate_email, EmailNotValidError

from plan_ilan.data_mining.staff.lookup_parameters import StaffLookup, StaffLookupAnswer
from plan_ilan.apps.web_site.models import DepartmentEnum

class StaffCrawler:
    """An object that crawls over different staff members web pages according to their attributes."""

    def __init__(self, url: Union[str, List[str]], department: DepartmentEnum, staff_lookup: StaffLookup,
                 **kwargs) -> None:
        self._urls = url if isinstance(url, List) else [url]
        self._department = department
        self.staff_lookup = staff_lookup
        self._has_photos = kwargs['has_photos'] if 'has_photos' in kwargs else True
        self._email_suffix = kwargs['email_suffix'] if 'email_suffix' in kwargs else ''
        self.__teachers_data = None

    @property
    def urls(self) -> List[str]:
        """Return the url of the crawled web page"""
        return self._urls

    @property
    def department(self):
        """Returns the department of the crawled web page"""
        return self._department

    def get_teachers_data(self) -> List[StaffLookupAnswer]:
        if self.__teachers_data is None:
            self.crawl()
        return self.__teachers_data

    def crawl(self) -> int:
        """"The main method for this interface, crawls over the staff members web page.
         returns the amount of valid entries found."""
        self.__teachers_data = []
        for url in self.urls:
            response = requests.get(url)
            if not response:
                continue
            soup = BeautifulSoup(response.text, 'html.parser')
            persons = self.staff_lookup.persons.get_tags(soup)
            for person in persons:
                teacher_data = StaffLookupAnswer()
                details_url = self.staff_lookup.details_url.get_single_tag(person)
                if details_url is None:
                    continue
                details_url = details_url['href'].strip()
                teacher_data.title = self.staff_lookup.params.title.get_values_from_tag(person)
                teacher_data.name = self.staff_lookup.params.name.get_values_from_tag(person)
                if not teacher_data.is_valid:
                    continue
                teacher_response = requests.get(details_url)
                if not teacher_response:
                    continue
                teacher_web_page = BeautifulSoup(teacher_response.text, 'html.parser')
                teacher_data.email = self.staff_lookup.params.email.get_values_from_tag(teacher_web_page)
                if not teacher_data.email:
                    teacher_data.email = ''
                mail_valid, value = self._validate_email(teacher_data.email)
                if mail_valid:
                    teacher_data.email = value
                else:
                    # todo: add logging
                    print(teacher_data.email, value)
                teacher_data.phone = self.staff_lookup.params.phone.get_values_from_tag(teacher_web_page)
                teacher_data.website = self.staff_lookup.params.website.get_values_from_tag(teacher_web_page)
                teacher_data.office = self.staff_lookup.params.office.get_values_from_tag(teacher_web_page)
                image_tag = self.staff_lookup.params.photo.get_single_tag(teacher_web_page)
                image_url = urljoin(url, image_tag['src'])
                teacher_data.photo = Image.open(requests.get(image_url, stream=True).raw)
                self.__teachers_data.append(teacher_data)
        return len(self.__teachers_data)

    def _validate_email(self, email: str) -> Tuple[bool, Union[Dict, str]]:
        """Validate the given email address
        Args:
            email:str
                The email to validate
        Returns:
            tuple -
                A tuple of size 2:
                1) outcome - indicates whether the email address is valid
                2) value - if outcome is true returns the email address in normalized form, else
                           returns a human readable error message.
        """
        try:
            if '@' not in email:
                email = f'{email}{self._email_suffix}'
            valid = validate_email(email, timeout=10)
            outcome = True
            value = valid.email
        except EmailNotValidError as e:
            outcome = False
            value = str(e)
        return outcome, value


import json
import logging
import os
import random
import re
import time
from datetime import datetime

import boto3
import requests
from dateutil import tz
from dateutil.parser import parse
from selectolax.parser import HTMLParser
from utils import headers

logger = logging.getLogger()
logger.setLevel(logging.INFO)

dynamodb = boto3.resource('dynamodb')
table = dynamodb.Table(os.environ['VLR_MATCHES_TABLE'])

# vlr match events cache
vlr_events_cache = {}

def insert(table, matches):
    '''
    put items into specified DynamoDB table.
    '''
    with table.batch_writer() as batch:
        for match in matches:
            logger.info('put match info into the table: {}'.format(match))
            batch.put_item({k: v for k, v in match.items()})

def sleep():
    '''
    sleep for 1~10 secs (randomly)
    '''
    sec = random.randint(1, 10)
    time.sleep(sec)

def get_event_from_cache(event_url_path):
    global vlr_events_cache

    result = ''

    if event_url_path in vlr_events_cache:
        result = vlr_events_cache[event_url_path]

    return result

def scrape_event(event_url_path):
    '''
    scrape event page of url_path
    '''
    global vlr_events_cache

    url = 'https://www.vlr.gg{}'.format(event_url_path)
    logger.info('get event info: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    event_id = int(event_url_path.split('/')[2])

    event_name = html.css_first('.wf-title').text().strip()
    event_name = event_name.replace('\t', '').replace('\n', '')

    country_flag = html.css_first(
        '.event-desc-item-value .flag').attributes['class']
    country_flag = country_flag.replace(' mod-', '_').replace('flag_', '')

    data = {
        'event_id': event_id,
        'event_name': event_name,
        'country_flag': country_flag
    }

    # caching
    vlr_events_cache[event_url_path] = data

    return data

def scrape_match(match_url_path):
    '''
    scrape match page of url_path
    '''
    global vlr_events_cache

    url = 'https://www.vlr.gg{}'.format(match_url_path)
    logger.info('get match info: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    match_id = int(match_url_path.split('/')[1])

    match_name = html.css_first('.match-header-event-series').text()
    match_name = match_name.replace('\t', '').replace('\n', '')

    start_time = html.css_first('.moment-tz-convert').attributes['data-utc-ts']
    with_timezone = ' '.join([start_time, 'EST'])

    tzinfo = {'EST': tz.gettz('America/New_York'),
              'CST': tz.gettz('America/Chicago')}
    start_time_est = parse(with_timezone, tzinfos=tzinfo)
    start_time_utc = start_time_est.astimezone(tz.gettz('Etc/GMT'))
    start_time_utc = datetime.strftime(start_time_utc, '%Y-%m-%dT%H:%M:%S%z')

    teams = html.css('.wf-title-med')
    teams = [t.text().replace('\t', '').replace('\n', '') for t in teams]

    best_of = html.css('.match-header-vs-note')[-1].text()
    best_of = best_of.replace('Bo', '').replace(' Maps', '')
    best_of = best_of.replace('\t', '').replace('\n', '')
    best_of = int(best_of)

    event_url_path = html.css_first('a.match-header-event').attributes['href']

    if event_url_path in vlr_events_cache:
        logger.info('get event info from cache: {}'.format(event_url_path))
        event_info = vlr_events_cache[event_url_path]
    else:
        logger.info('get event info from website: {}'.format(event_url_path))
        event_info = scrape_event(event_url_path)

    data = {
        'match_id': match_id,
        'event_name': event_info['event_name'],
        'event_country_flag': event_info['country_flag'],
        'start_time': start_time_utc,
        'best_of': best_of,
        'match_name': match_name,
        'teams': teams
    }
    return data

def scrape_matches(page: str = 1):
    '''
    scrape /matches page
    '''
    url = 'https://www.vlr.gg/matches?page={}'.format(page)
    logger.info('fetch matches list from: {}'.format(url))

    resp = requests.get(url, headers=headers)
    html = HTMLParser(resp.text)

    matches = []

    for item in html.css('a.wf-module-item'):
        match_url_path = item.attributes['href']

        sleep()
        match_detail = scrape_match(match_url_path)

        item = {
            'id': match_detail['match_id'],
            'eventName': match_detail['event_name'],
            'eventCountryFlag': match_detail['event_country_flag'],
            'startTime': match_detail['start_time'],
            'bestOf': match_detail['best_of'],
            'matchName': match_detail['match_name'],
            'teams': [{'title': team} for team in match_detail['teams']],
            'pagePath': match_url_path
        }
        logger.info('add match to the list: {}'.format(item))
        matches.append(item)

    return matches

def lambda_handler(event, context):
    records = event['Records']
    match_list = []

    for record in records:
        body = json.loads(record['body'])
        page = str(body['page'])

        matches = scrape_matches(page)
        match_list.extend(matches)

    insert(table, match_list)

    return {
        'matches_count': len(match_list)
    }


import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import csv
import random

linkas = input("Autopliuso masinu linkas: ")

def randomlaikas():
    _sleep = random.randint(3,10)
    time.sleep(_sleep)

def scrape_phone_numbers():
    driver = webdriver.Chrome(executable_path=ChromeDriverManager().install())
    driver.get(linkas)
    time.sleep(4)
    accept_cookies = driver.find_element(By.XPATH, '//button[text()="Sutinku"]')
    accept_cookies.click()
    time.sleep(3)

    results = []
    num_cars = int(input("Kiek nuskreipinti automobiliu: "))
    current_window_handle = driver.current_window_handle

    while len(results) < num_cars:
        cars = driver.find_elements(By.XPATH, '//div[@class="announcement-title"]')

        for i, car in enumerate(cars):
            if i == num_cars:
                break
            current_window_handle = driver.current_window_handle
            car.click()
            time.sleep(3)
            for handle in driver.window_handles:
                if handle != current_window_handle:
                    driver.switch_to.window(handle)
            html = driver.page_source
            soup = BeautifulSoup(html, "html.parser")
            phone = soup.find("div", class_="button seller-phone-number js-phone-number")
            if phone is not None:
                phone_number = phone.text.replace(" ", "")
                if "+370" in phone_number and phone_number not in results:
                    results.append(phone_number)
            driver.close()
            driver.switch_to.window(current_window_handle)

        try:
            next_page = driver.find_element(By.XPATH, '//a[@rel="next" and @class="next"]')
            next_page.click()
            time.sleep(3)
        except Exception as e:
            break

    with open("phones.csv", "w", newline="") as f:
        writer = csv.writer(f)
        writer.writerow(["Phone Number"])
        for phone_number in results:
            phone_number = phone_number.strip('"\n')
            if "+370" not in phone_number:
                continue
            writer.writerow([phone_number])
        print(results)

scrape_phone_numbers()


import qiskit as qk
from qiskit import QuantumProgram

qp = QuantumProgram()
qr = qp.create_quantum_register('qr', 2)
cr = qp.create_classical_register('cr',2)
qc = qp.create_circuit('qc', [qr], [cr])
circuit = qp.get_circuit('qc')
quantum_r = qp.get_quantum_register('qr')
classical_r = qp.get_classical_register('cr')

circuit.x(quantum_r[0])
circuit.y(quantum_r[1])

circuit.cx(quantum_r[0], quantum_r[1])

circuit.measure(quantum_r[0], classical_r[0])
circuit.measure(quantum_r[1], classical_r[1])

backend = 'local_qsam_simulator'
circuits = ['qc']

result = qp.execute(circuit, backend, wait=2, timeout = 240)
print(result)
result.get_counts('qc')
out = result.get_ran_qasm('qc')
print(out)


# -*- coding: utf-8 -*-
"""
Created on Sun Apr 25 19:52:38 2021

@author: Eliu
"""

#pip install keyboard

import keyboard
from time import sleep

def free_fire(cx, cy, w):
    res_x = 640
    res_y = 480
    time_release = 0.1
    sleep(0.001)

    no_move = w/3 #Divisor hace efecto en la sensibilidad
    if cx>(res_x/2)+no_move and cx !=0:
        print("IZQUIERDA")
        keyboard.press('a')
        #sleep(time_release)
        keyboard.press_and_release('d')

    elif cx<(res_x/2)-no_move:
        print("DERECHA")
        keyboard.press('d')
        #sleep(time_release)
        keyboard.press_and_release('a')

    else:
        print('CENTER')
        keyboard.press_and_release('d')
        keyboard.press_and_release('a')

    '''
    keyboard.press('w')
    sleep(1)
    keyboard.press_and_release('w')
    '''

import cv2

face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0) #My camer 640x480

while True:
    _, img = cap.read()
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.1,4)
    for (x,y,w,h) in faces:
        cv2.rectangle(img, (x,y), (x+w, y+h), (255,0,0), 2)
        cx = x+(w/2)
        cy = y+(h/2)
        free_fire(cx, cy, w)
        print(str(cx)+'    '+str(cy))
    cv2.imshow('img', img)
    k = cv2.waitKey(30)
    if k == 27:
        break
cap.release()


import pandas as pd
from tqdm import tqdm
import utilities as ut
import matplotlib.pyplot as plt
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import f1_score, recall_score, plot_confusion_matrix, roc_auc_score, classification_report,\
    accuracy_score, precision_score

if __name__ == '__main__':

    # =================================================================================
    # ==========================load prognostics data==================================
    data_path = r"/ML2/project/extra_cell_lymphoma_dev/prognostics_dev.xlsx"  # enter path
    df = ut.load_data_and_transform(data_path, "prognostics")

    # =================================================================================
    # ===========================data exploration&label handling=======================
    # we saw that their are patient w/o label.
    # in addition we saw that there is corr between missing labels and na in more than 4 features.
    # thus we decided to delete rows w/o labels.
    print(df.isnull().sum(axis=0))
    print(df.describe())
    for i in range(2,27,9): # features distribution
        ut.draw_multiple_histograms(df[df.columns[i:i+9]], 2, 2)
    ut.heatmap(df)
    ut.split_pos_and_neg(df)
    # we decided to delete number of features according to their distributions:
    # 1. Diagnosis - obviously all the patients has the same diagnosis so this is redundant
    # 2. Treatment - half of the data is missing and the other half has the same treatment.
    # 3. Biopsy Type - the major of the feature is pre-treatment
    # these features from the future thus can't be part of the model
    # 1. Included in survival analysis
    # 2. all progressions
    # 3. follow ups
    curr_df = ut.preprocessing_dataframe(df)
    print(curr_df.describe())
    # ==================================================================================================================
    # ============================= baseline model imputed missing values by traditional way ===========================
    baseline_df_trad = curr_df.copy()
    baseline_df_trad = ut.fill_missing_values_traditional_way(baseline_df_trad)
    baseline_df_trad = baseline_df_trad.round(
        {'Age': 0, 'IPI Group': 0, 'Ann Arbor Stage': 0, 'ECOG Performance Status': 0})
    X_train_trad, X_val_trad, y_train_trad, y_val_trad = ut.train_test_split_and_rename(baseline_df_trad,
                                                                                baseline_df_trad[
                                                                                'Number of Extranodal Sites'
                                                                                ])

    rf0 = RandomForestClassifier()
    log_r0 = LogisticRegression()
    knn0 = KNeighborsClassifier()
    mlp0 = MLPClassifier()
    models = [rf0, log_r0, knn0, mlp0]
    for clf in models:
        mean_auc = ut.cross_validation(10,X_train_trad,y_train_trad,clf)
        if clf == rf0:
            baseline_auc = mean_auc

    # ==================================================================================================================
    # =============================== baseline model with missing values imputed by KNN ================================
    curr_df = ut.fill_missing_values_by_KNN(curr_df)
    curr_df = curr_df.round({'Age': 0, 'IPI Group': 0, 'Ann Arbor Stage': 0, 'ECOG Performance Status': 0})
    # split train labels and validation
    X_train, X_val, y_train, y_val = ut.train_test_split_and_rename(curr_df, curr_df['Number of Extranodal Sites'])
    rf = RandomForestClassifier()
    log_r = LogisticRegression()
    knn = KNeighborsClassifier()
    mlp = MLPClassifier()
    models = [rf, log_r, knn, mlp]
    for clf in models:
        ut.cross_validation(10,X_train,y_train,clf)
    # =====================================================================================================
    # ============================================grid_search==============================================
    # rf = RandomForestClassifier()
    # rf_param_grid = {
    #      'max_depth': [10,20,50,80,110],
    #      'max_features': ['auto', 'sqrt'],
    #      'min_samples_leaf': [1,2,6,10],
    #      'min_samples_split': [2,6,10],
    #      'n_estimators': [10,100,600,1000]}
    # train_size = 0.8
    # # In the first step we will split the data in training and remaining dataset
    # X_train_grid, X_rem, y_train_grid, y_rem = X_train_trad, X_val_trad, y_train_trad, y_val_trad
    #
    # # Now since we want the valid and test size to be equal (10% each of overall data).
    # # we have to define valid_size=0.5 (that is 50% of remaining data)
    # test_size = 0.5
    # X_val_grid, X_test_grid, y_val_grid, y_test_grid = ut.train_test_split(X_rem, y_rem, test_size=0.5)
    #
    # best_param = ut.run_grid_search(rf, rf_param_grid, X_train_grid, y_train_grid, X_val_grid, y_val_grid)
    #
    # # rf_grid = RandomForestClassifier(**best_param)
    # # rf_grid.fit(X_train_grid,y_train_grid)
    # # plot_roc_curve(rf_grid, X_test_grid,y_test_grid)
    # # plt.show()
    # ==========================================================================================================
    # the two Imputing methods yield the same results, thus we will continue with the traditional baseline
    # In addition, after gridsearch we found the best parameters are the default

    X_train, X_val, y_train, y_val = X_train_trad, X_val_trad, y_train_trad, y_val_trad
    # ==========================================================================================================
    # ===================================load genetic data======================================================
    data_path = r"/ML2/project/extra_cell_lymphoma_dev/genomics_dev.xlsx"  # enter  path
    genomic_df = ut.load_data_and_transform(data_path, "genomics")
    X_train, y_train, X_val, y_val = ut.join_genomics(genomic_df, X_train, y_train, X_val, y_val)
    # ==========================================================================================================
    # ===================================drop features by domain knowledge =====================================
    X_train = X_train.drop(['ECOG Performance Status', 'IPI Range', 'dbGaP_accession_phs000178',
                            'dbGaP_accession_phs001444', 'Gene_Expression_Subgroup_ABC', 'Gene_Expression_Subgroup_GCB',
                            'Gene_Expression_Subgroup_Unclass', 'Genetic_Subtype_BN2', 'Genetic_Subtype_EZB',
                            'Genetic_Subtype_MCD', 'Genetic_Subtype_N1', 'Genetic_Subtype_Other'], axis=1)
    X_val = X_val.drop(['ECOG Performance Status', 'IPI Range', 'dbGaP_accession_phs000178',
                        'dbGaP_accession_phs001444', 'Gene_Expression_Subgroup_ABC', 'Gene_Expression_Subgroup_GCB',
                        'Gene_Expression_Subgroup_Unclass', 'Genetic_Subtype_BN2', 'Genetic_Subtype_EZB',
                        'Genetic_Subtype_MCD', 'Genetic_Subtype_N1', 'Genetic_Subtype_Other'], axis=1)
    # =========================================================================================================
    # ===================================run basic model with genomic data=====================================
    print("=================run basic model with all genomic data=================")
    rf2 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf2, X_train, y_train, X_val, y_val)
    print(f"for the baseline model with all the genomic data the AUC is: {auc}")

    # =========================================================================================================
    # ===================================we will try reducing dimension by using pca===========================
    print("=================run model with reduced dimensions by PCA=================")
    X_train_pca, X_val_pca = ut.get_reduced_dimensions_df(X_train, X_val)
    y_train_pca = y_train.reindex(X_train_pca.index)
    y_val_pca = y_val.reindex(X_val_pca.index)
    print(f"Reduced dimensions to: {X_train_pca.shape}")
    rf3 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf3, X_train_pca, y_train_pca, X_val_pca, y_val_pca)
    print(f"for the model with reduced dimensions by PCA the AUC is: {auc}")

    # =========================================================================================================
    # ===================================We will try feature selection by Domain knowledge====================
    print("=================run model with features selected by Domain knowledge=================")
    relevant_features = ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio',
                         'IPI Group', 'JAM2', 'JAM3', 'MYC', 'POU2AF1', 'EZH2',
                         'BCL2', 'AQP9',
                         'LMBR1L', 'FGF20', 'TANK', 'CRP', 'ORM1', 'JAK1', 'BACH1', 'MTCP1', 'IFITM1', 'TNFSF10',
                         'FGF12', 'RFX5', 'LAP3']
    print(f"The selected features are:{relevant_features}")
    X_train_fs = X_train[relevant_features]
    X_val_fs = X_val[relevant_features]

    rf4 = RandomForestClassifier()
    auc = ut.run_model_and_draw_ROC(rf4, X_train_fs, y_train, X_val_fs, y_val)
    print(f"for the model with features selected by Domain knowledge the AUC is: {auc}")

    # =========================================================================================================
    # =================================== lets try feature selection using forward selection ==================
    def forward_selection(data, target, X_val, y_val,auc, significance_level=0.01):
        initial_features = data.columns.tolist()
        best_features = ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio',
                         'IPI Group']
        scores = [auc,auc,auc]
        best_features_and_scores = {}
        while (len(initial_features) > 0):
            print(best_features_and_scores)
            remaining_features = list(set(initial_features) - set(best_features))
            new_pval = pd.Series(index=remaining_features)
            for new_column in tqdm(remaining_features):
                new_pval[new_column] = 0
                for i in range(3):
                    model = RandomForestClassifier().fit(data[best_features + [new_column]], target)
                    new_pval[new_column] += roc_auc_score(y_val,
                                                          model.predict_proba(X_val[best_features + [new_column]])[:,
                                                          1])
                new_pval[new_column] = new_pval[new_column] / 3
                print(f"{new_column}:", new_pval[new_column])
            max_score = new_pval.max()
            if (max_score - scores[-1] > significance_level):
                best_features.append(new_pval.idxmax())
                scores.append(max_score)
                best_features_and_scores[new_pval.idxmax()] = max_score
            else:
                best_features.append(new_pval.idxmax())
                scores.append(scores[-1])
                if scores[-1] == scores[-2] and scores[-1] == scores[-3]:
                    best_features = best_features[:len(best_features) - 2]
                    print(f"scores: {scores}")
                    break
        return best_features, best_features_and_scores

    # =========================================================================================================
    # ===================3 experiments for the feature selection===============================================
    for experiment in range(1,4):
        best_features, best_features_and_scores = forward_selection(X_train, y_train, X_val, y_val, baseline_auc)
        print(f"best features : {best_features}")
        print(f"best features for exp. {experiment}: {best_features_and_scores}")
        ut.open_file_and_save(f"exp.{experiment}", best_features_and_scores)

    # after finding the best features plot roc curves that compares the baseline model to the one
    # with the best genetics features

    print("=================run model with features selected by Forward Selection=================")

    print(f"best features: {best_features}")
    rf_basic = RandomForestClassifier()
    rf_genetic = RandomForestClassifier()
    models = [rf_basic, rf_genetic]
    relevant_features = {rf_basic: ['Gender', 'Age', 'Ann Arbor Stage', 'LDH Ratio', 'IPI Group'],
                         rf_genetic: best_features}
    ut.plot_multiple_ROC_curves(X_train, y_train, X_val, y_val, models, relevant_features)
    X_train = X_train[best_features]
    X_val = X_val[best_features]

    # =========================================================================================================
    # =====================================CM and scores=============================================
    rf = RandomForestClassifier()
    rf.fit(X_train, y_train)
    plot_confusion_matrix(rf, X_val, y_val)
    plt.show()
    y_pred_test = rf.predict(X_val)
    # accuracy: (tp + tn) / (p + n)
    accuracy = accuracy_score(y_val, y_pred_test)
    print(f"Accuracy: {accuracy}")
    # precision tp / (tp + fp)
    precision = precision_score(y_val, y_pred_test)
    print(f"Precision: {precision}")
    # recall: tp / (tp + fn)
    recall = recall_score(y_val, y_pred_test)
    print(f"Recall: {recall}")
    # f1: 2 tp / (2 tp + fp + fn)
    f1 = f1_score(y_val, y_pred_test)
    print(f"F1 score: {f1}")
    # View the classification report for test data and predictions
    print(classification_report(y_val, y_pred_test))
    ut.plot_train_test_ROC_curves(X_train, y_train, X_val, y_val, rf)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 18 08:53:40 2020

@author: Mathew
"""
# These are the packages we are using.

from skimage.io import imread
import matplotlib.pyplot as plt
from skimage import filters,measure

image=imread("/Users/Mathew/Desktop/Axioscan/apt.tif")

# This is just to show a small region of the image, since it's difficult to see what's happening with the full image.
region=image[500:1000,500:1000]
plt.imshow(region,cmap="Blues")
plt.show()

# Determine a threshold using the Otsu method - Note, if I was comparing cases etc., I'd keep the threhsold constant, and not use Otsu for all of them.
threshold=filters.threshold_otsu(image)
print(threshold)            # This prints the threshold that Otsu has determined.

filtered=image>threshold    # Apply the threshold to the image. If going for constant threshold for all images, could just replace this with a number.

region_filtered=filtered[:,:]   # Can set a region to look at.
plt.imshow(region_filtered,cmap="Blues")
plt.show()

# Now find the different features in the thresholded image.
label_image=measure.label(region_filtered)
plt.imshow(label_image)
plt.show()

# Measure parameters of labelled regions.
table=measure.regionprops_table(label_image,properties=('area','centroid','orientation','major_axis_length','minor_axis_length'))

# Get the area and length data.
areas=table['area']
lengths=table['major_axis_length']

number=len(areas)  # Count the number of features detected.

print(number)

# Plot some histograms.

plt.hist(areas, bins = 50,range=[0,100], rwidth=0.9,color='#607c8e')
plt.xlabel('Area (pixels)')
plt.ylabel('Number of Features')
plt.title('Area of features')
plt.show()

plt.hist(lengths, bins = 50,range=[0,200], rwidth=0.9,color='#607c8e')
plt.xlabel('Length (pixels)')
plt.ylabel('Number of Features')
plt.title('Length')
plt.show()

# For control:

imagecont=imread("/Users/Mathew/Desktop/Axioscan/noapt.tif")

filteredcont=imagecont>threshold    # Apply the threshold to the image. If going for constant threshold for all images, could just replace this with a number.

label_image_cont=measure.label(filteredcont)

# Measure parameters of labelled regions.
tablecont=measure.regionprops_table(label_image_cont,properties=('area','centroid','orientation','major_axis_length','minor_axis_length'))

# Get the area and length data.
areascont=tablecont['area']
lengthscont=tablecont['major_axis_length']

numbercont=len(areascont)  # Count the number of features detected.

print(numbercont)

# Plot some histograms.

plt.hist(areascont, bins = 50,range=[0,100], rwidth=0.9,color='#607c8e')
plt.xlabel('Area (pixels)')
plt.ylabel('Number of Features')
plt.title('Area of features (control)')
plt.show()

plt.hist(lengthscont, bins = 50,range=[0,200], rwidth=0.9,color='#607c8e')
plt.xlabel('Length (pixels)')
plt.ylabel('Number of Features (control)')
plt.title('Length')
plt.show()


# 20221122 - Python - Python OOP - Decorators
# Note 01 - Execution Time

import time

start = time.time()
time.sleep(3)
end = time.time()

print(f'The time between start and end is {end - start} seconds!')


from pluginfiles import HelperLibrary
from pluginfiles.plugin import Plugin
import numpy as np
import math
import time
from PIL import Image, ImageFile

class LaplacianEdgeDetectionFilter(Plugin):
    kernal = None
    filteredImage = None
    masksize = None
    weight = None
    img_data = None

    def setkernal(self):
        self.kernal = [[0, 1, 0], [1, -4, 1], [0, 1, 0]]

    def applykernalweight(self):
        for r, row in enumerate(self.kernal):
            for c, col in enumerate(row):
                self.kernal[r][c] = self.weight * col

    def negateImage(self):
        for i in range(self.filteredImage.shape[0]):
            for j in range(self.filteredImage.shape[1]):
                if self.filteredImage[i][j] == 255:
                    self.filteredImage[i][j] = 0
                else:
                    self.filteredImage[i][j] = 255

    def filterComputation(self, window_slice):
        w = window_slice.shape
        tempArray = []
        for i in range(w[0]):
            for j in range(w[1]):
                window_pixel = window_slice[i, j]
                kernal_value = self.kernal[i][j]
                temp = window_pixel * kernal_value
                tempArray.append(temp)
        result = np.sum(tempArray)
        return result

    def run(self, filename, raw_img, definition_path):
        operationStartTime = time.time()
        params = HelperLibrary.readDefinitionFile(definition_path)
        self.masksize = int(params["maskSize"])
        self.weight = float(params["filterWeight"])
        self.img_data = np.array(raw_img)
        self.filteredImage = np.empty_like(self.img_data)
        self.setkernal(self)
        self.applykernalweight(self)
        S = self.img_data.shape
        kernal_height = len(self.kernal)
        kernal_width = len(self.kernal[0])

        R = S[0] + kernal_height - 1
        C = S[1] + kernal_width - 1
        Z = np.zeros((R, C))
        t1 = np.int((kernal_height - 1) / 2)
        t2 = np.int((kernal_width - 1) / 2)
        for i in range(S[0]):
            for j in range(S[1]):
                Z[i + t1, j + t2] = self.img_data[i, j]
        for i in range(S[0]):
            for j in range(S[1]):
                window_slice = np.array(Z[i:i + kernal_height, j:j + kernal_width])
                result = self.filterComputation(self, window_slice)
                self.filteredImage[i, j] = result
        totalOperation = time.time() - operationStartTime
        self.negateImage(self)
        return self.filteredImage