from fastapi import Depends, FastAPI, UploadFile, Request
from fastapi_utils.cbv import cbv
from fastapi_utils.inferring_router import InferringRouter
from fastapi.responses import JSONResponse
from typing import Union
from inference import make_predictions
import pandas as pd
import json
from common.db import write_data, read_data
import time
from common.config import config
import shutil

def validate_headers():
    return {'status': True}

app = FastAPI()

router = InferringRouter()

def make_data(dates, predictions):
    print(predictions)
    predictions_df = pd.DataFrame(predictions[:, 1].reshape(-1, 1),
                                  columns=['Rainfall predicted'], index=dates.index)
    return dates.join(predictions_df)

def check_date_exists(prediction_date):
    query = "SELECT prediction from rainfall_predicted_value where date = '"+prediction_date+"'"
    prediction  = read_data(query)
    return prediction

@cbv(router)
class Api:
    valid_request: dict = Depends(validate_headers)
    response_code: int = 200
    response_data: dict = {'data': [], 'msg': ''}

    # def __init__(self):
    #     if not self.valid_request['status']:
    #         self.response_code = 401
    #         self.response_data['msg'] = 'Un-authorized Access'
    #         self.respond_api()

    def save_test_data(self, csv_file):
        t = int(time.time())
        file_name = "weather-"+str(t)+".csv"
        file_location = config['DATA']['DATA_SAVE_PATH']+file_name
        csv_file.to_csv(file_location, sep=',', index=False)
        q = "INSERT INTO rainfall_test_data(created_on, file_location, processed) VALUES ('2022-06-19','"+file_name+"' , 0);"
        write_data(q)

    @router.post("/predictions")
    async def predictions(self, request: Request, file: Union[UploadFile, None] = None):
        form = await request.form()
        contents = form["upload_file"].file
        data = pd.read_csv(contents)
        print(data.head())
        predictions = make_predictions(data.iloc[:, :-1])
        dataframe = make_data(data[['Date']], predictions)
        self.response_data = dataframe.to_json(orient='records')
        self.save_test_data(data)
        return self.respond_api()

    @router.post("/predict_rain")
    async def get_rain_prediction(self, request: Request):
        body = await request.body()
        json_body = json.loads(body)
        date_exists = check_date_exists(json_body['Date'])
        if len(date_exists) == 0:
            data = pd.json_normalize(json_body)
            predictions = make_predictions(data)
            self.response_data = {
                "msg": "There is a "+str(predictions[0 : 1]*100)+"% of chance of rain on "+json_body['Date']
            }
        else:
            self.response_data = {
                "msg": "There is a "+str(date_exists['prediction']*100)+"% of chance of rain on "+json_body['Date']
            }
        print(self.response_data)
        return self.respond_api()

    def respond_api(self):
        return JSONResponse(content=self.response_data, status_code=self.response_code)

app.include_router(router)


#!venv/bin/python3

from Context import Context, ContextFromJSON
import sys
import json
import argparse

parser = argparse.ArgumentParser(description='Run a JSON5 input')
parser.add_argument('input', help='JSON5 input file.')
parser.add_argument('output', help='JSON output filename. If omitted, prints to console.', nargs='?')
args = parser.parse_args()

fn = args.input
mgr = ContextFromJSON(fn)

if args.output:
    with open(args.output, 'w') as f:
        json.dump(mgr.jsonify(), f)
else:
    mgr.print_dump()


'''
Napiš funkci která pro zadaný počet N vygeneruje N náhodných čísel, každé
vygenerované číslo vypíše spolu s informací zda-li je liché či nikoliv.
Tuto funkci si pojmenuj jednoduše lichost_nahodnych_cisel(pocet).
'''
import random
def lichost_nahodnych_cisel(pocet):
    '''
    Vypis n nahodnych cisel, pro kazde rozhodni, zda je sude.
    # INPUT
    pocet - pocet generovanych cisel, integer
    # OUTPUT
    tisk do konzole nahodne cislo, True/False
    '''
    for i in range(pocet):
        # vygeneruj nahodne cislo
        cislo = random.randrange(100)

        # kdyz je sude
        if cislo%2 == 0:
            print(f'Pseudonahodne cislo {cislo} je sude.')
        # kdyz je liche
        else:
            print(f'Pseudonahodne cislo {cislo} je liche.')

lichost_nahodnych_cisel(10)


import networkx as nx
import os
from ParseFile import *
import pandas as pd
import numpy as np
import networkx as nx
from collections import Iterable
from grakel import Graph
from grakel import GraphKernel
def reNodeName(baseGraph,graph):
    basenodename_List=baseGraph.nodes()
    basenodename_number_List=[]
    for element in basenodename_List:
        if isinstance(element,int):
            basenodename_number_List.append()
    maxnum=max(basenodename_number_List)#找到最大节点的数，在这个基础上进行增加
def getfilePath(path,fileList):
    for item in os.listdir(path):
        current=os.path.join(path,item)
        if os.path.isfile(current):
            fileList.append(current)
        else:
            getfilePath(current,fileList)

def getpairFile(basefileList,targetfileList):
    #只处理文件数目没有变化
    pairfile=[]
    for base in basefileList:
        base_split=base.split("\\")
        base_name=base_split[len(base_split)-1]
        for target in targetfileList:
            target_split=target.split("\\")
            target_name=target_split[len(target_split)-1]
            if base_name==target_name:
                list=[base,target]
                pairfile.append(list)
    return pairfile
def getpairMethodGraph(basefileList,targetfileList):
    # {文件名：{（函数名_v1.0,函数名_v1.1）:[methodgraph1,methodgraph2.......}}
    pairfileList=getpairFile(basefileList,targetfileList)
    pairMethod = {}
    for paifile in pairfileList:
        filename=paifile[0]
        base=paifile[0]
        target=paifile[1]
        baseMethodGraph=ParseFile(base)
        targetMethodGraph=ParseFile(target)
        baseMethodGraphList=baseMethodGraph.getMethodGraph()
        targetMethodGraphList=targetMethodGraph.getMethodGraph()
        #两个文件的函数数目一样的时候
        if len(baseMethodGraphList)==len(targetMethodGraphList):
            tempdic={}
            for B,T in itertools.product(baseMethodGraphList,targetMethodGraphList):
                #为对应的函数名
                if B[0]==T[0]:
                    filename1=B[0]+"_"+baseMethodGraph.getVersion()
                    filename2=B[0]+"_"+targetMethodGraph.getVersion()
                    Bgraph=B[2]

                    Tgraph=T[2]

                    key=(filename1,filename2)
                    value=[Bgraph,Tgraph]
                    tempdic[key]=value

        else:
            pass
        pairMethod[filename]=tempdic
    return pairMethod

def addRoot(pairList):
    """
    #为文件添加根结点root，将两个版本对应得文件用root连接起来
    :param pairList:
    :return:
    """
    graph2graph=[]
    root="Root"
    for pair in pairList:
        g=nx.DiGraph()
        g1=ParseFile(pair[0])
        g2=ParseFile(pair[1])
        basefileGraph=g1.connectFile()
        targetfileGraph=g2.connectFile()
        basefileGraph_node=g1.getfileName()+"_"+g1.getVersion()#v1.0的文件节点
        targetfileGraph_node=g2.getfileName()+"_"+g2.getVersion()#v1.1的文件节点
        g.add_edges_from(basefileGraph.edges(data=True))
        g.add_edges_from(targetfileGraph.edges(data=True))
        g.add_edge(root,basefileGraph_node,{"connecting":"include"})
        g.add_edge(root, targetfileGraph_node, {"connecting": "include"})
    return g,basefileGraph_node,targetfileGraph_node

def writeCSV(G,sim,resiltUrl):
    label=list(G.nodes())
    dataframe=pd.DataFrame(sim,columns=label,index=label)
    dataframe.to_csv(resiltUrl)
def unDirection(G):
    #将图中的节点进行反向处理
    #(node ,node ,eges)
    for edge in G.edges(data=True):
        u=edge[0]
        v=edge[1]
        e=G.get_edge_data(u,v)
        G.remove_edge(u,v)
        G.add_edge(v,u,e)
    return G
def list2dic(list):
    dic={}
    for tup in list:
        pnode=tup[0]
        indic=tup[1]
        dic[pnode]=indic
    return dic
def getadjlist(graph):
    num=len(graph.nodes())#结点的个数
    tabel=[]
    #建立索引表
    for node in graph.nodes():
        tabel.append(node)
    #建立邻接矩阵
    adjlist=np.zeros(shape=(num,num))
    node_name={}
    edge_label={}
    #返回3元组（u,v,attri）
    for dic1 in graph.edges(data=True):
        u=dic1[0]
        u_index=tabel.index(u)
        v=dic1[1]
        v_index=tabel.index(v)
        #邻接矩阵添1
        adjlist[u_index][v_index]=1
        #添加结点结点名字标签
        node_name[u_index] = u
        node_name[v_index] = v
        #添加边关系:connecting
        attr=dic1[2]
        edgename=[item for item in attr.values()][0]
        edge_label[(u_index,v_index)]=edgename
    return adjlist,node_name,edge_label
def getMethodSim(pairMethodGraph):
    sim={}
    for filekey in pairMethodGraph.keys():
        file=pairMethodGraph[filekey]
        _sim={}
        for keytupe in file.keys():
            keytupe=tuple(keytupe)
            method=file[keytupe]
            basegraph=method[0]
            targetgraph=method[1]
            adj1, node_label1, edge_label1 = getadjlist(basegraph)
            adj2, node_label2, edge_label2 = getadjlist(targetgraph)
            sp_kernal = GraphKernel(kernel={"name": "shortest_path"}, normalize=True)
            g1 = Graph(adj1, node_label1, edge_label1)
            g2 = Graph(adj2, node_label2, edge_label2)
            tp = sp_kernal.fit_transform([g1])
            sim = sp_kernal.transform([g2])
            _sim[keytupe]=sim
        sim[file] = _sim


""" System Module """
from django.shortcuts import render, redirect
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.conf import settings
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.views.generic import View, DetailView
from django.core.exceptions import ObjectDoesNotExist
import stripe
from user_profile.models import Address
from .forms import CheckoutForm
from .models import Order, Payment

stripe.api_key = settings.STRIPE_SECRET_KEY

def is_valid_form(values):
    """
    Validate form if fields are empty
    """
    valid = True
    for field in values:
        if field == '':
            valid = False
    return valid

class CheckoutView(View):
    """
    A view to return checkout page
    """
    def get(self, *args, **kwargs):
        """
        Get the checkout form and display in page
        """
        form = CheckoutForm()
        if self.request.user.is_authenticated:
            customer = self.request.user
            order, created = Order.objects.get_or_create(
                customer=customer, ordered=False)
            items = order.orderbox_set.all()
            cart_items = order.get_cart_items
            context = {
                'form': form,
                'items': items,
                'order': order,
                'cart_items': cart_items,
            }

            shipping_address_qs = Address.objects.filter(
                customer=customer,
                address_type='S',
                default=True
                )
            if shipping_address_qs.exists():
                context.update(
                    {'default_shipping_address': shipping_address_qs[0]}
                    )

            billing_address_qs = Address.objects.filter(
                customer=customer,
                address_type='B',
                default=True
                )
            if billing_address_qs.exists():
                context.update(
                    {'default_billing_address': billing_address_qs[0]}
                    )

            return render(self.request, 'order/checkout.html', context)

    def post(self, *args, **kwargs):
        """
        Create an order and link to shipping address
        """
        form = CheckoutForm(self.request.POST or None)
        try:
            order = Order.objects.get(
                customer=self.request.user, ordered=False)
            if form.is_valid():

                # Create default shipping address
                use_default_shipping = form.cleaned_data.get(
                    'use_default_shipping')

                # if default shipping address checked
                if use_default_shipping:
                    # creates shipping address queryset
                    address_qs = Address.objects.filter(
                        customer=self.request.user,
                        address_type='S',
                        default=True
                    )
                    if address_qs.exists():
                        shipping_address = address_qs[0]
                        order.shipping_address = shipping_address
                        order.save()
                    else:
                        messages.info(
                            self.request, 'No default shipping address')
                        return redirect('checkout')
                else:
                    shipping_address1 = form.cleaned_data.get(
                        'shipping_address1')
                    shipping_address2 = form.cleaned_data.get(
                        'shipping_address2')
                    shipping_county = form.cleaned_data.get(
                        'shipping_county')
                    shipping_country = form.cleaned_data.get(
                        'shipping_country')
                    shipping_eircode = form.cleaned_data.get(
                        'shipping_eircode')

                    if is_valid_form([
                                     shipping_address1,
                                     shipping_country,
                                     shipping_eircode
                                     ]):
                        shipping_address = Address(
                            customer=self.request.user,
                            address1=shipping_address1,
                            address2=shipping_address2,
                            county=shipping_county,
                            country=shipping_country,
                            eircode=shipping_eircode,
                            address_type='S'
                            )
                        shipping_address.save()
                        order.shipping_address = shipping_address
                        order.save()

                        set_default_shipping = form.cleaned_data.get(
                            'set_default_shipping')
                        if set_default_shipping:
                            shipping_address.default = True

                            address_qs = Address.objects.filter(
                                customer=self.request.user,
                                address_type='S')

                            if address_qs.exists():
                                address_qs.update(default=False)

                            shipping_address.save()
                    else:
                        messages.info(
                            self.request, "Please fill in the \
                            required shipping address fields")
                        return redirect('checkout')

                # Creates default billing address
                use_default_billing = form.cleaned_data.get(
                    'use_default_billing')
                same_billing_address = form.cleaned_data.get(
                    'same_billing_address')

                if same_billing_address:
                    billing_address = shipping_address
                    billing_address.pk = None
                    billing_address.save()
                    billing_address.address_type = 'B'
                    billing_address.save()
                    order.billing_address = billing_address
                    order.save()

                # if default billing address checked
                elif use_default_billing:
                    # creates billing address queryset
                    address_qs = Address.objects.filter(
                        customer=self.request.user,
                        address_type='B',
                        default=True
                    )
                    if address_qs.exists():
                        billing_address = address_qs[0]
                        order.billing_address = billing_address
                        order.save()
                    else:
                        messages.info(
                            self.request, 'No default billing address')
                        return redirect('checkout')
                else:
                    billing_address1 = form.cleaned_data.get(
                        'billing_address1')
                    billing_address2 = form.cleaned_data.get(
                        'billing_address2')
                    billing_county = form.cleaned_data.get(
                        'billing_county')
                    billing_country = form.cleaned_data.get(
                        'billing_country')
                    billing_eircode = form.cleaned_data.get(
                        'billing_eircode')

                    if is_valid_form([
                                      billing_address1,
                                      billing_country,
                                      billing_eircode
                                      ]):
                        billing_address = Address(
                            customer=self.request.user,
                            address1=billing_address1,
                            address2=billing_address2,
                            county=billing_county,
                            country=billing_country,
                            eircode=billing_eircode,
                            address_type='B'
                        )
                        billing_address.save()
                        # Set billing address to order
                        order.billing_address = billing_address
                        order.save()
                        # Set default billing address
                        set_default_billing = form.cleaned_data.get(
                            'set_default_billing')
                        if set_default_billing:
                            billing_address.default = True
                            billing_address.save()
                    else:
                        messages.info(
                            self.request, "Please fill in the \
                                required billing address fields")
                        return redirect('checkout')

                return redirect('checkout_summary')

        except ObjectDoesNotExist:
            messages.error(self.request, "You do not have an active order")
            return redirect("cart")

def checkout_summary(request):
    """"
    Creates a checkout sumary page with all user details
    """
    order = Order.objects.get(
        customer=request.user, ordered=False)
    items = order.orderbox_set.all()
    cart_items = order.get_cart_items
    shipping = order.shipping()

    context = {
        'items': items,
        'order': order,
        'cart_items': cart_items,
        'shipping': shipping,
    }
    return render(request, "order/checkout_summary.html", context)

class PaymentView(View):
    """
    Create a payment view
    """
    def get(self, *args, **kwargs):
        """
        Get the payment url
        """
        return render(self.request, "order/payment.html")

    def post(self, *args, **kwargs):
        """
        Post form with order details and payment
        """
        order = Order.objects.get(
            customer=self.request.user, ordered=False)
        token = self.request.POST.get('stripeToken')
        amount = int(order.get_cart_total * 100)

        try:
            # create charge in euro
            charge = stripe.Charge.create(
                amount=amount,
                currency="eur",
                source=token,
            )
            # create stripe payment and save
            payment = Payment()
            payment.stripe_charge_id = charge['id']
            payment.customer = self.request.user
            payment.amount = order.get_cart_total
            payment.save()
            # create and save order
            order.ordered = True
            order.payment = payment
            order.save()
            messages.success(
                self.request,
                "Your order was successful! You will receive a\
                    confirmation email.")
            return redirect("success", order.pk)

        except stripe.error.CardError as e:
            body = e.json_body
            err = body.get('error', {})
            messages.error(self.request, f"{err.get('message')}")
            return redirect("/")

        except stripe.error.RateLimitError as e:
            # Too many requests made to the API too quickly
            messages.error(self.request, "Rate Limited Error")
            return redirect("/")

        except stripe.error.InvalidRequestError as e:
            # Invalid parameters were supplied to Stripe's API
            messages.error(self.request, "Invalid parameter")
            return redirect("/")

        except stripe.error.AuthenticationError as e:
            # Authentication with Stripe's API failed
            # (maybe you changed API keys recently)
            messages.error(self.request, "Not authenticated")
            return redirect("/")

        except stripe.error.APIConnectionError as e:
            # Network communication with Stripe failed
            messages.error(self.request, "Network error")
            return redirect("/")

        except stripe.error.StripeError as e:
            # Display a very generic error to the user, and maybe send
            # yourself an email
            messages.error(self.request, "Something went wrong, \
                you were not charged. Try again.")
            return redirect("/")

        except Exception as e:
            # Something else happened, completely unrelated to Stripe
            messages.error(self.request, "A serious error occurred.")
            return redirect("/")

@login_required
def success(request, pk):
    """
    Render a success page and send a purchase confirmation email
    """
    order = Order.objects.get(pk=pk)
    if order.customer == request.user:
        template = render_to_string(
            'order/email_template.html', {'name': request.user})
        email = EmailMessage(
            'Thanks for your purchase on The Regional Taste',
            template,
            settings.EMAIL_HOST_USER,
            [request.user.email]
        )
        email.fail_silently = False
        email.send()

        context = {'order': order}

        return render(request, 'home/index.html', context)

    else:
        messages.error(request, "Sorry, you cannot access this data.")
        return redirect("/")

class OrderDetailView(DetailView):
    """
    Creates past order details view
    """
    model = Order

    def get_queryset(self, *args, **kwargs):
        order = Order.objects.filter(id=self.kwargs['pk'])
        return order


"""
Given that integers are being read from a data stream.
Find median of all the elements read so far starting from the first integer
till the last integer. This is also called Median of Running Integers.
The data stream can be any source of data,
example: a file, an array of integers, input stream etc.
"""
import sys
from typing import Union
from heapq import heappushpop, heappush

from kth_smallest_element import left_child_index, parent_index  # type: ignore

class RunningMedian:
    """
    1. Create two heaps.
        - One max heap to maintain elements of lower half and
        - One min heap to maintain elements of higher half.
    2. Take initial value of median as 0.
    3. For every newly read element:
        i. If both heaps have same size:
            - If new element is greater than the median, then insert the new element
                into min_heap and set median to the top of min heap.
            - Else, insert it in the max heap and set median to the top of max heap
        ii. If max heap has more elements:
            - If new element is greater than the median, then insert it in the min heap
                and set median to avg of top elements.
            - Else, insert top of max heap to min heap and new element to max heap and
                and set median to avg of top elements.
        iii. If min heap has more elements:
            - If new element is greater than the median, insert top of min heap to
                max heap and new element to min heap and set median to avg of top elements.
            - Else, insert it in the max heap and set median to avg of top elements.

    Time Complexity: O(n*log(n))
    Space Complexity: O(n)
    """

    def __init__(self):
        self.max_heap: list = []
        self.max_heap_size: int = 0
        self.min_heap: list = []
        self.min_heap_size: int = 0
        self.current_median: Union[float, int] = 0

    def min_heapify_top_down(self, index: int = 0) -> None:
        """
        Time Complexity: O(log(n))
        """
        left = left_child_index(index)
        element: int = self.min_heap[index]

        while left < self.max_heap_size:
            right = left + 1
            min_index = index

            if self.min_heap[left] < self.min_heap[min_index]:
                min_index = left
            if (
                right < self.min_heap_size
                and self.min_heap[right] < self.min_heap[min_index]
            ):
                min_index = right

            if index != min_index:
                self.min_heap[index], self.min_heap[min_index] = (
                    self.min_heap[min_index],
                    self.min_heap[index],
                )
                index = min_index
                left = left_child_index(index)
            else:
                break

        self.min_heap[index] = element

    def min_heapify_bottom_up(self, index: int = -1) -> None:
        """
        Time Complexity: O(log(n))
        """
        if index == -1:
            index = self.min_heap_size - 1

        element: int = self.min_heap[index]

        while index > 0:
            parent = parent_index(index)

            if self.min_heap[parent] > element:
                self.min_heap[index] = self.min_heap[parent]
                index = parent
            else:
                break

        self.min_heap[index] = element

    def _add_to_min_heap(self, element: int):
        self.min_heap_size += 1
        self.min_heap.append(element)
        self.min_heapify_bottom_up()

    def max_heapify_top_down(self, index: int = 0) -> None:
        """
        Time Complexity: O(log(n))
        """
        left = left_child_index(index)
        element: int = self.max_heap[index]

        while left < self.max_heap_size:
            right = left + 1
            max_index = index

            if self.max_heap[left] > self.max_heap[max_index]:
                max_index = left
            if (
                right < self.max_heap_size
                and self.max_heap[right] > self.max_heap[max_index]
            ):
                max_index = right

            if index != max_index:
                self.max_heap[index] = self.max_heap[max_index]
                index = max_index
                left = left_child_index(index)
            else:
                break

        self.max_heap[index] = element

    def max_heapify_bottom_up(self, index: int = -1) -> None:
        """
        Time Complexity: O(log(n))
        """
        if index == -1:
            index = self.max_heap_size - 1

        element: int = self.max_heap[index]

        while index > 0:
            parent = parent_index(index)

            if self.max_heap[parent] < element:
                self.max_heap[index] = self.max_heap[parent]
                index = parent
            else:
                break

        self.max_heap[index] = element

    def _add_to_max_heap(self, element: int):
        self.max_heap_size += 1
        self.max_heap.append(element)
        self.max_heapify_bottom_up()

    def _set_median(self):
        """
        Time Complexity: O(1)
        """
        if self.max_heap_size == self.min_heap_size:
            self.current_median = (self.max_heap[0] + self.min_heap[0]) / 2
        elif self.max_heap_size < self.min_heap_size:
            self.current_median = self.min_heap[0]
        else:
            self.current_median = self.max_heap[0]

    def get_median(self, new_element: int) -> Union[float, int]:
        if self.max_heap_size > self.min_heap_size:
            if new_element > self.current_median:
                self._add_to_min_heap(new_element)
            else:
                self._add_to_min_heap(self.max_heap[0])
                self.max_heap[0] = new_element
                self.max_heapify_top_down()

        elif self.max_heap_size == self.min_heap_size:
            if new_element > self.current_median:
                self._add_to_min_heap(new_element)
            else:
                self._add_to_max_heap(new_element)
        else:
            if new_element > self.current_median:
                self._add_to_max_heap(self.min_heap[0])
                self.min_heap[0] = new_element
                self.min_heapify_top_down()
            else:
                self._add_to_max_heap(new_element)

        self._set_median()
        return self.current_median

class RunningMedianUsingInbuiltHeapq:
    def __init__(self):
        self.min_heap, self.max_heap = [], []

    def add_num(self, num: int) -> None:
        """
        The negative signs are to represent a max heap using min heap
        """
        if len(self.min_heap) == len(self.max_heap):
            heappush(self.max_heap, -heappushpop(self.min_heap, -num))
        else:
            heappush(self.min_heap, -heappushpop(self.max_heap, num))

    def find_median(self) -> float:
        if len(self.max_heap) > len(self.min_heap):
            return float(self.max_heap[0])
        return (self.max_heap[0] - self.min_heap[0]) / 2.0

if __name__ == "__main__":
    matrix: list = [
        [5, 15, 10, 20, 3],
        [2, 1, 5, 7, 2, 0, 5],
        list(range(1, 11)),
        list(range(-1, -6, -1)),
        [6, 10, 2, 6, 5, 0, 6, 3, 1, 0, 0],
    ]

    for arr in matrix:
        print("current array:", arr)
        run_med = RunningMedian()

        for element in arr:
            print(run_med.get_median(element))

    print("method 2")

    for arr in matrix:
        print("current array:", arr)
        run_med = RunningMedianUsingInbuiltHeapq()

        for element in arr:
            run_med.add_num(element)
            print(run_med.find_median())


from respect_validation.Exceptions import ValidationException

class RomanException(ValidationException):

    _default_templates = {
        'default': {
            'standard': '{name} must be a valid Roman numeral',
        },
        'negative': {
            'standard': '{name} must not be a valid Roman numeral',
        }
    }


import unittest

from troposphere import Ref, Template
from troposphere.s3 import Bucket

from s3_bucket import create_bucket, add_bucket_output

class BucketTest(unittest.TestCase):
    def test_should_create_s3_bucket(self):
        # GIVEN
        template = Template()

        # WHEN
        s3_bucket = create_bucket(template)

        # THEN
        self.assertEqual(s3_bucket.title, "TestBucket")
        self.assertEqual(s3_bucket.BucketName, "my-test-bucket")

    def test_should_add_output_for_bucket(self):
        # GIVEN
        template = Template()
        s3_bucket = Bucket("TestBucket",
                           BucketName="my-test-bucket")

        # WHEN
        output = add_bucket_output(template, s3_bucket)

        # THEN
        self.assertEqual(output.title, "BucketName")
        self.assertEqual(output.Value, Ref(s3_bucket))
        self.assertEqual(output.Description, "Name of S3 bucket")

if __name__ == '__main__':
    unittest.main()


'''
Problem 2. Simple Array Sum. Given an array of integers, find the sum of its elements.
https://www.hackerrank.com/challenges/simple-array-sum/problem
'''
#!/bin/python3

import os
import sys

#
# Complete the simpleArraySum function below.
#
def simpleArraySum(ar):
    #
    # Write your code here.
    #
    results=0
    for xvalues in ar:
        results+=xvalues
    return results

if __name__ == '__main__':
    fptr = open(os.environ['OUTPUT_PATH'], 'w')
    ar_count = int(input())
    ar = list(map(int, input().rstrip().split()))
    result = simpleArraySum(ar)
    fptr.write(str(result) + '\n')
    fptr.close()


import time
import os
import requests
from constant import url
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from app_logger import log

def scrape_images(name, num_images=100):
    """
    scrape_image will download certain number of images from google and save them to a specific path.
    :param name: name of that object who's images we want to scrape.
    :param num_images: How many images we want to download (default num_images=1000)
    :return: images
    """
    try:
        chrome_options = Options()
        chrome_options.add_argument("--headless")
        driver_path = "chromedriver.exe"
        # Create a webdriver object and set the path to the chromedriver executable
        driver = webdriver.Chrome(executable_path=driver_path,chrome_options=chrome_options)

        # Navigate to Google Images
        driver.get(url)

        # Find the search box element and enter the search query
        search_box = driver.find_element_by_name("q")
        search_box.send_keys(name)
        search_box.send_keys(Keys.ENTER)
        time.sleep(1)

        # Wait for the images to load
        driver.implicitly_wait(10)

        # Find all the image elements on the page
        images = driver.find_elements_by_css_selector("img.rg_i")
        last_height = driver.execute_script(
            "return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight );"
        )
        count = 0
        img = []
        # Iterate through the images and download them
        while count < num_images:
            for image in images:
                if count >= num_images:
                    break
                # Get the image source
                if image.get_attribute("src") and "http" in image.get_attribute("src"):
                    img_src = image.get_attribute("src")

                    img.append(img_src)
                count = len(img)

            # Scroll down to the bottom of the page
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            try:
                load_more_button = driver.find_element_by_css_selector(".mye4qd")
                load_more_button.click()
            except:
                continue

            # Wait for the page to load
            time.sleep(1)

            new_height = driver.execute_script(
                "return Math.max( document.body.scrollHeight, document.body.offsetHeight, document.documentElement.clientHeight, document.documentElement.scrollHeight, document.documentElement.offsetHeight );"
            )

            # If the new height is the same as the last height, then we have reached the end of the page
            if new_height == last_height:
                break

            # Find all the image elements on the page
            images = driver.find_elements_by_css_selector("img.rg_i")

        # Close the webdriver
        driver.close()
        log().info("Successfully obtained the source links of images")
    except Exception as e:
        log().exception(e)
    try:
        counter = 1

        if not os.path.exists("images"):
            os.makedirs("images")
        if not os.path.exists(f"images/{name}"):
            os.makedirs(f"images/{name}")

        for i in img:
            response = requests.get(i)
            with open(f"images/{name}/image{counter}.jpg", "wb") as f:
                f.write(response.content)
            counter += 1

        log().info(f"Successfully Downloaded the images: {name}")
        return os.getcwd()
    except Exception as e:
        log().exception(e)


import face_recognition
import json
import logging
import math
from .constants import DEFAULT_TRESHOLD
from .path import Path
from .errors import ArgumentError, FaceError

logger = logging.getLogger(__name__)

class Recognizer:
    def __init__(self, treshold = DEFAULT_TRESHOLD):
        self.treshold = DEFAULT_TRESHOLD

    def _encode_image_path(self, path):
        # First get encodings from all images in the target path
        encodings = {}

        image_paths = [str(p) for p in Path(path).images()]

        for imgpath in image_paths:
            try:
                encoding = self._encoding_from_image(imgpath)
            except:
                continue

            encodings[imgpath] = encoding

        return encodings

    # From:
    # < https://github.com/ageitgey/face_recognition/wiki/Calculating-Accuracy-as-a-Percentage >
    def _face_distance_to_conf(self, face_distance):
        if face_distance > self.treshold:
            range_ = (1.0 - self.treshold)
            linear_val = (1.0 - face_distance) / (range_ * 2.0)
            return linear_val
        else:
            range_ = self.treshold
            linear_val = 1.0 - (face_distance / (range_ * 2.0))
            return linear_val + ((1.0 - linear_val) * math.pow((linear_val - 0.5) * 2, 0.2))

    # Load one image, get the encoding and return that, but only if there
    # is exactly *one* face
    def _encoding_from_image(self, path):
        logging.debug(f"Getting encoding for {path}")
        image = face_recognition.load_image_file(path)
        image_encodings = face_recognition.face_encodings(image)

        if len(image_encodings) != 1:
            raise FaceError(f"{path} has 0 or more than 1 face, skipping")

        return image_encodings[0]

    def encode_path(self, path, return_type = "json"):
        encodings = {}
        image_paths = [str(p) for p in Path(path).images()]
        logging.debug(f"Encoding {len(image_paths)} images")

        for image_path in image_paths:
            try:
                encoding = self._encoding_from_image(image_path)
            except:
                continue

            encodings[image_path] = encoding.tolist()

        logging.debug(f"Encoded {len(encodings.values())} images")

        # Return encodings in a key, for future-proofing this file format
        # And return as JSON or a dict
        ret = {
            "encodings" : encodings
        }

        if return_type == "json":
            return json.dumps(ret)
        elif return_type == "dict":
            return ret
        else:
            raise ArgumentError(f"Invalid return_type: {return_type}")

    def recognize(self,
        input_path,
        model_path = None,
        target_path = None,
        as_percentage = False
    ):
        if not any([model_path, target_path]):
            raise ArgumentError("Need either a model or a target path")

        # First get the encoding for the input image and check if we
        # have only one face
        logging.debug(f"Getting encoding for input image {input_path}")
        input_image = face_recognition.load_image_file(input_path)
        input_encoding = face_recognition.face_encodings(input_image)

        if len(input_encoding) != 1:
            raise FaceError("Found more than one face in this image, aborting")
        else:
            # Take the first face
            input_encoding = input_encoding[0]

        # If we have a model path, load those encodings, otherwise we need
        # to calculate the target encodings
        if model_path:
            with open(model_path) as f:
                encodings = json.load(f)["encodings"]
        elif target_path:
            encodings = self._encode_image_path(target_path)

        image_paths = list(encodings.keys())
        results = face_recognition.face_distance(list(encodings.values()), input_encoding)

        if as_percentage:
            logging.debug("Converting all face distances to percentages")
            results = [self._face_distance_to_conf(d) for d in results]

        results = dict(zip(image_paths, results))

        # Sort by value
        return dict(sorted(results.items(), key = lambda x:x[1]))


#!/usr/bin/python
"""
sectors command
A command to extract unallocated sectors from a container
"""
import fritutils.termout
import fritutils.fritlog
import sys
import os.path

logger = fritutils.fritlog.loggers['sectorsLog']

def seclist(Evidences):
    """
    list subcommand. Print a list of unallocated sectors.
    We assume that sector size is 512 bytes. We will have to support variable
    sector sizes in a near future.
    """
    for evi in Evidences:
        secList = evi.getUnallocatedSectors()
        if secList:
            fritutils.termout.printSuccess('Unallocated sectors on "%s":' % evi.fileName)
            for u in secList:
                scount = u['end'] - u['start'] +1
                ssize = fritutils.humanize(scount * 512)
                fritutils.termout.printNormal("\tStart: %d End: %s Count: %d Size: %s." % (u['start'],u['end'],scount,ssize))
        else:
            fritutils.termout.printWarning('No unallocated sectors found on "%s".' % evi.fileName)

def export(Evidences,options):
    """
    export unallocated sectors to .frit/extractions/unallocated/START-END/START-END.dd
    As the dump could be time consuming for a single sector, when the '--split'
    option is given, we first extract the whole range and then split it.
    """
    for evi in Evidences:
        secList = evi.getUnallocatedSectors()
        if secList:
            for u in secList:
                endPath = "sectors_%d-%d" % (u['start'],u['end'])
                exportFile = endPath + '.dd'
                exportPath = os.path.join(evi.sectorsExportDir, endPath)
                if not os.path.exists(exportPath):
                    logger.info('Export path "%s" does not exist. Creating.' % exportPath)
                    os.makedirs(exportPath)
                exportFilePath = os.path.join(exportPath, exportFile)
                if os.path.exists(exportFilePath):
                    logger.error('Sectors file "%s" already exists, not exporting.' % exportFilePath)
                    fritutils.termout.printWarning('Sectors file "%s" already exists, not exporting.' % exportFilePath)
                else:
                    fritutils.termout.printMessage('Starting to export sectors from %d to %d' % (u['start'],u['end']))
                    logger.info('Starting to dump sectors %d - %d from %s' % (u['start'],u['end'],evi.fileName))
                    # the real dump starts here
                    evi.ddump(u['start'],u['end']+1,exportFilePath)
                if options and '--split' in options:
                    endSplitPath = "sectors_%d-%d_splitted" % (u['start'],u['end'])
                    exportSplitPath = os.path.join(evi.sectorsExportDir, endSplitPath)
                    if not os.path.exists(exportSplitPath):
                        logger.info('Directory "%s" does not exists, creating !' % exportSplitPath)
                        os.makedirs(exportSplitPath)
                    fritutils.termout.printMessage('Starting to export splitted sectors from %d to %d' % (u['start'],u['end']))
                    logger.info('Starting to dump splitted sectors from %d to %d from %s' % (u['start'],u['end'],evi.fileName))
                    sectorCount = 0
                    rf = open(exportFilePath,'r')
                    for sector in range(u['start'],u['end']+1):
                        fname = 'sector-%d.dd' % sector
                        exportSplitFilePath = os.path.join(exportSplitPath,fname)
                        if os.path.exists(exportSplitFilePath):
                            logger.warning('File "%s" already exists, not exporting.' % exportSplitFilePath)
                        else:
                            ef = open(exportSplitFilePath,'w')
                            rf.seek(sectorCount*512)
                            data = rf.read(512)
                            ef.write(data)
                            ef.close()
                        sectorCount += 1
                    rf.close()

def factory(Evidences,args,options):
    """
    args are the sectors command arguments
    """
    logger.info('Starting sectors command.')
    validArgs = ('list','export')
    if not args or len(args) == 0:
        fritutils.termout.printWarning('sectors command need at least an argument. Exiting.')
        logger.error('No argument given.')
        sys.exit(1)
    elif args[0] not in validArgs:
        fritutils.termout.printWarning('sectors command need a valid argument (%s)' % ', '.join(validArgs))
        logger.error('"%s" in not a valid arguement. Exiting.' % args[0])
        sys.exit(1)
    else:
        if args[0] == 'list':
            logger.info('list arguement given. Starting "sectors list" command.')
            seclist(Evidences)
        elif args[0] == 'export':
            logger.info('export arguement given. Starting "sectors export" command.')
            export(Evidences,options)


def getPrimes():
    n = 1000
    a = [False,False] + [True]*(n-1)
    primes = []

    for i in range(2, n+1):
        if a[i]:
            primes.append(i)
            for j in range(2*i, n+1, i):
                a[j] = False
    return primes

def main():
    N = int(input())
    dataList = list(map(int, input().split()))
    primes = getPrimes()

    solve = 0
    for data in dataList:
        solve += primes.count(data)

    print(solve)

if __name__ == '__main__':
    main()


from djangae.test import TestCase
from django.contrib.auth.models import AnonymousUser
from django.http import Http404
from django.template import TemplateSyntaxError
from django.test import RequestFactory, override_settings
from django_webtest import WebTest

from tracker.checks import check_csp_is_not_report_only, check_session_csrf_enabled, check_csp_sources_not_unsafe
from tracker.site.models import Ticket, Project
from tracker.site.views import update_project_view, update_ticket_view, create_ticket_view, delete_ticket_view, \
    my_tickets_view, create_project_view
from tracker.site.factories import ProjectFactory, TicketFactory

class ProjectTest(TestCase):
    def setUp(self):
        self.project = ProjectFactory.create()
        self.user = self.project.created_by
        self.rf = RequestFactory()

    def test_projects_unauthenticated(self):
        r = self.client.get('/projects/')
        self.assertEquals(r.status_code, 200)

    def test_project_edit(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.get('/projects/%s/edit/' % self.project.id)
        request.user = self.user
        r = update_project_view(request, project_id=self.project.id)
        self.assertEqual(r.status_code, 200)
        r.render()
        self.assertIn(u'Edit', r.content.decode('utf8'))

    def test_project_create_ticket(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.get('/projects/%s/tickets/create' % self.project.id)
        request.user = self.user
        r = create_ticket_view(request, project_id=self.project.id)
        self.assertEqual(r.status_code, 200)
        r.render()
        self.assertIn(u'Submit', r.content.decode('utf8'))

    def test_project_create_get(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.get('/projects/create')
        request.user = self.user
        r = create_project_view(request)
        self.assertEqual(r.status_code, 200)
        r.render()
        self.assertIn(u'Submit', r.content.decode('utf8'))

    def test_project_create_post(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.post('/projects/create', data={'title': 'New project'})
        request.user = self.user
        r = create_project_view(request)
        self.assertEqual(r.status_code, 302)
        self.assertTrue(Project.objects.filter(title='New project'))

class TicketTest(TestCase):
    def setUp(self):
        self.ticket = TicketFactory.create()
        self.user = self.ticket.created_by
        self.project = self.ticket.project
        self.rf = RequestFactory()

    def test_my_tickets_not_authenticated(self):
        request = self.rf.get('/')
        request.user = AnonymousUser()
        r = my_tickets_view(request)
        self.assertEqual(r.status_code, 200)

    def test_my_tickets_authenticated(self):
        request = self.rf.get('/')
        request.user = self.user
        r = my_tickets_view(request)
        self.assertEqual(r.status_code, 200)

    def test_switch_ticket_ownership(self):
        project2 = ProjectFactory.create()
        self.assertNotEqual(project2.id, self.project.id)

        request = self.rf.get('/projects/%s/tickets/%s/edit' % (project2.id, self.ticket.id))
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request.user = self.user
        self.assertRaises(Http404, update_ticket_view, request, project_id=project2.id, ticket_id=self.ticket.id)

    def test_ticket_edit(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.get('/projects/%s/tickets/%s/edit' % (self.project.id, self.ticket.id))
        request.user = self.user
        r = update_ticket_view(request, project_id=self.project.id, ticket_id=self.ticket.id)
        self.assertEqual(r.status_code, 200)
        r.render()
        self.assertIn(u'Submit', r.content.decode('utf8'))

    def test_ticket_delete(self):
        # I'm unsure how to test with logged user using djangoae
        # Using request factory to bypass that.
        request = self.rf.post('/projects/%s/tickets/%s/delete' % (self.project.id, self.ticket.id))
        request.user = self.user
        r = delete_ticket_view(request, project_id=self.project.id, ticket_id=self.ticket.id)
        self.assertEqual(r.status_code, 302)
        self.assertFalse(Ticket.objects.filter(id=self.ticket.id).exists())

class ProjectWebTest(WebTest):
    def setUp(self):
        self.project = ProjectFactory.create()
        self.user = self.project.created_by

    def test_project_details(self):
        projects_list = self.app.get('/projects/', user=self.user.username)
        self.assertIn(self.project.title, projects_list)

        # Now click on the title to access details
        project_details = projects_list.click(self.project.title)

        # Not ideal as using string which can change.
        self.assertIn('Create ticket', project_details)

class CheckTests(TestCase):
    @override_settings(MIDDLEWARE_CLASSES=[])
    def test_check_missing_session_middleware(self):
        self.assertEqual(check_session_csrf_enabled(), ['SESSION_CSRF_DISABLED'])

    @override_settings(CSP_REPORT_ONLY=True)
    def test_check_missing_csp_report_only_setting(self):
        self.assertEqual(check_csp_is_not_report_only(), ['CSP_REPORT_ONLY_ENABLED'])

    @override_settings(MIDDLEWARE_CLASSES=['session_csrf.CsrfMiddleware'])
    def test_check_middleware_csrf_is_present(self):
        self.assertEqual(check_session_csrf_enabled(), [])

    @override_settings(CSP_REPORT_ONLY=False)
    def test_check_csp_report_is_missing(self):
        self.assertEqual(check_csp_is_not_report_only(), [])

    @override_settings(CSP_DEFAULT_SRC=("'self'", "*.gstatic.com"))
    def test_check_csp_sources_not_unsafe(self):
        self.assertEqual(check_csp_sources_not_unsafe(), ['CSP_STYLE_SRC_UNSAFE'])


import sys
sys.stdin = open('bj17144.txt','r')

dx = [1,0,-1,0]
dy = [0,1,0,-1]
up_dir = [3,0,1,2]
do_dir = [1,0,3,2]
def IS(y,x):
    if -1<y<R and -1<x<C:return True
    return False

def Dust(y,x):
    tmp = Arr[y][x] // 5
    for dir in range(4):
        nY = y + dy[dir]
        nX = x + dx[dir]
        if IS(nY, nX) and Arr[nY][nX] != -1:
            Map[nY][nX] += tmp
            Arr[y][x] -= tmp
    Map[y][x] += Arr[y][x]

def BFS(y,x,dir_list,check):
    hY = y
    hX = x
    for dir in dir_list:
        while IS(hY+dy[dir],hX+dx[dir]):
            nY = hY + dy[dir]
            nX = hX + dx[dir]
            if check == 'up' and nY == y+1:break
            if check == 'down' and nY == y-1:break
            Arr[hY][hX] = Arr[nY][nX]
            Arr[nY][nX] = 0
            hY, hX = nY, nX
            if (hY, hX) == (y, x+1):break

R, C, T = map(int, input().split())
Arr = [[int(x)for x in input().split()]for y in range(R)]
Result = 0
for a in range(R):
    if Arr[a][0] == -1:
        air = a
        break
while T:
    Map = [[0]*C for _ in range(R)]
    for i in range(R):
        for j in range(C):
            if Arr[i][j] and Arr[i][j] != -1:
                Dust(i,j)
    Arr = Map
    BFS(air,0,up_dir, 'up')
    BFS(air+1,0,do_dir, 'down')
    Arr[air][0] = Arr[air + 1][0] = -1
    T -= 1
for r in range(R):
    Result += sum(Arr[r])
print(Result+2)

algoritm/19하반기 코딩테스트/미세먼지 안녕!/bj17144.py


import unittest
from hw1 import *

class Testhw1(unittest.TestCase):

    def test_score_equal(self):
        "General tests for score"
        self.assertEqual(score([(2, 'Clubs'),(8, 'Diamonds')]), 10)
        self.assertEqual(score([(10, 'Hearts'),(3, 'Spades')]), 13)

    def test_score_equal_face_cards(self):
        "Tests for face card values to equal 10"
        self.assertEqual(score([(13, 'Clubs'),(11, 'Diamonds')]), 20)

    def test_score_equal_ace_value(self):
        "Tests for correct ace values 11 and 1 depending on current score"
        self.assertEqual(score([(1, 'Clubs'),(1, 'Diamonds')]), 12)

    def test_deal_perfect(self):
        "Tests case where there are twice as many cards in the deck as there are hands"
        deck = [(2, 'Clubs'),(8, 'Diamonds'),(13, 'Clubs'),(11, 'Diamonds')]
        deal(deck, ([],[]))
        self.assertEqual(deck, []) #Checks to see if deck is empty aka all cards have been dealt

    def test_deal_too_many(self):
        "Test to check if last card in deck is correct"
        deck = [(2, 'Clubs'),(8, 'Diamonds'),(13, 'Clubs'),(11, 'Diamonds'),(9,'Hearts')]
        deal(deck, ([],[]))
        self.assertEqual(deck, [(9,'Hearts')]) #Checks to see if last card left in deck is (9,'Hearts'), which shouldn't be dealt

    def test_deal_exception(self):
        "Tests deal() to see if it raises the correct exception"
        deck = [(2, 'Clubs')]
        with self.assertRaises(TooFewCardsError):
            deal(deck, ([],[]))

if __name__ == '__main__':
    unittest.main()


from nltk.corpus import wordnet as wn

# Get a definition of all the words in the list
# [List-of String] -> [List-of [List String [List-of String]]]
# First string in the return value is the word,
# and the second part is the definition
def flashCard(los):
    r = []
    for s in los:
        r.append([s, genDef(s)])
    return r

# Generate the dictionary definition for 1 string
# String -> [List-of String]
def genDef(s):
    r = []
    for syns in wn.synsets(s, pos='n'):
       r.append(syns.definition())
    return r


import os
import json
from pathlib import Path
import shutil

# Define paths
#base_path = Path("..\data")
base_path = Path("racuni")

metadata_path = base_path.joinpath("key")
image_path = base_path.joinpath("img")
# define metadata list
metadata_list = []

# parse metadata
for file_name in metadata_path.glob("*.json"):
  with open(file_name, "r", encoding="utf8") as json_file:
    # load json file
    data = json.load(json_file)
    # create "text" column with json string
    text = json.dumps(data)
    # add to metadata list if image exists
    if image_path.joinpath(f"{file_name.stem}.jpg").is_file():
      metadata_list.append({"text":text,"file_name":f"{file_name.stem}.jpg"})
      # delete json file

# write jsonline file
with open(image_path.joinpath('metadata.jsonl'), 'w') as outfile:
    for entry in metadata_list:
        json.dump(entry, outfile)
        outfile.write('\n')

# remove old meta data
shutil.rmtree(metadata_path)


import discord
from discord.ext import commands
import os
from __main__ import settings, send_cmd_help
import logging
from cogs.utils.dataIO import dataIO
from .utils import checks
import re

# Some magic values
ALL_CHANNELS = 'all'
MODE_INCLUSIVE = 'incl'  # deletes all messages matching
MODE_EXCLUSIVE = 'excl'  # delete all messages not matching
MODE_DISABLED = 'none'   # pattern disabled
MODES = [MODE_DISABLED, MODE_EXCLUSIVE, MODE_INCLUSIVE]

log = logging.getLogger('red.recensor')

DATA_PATH = "data/recensor/"
JSON_PATH = DATA_PATH + "regexen.json"

class ReCensor:

    # Data format:
    # {
    #  serverid (str): {
    #   channelid (str): {
    #     regex (str): mode (str from MODES)
    #     }
    #   }
    # }

    def __init__(self, bot):
        self.bot = bot
        self.regexen = dataIO.load_json(JSON_PATH)
        self.recache = {}
        bot.loop.create_task(self.compile_regexen())

    def _re_present(self, obj):
        """Determines if any patterns are set for a server or channel"""
        if type(obj) is discord.Server:
            server = obj
            if server.id in self.regexen:
                for relist in self.regexen[server.id].values():
                    if bool(relist):  # nonempty list
                        return True
                return False
            else:
                return False

        elif type(obj) is discord.Channel:
            server = obj.server
            channel = obj
            if channel.id in self.regexen[server.id]:
                return bool(self.regexen[server.id][channel.id])
            else:
                return False

        elif type(obj) is str:  # won't work with ALL_CHANNELS
            channel = self.bot.get_channel(obj)
            server = channel.server
            if channel.id in self.regexen[server.id]:
                return bool(self.regexen[server.id][channel.id])
            else:
                return False

    def _ls_excl(self, server):
        """returns a list of channel IDs with exclusive filters"""
        clist = []
        if type(server) is discord.Server:
            server = server.id
        if server in self.regexen:
            for c, relist in self.regexen[server].items():
                if MODE_EXCLUSIVE in relist.values():
                    clist.append(c)
        return clist

    # Background cache regexen for speed
    async def compile_regexen(self):
        for s, channels in self.regexen.items():
            for regex in channels:
                self.recache[regex] = re.compile(regex)

    @commands.group(name='recensor', pass_context=True)
    @checks.mod_or_permissions(manage_messages=True)
    async def recensor(self, ctx):
        """Configure regular expression censorship"""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @recensor.command(pass_context=True, name='list')
    async def _list(self, ctx, channel: discord.Channel=None):
        """Lists regexes used to filter messages.
        Channel listing includes global patterns."""
        server = ctx.message.server
        self.regexen = dataIO.load_json(JSON_PATH)
        if not self._re_present(server):
            await self.bot.say('There are no filter patterns set for this server.')
            return
        table = ' | '.join(['mode', 'pattern']) + '\n'  # header
        for c in self.regexen[server.id]:
            if c == ALL_CHANNELS and self._re_present(server):
                    table += '\nServer-wide:\n'
            elif (channel and channel.id == c) or not channel:
                if channel:
                    ch_obj = channel
                else:
                    ch_obj = self.bot.get_channel(c)
                if ch_obj is None:
                    table += '\n' + 'Channel ID %s (deleted):' % c + '\n'
                if self._re_present(ch_obj):
                    table += '\n#' + ch_obj.name + '\n'

            for regex, mode in self.regexen[server.id][c].items():
                table += ' | '.join([mode, regex]) + '\n'
        await self.bot.say('```py\n' + table + '```')

    @recensor.command(pass_context=True, name='add')
    async def _add(self, ctx, pattern: str, mode: str=MODE_INCLUSIVE, channel: discord.Channel=None):
        """Adds a pattern to filter messages. Mods, bot admins, and the bot's
        owner are not subjected to the filter.
        If the pattern contains spaces, it must be put in double quotes. Single quotes will not work.

        mode is one of:
        incl: Default, filter messages that match the pattern
        excl: filter non-matching, only one allowed per channel or server
        none: adds pattern to storage but doesn't apply filtering. Use recensor set to enable.

        To use channel, mode must also be specified. If channel is not specified,
        the filter is used across the entire server."""
        server = ctx.message.server

        # initialize
        self.regexen = dataIO.load_json(JSON_PATH)
        if server.id not in self.regexen:
            self.regexen[server.id] = {}

        if pattern.startswith("'"):
            await self.bot.say("Patterns cannot be specified within single quotes.")
            return

        if mode not in MODES:
            await self.bot.say('"%s" is not a valid mode. You must specify one of `%s`.' % (mode, '`, `'.join(MODES)))
            return
        if mode == MODE_EXCLUSIVE:
            if ALL_CHANNELS in self._ls_excl(server):
                await self.bot.say("There is already a server-wide exclusive filter. Remove or disable it first.")
                return
            if channel and channel.id in self._ls_excl(server):
                await self.bot.say("That channel already has an exclusive filter. Remove or disable it first.")
                return
        cid = channel.id if channel else ALL_CHANNELS
        if cid not in self.regexen[server.id]:
            self.regexen[server.id][cid] = {}
        self.regexen[server.id][cid][pattern] = mode
        await self.bot.say('Pattern added.')
        dataIO.save_json(JSON_PATH, self.regexen)

    @recensor.command(pass_context=True, name='set')
    async def _set(self, ctx, mode: str, channel: discord.Channel=None):
        """Lists regexes used to filter messages"""
        server = ctx.message.server
        self.regexen = dataIO.load_json(JSON_PATH)
        if not self._re_present(server):
            await self.bot.say('There are no patterns in the server to modify.')
            return
        if mode not in MODES:
            self.bot.reply('"%s" is not a valid mode. You must specify one of `%s`.') % (mode, '`, `'.join(MODES))

        if mode == MODE_EXCLUSIVE:
            if ALL_CHANNELS in self._ls_excl(server):
                await self.bot.say("There is already a server-wide exclusive filter. Remove or disable it first.")
                return
            if channel and channel.id in self._ls_excl(server):
                await self.bot.say("That channel already has an exclusive filter. Remove or disable it first.")
                return

        re_list = {}
        i = 1
        table = ' | '.join(['#'.ljust(4), 'mode', 'pattern']) + '\n'  # header
        for c in self.regexen[server.id]:
            if c == ALL_CHANNELS and self._re_present(server):
                    table += '\nServer-wide:\n'
            elif (channel and channel.id == c) or not channel:
                if channel:
                    ch_obj = channel
                else:
                    ch_obj = self.bot.get_channel(c)
                if ch_obj is None:
                    table += '\n' + 'Channel ID %s (deleted):' % c + '\n'
                if self._re_present(ch_obj):
                    table += '\n#' + ch_obj.name + '\n'

            for regex, oldmode in self.regexen[server.id][c].items():
                table += ' | '.join([str(i).ljust(4), oldmode, regex]) + '\n'
                re_list[str(i)] = (server.id, c, regex, oldmode)
                i += 1
        prompt = 'Choose the number of the pattern to set to `%s`:\n' % mode
        await self.bot.say(prompt + '```py\n' + table + '```')

        msg = await self.bot.wait_for_message(author=ctx.message.author, timeout=15)
        if msg is None:
            return
        msg = msg.content.strip()
        if msg in re_list:
            sid, cid, regex, _ = re_list[msg]
            self.regexen[sid][cid][regex] = mode
            await self.bot.say('Mode set.')
            dataIO.save_json(JSON_PATH, self.regexen)

    @recensor.command(pass_context=True, name='del')
    async def _del(self, ctx, channel: discord.Channel=None):
        """Lists regexes used to filter messages"""
        server = ctx.message.server
        self.regexen = dataIO.load_json(JSON_PATH)
        if not self._re_present(server):
            await self.bot.say('There are no filter patterns set for this server.')
            return
        re_list = {}
        i = 1
        table = ' | '.join(['#'.ljust(4), 'mode', 'pattern']) + '\n'  # header
        for c in self.regexen[server.id]:
            if c == ALL_CHANNELS and self._re_present(server):
                    table += '\nServer-wide:\n'
            elif (channel and channel.id == c) or not channel:
                if channel:
                    ch_obj = channel
                else:
                    ch_obj = self.bot.get_channel(c)
                if ch_obj is None:
                    table += '\n' + 'Channel ID %s (deleted):' % c + '\n'
                if self._re_present(ch_obj):
                    table += '\n#' + ch_obj.name + '\n'

            for regex, mode in self.regexen[server.id][c].items():
                table += ' | '.join([str(i).ljust(4), mode, regex]) + '\n'
                re_list[str(i)] = (server.id, c, regex)
                i += 1
        prompt = 'Choose the number of the pattern to delete:\n'
        await self.bot.say(prompt + '```py\n' + table + '```')
        msg = await self.bot.wait_for_message(author=ctx.message.author, timeout=15)
        if msg is None:
            return
        msg = msg.content.strip()
        if msg in re_list:
            sid, cid, regex = re_list[msg]
            del(self.regexen[sid][cid][regex])
            await self.bot.say('Pattern removed.')
        dataIO.save_json(JSON_PATH, self.regexen)

    def immune_from_filter(self, message):
        """Tests message to see if it is exempt from filter. Taken from mod.py"""
        user = message.author
        server = message.server
        admin_role = settings.get_server_admin(server)
        mod_role = settings.get_server_mod(server)

        if user.id == settings.owner:
            return True
        elif discord.utils.get(user.roles, name=admin_role):
            return True
        elif discord.utils.get(user.roles, name=mod_role):
            return True
        else:
            return False

    async def on_message(self, message):
        # Fast checks
        if message.channel.is_private:
            return
        if message.author.id == self.bot.user.id:
            return

        server = message.server
        sid = server.id
        can_delete = message.channel.permissions_for(server.me).manage_messages

        # Owner, admins and mods are immune to the filter
        if self.immune_from_filter(message) or not can_delete:
            return

        if sid in self.regexen:
            patterns = {}
            # compile list of patterns from global and channel
            for key in [ALL_CHANNELS, message.channel.id]:
                if key in self.regexen[sid]:
                    patterns.update(self.regexen[sid][key])
            # Iterate through patterns
            for regex, mode in patterns.items():
                # Skip disabled patterns
                if mode == MODE_DISABLED:
                    continue
                regex = self.recache[regex] if regex in self.recache else re.compile(regex)
                if (mode == MODE_EXCLUSIVE) != bool(regex.match(message.content)):  # xor
                    await self.bot.delete_message(message)

def check_folder():
    if not os.path.exists(DATA_PATH):
        log.debug('Creating folder: %s' % DATA_PATH)
        os.makedirs(DATA_PATH)

def check_file():
    if dataIO.is_valid_json(JSON_PATH) is False:
        log.debug('Creating json: %s' % JSON_PATH)
        dataIO.save_json(JSON_PATH, {})

def setup(bot):
    check_folder()
    check_file()
    n = ReCensor(bot)
    bot.add_cog(n)


#!/usr/bin/env python3
'''read json and write markdown table'''

import json
import argparse
from sys import stdout, stderr

def parseOptions():
    '''Parse the commandline options'''
    parser = argparse.ArgumentParser(description='None :)')
    parser.add_argument('--verbose','-v', dest='verbose', action='count', default=0)
    parser.add_argument('--inputfile','--input', '-i', default=None)
    args = parser.parse_args()
    return args
args = parseOptions()

with open(args.inputfile,"r") as f:
    jsonstring = f.read()
metrics = [  "cores", "ram", "instances", "gigabytes", "floating-ips", "fixed-ips"]

data = json.loads(jsonstring)
allvos={}

# print(json.dumps(data, sort_keys=True, indent=4, separators=(',', ': ')))

# collect available VOs
for site in data.keys():
    for vo in data[site].keys():
        if vo not in ["mteam.data.kit.edu"]:
            allvos[vo]=True

# Dump the tables

for metric in metrics:

    stdout.write(F"| {metric} |")
    for vo in allvos:
        stdout.write(F" {vo.split('.')[0]} |")
    print("")
    for vo in allvos:
        stdout.write("|-------------------------")
    print("|---|")
    for site in data.keys():
        stdout.write(F"| {site}| ")
        for vo in allvos:
            if data[site].get(vo):
                stdout.write(F"    {data[site].get(vo)[metric]}|")
            else:
                stdout.write(F"    | ")
        print("")
    print ("\n\n")


from flask import Flask,render_template,request,redirect
import requests
import pandas as pd
import os
import numpy as np
import bokeh
from bokeh.layouts import gridplot
from bokeh.plotting import figure, show, output_file
from bokeh.embed import components
bv = bokeh.__version__

app = Flask(__name__)

app.vars={}

@app.route('/')
def main():
	return redirect('/index_sheng')

@app.route('/index_sheng',methods=['GET','POST'])
def index_sheng():
    if request.method == 'GET':
        return render_template('Small Project.html')
    else:
        app.vars['ticker'] = request.form['ticker']
        app.vars['select'] = request.form.values()
        return redirect('/graph')

@app.route('/graph',methods=['GET','POST'])
def graph():
    ticker = app.vars['ticker'].upper()
    select = app.vars['select'][1]
    res = requests.get("https://www.quandl.com/api/v3/datasets/WIKI/"+str(ticker)+".json?start_date=2017-05-01&end_date=2017-06-01&frequency=daily&api_key=mRVPnQBf9hpYoxLWA_iA")
    df = pd.DataFrame(res.json()['dataset']['data'],columns=res.json()['dataset']['column_names']).set_index('Date')
    app.vars['desc'] = res.json()['dataset']['name'].split(',')[0]

    def datetime(x):
        return np.array(x, dtype=np.datetime64)

    plot = figure(x_axis_type="datetime", title=str(ticker)+" "+str(select))
    plot.grid.grid_line_alpha=0.5
    plot.xaxis.axis_label = 'Date'
    plot.yaxis.axis_label = 'Price'
    plot.line(datetime(df.index), df[select],color='#B2DF8A')
    #p.legend.location = "top_left"
    script,div = components(plot)

    return render_template('graph.html', bv=bv, ticker=app.vars['ticker'], script=script, div=div)

@app.errorhandler(500)
def error_handler(e):
	return render_template('error.html',ticker=app.vars['ticker'])

if __name__ == "__main__":
	port = int(os.environ.get("PORT", 5000))
	app.run(host='0.0.0.0', port=port,debug=False)
	#app.run(debug=True)


"""
    Projection layer:
    - adapted from code by Abhishek Karr for Learned Stereo Machine
        link:

    This layer projects 2D feature map onto a 3D feature tensor.
    Input:
        - features: 4D tensor of image features (Batch x W x H x F)
        - KRcam:    3D tensor of the camera coordinates (Batch x 3 x 4)

    Output:
        - grid      image features projected into a grid

"""
import torch
import torch.nn.functional as F
from torch import nn

def get_camera_params():
    fx = fy = 1  # -420
    x0 = y0 = 0  # im_dim/2

    R = torch.Tensor([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 1]])
    K = torch.Tensor([[fx, 0, x0], [0, fy, y0], [0, 0, 1]])
    return R, K

class Project2Dto3D(nn.Module):
    def __init__(self, res):
        super(Project2Dto3D, self).__init__()
        self.res = res
        self.nVox = self.res ** 3
        self.grid = self.generate_grid()

        self.R, self.K = get_camera_params()  # Projection Parameters

    def generate_grid(self):
        g_range = torch.linspace(-0.5, 0.5, self.res)
        grid = torch.stack(torch.meshgrid(g_range, g_range, g_range))
        grid = grid.view(3, -1)
        # convert to homogeneous coordinates
        grid = torch.cat((grid, torch.ones(1, self.nVox)), dim=0)
        return grid

    def forward(self, features, Rcam=None, depth=None, return_sdf=False):
        n_batch = features.shape[0]
        self.K, self.grid = self.K.cuda(), self.grid.cuda()

        # if no RCam, use an arbitary one for no rotation
        if Rcam is None:
            Rcam = self.R[None, :, :].repeat(n_batch, 1, 1).cuda()

        # Rotate grid using KRcam
        KRcam = self.K.matmul(Rcam)  # batch x 3 x 4
        b_grid = KRcam.matmul(self.grid)  # batch x 3 x XYZ

        if depth is not None:
            # calculate depth grid
            d_grid = (
                b_grid[:, 2:] - 1
            )  # scale from 0.5 to 1.5 instead of 1.5-2.5:
            d_grid = d_grid.view(
                -1, 1, self.res, self.res, self.res
            )  # batch x 1 x X x Y x Z
            d_grid = d_grid.permute(0, 1, 4, 3, 2)  # batch x 1 x Z x Y x X

        # remap grid to [-1, 1] range
        # z = 0 for 3D interplation of a 2D array
        b_grid[:, 2] = b_grid[:, 2] * 0.0
        b_grid = 2 * b_grid

        # remap from XYZ to DxHxW; and move xyz dim to last
        b_grid = b_grid.view(
            -1, 3, self.res, self.res, self.res
        )  # batch x 3 x X x Y x Z
        b_grid = b_grid.permute(0, 4, 3, 2, 1)  # batch x Z x Y x X x 3

        # bilinear using grid_sample
        features = features[:, :, None, :, :]
        f_bil = F.grid_sample(
            features,
            b_grid,
            mode="bilinear",
            padding_mode="zeros",
            align_corners=True
        )  # batch x F x Z x Y x X

        if depth is not None:
            depth = depth[:, :, None, :, :]
            d_bil = F.grid_sample(
                depth, b_grid, mode="bilinear",
                padding_mode="border",
                align_corners=True
            )  # batch x 1 x Z x Y x X

            # rescale depth according to Z valuei
            sdf = d_grid - (d_bil - 1.0)  # signed distance function for depth
            weighting = (sdf >= 0).float()
            f_bil = weighting * f_bil

        if return_sdf:
            return f_bil, sdf
        else:
            return f_bil

class Project3Dto2D(nn.Module):
    def __init__(self, res, samples=64):
        super(Project3Dto2D, self).__init__()
        self.res = res
        self.nPix = self.res ** 2
        self.S = samples

        # generate 2D grid
        self.grid = self.generate_img_grid()
        self.R, self.K = get_camera_params()
        self._generate_Xc()

    def generate_img_grid(self):
        g_range = torch.linspace(-0.5, 0.5, self.res)
        z_range = torch.linspace(-0.5, 0.5, self.S)
        grid = torch.stack(torch.meshgrid(g_range, g_range, z_range))
        grid = grid.view(3, -1)
        return grid

    def _generate_Xc(self):
        b_grid = self.grid[None, :, :]  # 1 x 3 x X*Y*Z
        b_grid = b_grid.view(1, 3, self.res ** 2, self.S)  # 1 x 3 x XY x Z
        b_grid = b_grid.permute(0, 3, 1, 2)  # 1 x S x 3 x XY

        # make into homogeneous coordinates
        hom_coord = torch.ones(1, self.S, 1, self.nPix)  # 1 x S x 1 x X*Y
        self.Xc = torch.cat((b_grid, hom_coord), dim=2)  # 1 x S x 4 x X*Y

    def forward(self, f_tensor, R=None, interp="bilinear"):
        n_batch, g_h, g_w, g_d, g_f = f_tensor.shape

        # -- Image2Cam -- precomputed to self.Xc
        Xc = self.Xc.cuda()  # 1 x S x 4 x X*Y

        # -- Cam2World --
        # Use arbitary R if no R is provided
        if R is None:
            R = self.R[None, :, :].cuda().repeat(n_batch, 1, 1)

        # construct {R^T | -R^Tt]
        tr = R[:, :, 3:4] * 0  # batch x 3 x 1
        Rt = R[:, :, :3].transpose(1, 2)  # batch x 3 x 3
        Rt = torch.cat((Rt, tr), dim=2)  # batch x 3 x 4

        Rt = Rt[:, None, ::].repeat(1, self.S, 1, 1)  # batch x S x 3 x 4
        Xw = Rt.matmul(Xc)  # batch x S x 3 x X*Y
        Xw = Xw.view(-1, self.S, 3, self.res, self.res)  # batch x S x 3 x X x Y
        Xw = Xw.permute(0, 1, 4, 3, 2)  # batch x S x Y x X x 3

        # -- Sample 3D -> dim of g_val batch x F x D x H x W
        Xw = Xw * 2  # range from [-0.5, 0.5] to [-1, 1]
        g_val = F.grid_sample(f_tensor, Xw, mode=interp, padding_mode="zeros", align_corners=True)

        # move Z to samples
        g_val = g_val.permute(0, 1, 3, 4, 2)  # batch x F x Y x X x S
        return g_val


import numpy as np
import matplotlib.pyplot as plt
import math as m

#Trapz rule is NOT L-stable! (While Backward Euler is L-Stable)
#Test equation: y'(t)=-2000*(y-cos(t)), y(0)=0

def Fun(t,x):
    return -2000*(x-m.cos(t))

tf=1.5
t0=0
ts=50
k=(tf-t0)/ts
t=np.linspace(t0,tf,ts+1)
y=np.zeros((ts+1)) #trapz
yE=np.zeros((ts+1)) #Backward Euler
y0=0
y[0]=y0
yE[0]=0

for i in range (0,ts):
    y[i+1]=(y[i]+1000*k*(-y[i]+m.cos(t[i])+m.cos(t[i+1])))/(1+k*1000)
    yE[i+1]=(y[i]+2000*k*m.cos(t[i+1]))/(1+k*2000)

plt.plot(t,y,marker='o',label='Trapezi')
plt.plot(t,yE,marker='d',label='Backward Euler')
plt.legend()
plt.show()


def increment_string(strng):
    original_length = 1
    number = 0
    for index, char in enumerate(strng):
        try:
            new_char = int(char)
            number = int(strng[index:])
            original_length = len(strng) - index
            strng = strng[:index]
        except ValueError:
            continue

    number += 1
    return strng + str(number).zfill(original_length)

print(increment_string())


import numpy as np
import pandas as pd
import librosa
import os
from python_speech_features import fbank
from multiprocessing import Pool,cpu_count
import argparse
import traceback
import os

parser = argparse.ArgumentParser(description='npy file creator')
parser.add_argument('--csv', type=str, help='Input csv filpath')
parser.add_argument('--partition', type=int, help='Number of splits to make',default=128)
parser.add_argument('--seg_len', type=int, help='signal length with Zero padding',default=3)
parser.add_argument('--rate', type=int, help='Sampling Rate',default=16000)
parser.add_argument('--keep_wav', type=str, help='"T" if the wav files has to be kept else "F"',default="F")

args = parser.parse_args()
SAMPLE_RATE=args.rate
csv_filepath=args.csv
seg_len=args.seg_len

def normalize_frames(m):
    return [(v - np.mean(v)) / (np.std(v)+1e-12) for v in m]
def read_audio(filename, sample_rate=SAMPLE_RATE):
    audio, sr = librosa.load(filename, sr=SAMPLE_RATE, mono=True)
    audio = audio.flatten()
    return audio
def pre_process_inputs(filename, target_sample_rate=SAMPLE_RATE,seg_len=seg_len):
    if not os.path.exists(filename):
        return False
    signal=read_audio(filename)
    signal=signal[:(SAMPLE_RATE*seg_len)]
    padding_len=(SAMPLE_RATE*seg_len)-len(signal)
    signal=np.concatenate((signal,np.array([0.0]*padding_len)))
    filter_banks, energies = fbank(signal, samplerate=target_sample_rate, nfilt=64, winlen=0.025)
    filter_banks = normalize_frames(filter_banks)
    flatten=[]
    for i in filter_banks:
        flatten.extend(list(i))
    np.save(filename.replace('.wav', '.npy'),flatten)
    if args.keep_wav.upper()=='F':
        os.remove(filename)
    return True
def pre_process_inputs_MP(data_split):
        return np.array(data_split.apply(pre_process_inputs).tolist())

if __name__ == '__main__':
    try:
        print('Processing!')
        data=pd.read_csv(csv_filepath)
        data_split=np.array_split(data['filename'],int(args.partition))
        pool=Pool(cpu_count())
        x=np.concatenate(pool.map(pre_process_inputs_MP,data_split))
        pool.close()
        pool.join()
    except Exception as err:
        print(traceback.format_exc())


import types

from subscription_manager.base_plugin import SubManPlugin

requires_api_version = "1.0"

class AllSlotsPlugin(SubManPlugin):
    """Plugin with hooks for all slots."""

    name = "all_slots"
    all_slots = True

    def __init__(self, conf=None):
        super(AllSlotsPlugin, self).__init__(conf)

    def __getattr__(self, attrname):
        if attrname.endswith("_hook"):
            # if we get asked for a hook method, create one based on
            # "handler", set a slot_name attribute on it, and bind it
            # to our class with correct attribute name
            def handler(self, conduit):
                conduit.log.debug(
                    "%s all_slots_handler: %s slot_name: %s" % (self.name, handler, handler.slot_name)
                )

            # add a slot_name attr to the handler method obj itself
            setattr(handler, "slot_name", attrname[:-5])

            # create a new instance of it, and bind it to self
            new_hook = types.MethodType(handler, self, AllSlotsPlugin)

            # make the accesor point to the new method
            setattr(self, attrname, new_hook)

            # might we well return what we set via now working
            # attr lookup
            return getattr(self, attrname)

        # not a hook name, so legit attr error
        raise AttributeError


from airflow import DAG

from airflow.operators.dummy_operator import DummyOperator
from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator
from airflow.providers.cncf.kubernetes.sensors.spark_kubernetes import SparkKubernetesSensor
from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
from airflow.operators.python_operator import PythonOperator
from airflow.models import Variable
import boto3

aws_access_key_id = Variable.get('aws_access_key_id')
aws_secret_access_key = Variable.get('aws_secret_access_key')
glue = boto3.client('glue', region_name='us-east-1',
                    aws_access_key_id=aws_access_key_id,
                    aws_secret_access_key=aws_secret_access_key)

from airflow.utils.dates import days_ago

def trigger_crawler_enem2020_microdados_func():
    glue.start_crawler(Name='enem2020_crawler_microdados')

with DAG(
    'desafio_final_edc_enem_2020',
    default_args={
        'owner': 'Claudio',
        'depends_on_past': False,
        'email': ['claudiopinho@yahoo.com.br'],
        'email_on_failure': False,
        'email_on_retry': False,
        'max_active_runs': 1,
    },
    description='Extração e processamento de dados do ENEM 2020',
    schedule_interval=None,
    start_date=days_ago(1),
    catchup=False,
    tags=['spark', 'kubernetes', 'batch', 'ENEM'],
) as dag:

    # extracao = KubernetesPodOperator(
    #     namespace='airflow',
    #     image="539445819060.dkr.ecr.us-east-1.amazonaws.com/extraction-edsup-2019:latest",
    #     cmds=["python", "/run.py"],
    #     name="extraction-edsup-2019",
    #     task_id="extraction-edsup-2019",
    #     image_pull_policy="Always",
    #     is_delete_operator_pod=True,
    #     in_cluster=True,
    #     get_logs=True,
    # )

    inicio = DummyOperator(task_id='inicio')

    converte_MICRODADOS_parquet = SparkKubernetesOperator(
        task_id='converte_MICRODADOS_parquet',
        namespace="airflow",
        application_file="dag_enem2020_converte_MICRODADOS_parquet.yaml",
        kubernetes_conn_id="kubernetes_default",
        do_xcom_push=True,
    )

    converte_MICRODADOS_parquet_monitor = SparkKubernetesSensor(
        task_id='converte_MICRODADOS_parquet_monitor',
        namespace="airflow",
        application_name="{{ task_instance.xcom_pull(task_ids='converte_MICRODADOS_parquet')['metadata']['name'] }}",
        kubernetes_conn_id="kubernetes_default",
    )

    trigger_crawler_enem2020_microdados = PythonOperator(
        task_id='trigger_crawler_enem2020_MICRODADOS',
        python_callable=trigger_crawler_enem2020_microdados_func,
    )

inicio >> [converte_MICRODADOS_parquet]
converte_MICRODADOS_parquet >> converte_MICRODADOS_parquet_monitor >> trigger_crawler_enem2020_microdados


import zipfile
import re

fh = open('channel.zip', 'rb')
z = zipfile.ZipFile(fh)
for name in z.namelist():
    outpath = "tmp/"
    z.extract(name, outpath)
fh.close()

nothings = []
nextFile = "90052"
sumN = int(nextFile)
while nextFile:
    nothings.append(nextFile)
    fd = open("tmp/" + nextFile + ".txt")
    #zip.getinfo("tmp/" + nextFile + ".txt")
    data = fd.read()
    print(data)
    try:
        nextFile = re.search(r'nothing is \d+', data).group()
    except:
        print(str(sumN))
        break
    #print (nextFile,end="")
    nextFile = re.search(r'\d+', nextFile ).group()
    sumN += int(nextFile)
    #print ("  ", nextFile)

comment = ""
fh = open('channel.zip', 'rb')
z = zipfile.ZipFile(fh)
for n in nothings:
    comment += (z.getinfo(n+".txt").comment).decode("utf-8")
fh.close()
print (comment)

##import os
##from os import walk
##
##f = []
##for (dirpath, dirnames, filenames) in walk("tmp/"):
##    f.extend(filenames)
##    break
##for x in f:
##    #print (x)
##    st = os.stat("tmp/"+x)
##    print (st)


import Schedule
import Weather
import Wechat
import time
import _thread

global server_tz
server_tz = eval(Weather.GetPosition('auto_ip')['tz'])
print('Server timezone：' + str(server_tz))

with open('info.txt', 'r') as f:
    info = eval(f.read())
    f.close()

for name, item in info.items():
    Schedule.SetDailySchedule(name, item['times'], item['loc'])

Wechat.LogInWechat()
_thread.start_new_thread(Wechat.run, ())
print('Wechat is running in a new thread!')

while True:
    Schedule.RunPending()
    time.sleep(5)

Wechat.LogoutWechat()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='ProCode',
            fields=[
                ('fstateid', models.IntegerField(serialize=False, primary_key=True)),
                ('fstatename', models.CharField(max_length=30)),
                ('fnexttime', models.IntegerField()),
            ],
        ),
    ]


import pygame

class Background():
    def __init__(self, WIN):
        self.bgimage = pygame.transform.scale(
            pygame.image.load('images/bg.jpg'), (900, 500))
        self.bgimage_flipped = pygame.transform.scale(
            pygame.image.load('images/bg_flipped.jpg'), (900, 500))

        self.rectBGimg = self.bgimage.get_rect()
        self.rectBGimg_flipped = self.bgimage_flipped.get_rect()

        self.bgY1 = 0
        self.bgX1 = 0

        self.bgY2 = 0
        self.bgX2 = self.rectBGimg.width

        self.moving_speed = 0.5
        self.WIN = WIN

    def update(self):
        self.bgX1 -= self.moving_speed
        self.bgX2 -= self.moving_speed
        if self.bgX1 <= -self.rectBGimg.width:
            self.bgX1 = self.rectBGimg.width
        if self.bgX2 <= -self.rectBGimg.width:
            self.bgX2 = self.rectBGimg.width

    def render(self):
        self.WIN.blit(self.bgimage, (self.bgX1, self.bgY1))
        self.WIN.blit(self.bgimage_flipped, (self.bgX2, self.bgY2))


import time

MOVE_OFFSETS = (
    (-1, -2), (1, -2), (-2, -1), (2, -1), (-2, 1), (2, 1), (-1, 2), (1, 2),
)

class Cell:
    def __init__(self, char, positions):
        self.current = False
        self.visited = False
        self.char = char
        self.knight_pos = positions

class Board:
    def __init__(self, dimension_):
        self.column = dimension_[0]
        self.row = dimension_[1]
        self.cell_size = len(str(self.column * self.row))
        self.board = {}

    @staticmethod
    def position(col, row):
        return str(col) + str(row)

    def init_board(self):
        for row in range(1, self.row + 1):
            for column in range(1, self.column + 1):
                positions = set([pos for pos in self.legal_moves_from(column, row)])
                self.board[self.position(column, row)] = Cell(self.format_char('_'), positions)

    def legal_moves_from(self, col, row):
        for row_offset, col_offset in MOVE_OFFSETS:
            move_row, move_col = row + row_offset, col + col_offset
            if 1 <= move_row <= self.row and 1 <= move_col <= self.column:
                yield self.position(move_col, move_row)

    def format_char(self, char):
        if char == '_':
            return '_' * self.cell_size
        else:
            return ' ' * (self.cell_size - len(char)) + char

    def write_current(self, curr_pos, next_pos=None):
        if next_pos is None:
            self.board[curr_pos].current = True
            self.board[curr_pos].char = self.format_char('X')
        else:
            self.board[next_pos].current = True
            self.board[curr_pos].current = False
            self.board[curr_pos].visited = True

            self.board[curr_pos].char = self.format_char('*')
            self.board[next_pos].char = self.format_char('X')

    def manage_knights(self, curr_pos, next_pos=None):
        if next_pos is None:
            self.write_knights(curr_pos)
        else:
            self.write_knights(curr_pos, '_')
            self.write_knights(next_pos)

    def possible_moves(self, position, length=True):
        positions = self.board[position].knight_pos
        possible_moves = set([move for move in positions
                              if not self.board[move].current
                              and not self.board[move].visited])
        if length:
            return len(possible_moves)
        else:
            possible_moves = sorted(possible_moves, key=lambda x: len(self.board[x].knight_pos))
            return possible_moves

    def write_knights(self, position, char=None):
        for pos in self.board[position].knight_pos:
            if not self.board[pos].current and not self.board[pos].visited:
                if char is not None:
                    self.board[pos].char = self.format_char(char)
                else:
                    self.board[pos].char = self.format_char(str(self.possible_moves(pos)))

    def border_line(self):
        if self.cell_size == 1:
            print(' ' + '-' * (self.column * (self.cell_size + 1) + 3))
        else:
            print(' ' * (self.cell_size - 1) + '-' * (self.column * (self.cell_size + 1) + 3))

    def print_board(self):

        # upper border line
        self.border_line()

        for row in range(self.row, 0, - 1):
            # print rows number
            if self.row >= 10 > row:
                print(f' {row}| ', end='')
            else:
                print(f'{row}| ', end='')

            # print table
            for column in range(1, self.column + 1):
                print(f'{self.board[self.position(column, row)].char} ', end='')

            print('|', end='')
            print('\r')

        # lower border line
        self.border_line()

        # print column number
        print(' ' * 2 if self.row < 10 else ' ' * 3, end='')
        for i in range(1, self.column + 1):
            if i < 10:
                print(' ' * self.cell_size, end='')
            else:
                print(' ' * (self.cell_size - 1), end='')

            print(f'{i}', end='')

class Game(Board):
    def __init__(self):
        self.next = None
        self.moves = 0
        self.dimension = self.get_dimension()
        super().__init__(self.dimension)
        self.board_size = self.row * self.column
        self.current = self.get_starter()
        self.type = self.get_game_type()

    def run(self):
        if not self.type:
            self.run_computer()
        else:
            if self.run_computer(True):
                self.run_player()

    def run_computer(self, check=False):
        self.init_board()
        self.board[self.current].char = self.format_char('1')
        self.board[self.current].visited = True
        start = time.time()
        have_solution = self.find_solution(self.current, 2)
        if have_solution:
            if not check:
                print("Here's the solution!")
                self.print_board()
            else:
                return True
        else:
            print('No solution exists!')

        end = time.time()

        # print(f'\n{end - start}')

    def find_solution(self, position, counter):
        knight_positions = self.possible_moves(position, False)
        if counter < self.board_size + 1:
            for k_pos in knight_positions:
                self.board[k_pos].char = self.format_char(str(counter))
                self.board[k_pos].visited = True
                if self.find_solution(k_pos, counter + 1):
                    return True
                self.board[k_pos].char = self.format_char('_')
                self.board[k_pos].visited = False

            return False

        return True

    def run_player(self):
        self.init_board()
        self.write_current(self.current)
        self.manage_knights(self.current)
        self.print_board()
        while True:
            if self.next is not None and self.possible_moves(self.next) == 0:
                break
            self.next = self.get_player()
            self.write_current(self.current, self.next)
            self.manage_knights(self.current, self.next)
            self.current = self.next
            self.print_board()

        if self.column * self.row == self.moves:
            print('\n\nWhat a great tour! Congratulations!')
        else:
            print('\n\nNo more possible moves!')
            print(f'Your knight visited {self.moves} squares!')

    def valid_data(self, data, pos_check=False, possible_check=False):
        # number of data
        if len(data) != 2:
            return False
        # is it number
        elif not data[0].isdigit() or not data[1].isdigit():
            return False
        # is it negative
        elif int(data[0]) < 1 or int(data[1]) < 1:
            return False
        # is it in dimension
        if pos_check:
            if int(data[0]) > self.dimension[0] or int(data[0]) < 1 \
                    or int(data[1]) > self.dimension[1] or int(data[1]) < 1:
                return False
        # is it in possible moves
        if possible_check:
            data = self.position(data[0], data[1])
            if data not in self.board[self.current].knight_pos \
                    or self.board[data].current or self.board[data].visited:
                return False

        return True

    def get_dimension(self):

        dimension = input('Enter your board dimensions:').split()
        if self.valid_data(dimension):
            return [int(dim) for dim in dimension]
        else:
            print('Invalid dimensions!')
            return self.get_dimension()

    def get_game_type(self):
        game_type = input('Do you want to try the puzzle? (y/n):')
        if game_type == 'y':
            return True
        elif game_type == 'n':
            return False
        else:
            print('Invalid option')
            return self.get_game_type()

    def get_starter(self):
        self.moves += 1
        move = input("Enter the knight's starting position:").split()
        if self.valid_data(move, True):
            return self.position(move[0], move[1])
        else:
            print('Invalid position!')
            return self.get_starter()

    def get_player(self, first_time=True):
        self.moves += 1
        move = input("\n\nEnter your next move:" if first_time else "Invalid move! Enter your next move:").split()
        if self.valid_data(move, True, True):
            return self.position(move[0], move[1])
        else:
            return self.get_player(False)

if __name__ == '__main__':
    knight = Game()
    knight.run()