# -*- coding: utf-8 -*-
import re
import numpy as np
#import gdal
from os.path import join,relpath
from glob import glob
import matplotlib.pyplot as plt
import seaborn as sns
import math
import networkx as nx
import csv
from PIL import Image
import shelve

#x, yの並び順はすべて(x, y)

def main():
    dem_shelve = 'dem_s_rough'
    with shelve.open(dem_shelve) as db:
        dem_rough_sl = db['dem']

    shelve_x_start = 2750
    shelve_y_start = 6000
    shelve_x_end = 3750
    shelve_y_end = 7000

    path_edge_shelve = 'path_edge'
    with shelve.open(path_edge_shelve) as db:
        path_edges = db['edge']
    len_edges = len(path_edges)
    path_nodes = np.empty((len_edges + 1, 2))
    i = 0
    for ed in path_edges:
        path_nodes[i] = np.array(ed[0])
        i += 1
    path_nodes[i] = np.array(path_edges[i - 1][1])
    #np.arrayの[[x0, y0], [x1, y1], .....]という形を想定

    seg_num = 10

    seg_path = split_path(path_nodes, seg_num)

    seg_fin_nodes = find_fin_node(seg_path)

    next_initial_pos = np.array(path_nodes[0][0])
    initial_pos = np.empty(seg_num, 2)
    target_pos = np.empty(seg_num, 2)
    final_pos = np.empty(seg_num, 2)
    estimate_pos = np.empty(seg_num, 2)
    next_initial_pos = np.empty(seg_num, 2)

    for i in range(seg_num):
        initial_pos[i] = next_initial_pos
        #実際の初期位置
        target_pos[i]= seg_fin_nodes[i]
        #実際の目標位置
        final_pos[i] = rover_run(initial_pos, seg_path[i], i)
        #実際に到着した位置
        final_node_in_dem_all = find_closest_node_in_dem_all(final_pos[i], shelve_x_start, shelve_y_start)
        #実際に到着した位置に最も近いノード
        real_skyline = make_real_skyline(final_node_in_dem_all)
        #実際に到着した位置に最も近いノードから見える実際のスカイライン
        estimate_pos[i] = position_estimate(target_pos, real_skyline, dem_rough_sl)
        #スカイラインから推定される位置
        next_initial_pos[i] = modify_error(estimate_pos, target_pos, final_pos, i)
        #位置推定をもとに修正後の実際の位置

    global grid_size
    grid_size = 5.05
    global max_len
    max_len = math.sqrt(2) * grid_size
    global max_ang
    max_ang = 15

def split_path(path_nodes, seg_num):
    num_edge = len(path_nodes)
    seg_len = num_edge // seg_num

    seg_path = np.empty(seg_num)

    for i in range(seg_num):
        if i != seg_num - 1:
            seg_path[i] = path_nodes[seg_len * i : seg_len * (i + 1)]
        else:
            seg_path[i] = path_nodes[seg_len * i:]

    return seg_path

def find_fin_node(seg_path):
    seg_num = len(seg_path)
    fin_nodes = np.empty((seg_num, 2))
    for i in range(seg_num):
        fin_nodes[i] = seg_path[i][len(seg_path[i])]

    return fin_nodes

def rover_run(initial_pos, nodes, seg_i):
    len_nodes = len(nodes)
    current_pos = initial_pos
    rover_trajectory = np.empty((len_nodes, 2))
    rover_trajectory[0] = current_pos

    sigma_len = 0.1
    sigma_ang = 1

    for i in range(len_nodes - 1):
        vector_move = nodes[i + 1] - nodes[i]
        len_error = 1 + np.random.normal(0, sigma_len)
        ang_error = math.radians(np.random.normal(0, sigma_ang))
        x_move = math.sqrt(len_error) * (vector_move[0] * math.cos(ang_error) - vector_move * math.sin(ang_error))
        y_move = math.sqrt(len_error) * (vector_move[1] * math.sin(ang_error) + vector_move * math.cos(ang_error))
        current_pos += np.array([x_move, y_move])
        rover_trajectory[i + 1] = current_pos

    shelvename = 'trajectory_' + str(seg_i)
    with shelve.open(shelvename) as db:
        db['trajectroy'] = rover_trajectory

    return current_pos

def find_closest_node_in_dem_all(pos, shelve_x_start, shelve_y_start):
    x = 0
    while x < pos[0]:
        x += 1
    if x - pos[0] > 1 / 2:
        x -= 1
    y = 0
    while y < pos[1]:
        y += 1
    if y - pos[1] > 1 / 2:
        y -= 1

    x += shelve_x_start
    y += shelve_y_start

    return np.array([x, y])

def make_real_skyline(node):
    sigma_azimuth = 1   #方位角の推定誤差
    sigma_photo = 0.001     #撮影時に生じるノイズ
    sigma_pixel = 0.001     #解像度による誤差

    x_center = node[0]
    y_center = node[1]

    real_skyline = skyline_with_error_azi(sigma_azimuth, x_center, y_center)
    real_skyline = add_noise(real_skyline, sigma_photo)
    real_skyline = add_noise(real_skyline, sigma_pixel)

    return real_skyline

def skyline_with_error_azi(sigma, x_center, y_center):
    dem = np.array(Image.open('\\Users\\Maenaka\\python\\20190204\\mars_spirit_landing_site\\dem.tif'))
    #dem = np.array(Image.open('/home/shuto/python/20190131/mars_spirit_landing_site/dem.tif'))

    delta_p = 1     #各方位角方向のインクリメント[m]
    max_k = 2000      #インクリメント回数
    radius_star = 3397200     #星の半径（ここでは火星）
    height_cam = 0.7    #カメラの高さ
    grid_size = 1.01
    k = 1
    azi = 0
    elevation_max = 0
    skyline = np.zeros(360)

    error = np.random.normal(0, sigma)

    while azi < 360:
        while k <= max_k:
            x_on_azi_line =  round(x_center + k * delta_p * math.cos((azi + error) / 360 * 2 * math.pi) / grid_size)
            y_on_azi_line =  round(y_center + k * delta_p * math.sin((azi + error) / 360 * 2 * math.pi) / grid_size)
            curv = (radius_star - math.sqrt(radius_star ** 2 - (k * delta_p) ** 2) ) / grid_size
            height_interest = (dem[y_on_azi_line][x_on_azi_line] - curv) / grid_size
            height_center = (dem[y_center][x_center] + height_cam) / grid_size
            elevation_interest = math.atan((height_interest - height_center) / (k * delta_p / grid_size))
            if elevation_max < elevation_interest:
                elevation_max = elevation_interest
            k += 1
        skyline[azi] = elevation_max
        elevation_max = 0
        azi += 1
        k = 0

    return skyline

def add_noise(skyline, sigma):
    real_skyline = np.zeros(360)
    azi = 0
    while azi < 360:
        noise = np.random.normal(0, sigma)
        real_skyline[azi] = skyline[azi] + noise
        azi += 1

    return real_skyline

def position_estimate(target_pos, real_skyline, dem):
    search_range = 10
    #探索する正方形領域の一辺の長さ（エッジの数）
    min_x = target_pos[0] - search_range // 2
    min_y = target_pos[1] - search_range // 2
    max_x = target_pos[0] + search_range // 2
    max_y = target_pos[1] + search_range // 2

    min_diff_skyline = 99999999

    for y in range(min_y, max_y):
        for x in range(min_x, max_x):
            rendered_skyline = dem[y][x][1:]
            diff_skyline = 0
            for azi in range(360):
                diff_skyline += (rendered_skyline[azi] - real_skyline[azi]) ** 2
            if min_diff_skyline > diff_skyline:
                min_diff_skyline = diff_skyline
                esitimate_pos = np.array([x, y])

    return esitimate_pos

def modify_error(estimate_pos, target_pos, final_pos, seg_i):
    sigma_len = 0.1
    sigma_ang = 1

    vector_move = target_pos - estimate_pos
    len_error = 1 + np.random.normal(0, sigma_len)
    ang_error = math.radians(np.random.normal(0, sigma_ang))
    x_move = math.sqrt(len_error) * (vector_move[0] * math.cos(ang_error) - vector_move * math.sin(ang_error))
    y_move = math.sqrt(len_error) * (vector_move[1] * math.sin(ang_error) + vector_move * math.cos(ang_error))

    vector_move += np.array([x_move, y_move])

    next_initial_pos = final_pos + vector_move

    return next_initial_pos

if __name__ == "__main__":
    main()


from fastapi import APIRouter
from fastapi.requests import Request
from fastapi.responses import Response
from piccolo.apps.user.tables import BaseUser
from piccolo_api.token_auth.tables import TokenAuth

from myk_trade.db.models.transactions import WalletModel

router = APIRouter()

@router.get("/")
async def get_profile(
    request: Request,
) -> dict:
    if request.user.is_authenticated is False:
        return Response(status_code=401, content="Unauthorized")

    user = (
        await BaseUser.select(
            exclude_secrets=True,
        )
        .where(
            BaseUser.id == request.user.user.id,
        )
        .first()
        .run()
    )
    return user

@router.get("/wallets")
async def get_wallets(
    request: Request,
) -> list:
    if request.user.is_authenticated is False:
        return Response(status_code=401, content="Unauthorized")

    wallets = (
        await WalletModel.select(
            exclude_secrets=True,
        )
        .where(
            WalletModel.user_id == request.user.user.id,
        )
        .run(nested=True)
    )
    return wallets

@router.get("/token")
async def get_token(
    request: Request,
) -> dict:
    if request.user.is_authenticated is False:
        return Response(status_code=401, content="Unauthorized")

    token = (
        await TokenAuth.select(
            TokenAuth.token,
            exclude_secrets=True,
        )
        .where(
            TokenAuth.user == request.user.user,
        )
        .first()
        .run()
    )
    return token  # {"token": token}

@router.post("/token")
async def create_token(
    request: Request,
) -> dict:
    if request.user.is_authenticated is False:
        return Response(status_code=401, content="Unauthorized")

    await TokenAuth.delete().where(
        TokenAuth.user == request.user.user,
    ).run()

    token = await TokenAuth.create_token(request.user.user.id)

    return {"token": token}


import sys, heapq
sys.stdin = open("더맵게_input.txt")

scoville = list(map(int, input().split()))
K = int(input())

answer = 0
heapq.heapify(scoville)
heapq.heappush(scoville, 1000000001)
while True:
    fir = heapq.heappop(scoville)
    if fir >= K:
        print(answer)
        break
    answer += 1
    sec = heapq.heappop(scoville)
    if sec == 1000000001:
        answer = -1
        print(answer)
        break
    new = fir + sec * 2
    heapq.heappush(scoville, new)


from django.utils.safestring import mark_safe
from django import template
from django.template.defaultfilters import striptags
from front.utils import states
from front.models import Startup

register = template.Library()

@register.filter(name='state_name')
def state_name_filter(value):
    try:
        return mark_safe(dict(states)[value])
    except KeyError:
        return mark_safe(value)

state_name_filter.is_safe = True

@register.filter(name='startup_name')
def startup_name_filter(value):
    try:
        startup = Startup.objects.get(owner__username=value)
        return mark_safe(startup.name)
    except:
        return value

startup_name_filter.is_safe = True


import sympy

print(sympy.isprime(5))
count = 0
lists = []
for i in range(100,1000):
    if sympy.isprime(i) is True:
        count += 1
        lists.append(i)

print("100부터 1000까지의 소수 갯수{}",count)
print("소수 리스트",lists)

print(list(sympy.primerange(1,100)))


# reverse a string.

# first way.
name = 'saurabh'
print("Reverse of this string is ",name[::-1])

print("-------------------------------------------")
# second way.
name = 'saurabh'
rev = list(name)
rev.reverse()
print(''.join(rev))
print("-------------------------------------------")

# third way.
name = 'saurabh'
i = len(name)-1
while i >= 0 :
    print(name[i],end='')
    i -= 1
print()
print("-------------------------------------------")

# fourth way.
# ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€ðŸ‘€
# two pointers approach.

# loigc to sahi h but string is immutable in python.
# name = 'saurabh'
# start = 0
# end = len(name)-1
# while start <= end :
#     name[start],name[end] = name[end],name[start]
#     start += 1
#     end -= 1
# print(name)
name = ['s','a','u','r','a','b','h']
start = 0
end = len(name)-1
while start <= end :
    name[start],name[end] = name[end],name[start]
    start += 1
    end -= 1
print("".join(name))
print("-------------------------------------------")


from django.shortcuts import render
from django.http import HttpResponse
from django.db.models import Q

from .models import FileUploadForm, FirmContacted
from django.db.models import Q
import json

from rest_framework.response import Response
from rest_framework.decorators import action
from rest_framework.pagination import LimitOffsetPagination
from rest_framework import viewsets
import operator
from functools import reduce

from .serializers import FirmContactedSerializer, FirmSerializer, EmailTemplateSerializer
from .models import Firm, EmailTemplate

from collections import OrderedDict

def to_boolean(string):
    return string == "True"

class EmailTemplateView(viewsets.ModelViewSet):
    pagination_class = None
    serializer_class = EmailTemplateSerializer
    queryset = EmailTemplate.objects.all()

class FirmContactedView(viewsets.ModelViewSet):
    serializer_class = FirmContactedSerializer
    queryset = FirmContacted.objects.all()

    @action(detail=False, methods=['POST'], name='/')
    def update_contacted(self, request):
        try:
            obj, created = FirmContacted.objects.update_or_create(**request.data, defaults={
                "user_id": request.data["user_id"],
                "orgnr": request.data["orgnr"],
                "contacted": not request.data["contacted"]
            })
        except Exception as e:
            return Response(f"Update or create failed: {e}")
        return Response("success")

class FirmView(viewsets.ModelViewSet):
    serializer_class = FirmSerializer
    pagination_class = LimitOffsetPagination
    queryset = Firm.objects.all()

    @action(detail=False, methods=['GET'], name='')
    def custom(self, request):

        queryset = Firm.objects.all().extra(select={'contacted': 'SELECT contacted FROM "app_firmcontacted" WHERE "app_firm".orgnr = "app_firmcontacted".orgnr'})

        if request.GET.get("email", None):
            queryset = queryset.exclude(email="[]")
        if request.GET.get("website", None):
            queryset = queryset.exclude(website="[]")
        if request.GET.get("telephone", None):
            queryset = queryset.exclude(telephone="[]")
        if request.GET.get("naering", None):
            nace_list = request.GET.get("naering").split("æsepæ")
            queryset = queryset.filter(nace__in=nace_list)
        if request.GET.get("employees", None):
            empl = json.loads(request.GET.get("employees"))
            queryset = queryset.filter(employees__gte=empl[0])
            queryset = queryset.filter(employees__lte=empl[1])
        if request.GET.get("municipality", None):
            print(request.GET.get("municipality"))
            municipalities = request.GET.get("municipality").split(",")
            print(municipalities)
            queryset = queryset.filter(municipality_nr__in=municipalities)
        print(queryset.query)
        count = queryset.count()
        queryset = LimitOffsetPagination().paginate_queryset(queryset=queryset, request=request)
        serializer = self.get_serializer(queryset, many=True)

        return Response(OrderedDict([
            ("results", serializer.data),
            ("count", count)
        ]))

def index(request):
    items = [f"doc{i}" for i in range(0)]
    if request.method == "POST":

        if request.FILES.get('file', False):

            form = FileUploadForm(request.POST, request.FILES)
            print(form)
            if form.is_valid():
                # Upload to S3 and get path
                path = "s3_path/document.csv"
                model = form.save(commit=False)
                print(model.__dict__)
                model.path = path
                model.save()

            file = request.FILES['file']
            items.append(file)
            return render(request, "app/index.html", {"upload_success": True, "documents": items})

    return render(request, "app/index.html", {"documents": items})


# Calculates the sum of all odd Fibonacci numbers that are less than or equal to num.
def sumOddFibs(num):
    sum = 0
    i = 0
    while sum <= num:
        res = fibo(i)
        i = i + 1
        if (res > num):
            break
        if (isOdd(res)):
            sum = sum + res
    return sum

# Finds nth fibonnaci number usign recursion
def fibo(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fibo(n - 1) + fibo(n - 2)

# Checks if num is odd
def isOdd(num):
    return num % 2 != 0

# Testing for values
test = {
    4: 5,
    75024: 60696,
    75025: 135721,
    1000: 1785,
    4000000: 4613732,
}

for key, value in test.items():
    res = sumOddFibs(key)
    assert res == value, f"fibo({key}) should equal {value}"
    print(f"fibo({key}) - Test Passed!")
    print(f"{key} -> {value} === {res} \n")


import requests
from requests_futures.sessions import FuturesSession
from urllib import parse
from bs4 import BeautifulSoup
import settings
import urllib
import db
from db import Gigs
from datetime import datetime, timedelta

CRAIGSLIST_SITE = 'sof'

def get_craigslist_post_details(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'lxml')
    post_content = soup.find('section', {'id': 'postingbody'})
    return post_content.text if post_content else None

def build_craigslist_data_object(soup, url, location, category):
    rows = soup.find('div', class_='content').find('div', class_='rows').find_all('p')
    data = []
    for row in rows:
        town = row.find('span', class_='pnr').find('small')
        _location = " ".join([location, town.text.strip()]) if town else location
        post_url = parse.urljoin(url, row.a['href'])
        data.append({
            'website': 'craigslist',
            'category': category,
            'website_supplied_id': row['data-pid'],
            'name': row.find('a', class_='hdrlnk').text.strip(),
            'url': post_url,
            'location': _location,
            'datetime': row.find('time')['datetime'],
            'details': get_craigslist_post_details(post_url) if settings.GET_CRAIGSLIST_POST_DETAILS else None
        })
    return data

def craigslist_searcher(locations, site=None):
    for location in locations:
        url = "https://{}.craigslist.org/search/{}/".format(location, (site or CRAIGSLIST_SITE))
        response = requests.get(url)
        soup = BeautifulSoup(response.content, 'lxml')
        db.insert_into_db(build_craigslist_data_object(soup, url, location, 'gigs, computer'))

def get_craigslist_cities():
    locations = {}
    cities_page = "https://www.craigslist.org/about/sites"
    response = requests.get(cities_page)
    soup = BeautifulSoup(response.content, 'lxml')
    countries = soup.find_all('h1')
    location_containers = soup.find_all('div', class_='colmask')
    for index, location in enumerate(location_containers):
        cities = [urllib.parse.urlparse(a['href']).hostname.split('.')[0] for a in location.find_all('a')]
        locations[countries[index].text] = cities
    return locations

def insert_callback(session, response):
    soup = BeautifulSoup(response.content, 'lxml')
    location = urllib.parse.urlparse(response.url).hostname.split('.')[0]
    data = build_craigslist_data_object(soup, response.url, location, 'gigs, computer')
    db.insert_into_db(data)

def async_requests(locations, site=None):
    session = FuturesSession()
    check_date = datetime.now() + timedelta(hours=-4)
    for location in locations:
        print(location)
        gig = Gigs.select().where(Gigs.location.contains(location)).order_by(Gigs.datetime.desc()).first()
        if (gig is None) or ((datetime.strptime(gig.datetime, '%Y-%m-%d %H:%M') < check_date)):
            url = "https://{}.craigslist.org/search/{}/".format(location, (site or CRAIGSLIST_SITE))
            future = session.get(url, background_callback=insert_callback)


import pygame
import UI
from funcs import canvas_flood_fill
import funcs
import sys

pygame.init()

screen_size = (1200, 850)
screen_w, screen_h = screen_size
canvas_size = (int(screen_size[0] / 1.2), int(screen_size[1] / 1.2))
flag = True

screen = pygame.display.set_mode(screen_size, 0, 16, pygame.HWACCEL)
clock = pygame.time.Clock()

canv_c, canv_r = 64, 64
canvas_keep_ratio = True
canvas_keep_rc = True
canv_px_w = canvas_size[0] // canv_c
canv_px_h = canvas_size[1] // canv_r
if canvas_keep_ratio:
    mn = min(canv_px_h, canv_px_w)
    canv_px_h, canv_px_w = mn, mn
    if not canvas_keep_rc:
        cw, ch = canvas_size
        cw, ch = cw - cw % mn, ch - ch % mn
        canvas_size = cw, ch
        canv_c, canv_r = cw // mn, ch // mn
        canv_c, canv_r = canv_c - canv_c % mn, canv_r - canv_r % mn
    canvas_size = mn * canv_c, mn * canv_r

main_p_r, main_p_c = 12, 8
p_square = 20
p_title_h = 30
basic_color_tooltip = UI.UITooltip(5, 2, p_square * main_p_c, p_square * main_p_r, main_p_r, main_p_c, title_h=p_title_h, text=('Main palette', UI.black), font=('arial', 15))

for i in range(0, len(UI.colors), main_p_c):
    for col, color in enumerate(UI.colors[i:(i + main_p_c)]):
        b_c = (*color, 255)
        basic_color_tooltip.set_item((i // main_p_c), col, UI.UIButton(0, 0, 20, 20, color=b_c))

custom_color_tooltip = UI.UITooltip(5, basic_color_tooltip.height + 22, 156, 26, 1, 6, title_h=20, text=('Custom palette', UI.black), font=('arial', 15))

current_color_indicator = UI.UITooltip(15, custom_color_tooltip.y + 40, 46, 36, 1, 1, title_h=20, text=('color', UI.black), font=('arial', 15))
current_color_indicator.set_item(0, 0, UI.Rectangle(7, current_color_indicator.y + 2, 42, 32, color=UI.black))

cw_icon = pygame.image.load('icons/pick_color.png')
pick_color_button = UI.UIButton(custom_color_tooltip.width - 10 - current_color_indicator.height, custom_color_tooltip.y + 40, current_color_indicator.height, current_color_indicator.height, image=cw_icon)
pick_color_button.state = False

def new_canvas(in_canv_args):
    return UI.Canvas(*in_canv_args)

size_slider = UI.UISlider(5, 450, 30, 170, min(canv_c // 8, 50), 1, slider_color=UI.red)

tool_square = 40
tool_r, tool_c = 4, 4
paint_tools_tooltip = UI.UITooltip(size_slider.x + size_slider.width + 10, 450, tool_square * tool_c, tool_square * tool_r, tool_r, tool_c, title_h=p_title_h, text=('Tools', UI.black), font=('arial', 15))

for i, item in enumerate(UI.icons):
    paint_tools_tooltip.set_item(i // paint_tools_tooltip.cols, i % paint_tools_tooltip.cols, UI.UIButton(0, 0, 24, 24, image=item))

curr_layer = 0
layer_square = 80
layers_tooltip = UI.UITooltip(screen_w - layer_square - 40, 2, int(layer_square * 1.2), layer_square, 2, 1, title_h=p_title_h, text=('Layers', UI.black), font=('arial', 15))

layers_tooltip.set_item(0, 0, UI.UIButton(0, 0, layer_square, layer_square, image=pygame.image.load('icons/default_image.png')))
layers_tooltip.set_item(1, 0, UI.UITooltip(0, 0, int(1.2 * layer_square) - 4, layer_square // 2 - 4, 1, 2, show_grid=False))
layer_buttons_tooltip = layers_tooltip.get_item(-1, 0)
layer_buttons_tooltip.set_item(0, 0, UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/visible.png')))
layer_buttons_tooltip.set_item(0, 1, UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/delete.png')))

add_layer_button = UI.UIButton(layers_tooltip.x, layers_tooltip.y + layers_tooltip.height, layer_square + 15, 20, text=('+    add layer', UI.black), font=('arial', 15), image=pygame.image.load('icons/button_1.png'))

old_mouse_x, old_mouse_y, mouse_x, mouse_y = 0, 0, 0, 0
draw_color = UI.black

selected_brush = pygame.image.load('brushes/circle16.png')
brush = selected_brush
selected_layer_button = layers_tooltip.get_item(0, 0)

pick_color_popup_w, pick_color_popup_h = 700, 350
pick_color_popup = UI.Popup((screen_w - pick_color_popup_w) // 2, (screen_h - pick_color_popup_h) // 2, pick_color_popup_w, pick_color_popup_h,
                            text=('  Pick a color', UI.black), font=('courier new', 20), title_h=30)
new_tt_y = pick_color_popup.y + 20
new_tt_square = 20
for i, key in enumerate(UI.colors2.keys()):
    color_lst = UI.colors2[key]
    new_tt = UI.UITooltip(pick_color_popup.x + 20, new_tt_y, new_tt_square * len(color_lst),
                          new_tt_square, 1, len(color_lst), title_h=20, font=('arial', 16),
                          text=(key, UI.black), bg_color=UI.white, border_w=2)
    pick_color_popup.add_item(new_tt, 20, new_tt_y - pick_color_popup.y + 20, name=f'palette{i}')
    for j, clr in enumerate(color_lst):
        new_tt.set_item(0, j, UI.UIButton(0, 0, 20, 20, color=clr))
    new_tt_y += 60

color_wheel_rect = UI.Rectangle(0, 0, 256, 256, image=pygame.image.load('icons/color_picker.png'), color=None)
pick_color_popup.add_item(color_wheel_rect, new_tt.width + 80, pick_color_popup_h - color_wheel_rect.height + 12, name='color_wheel')
old_color_ind = UI.UITooltip(color_wheel_rect.x + 20, color_wheel_rect.y - 84, 54, 64, 1, 1, title_h=20, text=('old', UI.black), font=('arial', 16))
pick_color_popup.add_item(old_color_ind, 20, -84, name='old_color_ind', reference_element=color_wheel_rect)
old_color_ind.set_item(0, 0, UI.Rectangle(0, 0, 62, 62, UI.white))
arrow_rect = UI.Rectangle(0, 0, 64, 64, image=pygame.image.load('icons/arrow_right.png'), color=None)
pick_color_popup.add_item(arrow_rect, old_color_ind.width + 20, 0, name='arrow', reference_element=old_color_ind)
new_color_ind = UI.UITooltip(arrow_rect.x + arrow_rect.width + 15, arrow_rect.y, 54, 64, 1, 1, title_h=20, text=('new', UI.black), font=('arial', 16))
pick_color_popup.add_item(new_color_ind, arrow_rect.width + 15, 0, name='new_color_ind', reference_element=arrow_rect)
new_color_ind.set_item(0, 0, UI.Rectangle(0, 0, 62, 62, UI.red))

sl_x = 40
for i, sl_name in enumerate(('R: ', 'G: ', 'B: ')):
    new_col_slider = UI.UISlider(new_color_ind.x + new_color_ind.width + sl_x + 20,
                                 new_color_ind.y, 40, color_wheel_rect.height + 40,
                                 255, 0, slider_color=UI.red, bg_color=UI.white, color=(80, 80, 80),
                                 text=(sl_name, UI.black), scale_h=color_wheel_rect.height - 15, title_h=20)
    pick_color_popup.add_item(new_col_slider, new_color_ind.width + sl_x + 20, 0,
                              f'slider{i}', reference_element=new_color_ind)
    sl_x += 65

ok_button = UI.UIButton(0, 0, 86, 24, color=UI.gray, text=('Ok', UI.black), font=('arial', 18), frame_col=UI.green, frame_w=2)
pick_color_popup.add_item(ok_button, pick_color_popup_w - ok_button.width - 20, pick_color_popup_h - 15, name='ok_button')
cancel_button = UI.UIButton(0, 0, 86, 24, color=UI.gray, text=('Cancel', UI.black), font=('arial', 18), frame_col=UI.red, frame_w=2)
pick_color_popup.add_item(cancel_button, -(cancel_button.width + 20), 0, name='cancel_button', reference_element=ok_button)

current_popup = None

ui_elements = [basic_color_tooltip, size_slider, paint_tools_tooltip, layers_tooltip, add_layer_button,
               custom_color_tooltip, current_color_indicator, pick_color_button]

canv_x = max(paint_tools_tooltip.x + paint_tools_tooltip.width, basic_color_tooltip.x + basic_color_tooltip.width)
canv_padding = (layers_tooltip.x - canv_x - canv_c * canv_px_w) // 2
canv_x += canv_padding
canv_args = (canv_x, 2, *canvas_size, canv_r, canv_c)
canvas = new_canvas(canv_args)
layers = [[canvas, True]]
selected_tool = None

def draw_layers(layers1, final_layer=-1):
    if final_layer < 0 or final_layer >= len(layers):
        final_layer = len(layers1) - 1
    if layers1:
        pygame.draw.rect(screen, UI.background, layers1[0][0].get_rectangle())
        for canv_obj in layers1[:final_layer + 1]:
            cv, do_draw = canv_obj
            if do_draw:
                cv.draw(screen)

def setup():
    global brush
    screen.fill(UI.white)
    canvas.draw(screen)
    pygame.display.flip()
    brush = selected_brush

def loop():
    global mouse_x
    global mouse_y
    global old_mouse_y
    global old_mouse_x
    global draw_color
    global selected_brush
    global brush
    global curr_layer
    global selected_layer_button
    global current_popup

    pygame.draw.rect(screen, UI.background, (layers_tooltip.x - 5, 0, layers_tooltip.width + 10, add_layer_button.x + 10))
    pygame.draw.rect(screen, UI.background, (paint_tools_tooltip.x - 5, paint_tools_tooltip.y - 5, paint_tools_tooltip.width + 10, paint_tools_tooltip.height + 10))

    draw_canvas = bool(layers)

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            pygame.quit()
        if event.type == pygame.KEYDOWN:
            pass
        if event.type == pygame.MOUSEBUTTONDOWN:
            if event.button == 4 and layers[curr_layer][1]:
                    layers[curr_layer][0].scale(-1, mouse_x, mouse_y)
                    layers[curr_layer][0].hide(screen, True)
                    draw_layers(layers, curr_layer)
                    pygame.display.update(layers[curr_layer][0].get_rectangle())
            if event.button == 5 and layers[curr_layer][1]:
                    layers[curr_layer][0].scale(1, mouse_x, mouse_y)
                    layers[curr_layer][0].hide(screen, True)
                    draw_layers(layers, curr_layer)
                    pygame.display.update(layers[curr_layer][0].get_rectangle())
            if current_popup is None:
                if basic_color_tooltip.mouse_hover():
                    for row in basic_color_tooltip.table:
                        for button in row:
                            if button.mouse_down() and isinstance(button, UI.UIButton):
                                button.tint(False)
                                draw_color = button.color
                                basic_color_tooltip.clicked = button
                                button.tint(True, amt=20)
                                break

                if paint_tools_tooltip.mouse_hover():
                    for i, row in enumerate(paint_tools_tooltip.table):
                        for j, button in enumerate(row):
                            if button.mouse_down() and isinstance(button, UI.UIButton):
                                paint_tools_tooltip.selected = button
                                ps = i * paint_tools_tooltip.cols + j
                                if ps == 0:
                                    brush = selected_brush
                                if ps == 1:
                                    brush = None
                                if ps == 2:
                                    if draw_canvas:
                                        draw_color = layers[curr_layer][0].bg_col
                                    brush = None
                                if ps == 3:
                                    brush = canvas_flood_fill
                                if ps == 9:
                                    brush = UI.LASSO
                                if ps == 11:
                                    brush = UI.MOVE_TOOL
                                paint_tools_tooltip.clicked = button
                                button.tint(True, amt=20)
                                break

                if layers_tooltip.mouse_hover():
                    for obj_i, obj in enumerate([x[0] for x in layers_tooltip.table]):
                        if obj.mouse_down():
                            layers_tooltip.clicked = obj
                            if isinstance(obj, UI.UIButton):
                                selected_layer_button = obj
                                curr_layer = obj_i // 2
                                draw_layers(layers, curr_layer)
                                obj.color = UI.gray
                                pygame.display.update(layers[curr_layer][0].get_rectangle())
                                break
                            elif isinstance(obj, UI.UITooltip):
                                visibility_toggle_button = obj.table[0][0]
                                delete_button = obj.table[0][1]
                                if visibility_toggle_button.mouse_down():
                                    layers[obj_i // 2][1] = not layers[obj_i // 2][1]
                                    draw_layers(layers, curr_layer)
                                    pygame.display.update(layers[obj_i // 2][0].get_rectangle())
                                    if layers[obj_i // 2][1]:
                                        visibility_toggle_button.image = pygame.image.load('icons/visible.png')
                                    else:
                                        visibility_toggle_button.image = pygame.image.load('icons/invisible.png')
                                    visibility_toggle_button.color = UI.gray
                                    obj.clicked = visibility_toggle_button
                                elif delete_button.mouse_down():
                                    delete_button.color = UI.gray
                                    if layers:
                                        if len(layers_tooltip.table) > 2:
                                            layers_tooltip.pop_item(obj_i - 1, 0)
                                            layers_tooltip.pop_item(obj_i - 1, 0)

                                            layers.pop((obj_i - 1) // 2)
                                            curr_layer = min(curr_layer, len(layers) - 1)

                                            selected_layer_button = layers_tooltip.get_item(curr_layer * 2, 0)

                                            old_add_layer_button_pos = add_layer_button.get_rectangle()
                                            add_layer_button.x, add_layer_button.y = layers_tooltip.x, layers_tooltip.y + layers_tooltip.height
                                            pygame.display.update(old_add_layer_button_pos)

                                            draw_layers(layers, curr_layer)

                                            pygame.display.update(canvas.get_rectangle())
                                    obj.clicked = delete_button
                                    break
                            else:
                                print(obj)

                if add_layer_button.mouse_hover():
                    layers_tooltip.append_item(
                        UI.UIButton(0, 0, layer_square, layer_square,
                                    image=pygame.image.load('icons/default_image.png')))
                    layers_tooltip.append_item(
                        UI.UITooltip(0, 0, int(1.2 * layer_square) - 4, layer_square // 2 - 4, 1, 2, show_grid=False))
                    new_tooltip_element = layers_tooltip.get_item(-1, 0)
                    new_tooltip_element.set_item(0, 0,
                                                 UI.UIButton(0, 0, 16, 16,
                                                             image=pygame.image.load('icons/visible.png')))
                    new_tooltip_element.set_item(0, 1,
                                                 UI.UIButton(0, 0, 16, 16, image=pygame.image.load('icons/delete.png')))
                    layers.append([new_canvas(canv_args), True])
                    curr_layer += 1
                    layers_tooltip.update_item_sizes()
                    old_add_layer_button_pos = add_layer_button.get_rectangle()
                    add_layer_button.x, add_layer_button.y = layers_tooltip.x, layers_tooltip.y + layers_tooltip.height
                    pygame.display.update(old_add_layer_button_pos)
                if pick_color_button.mouse_hover():
                    pick_color_button.onclick()
                    if pick_color_button.state:
                        current_popup = pick_color_popup
                        pick_color_popup.items['old_color_ind'].table[0][0].color = draw_color
                        pick_color_popup.draw(screen)
                    pygame.display.update(pick_color_popup.get_rectangle())
                if custom_color_tooltip.mouse_hover():
                    for item in custom_color_tooltip.table[0]:
                        if item.mouse_down():
                            draw_color = item.color
                            item.tint(True, amt=20)
                            custom_color_tooltip.clicked = item
                            custom_color_tooltip.selected = item

            elif current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    item = pick_color_popup.items[key]
                    if item.mouse_hover():
                        if 'palette' in key:
                            for color_button in item.table[0]:
                                if color_button.mouse_hover():
                                    color_button.tint(False)
                                    pick_color_popup.items['new_color_ind'].table[0][0].color = color_button.color
                                    r, g, b = color_button.color
                                    pick_color_popup.items['slider0'].set_value(r)
                                    pick_color_popup.items['slider1'].set_value(g)
                                    pick_color_popup.items['slider2'].set_value(b)
                                    color_button.tint(True, amt=20)
                                    item.clicked = color_button
                        elif 'color_wheel' in key:
                            c_w = pick_color_popup.items[key]
                            new_clr = list(c_w.image.get_at((mouse_x - c_w.x, mouse_y - c_w.y)))[:3]
                            pick_color_popup.items['new_color_ind'].table[0][0].color = new_clr
                        elif 'ok_button' in key or 'cancel_button' in key:
                            if 'ok_button' in key:
                                draw_color = pick_color_popup.items['new_color_ind'].table[0][0].color
                                if custom_color_tooltip.selected:
                                    custom_color_tooltip.selected.color = draw_color
                                    custom_color_tooltip.selected.old_color = draw_color
                            screen.fill(UI.white)
                            draw_layers(layers, curr_layer)
                            pygame.display.flip()
                            current_popup = None

        if event.type == pygame.MOUSEBUTTONUP:
            if basic_color_tooltip.clicked:
                basic_color_tooltip.clicked.tint(False)
                basic_color_tooltip.clicked = None
            if paint_tools_tooltip.clicked:
                paint_tools_tooltip.clicked.tint(False)
                paint_tools_tooltip.clicked = None
            if custom_color_tooltip.clicked:
                custom_color_tooltip.clicked.tint(False)
                custom_color_tooltip.clicked = None
            if layers_tooltip.clicked:
                if isinstance(layers_tooltip.clicked, UI.UITooltip):
                    if layers_tooltip.clicked.clicked:
                        layers_tooltip.clicked.clicked.tint(False)
                        layers_tooltip.clicked.clicked = None
                else:
                    layers_tooltip.clicked.tint(False)
            if current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    if 'palette' in key:
                        if pick_color_popup.items[key].clicked:
                            pick_color_popup.items[key].clicked.tint(False)
                            pick_color_popup.items[key].clicked = None

        if event.type == pygame.MOUSEMOTION:
            old_mouse_x, old_mouse_y = mouse_x, mouse_y
            mouse_x, mouse_y = pygame.mouse.get_pos()
            if basic_color_tooltip.mouse_hover():
                for row in basic_color_tooltip.table:
                    for button in row:
                        button.tint(button.mouse_hover())
                        if button.mouse_hover():
                            basic_color_tooltip.tinted = button
                            break
            elif basic_color_tooltip.tinted:
                basic_color_tooltip.tinted.tint(False)
                basic_color_tooltip.tinted = None
            if size_slider.mouse_down():
                size_slider.move_slider()
            if current_popup == pick_color_popup:
                for key in pick_color_popup.items.keys():
                    if 'palette' in key:
                        for obj in pick_color_popup.items[key].table[0]:
                            obj.tint(obj.mouse_hover())
                    elif 'slider' in key:
                        sl = pick_color_popup.items[key]
                        if sl.mouse_down():
                            sl.move_slider()
                            new_clr = [pick_color_popup.items[key].value for key in pick_color_popup.items.keys() if 'slider' in key]
                            pick_color_popup.items['new_color_ind'].table[0][0].color = new_clr
            if pygame.mouse.get_pressed()[1] and layers[curr_layer][1]:
                layers[curr_layer][0].pan(mouse_x - old_mouse_x, mouse_y - old_mouse_y)
                layers[curr_layer][0].hide(screen, True)
                draw_layers(layers, curr_layer)
                pygame.display.update(layers[curr_layer][0].get_rectangle())

    if not layers:
        draw_canvas = False

    if draw_canvas and current_popup is None:
        now_canvas, do_draw = layers[curr_layer]
        overlay_layers = [x[0] for x in layers if x[0] != now_canvas and x[1]]
        if do_draw:
            brush_size = size_slider.value
            if now_canvas.mouse_down():
                if old_mouse_x != mouse_x or old_mouse_y != mouse_y:
                    now_canvas.line_paint((old_mouse_x, old_mouse_y), (mouse_x, mouse_y), color=draw_color,
                                      size=(brush_size, brush_size), brush=brush, surface=screen, overlays=overlay_layers)
                    if brush in [UI.LASSO, UI.MOVE_TOOL]:
                        now_canvas.show_selected_set(screen)
                    if brush in [UI.MOVE_TOOL]:
                        now_canvas.hide(screen, True)
                        now_canvas.draw(screen)
            else:
                if brush in [UI.LASSO]:
                    if now_canvas.selected_pixel_set:
                        layers[curr_layer][0].lasso_selection()
                        now_canvas.show_selected_set(screen)
                    now_canvas.lasso_clear_flag = True
            pygame.display.update(now_canvas.get_rectangle())
    current_color_indicator.get_item(0, 0).color = draw_color

    for el in ui_elements:
        el.draw(screen)

    selected_layer_button.draw_frame(screen, UI.red, width=5)

    for el in ui_elements:
        pygame.display.update(el.get_rectangle())

    if current_popup is not None:
        current_popup.draw(screen)
        pygame.display.update(current_popup.get_rectangle())

    pygame.display.update((layers_tooltip.x - 5, layers_tooltip.y + layers_tooltip.height,
                           layers_tooltip.width + 10, layers_tooltip.row_size + 20))
    clock.tick(128)
    #print(clock)

setup()
while flag:
    loop()


from __future__ import annotations
from typing import Dict, List, Any, Tuple, TYPE_CHECKING, Callable

from src.fleetctrl.pooling.immediate.insertion import simple_insert
from src.fleetctrl.planning.VehiclePlan import VehiclePlan, BoardingPlanStop, PlanStop
import src.fleetctrl.pooling.batch.AlonsoMora.AlonsoMoraAssignment as AlonsoMoraAssignment

if TYPE_CHECKING:
    from src.fleetctrl.pooling.batch.BatchAssignmentAlgorithmBase import SimulationVehicleStruct
    from src.routing.NetworkBase import NetworkBase
    from src.fleetctrl.planning.PlanRequest import PlanRequest
    from src.simulation.Legs import VehicleRouteLeg

import logging
LOG = logging.getLogger(__name__)

def remove_non_routing_planstops_and_copy(veh_plan : VehiclePlan, veh_obj : SimulationVehicleStruct,
                                          routing_engine : NetworkBase, sim_time : int) -> VehiclePlan:
    """this funtion removes all planstops that are empty (i.e. not locked, not end locked and not boarding/charging processes)"""
    return veh_plan.copy_and_remove_empty_planstops(veh_obj, sim_time, routing_engine)

class V2RB():
    """ this class is a collection of feasible vehicle plan for a specific vehicle serving the same requests"""
    def __init__(self, routing_engine : NetworkBase, rq_dict : Dict[Any, PlanRequest], sim_time : int, rtv_key : tuple,
                 veh_obj : SimulationVehicleStruct, std_bt : int, add_bt : int, obj_function : Callable,
                 orig_veh_plans : List[VehiclePlan] = None, new_prq_obj : PlanRequest = None):
        """ this method initializes a v2rb. depending on the input parameters different initialization are possible
        a) new_prq_obj is given, but no orig_veh_plans: -> the most basic v2rb with only one request is created
        b) new_prq_obj and orig_veh_plans is given: -> the new request will be inserted in the given vehicle plans
        c) new_prq_obj is not given, but orig_veh_plans is given: -> a v2rb based on thes plans is created, but it will be forced to be feasible (no deletion of infeasile plans)
        :param routing_engine: routing_engine ref
        :param rq_dict: dict rid -> prq of at least all requests part of the v2rb
        :param sim_time: simulation time
        :param rtv_key: key of the v2rb (tuple; see key creation function)
        :param veh_obj: corresponding simulationvehiclestruct ref
        :param std_bt: const boarding time
        :param add_bt: additional boarding time
        :param obj_function: objective funtion to rate vehicle plans
        :param orig_veh_plans: optional; list of vehicle plans as input
        :param new_prq_obj: optional; new plan request obj"""
        self.veh_plans : List[VehiclePlan] = []
        self.cost_function_value = None
        self.rtv_key = rtv_key
        self.veh = veh_obj
        if new_prq_obj is not None and orig_veh_plans is None:  # init v2rb with one request
            if veh_obj.has_locked_vehplan():
                # init = [ps.copy() for ps in veh_obj.locked_planstops]
                # empty_plan = VehiclePlan(veh_obj, sim_time, routing_engine, init)
                empty_plan = veh_obj.locked_vehplan.copy()
            else:
                empty_plan = VehiclePlan(veh_obj, sim_time, routing_engine, [])
            self.veh_plans = [new_plan for new_plan in simple_insert(routing_engine, sim_time, veh_obj, empty_plan, new_prq_obj, std_bt, add_bt)]
        elif new_prq_obj is not None and orig_veh_plans is not None: # insert into lower plans with new requests
            for veh_plan in orig_veh_plans:
                copy_plan = remove_non_routing_planstops_and_copy(veh_plan, veh_obj, routing_engine, sim_time)# veh_plan.copy()
                self.veh_plans += [new_plan for new_plan in simple_insert(routing_engine, sim_time, veh_obj, copy_plan, new_prq_obj, std_bt, add_bt)]
        if self.isFeasible(): # check feasibility
            self.computeCostFunctionValue(sim_time, obj_function, veh_obj, routing_engine, rq_dict)
        elif orig_veh_plans is not None and new_prq_obj is None: # force v2rb to exist even no longer feasible
            self.veh_plans = [remove_non_routing_planstops_and_copy(o_plan, veh_obj, routing_engine, sim_time) for o_plan in orig_veh_plans]# [o_plan.copy() for o_plan in orig_veh_plans]
            self.computeCostFunctionValue(sim_time, obj_function, veh_obj, routing_engine, rq_dict)
        # else:
        #     LOG.debug(f"V2RB infeasible: {self.rtv_key}")
        # LOG.debug(f"new V2RB {self.rtv_key}")
        # LOG.debug("\n".join([str(x) for x in self.veh_plans]))

    def __str__(self):
        l = ["V2RB {} wit cfv {}".format(self.rtv_key, self.cost_function_value)]
        for vp in self.veh_plans:
            l.append(str(vp))
        return "\n".join(l)

    def isFeasible(self) -> bool:
        """ checks feasibility of v2rb
        :return: bool"""
        if len(self.veh_plans) > 0:
            return True
        else:
            return False

    def addRequestAndCheckFeasibility(self, new_prq_obj : PlanRequest, new_rtv_key : tuple, routing_engine : NetworkBase,
                                      obj_function : Callable, rq_dict : Dict[Any, PlanRequest], sim_time : int, veh_obj : SimulationVehicleStruct,
                                      std_bt : int, add_bt : int) -> V2RB:
        """ insert a new request into the v2rb and create to next one"""
        newV2RB = V2RB(routing_engine, rq_dict, sim_time, new_rtv_key, veh_obj, std_bt, add_bt, obj_function, orig_veh_plans=self.veh_plans, new_prq_obj=new_prq_obj)
        if newV2RB.isFeasible():
            return newV2RB
        else:
            return None

    def updateAndCheckFeasibility(self, routing_engine : NetworkBase, obj_function : Callable, veh_obj : SimulationVehicleStruct,
                                  rq_dict : Dict[Any, PlanRequest], sim_time : int, list_passed_VRLs : List[VehicleRouteLeg]=None,
                                  is_assigned : bool=False):
        """ checks the feasibility of the v2rb after vehicle state updates from last simulation time step"""
        if list_passed_VRLs is None:
            list_passed_VRLs = []
        new_veh_plans = []
        self.veh = veh_obj
        if not is_assigned:
            for i, veh_plan in enumerate(self.veh_plans):
                veh_plan.update_plan(veh_obj, sim_time, routing_engine, list_passed_VRLs=list_passed_VRLs,
                                     keep_time_infeasible=False)
                if veh_plan.is_feasible():
                    new_veh_plans.append(veh_plan)
        else:
            struct_feasible_veh_plans = []
            for i, veh_plan in enumerate(self.veh_plans):
                if is_assigned and i == 0:
                    veh_plan.update_plan(veh_obj, sim_time, routing_engine, list_passed_VRLs=list_passed_VRLs,
                                         keep_time_infeasible=True)
                    if veh_plan.is_feasible():
                        new_veh_plans.append(veh_plan)
                    elif veh_plan.is_structural_feasible():
                        struct_feasible_veh_plans.append(veh_plan)
                    else:
                        LOG.warning("(assigned) vehicle plan became structural infeasible! {}".format(veh_plan))
                else:
                    veh_plan.update_plan(veh_obj, sim_time, routing_engine, list_passed_VRLs=list_passed_VRLs,
                                         keep_time_infeasible=False)
                    if veh_plan.is_feasible():
                        new_veh_plans.append(veh_plan)
            if len(new_veh_plans) == 0:
                new_veh_plans = struct_feasible_veh_plans
        self.veh_plans = new_veh_plans
        self.computeCostFunctionValue(sim_time, obj_function, veh_obj, routing_engine, rq_dict)

    def computeCostFunctionValue(self, sim_time : int, obj_function : Callable, veh_obj : SimulationVehicleStruct,
                                 routing_engine : NetworkBase, rq_dict : Dict[Any, PlanRequest]):
        """ computes the costfunction values of all plans and sorts the list self.veh_plans accordingly"""
        #simulation_time, veh_obj, veh_plan, rq_dict, routing_engine
        if len(self.veh_plans) > 0:
            #LOG.debug("compute CFV {}".format(self.rtv_key))
            for veh_plan in self.veh_plans:
                #LOG.debug(f"utility input : {veh_obj} | {veh_plan} | {[(rid, rq) for rid, rq in rq_dict.items()]}")
                veh_plan.set_utility(obj_function(sim_time, veh_obj, veh_plan, rq_dict, routing_engine))
                #LOG.debug(f" -> utility {veh_plan.utility}")
            self.veh_plans = sorted(self.veh_plans, key = lambda x:x.utility)
            self.cost_function_value = self.veh_plans[0].utility

    def getBestPlan(self) -> VehiclePlan:
        """ returns the best vehicleplan of the v2rb"""
        return self.veh_plans[0]

    def createLowerV2RB(self, lower_key : tuple, sim_time : int, routing_engine : NetworkBase, obj_function : Callable,
                        rq_dict : Dict[Any, PlanRequest], std_bt : int, add_bt : int) -> V2RB:
        keep_rids = AlonsoMoraAssignment.getRidsFromRTVKey(lower_key)
        #only keep best route
        base_plan = self.veh_plans[0]
        new_plan_list = []
        for ps in base_plan.list_plan_stops:
            if ps.is_locked_end():
                new_plan_list.append(ps.copy())
                continue
            new_boarding_dict = {}
            new_max_trip_time_dict = {}
            new_earliest_pickup_time_dict = {}
            new_latest_pickup_time_dict = {}
            change_nr_pax = 0
            ept, lpt, mtt, lat = ps.get_boarding_time_constraint_dicts()
            for rid in ps.get_list_boarding_rids():
                if rid in keep_rids:
                    try:
                        new_boarding_dict[1].append(rid)
                    except:
                        new_boarding_dict[1] = [rid]
                    new_earliest_pickup_time_dict[rid] = ept[rid]
                    new_latest_pickup_time_dict[rid] = lpt[rid]
            change_nr_pax += sum([rq_dict[rid].nr_pax for rid in new_boarding_dict.get(1, [])])
            for rid in ps.get_list_alighting_rids():
                if rid in keep_rids:
                    try:
                        new_boarding_dict[-1].append(rid)
                    except:
                        new_boarding_dict[-1] = [rid]
                    new_max_trip_time_dict[rid] = mtt[rid]
            change_nr_pax -= sum([rq_dict[rid].nr_pax for rid in new_boarding_dict.get(-1, [])])
            if len(new_boarding_dict.keys()) > 0 or ps.is_locked():
                #LOG.warning("not considering new time constraints!!!!") # TODO #
                # new_ps = BoardingPlanStop(ps.get_pos(), boarding_dict=new_boarding_dict, max_trip_time_dict=new_max_trip_time_dict,
                #                           earliest_pickup_time_dict=new_earliest_pickup_time_dict, latest_pickup_time_dict=new_latest_pickup_time_dict,
                #                           change_nr_pax=change_nr_pax, duration=ps.get_duration_and_earliest_departure()[0], locked=ps.is_locked())
                new_ps = PlanStop(ps.get_pos(), boarding_dict=new_boarding_dict, max_trip_time_dict=new_max_trip_time_dict,
                                          earliest_pickup_time_dict=new_earliest_pickup_time_dict, latest_pickup_time_dict=new_latest_pickup_time_dict,
                                          change_nr_pax=change_nr_pax, duration=ps.get_duration_and_earliest_departure()[0], locked=ps.is_locked(),
                                          charging_power=ps.get_charging_power(), planstop_state=ps.get_state(), charging_task_id=ps.get_charging_task_id(),
                                          change_nr_parcels=ps.get_change_nr_parcels())
                # new_ps = ps.copy()
                # new_ps.boarding_dict = new_boarding_dict
                new_plan_list.append(new_ps)
        new_veh_plan = VehiclePlan(self.veh, sim_time, routing_engine, new_plan_list, external_pax_info=base_plan.pax_info.copy())
        return V2RB(routing_engine, rq_dict, sim_time, lower_key, self.veh, std_bt, add_bt, obj_function, orig_veh_plans=[new_veh_plan])


import boto3
import pandas as pd
import openai
import json
import numpy as np
from io import StringIO
from botocore.exceptions import ClientError
import os

from dotenv import load_dotenv

# Load environment variables from .env file
load_dotenv('.env')

# Set the OpenAI API key from the environment variable
openai.api_key = os.environ['API_KEY']

s3 = boto3.client('s3')
def get_embedding(text, model):
    embedding = openai.Embedding.create(
    input=text, model=model
    )["data"][0]["embedding"]
    return embedding

def cosine_similarity(a, b):
    return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))

def lambda_handler(event, context):
    print("New Event Recieved")
    print(event)

    Body=json.loads(event['body'])
    question = Body['question']
    folder_name = Body['folder_name']

    print(f"Folder Name - {folder_name}")
    print(f"Question - {question}")

    # Get the CSV file from S3
    try:
        print("Loading S3 Bucket")
        obj = s3.get_object(Bucket='csvfiles100', Key=f'{folder_name}/embedding.csv')
        print("S3 Bucket Loaded Successfully")
        print("Decoding Body")
        body = obj['Body'].read().decode('utf-8')
        csv_file = StringIO(body)
        print("Body Decoded Successfully")
    except ClientError as e:
        print(e)
        raise Exception("Error getting S3 object: {}".format(e))

    # Preprocess the data
    emb = pd.read_csv(csv_file)
    print("Dataframe Initialised")
    emb['embedding'] = emb['embedding'].apply(lambda x: '[]' if x is None or not isinstance(x, str) else x)
    emb['embedding'] = emb['embedding'].apply(eval)
    emb['embedding'] = emb['embedding'].apply(lambda x: [float(i) for i in x])
    question_vector = get_embedding(question, model='text-embedding-ada-002')
    emb["similarities"] = emb['embedding'].apply(lambda x: cosine_similarity(x, question_vector) if len(x) > 0 else 0)
    emb = emb.sort_values("similarities", ascending=False).head(4)
    # Generate prompt for OpenAI
    nontext = []
    for i, row in emb.iterrows():
        if isinstance(row['Combined'], str):
            nontext.append(row['Combined'])
    text = "\n".join(nontext)
    nontext = text
    prompt = f"""Answer the following question using only the context below.

    Context:
    {nontext}

    Q: {question}
    A:"""

    # Call OpenAI API to get the answer
    try:
        print("Calling OpenAI API")
        ans = openai.Completion.create(
            prompt=prompt,
            temperature=1,
            max_tokens=200,
            top_p=1,
            frequency_penalty=0,
            presence_penalty=0,
            model="text-davinci-003",
        )["choices"][0]["text"].strip(" \n")
        print(f"Ans -: {ans}")
        res = {"msg": ans, "status": "success"}
        print(res)
        return {
            'statusCode': 200,
            'body': json.dumps(res)
            }
    except Exception as e:
        return {
            'statusCode': 400,
            'body': json.dumps({'message': str(e)})
        }


# -*- coding: utf-8 -*-

import re
import json
import pycurl
from copy import copy
from os import path, fsync, getenv
from time import strftime
from io import BytesIO
from urllib.parse import urlencode

CON_TIMEOUT = 300
REQ_TIMEOUT = 3600
headers = {}
host_sistec = 'https://sistec.mec.gov.br'
encoding = 'iso-8859-1'
campus_encoding = 'utf-8'
logDir = u'%s' % getenv('SISTEC_DOWNLOAD_LOG', default='../log')
pydbg_path = u'%s/pycurl_debug_%s.log' % (logDir, strftime('%Y-%m-%d'))

def get_data_sistec(cod_campus, campus, perfil, tipos, downDir, qtdPerfis, cookies, file_log_path=None, level_debug=[False, False, False]):
    global host_sistec, encoding, campus_encoding
    sucesso = False
    campus = campus.decode(campus_encoding)
    campus_csv = path.join(downDir, campus + '.csv')
    file_csv = open(campus_csv, mode='w', encoding=encoding)
    sucesso, mensagem = write_csv(file_csv, campus, cod_campus, perfil, qtdPerfis, tipos, cookies, file_log_path, level_debug)
    file_csv.close()
    if sucesso:
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s O download da planilha do SISTEC (%s) foi efetuado com sucesso em:\t%s\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), campus.title(), campus_csv))
                file_log.flush()
                fsync(file_log.fileno())
        sucesso = True
    else:
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Não foi possível salvar a planilha do SISTEC (%s) em:\t%s\t%s\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), campus.title(), campus_csv, mensagem))
                file_log.flush()
                fsync(file_log.fileno())
        sucesso = False
    return sucesso

def write_csv(file_csv, no_campus, co_campus, perfil, qtdPerfis, tipos, cookies, file_log_path = None, level_debug = [False, False, False]):
    u"""
    Percorre o sistec, pegando os dados dos alunos nos ciclos de matricula,
    do 'campus' e os salva no arquivo 'file_csv'.

    :file_csv:           arquivo csv onde salvar os dados do SISTEC
    :no_campus:          nome do campus no site do SISTEC
    :co_campus:          codigo do campus no site do SISTEC
    :perfil:             perfil do usuário no site do SISTEC
    :tipos:              tipos de cursos [Presencial, EaD, FIC]
    :cookies:            cookies da sessão aberta para acesso ao site do SISTEC
                         [phpsessid, jsessionid, zde, perfil, noticias]
    :http_header_common: cabecalho comum das requisicoes http ao site do SISTEC
    :returns:            Sucesso, Mensagem
    """
    global headers, host_sistec, encoding, CON_TIMEOUT, REQ_TIMEOUT

    url_alterar_perfil = 'https://sistec.mec.gov.br/index/selecionarinstituicao/alterar/perfil'
    url_index          = 'https://sistec.mec.gov.br/index/index'
    url_turmas         = 'https://sistec.mec.gov.br/gridciclo/turmas'
    url_ciclo_common   = 'https://sistec.mec.gov.br/gridciclo/listaralunosacao/periodo//ciclo/'
    url_tempo_sessao   = 'https://sistec.mec.gov.br/tempo-sessao/get-tempo-sessao/'
    url_dados_ciclo    = 'https://sistec.mec.gov.br/admciclomatricula/dadosciclo/'

    header_cache              = 'Cache-Control: max-age=0'
    #header_dnt               = 'DNT: 1'
    header_connection         = 'Connection: keep-alive'
    header_upgrade            = 'Upgrade-Insecure-Requests: 1'
    header_content_type       = 'Content-type: application/x-www-form-urlencoded charset=' + encoding
    header_content_len        = 'Content-Length: 0'
    header_user_agent         = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36'
    header_accept_text        = 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
    header_accept_lang        = 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7'
    header_accept_json        = 'Accept: application/json'
    header_accept_enc         = 'Accept-Encoding: gzip, deflate, br'
    header_xreq_xml           = 'X-Requested-With: XMLHttpRequest'
    header_xreq_json          = 'X-Request: JSON'
    header_sec_fetch_site     = 'Sec-Fetch-Site: '
    header_sec_fetch_mode     = 'Sec-Fetch-Mode: '
    header_sec_fetch_user     = 'Sec-Fetch-User: '
    header_sec_fetch_dest     = 'Sec-Fetch-Dest: '
    header_sec_ch_ua          = 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"'
    header_sec_ch_ua_mobile   = 'sec-ch-ua-mobile: ?0'
    header_sec_ch_ua_platform = 'sec-ch-ua-platform: "Linux"'

    cookie_phpsessid = cookies[0]
    cookie_noticias = cookies[1]
    cookie_perfil = u'perfil_cookie=' + perfil
    cookie_usuario = u'co_usuario=' + co_campus

    tipo_presencial = tipos[0]
    tipo_ead = tipos[1]
    tipo_fic = tipos[2]

    info_json   = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir),   '_info.json')
    turmas_json = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir), '_turmas.json')
    alunos_json = path.join(getenv('SISTEC_DOWNLOAD_APP', default=path.curdir), '_alunos.json')

    # COLUNAS DO ARQUIVO CSV
    # 00: CO_ALUNO_IDENTIFICADO       # 01: CO_ALUNO                     # 02: NO_ALUNO                     # 03: NO_MAE_ALUNO             # 04: NO_NOME_SOCIAL
    # 05: NU_CPF                      # 06: DS_EMAIL                     # 07: CO_PESSOA_FISICA_ALUNO       # 08: DS_SENHA                 # 09: CO_MATRICULA
    # 10: CO_CICLO_MATRICULA          # 11: CO_STATUS_CICLO_MATRICULA    # 12: CO_CURSO                     # 13: NU_CARGA_HORARIA         # 14: DT_DATA_INICIO
    # 15: DT_DATA_FIM_PREVISTO        # 16: CO_UNIDADE_ENSINO            # 17: CO_PERIODO_CADASTRO          # 18: NO_CICLO_MATRICULA       # 19: ST_ATIVO
    # 20: CO_TIPO_OFERTA_CURSO        # 21: CO_TIPO_INSTITUICAO          # 22: CO_INSTITUICAO               # 23: CO_PORTFOLIO             # 24: CO_TIPO_NIVEL_OFERTA_CURSO
    # 25: CO_TIPO_PROGRAMA_CURSO      # 26: ST_CARGA                     # 27: DT_DATA_FINALIZADO           # 28: NU_VAGAS_OFERTADAS       # 29: NU_TOTAL_INSCRITOS
    # 30: ST_ETEC                     # 31: CO_POLO                      # 32: UAB                          # 33: ST_PREVISTO              # 34: NU_VAGAS_PREVISTAS
    # 35: CO_CURSO_SUPERIOR_CORRELATO # 36: NU_CARGA_HORARIA_ESTAGIO     # 37: NO_ARQUIVO                   # 38: NO_CAMINHO_ARQUIVO       # 39: ST_ESTAGIO
    # 40: ST_EXPERIMENTAL             # 41: NO_STATUS_MATRICULA          # 42: CO_PESSOA_FISICA             # 43: NU_RG                    # 44: SG_SEXO
    # 45: DT_NASCIMENTO               # 46: NO_PESSOA_FISICA             # 47: CO_PESSOA                    # 48: CO_CARGO                 # 49: DS_ORGAO_EXPEDIDOR
    # 50: SG_UF_ORG_EXPED             # 51: DS_CARGO                     # 52: CO_UNIDADE_ENSINO_IMPORTACAO # 53: CO_MATRICULA_RESPONSAVEL # 54: NO_SOCIAL
    # USAREMOS AS SEGUINTES COLUNAS:
    # 02; 05; 12; 14; 15; 18; 20; 31; 41

    file_csv.write('NO_ALUNO;'
                   'NU_CPF;'
                   'CO_CURSO;'
                   'DT_DATA_INICIO;'
                   'DT_DATA_FIM_PREVISTO;'
                   'NO_CICLO_MATRICULA;'
                   'CO_TIPO_OFERTA_CURSO;'
                   'CO_POLO;'
                   'NO_STATUS_MATRICULA'
                   '\n')
    file_csv.flush()
    fsync(file_csv.fileno())

    # =================
    # DOWNLOAD: ETAPA 1
    # =================
    """
    curl 'https://sistec.mec.gov.br/index/index' \
        -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
        -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
        -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
        -H 'Cache-Control: max-age=0' \
        -H 'Connection: keep-alive' \
        -H 'Content-Type: application/x-www-form-urlencoded' \
        -H 'Upgrade-Insecure-Requests: 1' \
        -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
        -H 'sec-ch-ua-mobile: ?0' \
        -H 'sec-ch-ua-platform: "Linux"' \
        -H 'Sec-Fetch-Dest: document' \
        -H 'Sec-Fetch-Mode: navigate' \
        -H 'Sec-Fetch-Site: same-origin' \
        -H 'Sec-Fetch-User: ?1' \
        -H 'Origin: https://sistec.mec.gov.br' \
        -H 'Referer: https://sistec.mec.gov.br/index/selecionarinstituicao/' \
        -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660637' \
        --data-raw 'tipo=1660670&acao=&qtdPerfis=14' \
        --compressed
    """

    if level_debug[0]:
        with open(file_log_path, 'a') as file_log:
            file_log.write(u'%s Acessando o peril do %s ...\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title()))
            file_log.flush()
            fsync(file_log.fileno())
    headers = {}
    http_header = [
        header_user_agent,
        header_accept_text,
        header_accept_lang,
        header_accept_enc,
        header_cache,
        header_connection,
        header_content_type,
        header_upgrade,
        header_sec_ch_ua,
        header_sec_ch_ua_mobile,
        header_sec_ch_ua_platform,
        header_sec_fetch_dest + 'document',
        header_sec_fetch_mode + 'navigate',
        header_sec_fetch_site + 'same-origin',
        header_sec_fetch_user + '?1',
        'Origin: ' + host_sistec,
        'Referer: ' + url_alterar_perfil,
        'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
    ]
    http_header[-1] = http_header[-1].encode(encoding)
    http_header[-2] = http_header[-2].encode(encoding)
    http_header[-3] = http_header[-3].encode(encoding)
    post_data = {
        'tipo': co_campus,
        'acao': '',
        'qtdPerfis': str(qtdPerfis)
    }
    postfields = urlencode(post_data)
    buff = BytesIO()
    c = pycurl.Curl()
    c.setopt(c.URL, url_index)
    c.setopt(c.SSL_VERIFYPEER, 1)
    c.setopt(c.SSL_VERIFYHOST, 2)
    c.setopt(c.TCP_NODELAY, 1)
    c.setopt(c.TCP_FASTOPEN, 1)
    c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
    c.setopt(c.TIMEOUT, REQ_TIMEOUT)
    # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
    c.setopt(c.WRITEFUNCTION, buff.write)
    c.setopt(c.HTTPHEADER, http_header)
    c.setopt(c.HEADERFUNCTION, _pycurl_header)
    c.setopt(c.CUSTOMREQUEST, 'POST')
    c.setopt(c.POSTFIELDS, postfields)
    if level_debug[2]:
        c.setopt(c.VERBOSE, 1)
        c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
    c.perform()
    c.close()
    if level_debug[1]:
        with open(file_log_path, 'a') as file_log:
            file_log.write(u'%s post_data:\t%s\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), post_data))
            file_log.flush()
            fsync(file_log.fileno())

    lines = []
    pagina_ciclos = 1
    total_paginas_ciclos = '#'

    while True:
        # =================
        # DOWNLOAD: ETAPA 2
        # =================
        """
        curl 'https://sistec.mec.gov.br/tempo-sessao/get-tempo-sessao/' \
            -X 'POST' \
            -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
            -H 'Accept: application/json' \
            -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
            -H 'Connection: keep-alive' \
            -H 'Content-Length: 0' \
            -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
            -H 'X-Request: JSON' \
            -H 'X-Requested-With: XMLHttpRequest' \
            -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
            -H 'sec-ch-ua-mobile: ?0' \
            -H 'sec-ch-ua-platform: "Linux"' \
            -H 'Sec-Fetch-Dest: empty' \
            -H 'Sec-Fetch-Mode: cors' \
            -H 'Sec-Fetch-Site: same-origin' \
            -H 'Origin: https://sistec.mec.gov.br' \
            -H 'Referer: https://sistec.mec.gov.br/index/index' \
            -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
            --compressed
        """
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Tempo de sessão do %s ... ' % (strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title()))
                file_log.flush()
                fsync(file_log.fileno())
        headers = {}
        http_header = [
            header_user_agent,
            header_accept_json,
            header_accept_lang,
            header_accept_enc,
            header_connection,
            header_content_len,
            header_content_type,
            header_xreq_json,
            header_xreq_xml,
            header_sec_ch_ua,
            header_sec_ch_ua_mobile,
            header_sec_ch_ua_platform,
            header_sec_fetch_dest + 'empty',
            header_sec_fetch_mode + 'cors',
            header_sec_fetch_site + 'same-origin',
            'Origin: ' + host_sistec,
            'Referer: ' + url_index,
            'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
        ]
        http_header[-1] = http_header[-1].encode(encoding)
        http_header[-2] = http_header[-2].encode(encoding)
        http_header[-3] = http_header[-3].encode(encoding)
        data_buffer = BytesIO()
        c = pycurl.Curl()
        c.setopt(c.URL, url_tempo_sessao)
        c.setopt(c.SSL_VERIFYPEER, 1)
        c.setopt(c.SSL_VERIFYHOST, 2)
        c.setopt(c.TCP_NODELAY, 1)
        c.setopt(c.TCP_FASTOPEN, 1)
        c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
        c.setopt(c.TIMEOUT, REQ_TIMEOUT)
        # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
        c.setopt(c.WRITEDATA, data_buffer)
        c.setopt(c.HTTPHEADER, http_header)
        c.setopt(c.HEADERFUNCTION, _pycurl_header)
        c.setopt(c.CUSTOMREQUEST, 'POST')
        if level_debug[2]:
            c.setopt(c.VERBOSE, 1)
            c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
        c.perform()
        c.close()
        tempo_sessao = data_buffer.getvalue().decode(encoding)
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s\n' % tempo_sessao)
                file_log.flush()
                fsync(file_log.fileno())

        """
        curl 'https://sistec.mec.gov.br/gridciclo/turmas' \
            -X 'POST' \
            -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
            -H 'Accept: application/json' \
            -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
            -H 'X-Requested-With: XMLHttpRequest' \
            -H 'X-Request: JSON' \
            -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
            -H 'Connection: keep-alive' \
            -H 'Content-Length: 0' \
            -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
            -H 'sec-ch-ua-mobile: ?0' \
            -H 'sec-ch-ua-platform: "Linux"' \
            -H 'Sec-Fetch-Dest: empty' \
            -H 'Sec-Fetch-Mode: cors' \
            -H 'Sec-Fetch-Site: same-origin' \
            -H 'Origin: https://sistec.mec.gov.br' \
            -H 'Referer: https://sistec.mec.gov.br/index/index' \
            -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
            --compressed

        pagina=1
        while (True):
            ...
            if pagina < totalPaginas:
                pagina = pagina + 1
                continue
            break
        """
        if level_debug[0]:
            with open(file_log_path, 'a') as file_log:
                file_log.write(u'%s Acessando a página %s/%s da lista de ciclos do %s ...\n' % (
                    strftime('[%Y-%m-%d %H:%M:%S]'), pagina_ciclos, total_paginas_ciclos, no_campus.title()))
                file_log.flush()
                fsync(file_log.fileno())
        headers = {}
        http_header = [
            header_user_agent,
            header_accept_json,
            header_accept_lang,
            header_accept_enc,
            header_xreq_xml,
            header_xreq_json,
            header_content_type,
            header_connection,
            header_content_len,
            header_sec_ch_ua,
            header_sec_ch_ua_mobile,
            header_sec_ch_ua_platform,
            header_sec_fetch_dest + 'empty',
            header_sec_fetch_mode + 'cors',
            header_sec_fetch_site + 'same-origin'
            'Origin: ' + host_sistec,
            'Referer: ' + url_index,
            'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
        ]
        http_header[-1] = http_header[-1].encode(encoding)
        http_header[-2] = http_header[-2].encode(encoding)
        http_header[-3] = http_header[-3].encode(encoding)
        post_data = {'pagina': pagina_ciclos}
        postfields = urlencode(post_data)

        count = 0
        while True:
            try:
                file_json = open(turmas_json, mode='wb')
                c = pycurl.Curl()
                c.setopt(c.URL, url_turmas)
                c.setopt(c.SSL_VERIFYPEER, 1)
                c.setopt(c.SSL_VERIFYHOST, 2)
                c.setopt(c.TCP_NODELAY, 1)
                c.setopt(c.TCP_FASTOPEN, 1)
                c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
                c.setopt(c.TIMEOUT, REQ_TIMEOUT)
                # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
                c.setopt(c.WRITEDATA, file_json)
                c.setopt(c.HTTPHEADER, http_header)
                c.setopt(c.HEADERFUNCTION, _pycurl_header)
                c.setopt(c.CUSTOMREQUEST, 'POST')
                c.setopt(c.POSTFIELDS, postfields)
                if level_debug[2]:
                    c.setopt(c.VERBOSE, 1)
                    c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
                c.perform()
                c.close()
                file_json.close()
                file_json_r = open(turmas_json, mode='rb')
                turmas_data = json.load(file_json_r)
                file_json_r.close()
                if level_debug[0]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(
                            u'%s %d tentativa(s): ...\n' %
                            (strftime('[%Y-%m-%d %H:%M:%S]'), count + 1))
                        file_log.flush()
                        fsync(file_log.fileno())
                if level_debug[1]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s turmas_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), turmas_data))
                        file_log.flush()
                        fsync(file_log.fileno())
                ciclos = turmas_data['dados']
            except Exception as erro:
                count = count + 1
                # tenta 10 vezes
                if count > 9:
                    return (False, u"%s" % erro)
                else:
                    continue
            else:
                break

        for i in range(len(ciclos)):

            # PEGA O CODIGO DO CICLO:
            co_ciclo = ciclos[i]['Código do Ciclo']

            # PEGA OS DADOS DO CURSO:
            """
            curl 'https://sistec.mec.gov.br/admciclomatricula/dadosciclo/' \
                -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
                -H 'Accept: application/json' \
                -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
                -H 'X-Requested-With: XMLHttpRequest' \
                -H 'X-Request: JSON' \
                -H 'Connection: keep-alive' \
                -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
                -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
                -H 'sec-ch-ua-mobile: ?0' \
                -H 'sec-ch-ua-platform: "Linux"' \
                -H 'Sec-Fetch-Dest: empty' \
                -H 'Sec-Fetch-Mode: cors' \
                -H 'Sec-Fetch-Site: same-origin' \
                -H 'Origin: https://sistec.mec.gov.br' \
                -H 'Referer: https://sistec.mec.gov.br/index/index' \
                -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660670' \
                --data-raw 'idCiclo=1949843' \
                --compressed
            """
            headers = {}
            http_header = [
                header_user_agent,
                header_accept_json,
                header_accept_lang,
                header_accept_enc,
                header_xreq_xml,
                header_xreq_json,
                header_connection,
                header_content_type,
                header_sec_ch_ua,
                header_sec_ch_ua_mobile,
                header_sec_ch_ua_platform,
                header_sec_fetch_dest + 'empty',
                header_sec_fetch_mode + 'cors',
                header_sec_fetch_site + 'same-origin',
                'Origin: ' + host_sistec,
                'Referer: ' + url_index,
                'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
            ]
            http_header[-1] = http_header[-1].encode(encoding)
            http_header[-2] = http_header[-2].encode(encoding)
            http_header[-3] = http_header[-3].encode(encoding)
            post_data = {'idCiclo': co_ciclo}
            postfields = urlencode(post_data)
            file_json = open(info_json, mode='wb')
            c = pycurl.Curl()
            c.setopt(c.URL, url_dados_ciclo)
            c.setopt(c.SSL_VERIFYPEER, 1)
            c.setopt(c.SSL_VERIFYHOST, 2)
            c.setopt(c.TCP_NODELAY, 1)
            c.setopt(c.TCP_FASTOPEN, 1)
            c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
            c.setopt(c.TIMEOUT, REQ_TIMEOUT)
            # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
            c.setopt(c.WRITEDATA, file_json)
            c.setopt(c.HTTPHEADER, http_header)
            c.setopt(c.HEADERFUNCTION, _pycurl_header)
            c.setopt(c.CUSTOMREQUEST, 'POST')
            c.setopt(c.POSTFIELDS, postfields)
            if level_debug[2]:
                c.setopt(c.VERBOSE, 1)
                c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
            c.perform()
            c.close()
            file_json.close()
            file_json_r = open(info_json, mode='rb')
            info_data = json.load(file_json_r)
            file_json_r.close()

            try:
                dados = info_data['mensagem']
            except KeyError as erro:
                return (False, u"%s" % erro)

            # PEGA O CODIGO DO CURSO
            co_curso = dados['co_curso']

            # PEGA O TIPO DE CURSO:
            tipo_curso = ciclos[i]['Tipo do Curso']

            # PEGA O NOME DO CURSO:
            nome_curso = ciclos[i]['Nome do Ciclo']

            # VERIFICA SE CURSO EH DO TIPO FIC:
            re_fic = re.compile(u'^.*FORMA[CÇ][AÃ]O\s*(INICIAL|CONTINUADA).*$')
            match_fic = re_fic.search(tipo_curso)

            # VERIFICA SE CURSO EH DO TIPO MULHERES MIL:
            re_mm = re.compile(u'^.*MULHERES\s*MIL.*$')
            match_mm = re_mm.search(tipo_curso)

            # VERIFICA SE CURSO EH DO TIPO EAD:
            re_ead = re.compile(u'^.*\s*DIST[AÂ]NCIA\s*$')
            match_ead = re_ead.search(nome_curso)

            # VERIFICA SE CURSO EH DO TIPO PRESENCIAL:
            re_presencial = re.compile(u'^.*\s*PRESENCIAL\s*$')
            match_presencial = re_presencial.search(nome_curso)

            if match_fic:
                if not tipo_fic:
                    continue
            elif match_mm:
                if not tipo_fic:
                    continue
            elif match_ead:
                co_cursos = [
                    338565, 338569, 338571, 338572, 338573, 338574, 338575, 338578, 338580,
                    338581, 338582, 338583, 362343, 362344, 362345, 362346, 362347
                ]
                if (tipo_presencial and int(co_curso) in co_cursos) or (tipo_ead and int(co_curso) not in co_cursos):
                    pass
                else:
                    continue
            elif match_presencial:
                if not tipo_presencial:
                    continue
            else:
                continue
                # return (
                #    False, u"não foi possível determinar o tipo do curso:\t%s" % nome_curso
                # )

            # PEGA AS DATAS DO INICO E DO FIM PREVISTO:
            dt_data_inicio = dados['dt_data_inicio']
            dt_data_fim_previsto = dados['dt_data_fim_previsto']

            # PEGA O NOME DO CICLO DE MATRICULA:
            no_ciclo_matricula = dados['no_ciclo_matricula']
            no_ciclo_matricula = no_ciclo_matricula.strip()

            # PEGA O CODIGO DO TIPO DA OFERTA DO CURSO:
            co_tipo_oferta_curso = dados['co_tipo_oferta_curso']
            if co_tipo_oferta_curso is None:
                co_tipo_oferta_curso = 0

            # PEGA O CODIGO DO POLO DO CURSO:
            d_campus = {
                u'CÂMPUS ÁGUAS LINDAS': 3647,
                u'CÂMPUS ANÁPOLIS': 699,
                u'CÂMPUS APARECIDA DE GOIÂNIA': 210,
                u'CÂMPUS CIDADE DE GOIÁS': 696,
                u'CÂMPUS FORMOSA': 2012,
                u'CÂMPUS GOIÂNIA': 212,
                u'CÂMPUS GOIÂNIA OESTE': 3646,
                u'CÂMPUS INHUMAS': 238,
                u'CÂMPUS ITUMBIARA': 244,
                u'CÂMPUS JATAÍ': 241,
                u'CÂMPUS LUZIÂNIA': 209,
                u'CÂMPUS SENADOR CANEDO': 3648,
                u'CÂMPUS URUAÇU': 161,
                u'CÂMPUS VALPARAÍSO': 3649
            }
            co_polo = d_campus[no_campus.upper()]

            line_common = []
            line_common.append(str(co_curso) + u';')
            line_common.append(dt_data_inicio + u';')
            line_common.append(dt_data_fim_previsto + u';')
            line_common.append(no_ciclo_matricula + u';')
            line_common.append(str(co_tipo_oferta_curso) + u';')
            line_common.append(str(co_polo) + u';')

            continue_alunos = 0
            pagina_alunos = 1
            while True:
                # =================
                # DOWNLOAD: ETAPA 3
                # =================
                """
                curl 'https://sistec.mec.gov.br/gridciclo/listaralunosacao/periodo//ciclo/2678641/coaluno/' \
                    -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36' \
                    -H 'Accept: application/json' \
                    -H 'Accept-Language: pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7' \
                    -H 'X-Requested-With: XMLHttpRequest' \
                    -H 'X-Request: JSON' \
                    -H 'Connection: keep-alive' \
                    -H 'Content-type: application/x-www-form-urlencoded; charset=UTF-8' \
                    -H 'sec-ch-ua: " Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"' \
                    -H 'sec-ch-ua-mobile: ?0' \
                    -H 'sec-ch-ua-platform: "Linux"' \
                    -H 'Sec-Fetch-Dest: empty' \
                    -H 'Sec-Fetch-Mode: cors' \
                    -H 'Sec-Fetch-Site: same-origin' \
                    -H 'Origin: https://sistec.mec.gov.br' \
                    -H 'Referer: https://sistec.mec.gov.br/index/index' \
                    -H 'Cookie: sistecNoticias=0; PHPSESSID=j8fhj595d4irervirp5cv1tombl0kcp7; perfil_cookie=GESTOR+DA+UNIDADE+DE+ENSINO; co_usuario=1660637' \
                    --data-raw 'registros=30&pagina=2' \
                    --compressed

                pagina=1
                while (True):
                    ...
                    if pagina < totalPaginas:
                        pagina = pagina + 1
                        continue
                    break
                """
                if level_debug[0]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s Acessando a lista de alunos (%s) do ciclo:\t%s ...\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), no_campus.title(), no_ciclo_matricula.title()))
                        #  file_log.write(u'%s url_ciclo = %s + %s + /coaluno/\n' % (
                            #  strftime('[%Y-%m-%d %H:%M:%S]'), url_ciclo_common, co_ciclo))
                        file_log.flush()
                        fsync(file_log.fileno())
                headers = {}
                url_ciclo = url_ciclo_common + str(co_ciclo) + '/coaluno/'

                http_header = [
                    header_user_agent,
                    header_accept_json,
                    header_accept_lang,
                    header_accept_enc,
                    header_xreq_xml,
                    header_xreq_json,
                    header_connection,
                    header_content_type,
                    header_sec_ch_ua,
                    header_sec_ch_ua_mobile,
                    header_sec_ch_ua_platform,
                    header_sec_fetch_dest + 'empty',
                    header_sec_fetch_mode + 'cors',
                    header_sec_fetch_site + 'same-origin',
                    'Origin: ' + host_sistec,
                    'Referer: ' + url_index,
                    'Cookie: ' + cookie_noticias + '; ' + cookie_phpsessid + '; ' + cookie_perfil + '; ' + cookie_usuario
                ]
                http_header[-1] = http_header[-1].encode(encoding)
                http_header[-2] = http_header[-2].encode(encoding)
                http_header[-3] = http_header[-3].encode(encoding)
                post_data = {'registros': 30, 'pagina': pagina_alunos}
                postfields = urlencode(post_data)
                file_json = open(alunos_json, mode='wb')
                c = pycurl.Curl()
                c.setopt(c.URL, url_ciclo)
                c.setopt(c.SSL_VERIFYPEER, 1)
                c.setopt(c.SSL_VERIFYHOST, 2)
                c.setopt(c.TCP_NODELAY, 1)
                c.setopt(c.TCP_FASTOPEN, 1)
                c.setopt(c.CONNECTTIMEOUT, CON_TIMEOUT)
                c.setopt(c.TIMEOUT, REQ_TIMEOUT)
                # c.setopt(c.CAINFO, path.join(path.curdir, 'curl-ca-bundle.crt'))
                c.setopt(c.WRITEDATA, file_json)
                c.setopt(c.HTTPHEADER, http_header)
                c.setopt(c.HEADERFUNCTION, _pycurl_header)
                c.setopt(c.CUSTOMREQUEST, 'POST')
                c.setopt(c.POSTFIELDS, postfields)
                if level_debug[2]:
                    c.setopt(c.VERBOSE, 1)
                    c.setopt(c.DEBUGFUNCTION, _pycurl_debug)
                c.perform()
                c.close()
                file_json.close()
                file_json_r = open(alunos_json, mode='rb')
                alunos_data = json.load(file_json_r)
                file_json_r.close()
                if level_debug[1]:
                    with open(file_log_path, 'a') as file_log:
                        file_log.write(u'%s url_ciclo:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), url_ciclo))
                        file_log.write(u'%s http_header:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), http_header))
                        file_log.write(u'%s post_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), post_data))
                        file_log.write(u'%s alunos_data:\t%s\n' % (
                            strftime('[%Y-%m-%d %H:%M:%S]'), alunos_data))
                        file_log.flush()
                        fsync(file_log.fileno())

                try:
                    alunos = alunos_data['dados']
                except KeyError:
                    try:
                        total_paginas_alunos = alunos_data['totalPaginas']
                    except Exception as erro:
                        return (False, u"%s" % erro)
                    # print(pagina_alunos)
                    # print(total_paginas_alunos)
                    if pagina_alunos < total_paginas_alunos:
                        pagina_alunos = pagina_alunos + 1
                        continue_alunos = 1
                    else:
                        continue_alunos = 2

                if continue_alunos == 1:
                    continue_alunos = 0
                    continue # while alunos
                elif continue_alunos == 2:
                    continue_alunos = 0
                    break # while alunos

                for j in range(len(alunos)):

                    line = copy(line_common)

                    # PEGA O NOME DO ALUNO:
                    nome_aluno = alunos[j]['Aluno']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s nome_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), nome_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    nome_aluno = nome_aluno.strip() + ';'
                    line.insert(0, nome_aluno)

                    # PEGA O CPF DO ALUNO:
                    cpf_aluno = alunos[j]['CPF']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s cpf_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), cpf_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    cpf_aluno = cpf_aluno.strip() + ';'
                    line.insert(1, cpf_aluno)

                    # PEGA O STATUS DO ALUNO:
                    status_aluno = alunos[j]['Status']
                    if level_debug[1]:
                        with open(file_log_path, 'a') as file_log:
                            file_log.write(u'%s status_aluno:\t%s\n' % (
                                strftime('[%Y-%m-%d %H:%M:%S]'), status_aluno))
                            file_log.flush()
                            fsync(file_log.fileno())
                    status_aluno = status_aluno.strip()
                    line.append(status_aluno)

                    # ADICIONA UMA LINHA DA PLANILHA NO ARRAY "lines":
                    line.append(u'\n')
                    line = u''.join(line)
                    try:
                        new_line = line.encode(encoding)
                    except UnicodeEncodeError:
                        new_line = limpa_linha(line, encoding)
                    line = new_line.decode(encoding)
                    if line not in lines:
                        lines.append(line)

                try:
                    total_paginas_alunos = alunos_data['totalPaginas']
                except Exception as erro:
                    return (False, u"%s" % erro)

                if pagina_alunos < total_paginas_alunos:
                    pagina_alunos = pagina_alunos + 1
                    continue  # while alunos
                else:
                    break  # while alunos

        try:
            total_paginas_ciclos = turmas_data['totalPaginas']
        except Exception as erro:
            return (False, u"%s" % erro)

        if pagina_ciclos < total_paginas_ciclos:
            pagina_ciclos = pagina_ciclos + 1
            continue  # while ciclos
        else:
            break  # while ciclos

    if lines:
        try:
            file_csv.writelines(lines)
            file_csv.flush()
            fsync(file_csv.fileno())
        except Exception as erro:
            return (False, u"%s" % erro)

    return (True, u'OK')

def limpa_linha(line, encoding):
    new_line = []
    for i in range(len(line)):
        try:
            new_line.append(line[i].encode(encoding))
        except UnicodeEncodeError:
            new_line.append(u'_'.encode(encoding))
    for i in range(len(new_line)):
        new_line[i] = new_line[i].decode(encoding)
    new_line = u''.join(new_line)
    new_line = new_line.encode(encoding)
    return (new_line)

def _pycurl_debug(debug_type, debug_msg):
    global pydbg_path
    with open(pydbg_path, 'a') as pydbg_file:
        pydbg_file.write(u'%s DEBUG(%d):\t%s\n' % (strftime('[%Y-%m-%d %H:%M:%S]'), debug_type, debug_msg))
        pydbg_file.flush()
        fsync(pydbg_file.fileno())

def _pycurl_header(header_line):
    global headers, encoding
    header_line = header_line.decode(encoding)
    if ':' not in header_line:
        return
    name, value = header_line.split(':', 1)
    name = name.strip()
    name = name.lower()
    value = value.strip()
    if name == 'set-cookie':
        cookie, dados = value.split(';', 1)
        cookie = cookie.strip()
        cookie_name, cookie_value = cookie.split('=', 1)
        cookie_name = cookie_name.strip()
        cookie_value = cookie_value.strip()
    if name not in headers:
        headers[name] = value
    return