import os
from datetime import date

import requests
import pandas as pd
from bs4 import BeautifulSoup

def main():
    url = 'https://covid19.gov.kn/src/stats2/'
    location = "Saint Kitts and Nevis"
    output_file = f'automated_sheets/{location}.csv'

    soup = BeautifulSoup(requests.get(url).content, 'html.parser')
    df = pd.read_html(str(soup.find('table')))[0]
    count = df.loc[df[0] == "No. of Persons Tested", 1].item()
    # print(count)

    date_str = date.today().strftime("%Y-%m-%d")
    df = pd.DataFrame({
        'Country': location,
        'Date': [date_str],
        'Cumulative total': count,
        'Source URL': url,
        'Source label': 'Ministry of Health',
        'Units': 'people tested'
    })

    if os.path.isfile(output_file):
        existing = pd.read_csv(output_file)
        if count > existing["Cumulative total"].max():
            df = pd.concat([df, existing]).sort_values('Date', ascending=False).drop_duplicates()
    df.to_csv(output_file, index=False)

if __name__=="__main__":
    main()


import random

def fresh_deck():
    suits = {"Spade", "Heart", "Diamond", "Club"}
    ranks = {"A", 2, 3, 4, 5, 6, 7, 8, 9, 10, "J", "Q", "K"}
    deck = []
    for i in suits:
        for j in ranks:
            deck.append({"suit":i, "rank":j})
    random.shuffle(deck)
    return deck

def hit(deck):
    if deck==[]:
        deck = fresh_deck()
    return (deck[0], deck[1:])

def count_score(cards):
    score = 0
    number_of_ace = 0
    for card in cards:
        if type(card['rank'])==int:
            score+=card['rank']
        elif card['rank']=="A":
            score+=11
            number_of_ace+=1
        else:
            score+=10

    # score가 21이 넘고 A가 있으면 score를 재조정함
    while score>21 and number_of_ace>0:
        score-=10
        number_of_ace-=1

    return score
deck = fresh_deck()
a=[]
(c, deck) = hit(deck)
a.append(c)
a.append({'suit': 'Club', 'rank': 'A'})
a.append({'suit': 'Club', 'rank': 'A'})
a.append({'suit': 'Club', 'rank': 'A'})

print(a)
print(count_score(a))


import datetime
import typing

from .models import Certificate
from .storage.base import StorageProtocol

def find_certificates_to_renew(
    storage: StorageProtocol, cert_fresh_days: int = 60
) -> typing.Iterator[typing.Tuple[Certificate, datetime.datetime]]:
    """Returns iterator of `domain name` and `valid after date` of stored certs."""
    now = datetime.datetime.now(datetime.timezone.utc)
    for cert_name, valid_after in storage.list_certificates():
        fresh_before = valid_after + datetime.timedelta(days=cert_fresh_days)
        if now > fresh_before:
            cert = storage.get_certificate(name=cert_name)
            assert cert
            yield (cert, valid_after)


from ratelimit import rate_limited
from commons.config import Config
from commons.utils import utils
import logging
import time
import pandas.io.sql as psql
import pandas as pd
from datetime import datetime, timedelta, date
from sqlalchemy import create_engine
import pandas.io.sql as psql
import pandas as pd
import numpy as np
from sqlalchemy import create_engine
from pytrends.request import TrendReq
from random import randint

@rate_limited(5, 1)
def get_info_google_trend(pytrends, df_to_db, standard, coin_row, pytrend_period):

    #Lower case everything
    coin_id = coin_row['id_cryptocompare']  # Exemple : 7605
    project = coin_row['coin_name'].lower()  # Exemple : ethereum
    symbol = coin_row['symbol'].lower()  # Exemple : eth

    logging.warning("__get_info_google_trend start for coin " + project)

    # Compare symbol and project name
    # wait a random amount of time to avoid bot detection
    time.sleep(randint(3, 8))
    #pytrends.build_payload([project, symbol], cat=0, timeframe='today 1-m', geo='', gprop='')

    # Modif CSA pour ML
    pytrends.build_payload([project, symbol], cat=0, timeframe=pytrend_period, geo='', gprop='')

    # Keep the most significant
    df_extract_trend = pytrends.interest_over_time()
    max_project = df_extract_trend[project].max()
    max_symbol = df_extract_trend[symbol].max()

    significant = ''

    # If the two results are at 100, take the oldest
    if max_project == max_symbol:
        date_project = df_extract_trend[project].idxmax()
        date_symbol = df_extract_trend[symbol].idxmax()
        significant = project if date_project > date_symbol else symbol
    else:
        significant = project if max_project > max_symbol else symbol

    # Result for stand alone
    df_result_coin = df_extract_trend[[significant]]
    df_result_coin.columns = ['value_standalone']

    # Result vs Standard
    # wait a random amount of time to avoid bot detection
    time.sleep(randint(8, 13))
    pytrends.build_payload([significant, standard], cat=0, timeframe=pytrend_period, geo='', gprop='')
    df_google_trend_compared = pytrends.interest_over_time()
    df_google_trend_compared = df_google_trend_compared[[significant]]
    df_google_trend_compared.columns = ['value_compared_to_standard']

    df_result_coin = df_result_coin.join(df_google_trend_compared)
    df_result_coin['id_cryptocompare'] = coin_id

    df_result_coin.reset_index(inplace=True)
    df_result_coin.rename(columns={'date': 'timestamp'}, inplace=True)

    # append aggregated result coin to one specific dataframe
    if df_to_db is None:
        df_to_db = df_result_coin
    else:
        # df_to_db = df_to_db.append(df_result_coin, ignore_index=True)
        df_to_db = pd.concat([df_to_db, df_result_coin])

    logging.warning("__get_info_google_trend end  for coin " + project)
    return df_to_db


# Import base streamlit dependency
import streamlit as st
# Import pandas to load the analytics data
import pandas as pd
# Import subprocess to run tiktok script from command line
from subprocess import call
# Import plotly for viz
import plotly.express as px
# Import numpy for math
import numpy as np
# Import time
import time
# Import base64 for encoding
import base64
# Import zipfile for zipping
import zipfile

# Set page width to wide
st.set_page_config(layout='wide')

# Set Page title
st.title('Instagram Analytics by Voomo')

# Create sidebar
st.sidebar.markdown("

Instagram Analytics

", unsafe_allow_html=True)
st.sidebar.markdown("This dashboard allows you to analyse Multiple Instgram accounts using Python and Streamlit.")
st.sidebar.markdown("To get started

Enter the Usernames you wish to analyse separated by (' , ')

Hit Get Data.

Get analyzing

",unsafe_allow_html=True)

def filedownload(df,stats):
    complete = df.to_csv('./output/csv/complete.csv',index=False)
    stats_file = stats.to_csv('./output/csv/statistics.csv',index=False)
    fig.write_image("./output/images/histogram.png")
    scatter1.write_image("./output/images/scatter.png")
    followers_Bar.write_image("./output/images/followers.png")
    postsNum_Bar.write_image("./output/images/posts.png")
    er_Bar.write_image("./output/images/er.png")
    zip_file = zipfile.ZipFile('./output/zip/output.zip', 'w')
    zip_file.write('./output/csv/complete.csv')
    zip_file.write('./output/csv/statistics.csv')
    zip_file.write('./output/images/histogram.png')
    zip_file.write('./output/images/scatter.png')
    zip_file.write('./output/images/followers.png')
    zip_file.write('./output/images/posts.png')
    zip_file.write('./output/images/er.png')
    zip_file.close()
    with open("./output/zip/output.zip", "rb") as f:
        btn = st.download_button(
            label="Download ZIP",
            data=f,
            file_name="report.zip",
            mime="application/zip"
        )
        return btn

# Input
usernames = st.text_input('Search for a accounts here', value="")

if st.button('Get Data'):
    # Run get data function here

    call(['python', 'instagram.py', usernames])
    with st.spinner('Wait for it...'):
        time.sleep(5)
    st.success('Done!')

    # Load in existing state data to test it out
    stats = pd.read_csv('stats.csv')

    stats0 = stats.sort_values(by=['# Followers'])
    followers_Bar = px.bar(stats0 , y= 'Full Name', x='# Followers',  orientation='h', height=500, width=800, title='Who is Followed the most?',color_discrete_sequence= px.colors.sequential.Plasma)

    stats0 = stats.sort_values(by=['# Uploads'])
    postsNum_Bar = px.bar(stats0 , x= 'Full Name', y='# Uploads', orientation='v', height=500, width=800, title='Who has the most posts?',color_discrete_sequence= px.colors.sequential.Plasma)

    stats0 = stats.sort_values(by=['Average Engagment Rate'])
    er_Bar = px.bar(stats0 , x= 'Full Name', y='Average Engagment Rate',  orientation='v', height=500, width=800, title='Who has the highest Engagement Rate?',color_discrete_sequence= px.colors.sequential.Plasma)

    # Load in existing data to test it out
    df = pd.read_csv('results.csv')
    df['date']= pd.to_datetime(df['Date'])

    # lucky hour
    alllikes = []
    max_like = []
    hours = []
    lucky_hour = []
    for user in df['Username'].unique():
        alllikes = df[df['Username'] == user]['Likes'].tolist()
        max_like.append(max(alllikes))
        hours = df[df['Username'] == user]['Time'].tolist()
        lucky_hour.append(hours[np.argmax(alllikes)])

    # Plotly viz here
    fig = px.histogram(stats, x='Full Name', hover_data=['Full Name'], y='# Followers', height=500,title='Histogram between # of followers')
    st.plotly_chart(fig, use_container_width=True)

    # Split columns
    left_col, right_col = st.columns(2)

    # First Chart - video stats
    scatter1 = px.scatter(df, x='Date', y='Likes', hover_data=['Username'], color='Username',title='Date vs Likes',symbol='Username',color_discrete_sequence= px.colors.sequential.Plasma)
    left_col.plotly_chart(scatter1, use_container_width=True)
    left_col.plotly_chart(er_Bar, use_container_width=True)

    # Second Chart
    right_col.plotly_chart(followers_Bar, use_container_width=True)
    right_col.plotly_chart(postsNum_Bar, use_container_width=True)

    filedownload(df,stats)
    # Show tabular dataframe in streamlit


# -*- coding: utf-8 -*-

"""
Created on Sat Nov 7 12:47:39 2020

@author: matthieufuteral-peter
"""

import argparse
import os
from sklearn.metrics import classification_report

from utils.utils import *

def pil_loader(path):
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        with Image.open(f) as img:
            return img.convert('RGB')

def missing_pred(model, val_images, val_labels):
    '''
    Evaluate the model performances on the dev set

    Parameters:
        model (nn.Module pytorch model): model to evaluate performance
        val_images (dict): keys are categories, values are lists of img path by category
        val_labels (list): labels of images

    Return:
        y_true (list): list of ground truth labels
        y_pred (list): list of predicted labels
        miss (list): list of images with prediction error
        n_miss (int): number of prediction errors
        miss_prob (list): list of probability of the prediction if mistake
    '''
    y_true = []
    y_pred = []
    miss_prob = []
    miss = []
    n_miss = 0
    for cat, imgs in val_images.items():
        for img in imgs:
            path_img = os.path.join(val_dir, cat, img)
            data = data_transforms_dev(pil_loader(path_img)).unsqueeze(0).to(device)

            output = F.softmax(model(data), dim=-1)
            pred = torch.argmax(output, dim=-1).item()

            if pred != val_labels[cat]:
                n_miss += 1
                miss.append(pil_loader(path_img))
                miss_prob.append(torch.max(output).item())

            y_pred.append(pred)
            y_true.append(val_labels[cat])

    return y_true, y_pred, miss, n_miss, miss_prob

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='RecVis A3 evaluation script')
    parser.add_argument('--data', type=str, default='cropped_birds', metavar='D',
                        help="folder where data is located. test_images/ need to be found in the folder")
    parser.add_argument('--model', type=str, metavar='M',
                        help="model architecture, e.g. resnet"),
    parser.add_argument('--cfg', type=str, default='vit_large_patch16_224'),
    parser.add_argument('--experiment', type=str, default='semi_supervized_experiment')
    parser.add_argument('--RUN_ID', type=str, help='id of the model', required=True),
    parser.add_argument('--size', type=int, default=224, help='size of the images'),
    args = parser.parse_args()

    use_cuda = torch.cuda.is_available()
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print('Using GPU : {}'.format(use_cuda))

    path = os.getcwd()
    path_experiments = os.path.join(path, args.experiment)
    path_id = os.path.join(path_experiments, args.RUN_ID)
    path_model = os.path.join(path_id, 'model.pt')
    val_dir = os.path.join(args.data, 'val_images')
    data_transforms_dev = data_transformation(model=args.model, size=args.size, train=0)

    # Load model
    model = load_model(path_model=path_model,
                       model_type=args.model,
                       cfg=args.cfg,
                       use_cuda=1,
                       dropout=0.0,
                       load_weights=1)
    # Testing mode
    model.eval()

    # Class to index
    val_index = datasets.ImageFolder(args.data + '/val_images', transform=data_transforms_dev).class_to_idx
    # Index to class
    index_to_class = {v: k for k, v in val_index.items()}
    val_images = {cat: [img for img in os.listdir(os.path.join(val_dir, cat)) if 'jpg' in img] for cat in
                  os.listdir(val_dir)}

    y_true, y_pred, miss_pred, n_miss, miss_prob = missing_pred(model, val_images, val_index)

    print(1 - n_miss/176)
    print(miss_prob)

    print(classification_report(y_true, y_pred, target_names=list(val_index.keys())))

    # Display the images where the model made a mistake
    fig = plt.figure(figsize=(20, 20))
    gs = fig.add_gridspec(8, 1)
    for idx in range(n_miss):
        axs = fig.add_subplot(gs[idx])
        axs.set_title('Pred : {} - Ground Truth : {}'.format(index_to_class[y_pred[idx]], index_to_class[y_true[idx]]))
        axs.imshow(miss_pred[idx])
        axs.set_axis_off()

    plt.show()


import os.path
import unittest

import mock  # pip install mock  # Python 2
from unittest import mock  # Python 3

import my_module
from my_module import my_remove

class NoMockingTestCase(unittest.TestCase):
    def test_provided_extension_should_be_used(self):
        filename = 'file.md'
        open(filename, 'w').close()
        self.assertTrue(os.path.isfile(filename))
        my_remove(filename)
        self.assertFalse(os.path.isfile(filename))

    def test_when_extension_is_missing_then_use_default_one(self):
        filename = 'file.txt'
        open(filename, 'w').close()
        self.assertTrue(os.path.isfile(filename))
        my_remove('file')
        self.assertFalse(os.path.isfile(filename))

class PatchingTestCase(unittest.TestCase):
    @mock.patch('my_module.os')
    def test_provided_extension_should_be_used(self, os_mock):
        my_remove('file.md')
        os_mock.remove.assert_called_once_with('file.md')

    @mock.patch('my_module.os')
    def test_when_extension_is_missing_then_use_default_one(self, os_mock):
        my_remove('file')
        os_mock.remove.assert_called_once_with('file.txt')

class ManualPatching(unittest.TestCase):
    def setUp(self):
        self.os_mock = mock.Mock()
        self.real_os = my_module.os
        my_module.os = self.os_mock

    def tearDown(self):
        my_module.os = self.real_os

    def test_provided_extension_should_be_used(self):
        my_remove('file.md')
        self.os_mock.remove.assert_called_once_with('file.md')

if __name__ == "__main__":
    unittest.main()


from mtcnn import MTCNN
import cv2
from os import listdir
from os.path import isfile, join, isdir, basename
import pprint
import csv

myPath = "./test"
directories = [join(myPath, d) for d in listdir(myPath) if isdir(join(myPath, d))]

pp = pprint.PrettyPrinter()
faceList = []
with open('eggs.csv', 'w', newline='') as csvfile:
    spamWriter = csv.writer(csvfile, delimiter=' ',
                            quotechar='|', quoting=csv.QUOTE_MINIMAL)
    spamWriter.writerow(['filename', 'subject_id', 'xmin', 'ymin', 'width', 'height'])

detector = MTCNN()
for directory in directories:
    images = [f for f in listdir(directory) if isfile(join(directory, f))]
    for imageFile in images:
        subjectID = basename(directory).replace("n", "")
        fileName = imageFile
        print(imageFile)
        img = cv2.cvtColor(cv2.imread(directory + "/" + imageFile), cv2.COLOR_BGR2RGB)
        faceData = detector.detect_faces(img)
        print(faceData)
        if len(faceData) == 0:
            continue
        box = faceData[0]['box']
        face = {
            'filename': fileName,
            'subject_id': subjectID,
            'xmin': box[0],
            'ymin': box[1],
            'width': box[2],
            'height': box[3]
        }
        with open("eggs.csv", 'a', newline='') as csvfile:
            spamWriter = csv.writer(csvfile, delimiter=' ',
                                    quotechar='|', quoting=csv.QUOTE_MINIMAL)
            spamWriter.writerow(face.values())
        # row = face.keys()
        # rowValue = face.values()
        # faceList.append({
        #     'filename': fileName,
        #     'subject_id': subjectID,
        #     'xmin': box[0],
        #     'ymin': box[1],
        #     'width': box[2],
        #     'height': box[3]
        # })
#
# # faceList = [face for face in faceList if len(face) != 0]
# pp = pprint.PrettyPrinter()
# pp.pprint(faceList)


import yaml
import component
import config
from collections import defaultdict

yaml_file_addr = config.WORKFLOW_YAML_ADDR

def parse(workflow_name):
    data = yaml.load(open(yaml_file_addr[workflow_name]), Loader=yaml.FullLoader)
    print(data)
    global_input = dict()
    start_functions = []
    nodes = dict()
    parent_cnt = dict()
    foreach_functions = set()
    merge_funtions = set()
    total = 0

    next_function_node = defaultdict(list) #?记录某个函数的后续节点是什么,一个函数可能有多个后续节点
    ##!注:这里的后续节点注包括直接相连的边，比如说A->B,B->C,B->D,此时A的后续节点只有B
    #? B的后续节点有C和D
    prev_function_node  =dict() # defaultdict(list) ##!记录某个函数的前驱节点是什么，
    #? 此时C和D的后续节点为B

    ###todo gloabl_input还需理解
    if 'global_input' in data:
        for key in data['global_input']:
            parameter = data['global_input'][key]['value']['parameter']
            #global_input[parameter] = data['global_input'][key]['size']
            global_input[key] = data['global_input'][key]['size'] #为科学计算服务
    functions = data['functions']
    parent_cnt[functions[0]['name']] = 0
    for function in functions:
        #print('*********functon:',function,'\n')
        name = function['name'] #?name为某个函数名字
        print('------------name:----------',name) #这个应该是start / count /merge
        source = function['source']
        runtime = function['runtime']
        input_files = dict()
        output_files = dict()
        next = list()
        nextDis = list()
        send_byte = 0
        if 'input' in function:
            for key in function['input']:
                print('**********start********')
                #当'input'在funcion有时，表示这个function需要外部输入，即这个function不是DAG的起始点函数
                print('key:',key,'\n\n')
                print('function:', function['input'][key]['value']['function'],'\n') #记录这个input来着哪个函数
                print('parameter:', function['input'][key]['value']['parameter']) #记录参数，在wordcount是一个filename
                print('size:', function['input'][key]['size'])#记录大小
                print('arg:', key)#这个未知
                print('type:', function['input'][key]['type']) #记录type
                print('*********end*************\n\n\n')
                input_files[key] = {'function': function['input'][key]['value']['function'],
                                    'parameter': function['input'][key]['value']['parameter'],
                                    'size': function['input'][key]['size'], 'arg': key,
                                    'type': function['input'][key]['type']}
                ###todo:这里记录input_files[key]是干啥
                #!12-10更新，input_file[key]记录某个函数的输入
                #!12-10 更新，这个函数的输入存在两种情况
                #!入度为0，它的输入来着外部输入
                #! 入度不为0，它的输入来自它的前驱节点
        if 'output' in function:
            print('the function has output:',name,'\n\n')
            for key in function['output']:
                output_files[key] = {'size': function['output'][key]['size'], 'type': function['output'][key]['type']} #这个是每个函数的输出
                send_byte += function['output'][key]['size']
        send_time = send_byte / config.NETWORK_BANDWIDTH
        conditions = list()
        #看function是否存在'next'
        #目前只有count和start有next
        #表示它们有后续节点，
        ###在wordcount的workflow中，是start->count->merge
        ##todo:这里需要记录一些东西
        ###todo:比如说记录start的next为count,count的后续节点为merge,我需要增加的点
        ###todo:也可能需要记录count的前驱节点为start,merge的前驱节点为count
        ###todo:这是我需要加上的点
        if 'next' in function:
            foreach_flag = False
            if function['next']['type'] == 'switch':
                conditions = function['next']['conditions']
            elif function['next']['type'] == 'foreach':
                foreach_flag = True
            for n in function['next']['nodes']:
                ###这一步记录next有什么函数，因为一个函数的后续节点可能有多个
                #? 记录前驱节点和后续节点
                #?在count中，它的function['next']['nodes] = merge
                next_function_node[name].append(n) ###todo:这里用set是不是更合适？(by lambda,或许需要修改)
                prev_function_node[n] = name #?每个节点的前驱节点只有一个

                if name in foreach_functions:
                    merge_funtions.add(n)
                ###todo:加入merge_function是不是很理解
                if foreach_flag:
                    foreach_functions.add(n) #把n加入foreach_functions
                next.append(n)#next是一个list,可以重复加入相同的 数据,next记录这个函数的后续节点
                ###n是一个函数名
                nextDis.append(send_time)#todo:这里的send_time加入nextDis还是不理解
                ###存在next,表示它有后续节点
                ###todo 12-09, nextDis是什么，
                #! send_time是发送数据的时间
                ##todo:不是很理解，parent_cnt[n]这里是记录这个函数n有多少个父节点
                ###todo:在FaasFlow的设计中，只有n所有的父节点都运行完了，n才可以运行
                ###*
                if n not in parent_cnt:
                    parent_cnt[n] = 1
                else:
                    parent_cnt[n] = parent_cnt[n] + 1
        current_function = component.function(name, [], next, nextDis, source, runtime,
                                              input_files, output_files, conditions)
        #得到current_function，它记录了它的next节点有哪些，以及输入输出文件，
        ###todo:conditions还不是很理解(by lambda)
        if 'scale' in function:
            current_function.set_scale(function['scale'])
        if 'mem_usage' in function:
            current_function.set_mem_usage(function['mem_usage'])
        if 'split_ratio' in function:
            current_function.set_split_ratio(function['split_ratio'])
        total = total + 1 #! total记录了整个workflow有几个节点
        nodes[name] = current_function #记录当前这个函数name的current_function
    for name in nodes:
        print('\n*********name:',name,'\n')
        if name not in parent_cnt or parent_cnt[name] == 0:
            parent_cnt[name] = 0
            start_functions.append(name) #找到哪些没有前驱节点的，即该节点的入度为0
            #! start_functions是一个list,可以有多个
        for next_node in nodes[name].next:
            print('\n***next_node:',next_node,'\n')
            nodes[next_node].prev.append(name) #?记录每个节点的前驱节点函数
    return component.workflow(workflow_name, start_functions, nodes, global_input, total, parent_cnt, foreach_functions, merge_funtions)


import numpy as np
import pandas as pd
import xarray as xr
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import cartopy.crs as ccrs
from cartopy.feature import NaturalEarthFeature, LAND, COASTLINE, ShapelyFeature
import matplotlib
import matplotlib.cm as cm
import cartopy.io.shapereader as shpreader
import xarray as xr
import cmocean
import matplotlib.pyplot as plt
from matplotlib.patches import Path, PathPatch

cmap_rain = cmocean.cm.rain

def cut2shapefile(plot_obj, shape_obj):

    """
    plot_obj: axis where plot is being made. ex: ax
    shape_obj: basemap shapefile. ex: m.nordeste_do_brasil when shape is read with m.readshapefile(path/to/nordeste_do_brasil, nordeste_do_brasil)
    """

    x0,x1 = plot_obj.get_xlim()
    y0,y1 = plot_obj.get_ylim()

    edges = [(x0, y0), (x1, y0), (x1, y1), (x0, y1), (x0, y0)]
    edge_codes = [Path.MOVETO] + (len(edges) - 1) * [Path.LINETO]

    verts = shape_obj[0] + [shape_obj[0][0]]
    codes = [Path.MOVETO] + (len(verts) - 1) * [Path.LINETO]

    path = Path(verts+edges, codes+edge_codes)

    patch = PathPatch(path, facecolor='white', lw=0)
    plot_obj.add_patch(patch)

shp=shpreader.Reader('../data/shapes/semiarido_nordestino_single.shp')
shape_feature=ShapelyFeature(shp.geometries(),ccrs.PlateCarree())

caatinga=shpreader.Reader('../data/shapes/caatinga.shp')
caatinga_feature=ShapelyFeature(caatinga.geometries(),ccrs.PlateCarree())

for f in shp.records():
        if f.attributes["nome"] == "Bahia":
            a = (f.geometry)
            x, y = a.exterior.coords.xy
            t = [[(a, s) for a, s in zip(x, y)]]

ds=xr.open_dataset("../data/noah_mon_glue.nc")

grain=xr.open_dataset("out_vars.nc")

precip=xr.open_dataset('precip.mon.ltm.0.25x0.25.v2020.nc')
precip.coords['lon'] = (precip.coords['lon'] + 180) % 360 - 180
precip=precip.sortby(precip.lon)
precip=precip.sortby(precip.lat)
precip=precip.sel(lat=slice(-15.75,-2.25),lon=slice(-46.75,-34.25))
precip=precip.mean(dim='time').precip

lon=np.linspace(-49.875,-30.125, num=80)
lat=np.linspace(-19.875,-0.125, num=80)
grain.coords["lat"] = (("south_north"), lat)
grain.coords["lon"] = (("west_east"), lon)
grain.coords["Times"] = pd.date_range(start='1/1/2001', periods=367, freq='M')
grain=grain.rename({'south_north': 'lat','west_east': 'lon', 'Times': 'time'})
grain=grain.sel(time=slice("2001-01-01", "2001-01-31"))

grain=grain.isel(lon=slice(14,64),lat=slice(14,72))

def rgb2hex(r,g,b):
    return "#{:02x}{:02x}{:02x}".format(r,g,b)

fig=plt.figure(figsize=(13.9,11))

gs=gridspec.GridSpec(2,4)
# gs.update(wspace=0.5)
ax=[1,1,1]
ax[0]=fig.add_subplot(gs[0,:2],projection=ccrs.PlateCarree())
ax[1]=fig.add_subplot(gs[0,2:],projection=ccrs.PlateCarree())
ax[2]=fig.add_subplot(gs[1,1:3],projection=ccrs.PlateCarree())

xticks = [-48, -44, -40, -36, -32]
yticks = [-18,-14, -10, -6, -2]

xticks = [-47,-45, -43, -41,-39, -37, -35]
yticks = [-15,-13, -11,-9, -7, -5,-3]
yticklabels=['{}S'.format(i) for i in yticks]
xticklabels=['{}W'.format(i) for i in xticks]

ax[0].set_title('Vegetation Type',fontsize=14)
ax[1].set_title('Soil Type',fontsize=14)
ax[2].set_title('Annual Precipitation (mm/day)',fontsize=14)

# Barra de cores
veg_custom_map = [rgb2hex(0,100,0), #darkgreen
              rgb2hex(0,175,0), #green
              rgb2hex(151,60,65), #bordeaux
              rgb2hex(228,151,95), #orange
              rgb2hex(255,212,177), #lightorange
              rgb2hex(3,253,3), #lightgreen
              rgb2hex(181,224,251), #n sei
              rgb2hex(168,168,168), #grey
              rgb2hex(248,159,207), #pink
              rgb2hex(244,237,200)] #sand
veg_colormap = matplotlib.colors.ListedColormap(veg_custom_map)
levels = [2,4,6,8,9,10,12,13,14,16,17]
norm = matplotlib.colors.BoundaryNorm(boundaries=levels, ncolors=10)
ax[0].pcolor(grain.lon,grain.lat,grain.IVGTYP[0,:,:],transform=ccrs.PlateCarree(),cmap=veg_colormap,norm=norm)
cbar = fig.colorbar(cm.ScalarMappable(norm=norm,cmap=veg_colormap),ticks=[1,3,5,7,8.5,9.5,11,12.5,13.5,15,16.5],ax=ax[0])
cbar.ax.set_yticklabels(['Evergreen Broad', 'Deciduous Broad', 'Closed Shrublands',
                         'Woody Savannas', 'Savannas','Grasslands','Croplands',
                         'Urban', 'Crop/veg','Barren Land'],fontsize=10)

soil=grain.ISLTYP.where(grain.ISLTYP!=14)

# Barra de cores
soil_custom_map = ["#FCFFC9","#EDD788","#E3A946","#D67500","#AB4A3D","#73243C","#1D0B14"]
soil_colormap = matplotlib.colors.ListedColormap(soil_custom_map)
levels = [1,2,3,6,7,9,12,13]
norm = matplotlib.colors.BoundaryNorm(boundaries=levels, ncolors=8)
ax[1].pcolor(grain.lon,grain.lat,soil[0,:,:],transform=ccrs.PlateCarree(),cmap=soil_colormap,norm=norm)
cbar = fig.colorbar(cm.ScalarMappable(norm=norm,cmap=soil_colormap),ticks=[0.5,1.5,2.5,4.5,6.5,8,10.5,12.5],ax=ax[1])
cbar.ax.set_yticklabels(['Sand', 'Loamy Sand', 'Sandy Loam',
                         'Loam', 'Sandy Clay Loam','Clay Loam','Clay'],fontsize=10)

states = NaturalEarthFeature(category='cultural', scale='50m', facecolor='none',
                                name='admin_1_states_provinces_lines')#estados

levels = [1,2,3,6,7,9,12,13]
norm = matplotlib.colors.Normalize(vmin=0,vmax=3)
ax[2].pcolor(precip.lon,precip.lat,precip,transform=ccrs.PlateCarree(),cmap=cmap_rain)
cbar = fig.colorbar(cm.ScalarMappable(norm=norm,cmap=cmap_rain),ax=ax[2])

states = NaturalEarthFeature(category='cultural', scale='50m', facecolor='none',
                                name='admin_1_states_provinces_lines')

ax[0].add_feature(caatinga_feature,edgecolor='grey',linewidth=1,linestyle='-',facecolor='none')
ax[1].add_feature(caatinga_feature,edgecolor='grey',linewidth=1,linestyle='-',facecolor='none')
ax[2].add_feature(caatinga_feature,edgecolor='grey',linewidth=1,linestyle='-',facecolor='none')

for n in [0,1,2]:
    ax[n].gridlines(xlocs=xticks, ylocs=yticks,alpha=0.3)
    ax[n].set_xticks(xticks)
    ax[n].set_xticklabels(xticklabels, fontsize=10)
    ax[n].set_yticks(yticks)
    ax[n].set_yticklabels(yticklabels, fontsize=10)
    cut2shapefile(ax[n], t)
    ax[n].coastlines(alpha=1)
    ax[n].add_feature(states, edgecolor='black', alpha=1,linestyle='-')#estados
    ax[n].add_feature(shape_feature,edgecolor='black',linewidth=1,linestyle='-',facecolor='none')
     #sab

fig.text(0.03,0.96,'a)',weight='bold',fontsize=13)
fig.text(0.52,0.96,'b)',weight='bold',fontsize=13)
fig.text(0.26,0.47,'c)',weight='bold',fontsize=13)

fig.subplots_adjust(left=0,right=0.94,bottom=0.03,top=0.95,hspace=0.15)
plt.savefig ('studyarea.jpg', dpi=1000)

print(grain.variables)


import numpy as np
from sklearn.neighbors import KDTree
import random
import matplotlib.pyplot as plt
import csv

class music_space:

    def __init__(self, path):
        self.feature_vectors = np.load(path + "MSD_features_2.npy")
        self.song_IDs = np.load(path + "MSD_new_song_IDs_2.npy", allow_pickle=True)
        self.KD_TREE = KDTree(data= self.feature_vectors, leaf_size=1 ,metric="minkowski") #tune leaf size??

    def vector_to_songID(self, song_vector):
        """returns the song_ID for the given song_vector"""

        distances = np.linalg.norm(self.feature_vectors - song_vector, axis=1)
        min_distance_index = np.argmin(distances)
        song_id = self.song_IDs[min_distance_index]

        return song_id

    def restrict_to_songs(self, songs):
        A = self.song_IDs
        B = songs
        i = np.in1d(
                self.song_IDs,
                songs )
        self.song_IDs = self.song_IDs[i]
        self.feature_vectors = self.feature_vectors[i]

    def songID_to_vector(self, song_id):
        """returns the song_vector for the given song_id"""

        index = np.where(self.song_IDs == song_id)[0]
        if index.size == 0:
            return None
        else:
            vector = self.feature_vectors[index[0]]
            return vector

    def NN(self, song_vector):
        """returns the nearest neighbour of a song vector"""

        dist, index = self.KD_TREE.query(X = song_vector.reshape(1,-1), k = 1)
        print(f"distance = {dist[0][1]}")
        return(self.feature_vectors[index[0][1]])

    def __select_frequent_terms(self):
        with open('music_space/embeddings/term_frequency.csv', 'r', newline='') as file:
            reader = csv.reader(file)
            D = [int(row[0]) for row in reader]
            self.feature_vectors = self.feature_vectors[:, D[:40]]

    def get_random_song(self):
        i = random.randint(0,self.feature_vectors.shape[0]-1)
        return self.song_IDs[i]


from dash import dcc
from dash import html
import plotly.express as px
import pandas as pd

singapore = pd.read_csv('data/singapore.csv')
singapore = px.line(x=singapore['year'], y=singapore.inflation,
                    labels={"x": "Year", "y": "Inflation Rate",}, title='Local Rates')
singapore.update_layout(template='simple_white', height=600, margin={
                        "r": 150, "t": 50, "l": 150, "b": 50},
                        title_text='Local Rates', title_x=0.5)

world = pd.read_csv('data/world.csv')
world = px.choropleth(world, locations="country",
                      color=world['inflation'],
                      locationmode='country names', hover_name="country",
                      animation_frame=world['year'],
                      color_continuous_scale=px.colors.sequential.matter)
world.update_layout(title_text='Worldwide Rates', title_x=0.5)

layout = html.Div([
    html.H4(children='Impact of Inflation on Oil and Gas Industry'),
    dcc.Graph(id='dist-chart', figure=singapore),
    dcc.Graph(id='map', figure=world),],
    style={'color': 'blue', 'textAlign': 'center', 'paddingTop': '20px'})


import unittest
from textToJsonWidget import TextToJsonWidget
from PySide6.QtTest import QTest
from PySide6.QtWidgets import QApplication

class TestTextToJsonWidget(unittest.TestCase):
    def setUp(self) -> None:
        super(TestTextToJsonWidget, self).setUp()
        if isinstance(QApplication.instance(), type(None)):
            self.app = QApplication()
        else:
            self.app = QApplication.instance()
        self._label = "label"

    def tearDown(self) -> None:
        del self.app
        return super(TestTextToJsonWidget, self).tearDown()

    def test_update_string_value(self):
        # setup for str based widget
        index = "index"
        value = "value"
        data = {index: value}
        widget = TextToJsonWidget(self._label, data, index)

        # test setup of str widget
        self.assertEqual(self._label, widget.labelWidget.text())
        self.assertEqual(data, widget.jsonBlob)
        self.assertEqual(str, widget.dataType)
        self.assertEqual(value, widget.textBox.text())

        # typing in the text box will append to existing string
        QTest.keyClicks(widget.textBox, "new")
        self.assertEqual("valuenew", widget.textBox.text())
        self.assertEqual({index: "valuenew"}, widget.jsonBlob)

    def test_update_int_value(self):
        # setup for int based widget
        index = "index"
        value = 1
        data = {index: value}
        widget = TextToJsonWidget(self._label, data, index, int)

        # test setup of int widget
        self.assertEqual(self._label, widget.labelWidget.text())
        self.assertEqual(data, widget.jsonBlob)
        self.assertEqual(int, widget.dataType, int)
        self.assertEqual(str(value), widget.textBox.text())

        # Using a string when a number will not change text
        QTest.keyClicks(widget.textBox, "k")
        self.assertEqual(str(value), widget.textBox.text())
        # using a number will change the string text and the json int value
        QTest.keyClicks(widget.textBox, "1")
        self.assertEqual(str(value) + "1", widget.textBox.text())
        self.assertEqual({index: 11}, widget.jsonBlob)

    def test_new_string_value(self):
        # setup for str based widget where value is added
        index = "newIndex"
        data = {}
        widget = TextToJsonWidget(self._label, data, index)

        # test setup of new str widget
        self.assertEqual(widget.labelWidget.text(), self._label)
        self.assertEqual(widget.jsonBlob, data)
        self.assertEqual(widget.dataType, str)
        self.assertEqual(widget.textBox.text(), "")

        # typing in the text box will create the key-value pair based on input
        QTest.keyClicks(widget.textBox, "new")
        self.assertEqual("new", widget.textBox.text())
        self.assertEqual({index: "new"}, widget.jsonBlob)

    def test_no_data(self):
        index = "index"
        widget = TextToJsonWidget(self._label, None, index)

        # text box data
        self.assertEqual("", widget.textBox.text())
        QTest.keyClicks(widget.textBox, "fail")
        self.assertEqual("", widget.textBox.text())

        self.assertTrue(isinstance(widget.jsonBlob, type(None)))

        data = {"index": "value"}
        widget.set_data(data)
        self.assertEqual("value", widget.textBox.text())
        self.assertTrue(widget.isEnabled())

        widget.set_data(None)
        self.assertEqual("", widget.textBox.text())
        self.assertFalse(widget.isEnabled())

    def test_reset_data(self):
        index = "index"
        value = "value"
        data = {index: value}
        widget = TextToJsonWidget(self._label, data, index, reset_value="newone")

        self.assertEqual(value, widget.textBox.text())

        widget.reset_data()

        # typing in the text box will append to existing string
        self.assertEqual("newone", widget.textBox.text())
        self.assertEqual({index: "newone"}, widget.jsonBlob)

if __name__ == '__main__':
    unittest.main()


from openupgradelib import openupgrade

def m2m_columns_and_tables_renamed(env):
    openupgrade.rename_columns(
        env.cr,
        {"crm_lead_tag_crm_reveal_rule_rel": [("crm_lead_tag_id", "crm_tag_id")]},
    )

    openupgrade.rename_tables(
        env.cr,
        [("crm_lead_tag_crm_reveal_rule_rel", "crm_reveal_rule_crm_tag_rel")],
    )

@openupgrade.migrate()
def migrate(env, version):
    m2m_columns_and_tables_renamed(env)


#!/usr/bin/env python3

class Node:
    def __init__(self, x):
        self.val = x
        self.next = None

def get_number(node):
    number = ''

    while node:
        number += str(node.val)
        node = node.next

    return int(number[::-1])

def create_linked_list(number):
    mem = None
    first = None

    for n in number:
        print(n)
        node = Node(int(n))

        if mem:
            mem.next = node
        else:
            first = node

        mem = node

    return first

def add(node1, node2):
    number1 = get_number(node1)
    number2 = get_number(node2)

    result = str(number1 + number2)

    result_node = create_linked_list(result)

    return result_node

if __name__ == '__main__':
    l1 = [2, 4, 3]
    l2 = [5, 6, 4]

    mem1 = None
    mem2 = None

    for i in range(len(l1)):
        node1 = Node(l1[i])
        node2 = Node(l2[i])

        if (mem1 and mem2):
            mem1.next = node1
            mem2.next = node2
        else:
            first1 = node1
            first2 = node2

        mem1 = node1
        mem2 = node2

    print(add(first1, first2))


import copy
import importlib
import json
import multiprocessing
import os
import shutil
import tarfile
import time
import traceback
import zipfile

from clade import Clade

import klever.core.utils
import klever.core.session
import klever.core.components
from klever.core.cross_refs import CrossRefs
from klever.core.progress import PW
from klever.core.coverage import JCR

JOB_FORMAT = 1
JOB_ARCHIVE = 'job.zip'
NECESSARY_FILES = [
    'job.json',
    'tasks.json',
    'verifier profiles.json'
]
DEFAULT_ARCH = 'x86-64'
DEFAULT_ARCH_OPTS = {
  'ARM': {
    'CIF': {
      'cross compile prefix': 'arm-unknown-eabi-'
    },
    'CIL': {
      'machine': 'gcc_arm_32'
    },
    'Clade': {
      'preset': 'klever_linux_kernel_arm'
    }
  },
  'ARM64': {
    'CIF': {
      'cross compile prefix': 'aarch64_be-unknown-linux-gnu-'
    },
    # As above.
    'CIL': {
      'machine': 'gcc_arm_64'
    },
    'Clade': {
      'preset': 'klever_linux_kernel_arm'
    }
  },
  'x86-64': {
    'CIF': {
      'cross compile prefix': ''
    },
    'CIL': {
      'machine': 'gcc_x86_64'
    },
    # Hey! Everybody will use Linux kernel specific preset for Clade. Let's hope that it does not matter since at the
    # moment this is used just for getting cross references.
    'Clade': {
      'preset': 'klever_linux_kernel'
    }
  }
}
CODE_COVERAGE_DETAILS_MAP = {
    '0': 'Original C source files',
    '1': 'C source files including models',
    '2': 'All source files'
}

def start_jobs(core_obj, vals):
    core_obj.logger.info('Check how many jobs we need to start and setup them')

    core_obj.logger.info('Extract job archive "{0}" to directory "{1}"'.format(JOB_ARCHIVE, 'job'))
    with zipfile.ZipFile(JOB_ARCHIVE) as ZipFile:
        ZipFile.extractall('job')

    for configuration_file in NECESSARY_FILES:
        path = klever.core.utils.find_file_or_dir(core_obj.logger, os.path.curdir, configuration_file)
        with open(path, 'r', encoding='utf-8') as fp:
            try:
                json.load(fp)
            except json.decoder.JSONDecodeError as err:
                raise ValueError("Cannot parse JSON configuration file {!r}: {}".format(configuration_file, err)) \
                    from None

    common_components_conf = __get_common_components_conf(core_obj.logger, core_obj.conf)
    core_obj.logger.info("Start results arranging and reporting subcomponent")

    core_obj.logger.info('Get project')
    if 'project' in common_components_conf:
        project = common_components_conf['project']
    else:
        raise KeyError('Specify attribute "project" within job.json')
    core_obj.logger.debug('Project is "{0}"'.format(project))

    # Save bases for components.
    common_components_conf['specifications dir'] = os.path.abspath(
        klever.core.utils.find_file_or_dir(core_obj.logger, os.path.curdir, 'specifications'))
    common_components_conf['specifications base'] = os.path.abspath(
        klever.core.utils.find_file_or_dir(core_obj.logger, os.path.curdir,
                                           os.path.join('specifications',
                                                        '{0}.json'.format(common_components_conf['project']))))
    common_components_conf['verifier profiles base'] = os.path.abspath(
        klever.core.utils.find_file_or_dir(core_obj.logger, os.path.curdir, 'verifier profiles.json'))
    common_components_conf['program fragments base'] = os.path.abspath(
        klever.core.utils.find_file_or_dir(core_obj.logger, os.path.curdir, 'fragmentation sets'))

    common_components_conf['code coverage details'] = CODE_COVERAGE_DETAILS_MAP[
        common_components_conf['code coverage details']]

    subcomponents = []
    try:
        queues_to_terminate = []

        pc = PW(core_obj.conf, core_obj.logger, core_obj.ID, core_obj.callbacks, core_obj.mqs, vals,
                separate_from_parent=False, include_child_resources=True,
                total_subjobs=(len(common_components_conf['sub-jobs']) if 'sub-jobs' in common_components_conf else 0))
        pc.start()
        subcomponents.append(pc)

        # TODO: split collecting total code coverage for sub-jobs. Otherwise there is too much redundant data.
        if 'collect total code coverage' in common_components_conf and \
                common_components_conf['collect total code coverage']:
            def after_process_finished_task(context):
                coverage_info_file = os.path.join(context.conf['main working directory'], context.coverage_info_file)
                if os.path.isfile(coverage_info_file):
                    context.mqs['req spec ids and coverage info files'].put({
                        'sub-job identifier': context.conf['sub-job identifier'],
                        'req spec id': context.req_spec_id,
                        'coverage info file': coverage_info_file
                    })

            def after_launch_sub_job_components(context):
                context.logger.debug('Put "{0}" sub-job identifier for finish coverage'.format(context.id))
                context.mqs['req spec ids and coverage info files'].put({
                    'sub-job identifier': context.common_components_conf['sub-job identifier']
                })

            cr = JCR(common_components_conf, core_obj.logger, core_obj.ID, core_obj.callbacks, core_obj.mqs, vals,
                     separate_from_parent=False, include_child_resources=True, queues_to_terminate=queues_to_terminate)
            # This can be done only in this module otherwise callbacks will be missed
            klever.core.components.set_component_callbacks(core_obj.logger, Job,
                                                           [after_launch_sub_job_components,
                                                            after_process_finished_task])
            cr.start()
            subcomponents.append(cr)

        if 'extra results processing' in common_components_conf:
            ra = REP(common_components_conf, core_obj.logger, core_obj.ID, core_obj.callbacks, core_obj.mqs, vals,
                     separate_from_parent=False, include_child_resources=True, queues_to_terminate=queues_to_terminate)
            ra.start()
            subcomponents.append(ra)

        if 'sub-jobs' in common_components_conf:
            core_obj.logger.info('Decide sub-jobs')
            sub_job_solvers_num = klever.core.utils.get_parallel_threads_num(core_obj.logger, common_components_conf,
                                                                             'Sub-jobs processing')
            core_obj.logger.debug('Sub-jobs will be decided in parallel by "{0}" solvers'.format(sub_job_solvers_num))
            __solve_sub_jobs(core_obj, vals, common_components_conf,
                             subcomponents + [core_obj.uploading_reports_process])
        else:
            job = Job(
                core_obj.conf, core_obj.logger, core_obj.ID, core_obj.callbacks, core_obj.mqs,
                vals,
                cur_id='Job',
                work_dir=os.path.join(os.path.curdir, 'job'),
                separate_from_parent=True,
                include_child_resources=False,
                components_common_conf=common_components_conf)
            klever.core.components.launch_workers(core_obj.logger, [job], subcomponents +
                                                  [core_obj.uploading_reports_process])
            core_obj.logger.info("Finished main job")
    except Exception:
        for p in subcomponents:
            if p.is_alive():
                p.terminate()
        raise

    # Stop queues
    for queue in queues_to_terminate:
        core_obj.logger.info('Terminate queue {!r}'.format(queue))
        core_obj.mqs[queue].put(None)
    # Stop subcomponents
    core_obj.logger.info('Jobs are solved, waiting for subcomponents')
    for subcomponent in subcomponents:
        subcomponent.join()
    core_obj.logger.info('Jobs and arranging results reporter finished')

def __get_common_components_conf(logger, conf):
    logger.info('Get components common configuration')

    with open(klever.core.utils.find_file_or_dir(logger, os.path.curdir, 'job.json'), encoding='utf-8') as fp:
        components_common_conf = json.load(fp)

    # Add architecture specific options. At the moment there are only default options but one may add dedicated
    # configuration files to jobs.
    if 'architecture' not in components_common_conf:
        components_common_conf['architecture'] = DEFAULT_ARCH
    if components_common_conf['architecture'] not in DEFAULT_ARCH_OPTS:
        raise ValueError("Klever does not support architecture {!r} yet, available options are: {}"
                         .format(components_common_conf['architecture'], ', '.join(DEFAULT_ARCH_OPTS.keys())))
    components_common_conf.update(DEFAULT_ARCH_OPTS[components_common_conf['architecture']])
    if 'cross compile prefix' in components_common_conf:
        components_common_conf['CIF']['cross compile prefix'] = components_common_conf['cross compile prefix']

    # Add complete Klever Core configuration itself to components configuration since almost all its attributes will
    # be used somewhere in components.
    components_common_conf.update(conf)

    if components_common_conf['keep intermediate files']:
        logger.debug('Create components common configuration file "components common conf.json"')
        with open('components common conf.json', 'w', encoding='utf-8') as fp:
            json.dump(components_common_conf, fp, ensure_ascii=False, sort_keys=True, indent=4)

    return components_common_conf

def __solve_sub_jobs(core_obj, vals, components_common_conf, subcomponents):
    def constructor(number):
        # Sub-job configuration is based on common sub-jobs configuration.
        sub_job_components_common_conf = copy.deepcopy(components_common_conf)
        del sub_job_components_common_conf['sub-jobs']
        sub_job_concrete_conf = klever.core.utils.merge_confs(sub_job_components_common_conf,
                                                              components_common_conf['sub-jobs'][number])

        job = SubJob(
            core_obj.conf, core_obj.logger, core_obj.ID, core_obj.callbacks, core_obj.mqs,
            vals,
            cur_id='Sub-job-{0}'.format(number),
            work_dir='sub-job-{0}'.format(number),
            attrs=[{
                'name': 'Sub-job identifier',
                'value': str(number),
                # Sub-jobs are intended for combining several relatively small jobs together into one large job. For
                # instance, this abstraction is useful for testing and validation. But most likely most of users do not
                # need even to know about them.
                # From ancient time we tried to assign nice names to sub-jobs to distinguish them, in particular to be
                # able to compare corresponding verification results. These names were based on sub-job configurations,
                # e.g. they included commit hashes, requirement specification identifiers, module names, etc. Such the
                # approach turned out to be inadequate since we had to add more and more information to sub-job names
                # that involves source code changes and results in large working directories that look like these names.
                # After all we decided to use sub-job ordinal numbers to distinguish them uniquely (during a some time
                # old style names were used in addition to these ordinal numbers). The only bad news is that in case of
                # any changes in a global arrangement of sub-jobs, such as a new sub-job is added somewhere in the
                # middle or an old sub-job is removed, one is not able to compare verification results as it was with
                # pretty names since correspondence of ordinal numbers breaks.
                'compare': True,
            }],
            separate_from_parent=True,
            include_child_resources=False,
            components_common_conf=sub_job_concrete_conf
        )

        return job

    core_obj.logger.info('Start job sub-jobs')
    sub_job_solvers_num = klever.core.utils.get_parallel_threads_num(core_obj.logger, components_common_conf,
                                                                     'Sub-jobs processing')
    core_obj.logger.debug('Sub-jobs will be decided in parallel by "{0}" solvers'.format(sub_job_solvers_num))

    subjob_queue = multiprocessing.Queue()
    # Initialize queue first
    core_obj.logger.debug('Initialize workqueue with sub-job identifiers')
    for num in range(len(components_common_conf['sub-jobs'])):
        subjob_queue.put(num)
    subjob_queue.put(None)

    # Then run jobs
    core_obj.logger.debug('Start sub-jobs pull of workers')
    klever.core.components.launch_queue_workers(core_obj.logger, subjob_queue, constructor, sub_job_solvers_num,
                                                components_common_conf['ignore failed sub-jobs'], subcomponents)

SINGLE_ENV_NAME = 'base'

def _vrp_callback(context):
    context.mqs['verification statuses'].put({
        'program fragment id': context.program_fragment_id,
        'environment model': context.envmodel,
        'req spec id': context.req_spec_id,
        'verdict': context.verdict,
        'sub-job identifier': context.conf['sub-job identifier'],
        'ideal verdicts': context.conf['ideal verdicts'],
        'data': context.conf.get('data')
    })

def _vtg_plugin_callback(context):
    context.mqs['verification statuses'].put({
        'program fragment id': context.task.fragment,
        'req spec id': context.task.rule if hasattr(context.task, 'rule') else context.task.rule_class,
        'environment model': context.task.envmodel if hasattr(context.task, 'envmodel') else SINGLE_ENV_NAME,
        'verdict': 'non-verifier unknown',
        'sub-job identifier': context.conf['sub-job identifier'],
        'ideal verdicts': context.conf['ideal verdicts'],
        'data': context.conf.get('data')
    })

class REP(klever.core.components.Component):

    def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, cur_id=None, work_dir=None, attrs=None,
                 separate_from_parent=True, include_child_resources=False, queues_to_terminate=None):
        super().__init__(conf, logger, parent_id, callbacks, mqs, vals, cur_id, work_dir, attrs,
                         separate_from_parent, include_child_resources)
        self.data = {}

        self.mqs['verification statuses'] = multiprocessing.Queue()
        queues_to_terminate.append('verification statuses')
        self.__set_callbacks()

    def process_results_extra(self):
        os.mkdir('results')

        while True:
            verification_status = self.mqs['verification statuses'].get()

            if verification_status is None:
                self.logger.debug('Verification statuses message queue was terminated')
                self.mqs['verification statuses'].close()
                break

            id_suffix, verification_result = self.__match_ideal_verdict(verification_status)
            sub_job_id = verification_status['sub-job identifier']

            if self.conf['extra results processing'] == 'testing':
                # For testing jobs there can be several verification tasks for each sub-job, so for uniqueness of
                # tasks and directories add identifier suffix in addition.
                test_id = os.path.join(sub_job_id, id_suffix)
                self.logger.info('Ideal/obtained verdict for test "%s" is "%s"/"%s"%s',
                    test_id, verification_result['ideal verdict'], verification_result['verdict'],
                    ' ("{0}")'.format(verification_result['comment'])
                    if verification_result['comment'] else '')
                results_dir = os.path.join('results', test_id)
                data = {
                    'type': 'testing',
                    'test': test_id
                }
                data.update(verification_result)
            elif self.conf['extra results processing'] == 'validation':
                # For validation jobs we can't refer to sub-job identifier for additional identification of verification
                # results because of most likely we will consider pairs of sub-jobs before and after corresponding bug
                # fixes.
                bug_id, verification_result = self.__process_validation_results(verification_result,
                                                                                verification_status['data'], id_suffix)
                # For validation jobs sub-job identifiers guarantee uniqueness for naming directories since there is
                # the only verification task for each sub-job.
                results_dir = os.path.join('results', sub_job_id)
                data = {
                    'type': 'validation',
                    'bug': bug_id
                }
                data.update(verification_result)
            else:
                raise NotImplementedError('Extra results processing {!r} is not supported'
                                          .format(self.conf['extra results processing']))

            os.makedirs(results_dir)

            klever.core.utils.report(
                self.logger,
                'patch',
                {
                    'identifier': self.parent_id,
                    'data': data
                },
                self.mqs['report files'],
                self.vals['report id'],
                self.conf['main working directory'],
                results_dir
            )

    main = process_results_extra

    def __set_callbacks(self):

        def after_plugin_fail_processing(context):
            _vtg_plugin_callback(context)

        def after_process_failed_task(context):
            _vrp_callback(context)

        def after_process_single_verdict(context):
            _vrp_callback(context)

        klever.core.components.set_component_callbacks(
            self.logger,
            type(self),
            (
                after_plugin_fail_processing,
                after_process_single_verdict,
                after_process_failed_task
            )
        )

    def __match_ideal_verdict(self, verification_status):
        def match_attr(attr, ideal_attr):
            if ideal_attr and ((isinstance(ideal_attr, str) and attr == ideal_attr) or
                               (isinstance(ideal_attr, list) and attr in ideal_attr)):
                return True

            return False

        program_fragment_id = verification_status['program fragment id']
        req_spec_id = verification_status['req spec id']
        envmodel_id = verification_status.get('environment model')
        ideal_verdicts = verification_status['ideal verdicts']
        comparison_list = [(program_fragment_id, 'program fragments'),
                           (req_spec_id, 'requirements specification'),
                           (envmodel_id, 'environment model')]

        matched_ideal_verdict = None
        ideal_verdicts = sorted(ideal_verdicts, key=lambda x: len(x.keys()), reverse=True)
        for verdicts in ideal_verdicts:
            matched = True
            for value, attribute in comparison_list:
                if attribute in verdicts:
                    matched = match_attr(value, verdicts[attribute])
                    if not matched:
                        break

            if matched:
                matched_ideal_verdict = verdicts
                break

        if not matched_ideal_verdict:
            raise ValueError(
                'Could not match ideal verdict for program fragment "{0}", environment model {1} and requirements '
                'specification "{2}"'.format(program_fragment_id, envmodel_id, req_spec_id))

        # This suffix will help to distinguish sub-jobs easier.
        if envmodel_id == SINGLE_ENV_NAME:
            id_suffix = os.path.join(program_fragment_id, req_spec_id)\
                if program_fragment_id and req_spec_id else ''
        else:
            id_suffix = os.path.join(program_fragment_id, req_spec_id, envmodel_id) \
                if program_fragment_id and req_spec_id and envmodel_id else ''

        return id_suffix, {
            'verdict': verification_status['verdict'],
            'ideal verdict': matched_ideal_verdict['ideal verdict'],
            'comment': matched_ideal_verdict.get('comment')
        }

    def __process_validation_results(self, verification_result, data, id_suffix):
        # Relate verification results on commits before and after corresponding bug fixes if so.
        # Data (variable "self.data") is intended to keep verification results that weren't bound still. For such the
        # results we will need to update corresponding data sent before.

        # Verification results can be bound on the basis of data (parameter "data").
        if not data or 'bug identifier' not in data:
            raise KeyError('Bug identifier is not specified for some sub-job of validation job')

        # Identifier suffix clarifies bug nature without preventing relation of verification results, so, just add it
        # to bug identifier. Sometimes just this concatenation actually serves as unique identifier, e.g. when a bug
        # identifier is just a commit hash, while an identifier suffix contains a program fragment and a requirements
        # specification.
        bug_id = os.path.join(data['bug identifier'], id_suffix)

        bug_verification_result = None
        bug_fix_verification_result = None
        if verification_result['ideal verdict'] == 'unsafe':
            bug_verification_result = verification_result

            if bug_id in self.data:
                bug_fix_verification_result = self.data[bug_id]
        elif verification_result['ideal verdict'] == 'safe':
            bug_fix_verification_result = verification_result

            if bug_id in self.data:
                bug_verification_result = self.data[bug_id]
        else:
            raise ValueError('Ideal verdict is "{0}" (either "safe" or "unsafe" is expected)'
                             .format(verification_result['ideal verdict']))

        validation_status_msg = 'Verdict for bug "{0}"'.format(bug_id)

        new_verification_result = {}

        if bug_verification_result:
            new_verification_result.update({'before fix': bug_verification_result})
            validation_status_msg += ' before fix is "{0}"{1}'.format(
                bug_verification_result['verdict'],
                ' ("{0}")'.format(bug_verification_result['comment'])
                if bug_verification_result['comment']
                else '')

        if bug_fix_verification_result:
            new_verification_result.update({'after fix': bug_fix_verification_result})
            if bug_verification_result:
                validation_status_msg += ','
            validation_status_msg += ' after fix is "{0}"{1}'.format(
                bug_fix_verification_result['verdict'],
                ' ("{0}")'.format(
                    bug_fix_verification_result['comment'])
                if bug_fix_verification_result['comment'] else '')

        self.logger.info(validation_status_msg)

        if bug_id in self.data:
            # We don't need to keep previously obtained verification results since we found both
            # verification results before and after bug fix.
            del self.data[bug_id]
        else:
            # Keep obtained verification results to relate them later.
            self.data.update({bug_id: verification_result})

        return bug_id, new_verification_result

class Job(klever.core.components.Component):
    CORE_COMPONENTS = [
        'PFG',
        'VTG',
        'VRP'
    ]

    def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, cur_id=None, work_dir=None, attrs=None,
                 separate_from_parent=True, include_child_resources=False, components_common_conf=None):
        super().__init__(conf, logger, parent_id, callbacks, mqs, vals, cur_id, work_dir, attrs,
                                  separate_from_parent, include_child_resources)
        self.common_components_conf = components_common_conf

        if work_dir:
            self.common_components_conf['additional sources directory'] = os.path.join(os.path.realpath(work_dir),
                                                                                'additional sources')

        self.clade = None
        self.components = []
        self.component_processes = []

    def decide_job_or_sub_job(self):
        self.logger.info('Decide job/sub-job "%s"', self.id)

        # This is required to associate verification results with particular sub-jobs.
        # Skip leading "/" since this identifier is used in os.path.join() that returns absolute path otherwise.
        self.common_components_conf['sub-job identifier'] = self.id[1:]

        self.logger.info('Get specifications set')
        if 'specifications set' in self.common_components_conf:
            spec_set = self.common_components_conf['specifications set']
        else:
            raise KeyError('Specify attribute "specifications set" within job.json')
        self.logger.debug('Specifications set is "%s"', spec_set)

        # Check that specifications set is supported.
        with open(self.common_components_conf['specifications base'], encoding='utf-8') as fp:
            req_spec_base = json.load(fp)
        spec_set = self.common_components_conf['specifications set']
        if spec_set not in req_spec_base['specification sets']:
            raise ValueError("Klever does not support specifications set {!r} yet, available options are: {}"
                             .format(spec_set, ', '.join(req_spec_base['specification sets'])))

        # Check and set build base here since many Core components need it.
        self.__set_build_base()
        clade_conf = {"log_level": "ERROR"}
        self.clade = Clade(self.common_components_conf['build base'], conf=clade_conf)
        if not self.clade.work_dir_ok():
            raise RuntimeError(f'Build base "{self.common_components_conf["build base"]}" is not OK')

        self.__retrieve_working_src_trees()
        self.__get_original_sources_basic_info()
        self.__upload_original_sources()

        # Create directory where files will be cached and remember absolute path to it for components.
        os.mkdir('cache')
        self.common_components_conf['cache directory'] = os.path.realpath('cache')

        if self.common_components_conf['keep intermediate files']:
            self.logger.debug('Create components configuration file "conf.json"')
            with open('conf.json', 'w', encoding='utf-8') as fp:
                json.dump(self.common_components_conf, fp, ensure_ascii=False, sort_keys=True, indent=4)

        self.__get_job_or_sub_job_components()
        self.callbacks = klever.core.components.get_component_callbacks(self.logger, [type(self)] + self.components)
        self.launch_sub_job_components()

        self.clean_dir = True
        self.logger.info("All components finished")
        if self.conf.get('collect total code coverage', None):
            self.logger.debug('Waiting for a collecting coverage')
            while not self.vals['coverage_finished'].get(self.common_components_conf['sub-job identifier'], True):
                time.sleep(1)
            self.logger.debug("Coverage collected")

    main = decide_job_or_sub_job

    def __set_build_base(self):
        if 'build base' not in self.common_components_conf:
            raise KeyError("Provide 'build base' configuration option to start verification")

        common_advice = 'please, fix "job.json" (attribute "build base")'
        common_advice += ' or/and deployment configuration file (attribute "Klever Build Bases")'

        # Try to find specified build base either in normal way or additionally in directory "build bases" that is
        # convenient to use when working with many build bases.
        try:
            build_base = klever.core.utils.find_file_or_dir(self.logger,
                                                            self.common_components_conf['main working directory'],
                                                            self.common_components_conf['build base'])
        except FileNotFoundError:
            self.logger.warning('Failed to find build base:\n%s', traceback.format_exc().rstrip())
            try:
                build_base = klever.core.utils.find_file_or_dir(
                    self.logger, self.common_components_conf['main working directory'],
                    os.path.join('build bases', self.common_components_conf['build base']))
            except FileNotFoundError:
                self.logger.warning('Failed to find build base:\n%s', traceback.format_exc().rstrip())
                raise FileNotFoundError(
                    'Specified build base "{0}" does not exist, {1}'.format(self.common_components_conf['build base'],
                                                                            common_advice)) from None

        # Extract build base from archive. There should not be any intermediate directories in archives.
        if os.path.isfile(build_base) and (tarfile.is_tarfile(build_base) or zipfile.is_zipfile(build_base)):
            if tarfile.is_tarfile(build_base):
                self.logger.debug('Build base "%s" is provided in form of TAR archive', build_base)
                with tarfile.open(build_base) as TarFile:
                    TarFile.extractall('build base')
            else:
                self.logger.debug('Build base "%s" is provided in form of ZIP archive', build_base)
                with zipfile.ZipFile(build_base) as zfp:
                    zfp.extractall('build base')

            # Directory contains extracted build base.
            extracted_from = ' extracted from "{0}"'.format(os.path.realpath(build_base))
            build_base = 'build base'
        else:
            extracted_from = ''

        # We need to specify absolute path to build base since it will be used in different Klever components. Besides,
        # this simplifies troubleshooting.
        build_base = os.path.realpath(build_base)

        # TODO: fix after https://github.com/17451k/clade/issues/108.
        if not os.path.isdir(build_base):
            raise FileExistsError('Build base "{0}" {1} is not a directory, {2}'
                                  .format(build_base, extracted_from, common_advice))

        if not os.path.isfile(os.path.join(build_base, 'meta.json')):
            raise FileExistsError(
                'Directory "{0}"{1} is not a build base since it does not contain file "meta.json", {2}'
                .format(build_base, extracted_from, common_advice))

        self.common_components_conf['build base'] = build_base

        self.logger.debug('Klever components will use build base "%s"'
                          , self.common_components_conf['build base'])

    # Klever will try to cut off either working source trees (if specified) or maximum common paths of CC/CL input files
    # and LD/Link output files (otherwise) from referred file names. Sometimes this is rather optional like for source
    # files referred by error traces, but, say, for program fragment identifiers this is strictly necessary, e.g.
    # because of otherwise expert assessment will not work as expected.
    def __retrieve_working_src_trees(self):
        clade_meta = self.clade.get_meta()

        # Best of all if users specify working source trees in build bases manually themselves. It is a most accurate
        # approach.
        if 'working source trees' in clade_meta:
            work_src_trees = clade_meta['working source trees']
        # Otherwise try to find out them automatically as described above.
        else:
            in_files = []
            for cmd in self.clade.get_all_cmds_by_type("CC") + self.clade.get_all_cmds_by_type("CL"):
                if cmd['in']:
                    for in_file in cmd['in']:
                        # Sometimes some auxiliary stuff is built in addition to normal C source files that are most
                        # likely located in a place we would like to get.
                        if not in_file.startswith('/tmp') and in_file != '/dev/null':
                            in_files.append(os.path.join(cmd['cwd'], in_file))
            in_files_prefix = os.path.dirname(os.path.commonprefix(in_files))
            self.logger.info('Common prefix of CC/CL input files is "%s"', in_files_prefix)

            out_files = []
            for cmd in self.clade.get_all_cmds_by_type("LD") + self.clade.get_all_cmds_by_type("Link"):
                if cmd['out']:
                    for out_file in cmd['out']:
                        # Like above.
                        if not out_file.startswith('/tmp') and out_file != '/dev/null':
                            out_files.append(os.path.join(cmd['cwd'], out_file))
            out_files_prefix = os.path.dirname(os.path.commonprefix(out_files))
            self.logger.info('Common prefix of LD/Link output files is "%s"', out_files_prefix)

            # Meaningful paths look like "/dir...".
            meaningful_paths = []
            for path in (in_files_prefix, out_files_prefix):
                if path and path != os.path.sep and path not in meaningful_paths:
                    meaningful_paths.append(path)

            if meaningful_paths:
                work_src_trees = meaningful_paths
            # At least consider build directory as working source tree if the automatic procedure fails.
            else:
                self.logger.warning(
                    'Consider build directory "%s" as working source tree.'
                    'This may be dangerous and we recommend to specify appropriate working source trees manually!'
                    , clade_meta['build_dir'])
                work_src_trees = [clade_meta['build_dir']]

        # Consider minimal path if it is common prefix for other ones. For instance, if we have "/dir1/dir2" and "/dir1"
        # then "/dir1" will become the only working source tree.
        if len(work_src_trees) > 1:
            min_work_src_tree = min(work_src_trees)
            if os.path.commonprefix(work_src_trees) == min_work_src_tree:
                work_src_trees = [min_work_src_tree]

        self.logger.info(
            'Working source trees to be used are as follows:\n%s'
            , '\n'.join(['  {0}'.format(t) for t in work_src_trees]))
        self.common_components_conf['working source trees'] = work_src_trees

    def __refer_original_sources(self, src_id):
        klever.core.utils.report(
            self.logger,
            'patch',
            {
                'identifier': self.id,
                'original_sources': src_id
            },
            self.mqs['report files'],
            self.vals['report id'],
            self.conf['main working directory']
        )

    def __process_source_files(self):
        for file_name in self.clade.src_info:
            self.mqs['file names'].put(file_name)

        for _ in range(self.workers_num):
            self.mqs['file names'].put(None)

    def __process_source_file(self):
        while True:
            file_name = self.mqs['file names'].get()

            if not file_name:
                return

            src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'],
                                                                 file_name)

            if src_file_name != file_name:
                src_file_name = os.path.join('source files', src_file_name)

            new_file_name = os.path.join('original sources', src_file_name.lstrip(os.path.sep))
            os.makedirs(os.path.dirname(new_file_name), exist_ok=True)
            shutil.copy(self.clade.get_storage_path(file_name), new_file_name)

            cross_refs = CrossRefs(self.common_components_conf, self.logger, self.clade,
                                   file_name, new_file_name,
                                   self.common_components_conf['working source trees'], 'source files')
            cross_refs.get_cross_refs()

    def __get_original_sources_basic_info(self):
        self.logger.info('Get information on original sources for following visualization of uncovered source files')

        # For each source file we need to know the total number of lines and places where functions are defined.
        src_files_info = {}
        for file_name, file_size in self.clade.src_info.items():
            src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'],
                                                                 file_name)

            # Skip non-source files.
            if src_file_name == file_name:
                continue

            src_file_name = os.path.join('source files', src_file_name)

            src_files_info[src_file_name] = []

            # Store source file size.
            src_files_info[src_file_name].append(file_size['loc'])

            # Store source file function definition lines.
            func_def_lines = []
            funcs = self.clade.get_functions_by_file([file_name], False)

            if funcs:
                for _, func_info in list(funcs.values())[0].items():
                    func_def_lines.append(int(func_info['line']))

            src_files_info[src_file_name].append(sorted(func_def_lines))

        # Dump obtain information (huge data!) to load it when reporting total code coverage if everything will be okay.
        with open('original sources basic information.json', 'w') as fp:
            klever.core.utils.json_dump(src_files_info, fp, self.conf['keep intermediate files'])

    def __upload_original_sources(self):
        # Use Clade UUID to distinguish various original sources. It is pretty well since this UUID is uuid.uuid4().
        src_id = self.clade.get_uuid()
        # In addition, take into account a meta content as we like to change it manually often. In this case it may be
        # necessary to re-index the build base. It is not clear if this is the case actually, so, do this in case of
        # any changes in meta.
        src_id += '-' + klever.core.utils.get_file_name_checksum(json.dumps(self.clade.get_meta()))[:12]

        session = klever.core.session.Session(self.logger, self.conf['Klever Bridge'], self.conf['identifier'])

        if session.check_original_sources(src_id):
            self.logger.info('Original sources were uploaded already')
            self.__refer_original_sources(src_id)
            return

        self.logger.info(
            'Cut off working source trees or build directory from original source file names and convert index data')
        os.makedirs('original sources')
        self.mqs['file names'] = multiprocessing.Queue()
        self.workers_num = klever.core.utils.get_parallel_threads_num(self.logger, self.conf)
        subcomponents = [('PSFS', self.__process_source_files)]
        for _ in range(self.workers_num):
            subcomponents.append(('PSF', self.__process_source_file))
        self.launch_subcomponents(False, *subcomponents)
        self.mqs['file names'].close()

        self.logger.info('Compress original sources')
        klever.core.utils.ArchiveFiles(['original sources']).make_archive('original sources.zip')

        self.logger.info('Upload original sources')
        try:
            session.upload_original_sources(src_id, 'original sources.zip')
        # Do not fail if there are already original sources. There may be complex data races because of checking and
        # uploading original sources archive are not atomic.
        except klever.core.session.BridgeError:
            if "original sources with this identifier already exists." not in list(session.error.values())[0]:
                raise

        self.__refer_original_sources(src_id)

        if not self.conf['keep intermediate files']:
            shutil.rmtree('original sources')
            os.remove('original sources.zip')

    def __get_job_or_sub_job_components(self):
        self.logger.info('Get components for sub-job "%s"', self.id)

        self.components = [getattr(importlib.import_module('.{0}'.format(component.lower()), 'klever.core'), component)
                           for component in self.CORE_COMPONENTS]

        self.logger.debug('Components to be launched: "%s"',
            ', '.join([component.__name__ for component in self.components]))

    def launch_sub_job_components(self):
        """Has callbacks"""
        self.logger.info('Launch components for sub-job "%s"', self.id)

        for component in self.components:
            p = component(self.common_components_conf, self.logger, self.id, self.callbacks, self.mqs,
                          self.vals, separate_from_parent=True)
            self.component_processes.append(p)

        klever.core.components.launch_workers(self.logger, self.component_processes)

class SubJob(Job):

    def decide_sub_job(self):
        try:
            self.decide_job_or_sub_job()
            self.vals['subjobs progress'][self.id] = 'finished'
        except Exception:
            self.vals['subjobs progress'][self.id] = 'failed'
            raise

    main = decide_sub_job


# -*- coding: utf-8 -*-

""" this is a utility to make calls into the header and footer server, and return
    strings representing the header and footer...
"""
import sys
import requests
from repoze.lru import lru_cache

import logging
log = logging.getLogger(__file__)

def clean_str(s):
    ret_val = decode(s)
    return ret_val

def decode(str, codec='utf-8'):
    try:
        ret_val = str.decode(codec)
    except:
        ret_val = str
    return ret_val

def encode(str, codec='utf-8'):
    try:
        ret_val = str.encode(codec)
    except:
        ret_val = str
    return ret_val

def append_get_param(params, param_name, param_val):
    """ append a new url GET param to a string of such params, ala "?x=VAL&y=VAL&etc=..."
        NOTE: stupid crap you have to jump thru for utf-8 strings
    """
    if params is None:
        params = u""
    ret_val = params

    if param_name and param_val:
        ret_val = u"{}&{}={}".format(params, param_name, decode(param_val.replace(" ", "%20")))
    return ret_val

def wget_stuff(domain, port, path, is_mobile, params, def_val=""):
    ret_val = def_val

    if is_mobile:
        path = u"m/{}".format(path)

    url = u"http://{}:{}/{}?client_utils{}".format(domain, port, path, params)
    log.info(url)
    response = requests.get(url)
    html = response.text
    if html and len(html) > 0:
        ret_val = html.strip()

    return ret_val

def wget_header(domain="localhost", port="14441", path="header.html", is_mobile=False,
                title=None, header=None, sub_header=None, second_header=None,
                icon_cls=None, icon_url=None,
                onload=None,
                def_val=""):
    """ utility class curl a page header from the system
    """
    params = u""
    if title: params = append_get_param(params, 'title', title)
    if header: params = append_get_param(params, 'header', header)
    if sub_header: params = append_get_param(params, 'sub_header', sub_header)
    if second_header: params = append_get_param(params, 'second_header', second_header)
    if icon_cls: params = append_get_param(params, 'icon_cls', icon_cls)
    if icon_url: params = append_get_param(params, 'icon_url', icon_url)
    if onload: params = append_get_param(params, 'onload', onload)
    html = wget_stuff(domain, port, path, is_mobile, params, def_val)
    html = decode(html)
    return html

def wget_footer(domain="localhost", port="14441", path="footer.html", is_mobile=False, def_val=""):
    """ utility class curl a page header from the system
    """
    params = ""
    html = wget_stuff(domain, port, path, is_mobile, params, def_val)
    return html

@lru_cache(10000, timeout=600)
def cached_wget_header(domain="localhost", port="14441", path="header.html", is_mobile=False,
                title=None, header=None, sub_header=None, second_header=None,
                icon_cls=None, icon_url=None,
                onload=None,
                def_val=""):
    """ http://docs.repoze.org/lru/api.html#repoze.lru.lru_cache """
    return wget_header(domain, port, path, is_mobile, title, header, sub_header, second_header, icon_cls, icon_url, onload, def_val)

@lru_cache(10000, timeout=600)
def cached_wget_footer(domain="localhost", port="14441", path="footer.html", is_mobile=False, def_val=""):
    return wget_footer(domain, port, path, is_mobile, def_val)

def main():
    if len(sys.argv) >= 2:
        html = wget_header(header=sys.argv[1])
    else:
        html = wget_header()
    print(html)

if __name__ == '__main__':
    main()


import pytest

from lord import scrap, loader
from lord import Hero, Jogo, Baralho
import lord

@pytest.fixture
def jogo_com_um_jogador():
    jogo = Jogo()
    jogo.novo_jogador('Paulo')
    return jogo

@pytest.fixture
def deck_de_missao():
    # http://hallofbeorn.com/LotR/Scenarios/A-Shadow-of-the-Past
    texto = """
    Setup: Set Buckleberry Ferry aside, out of play. Add 1 Black Rider to the
    staging area and make Bag End the active location. Shuffle the encounter
    deck.
    """
    deck = Baralho()
    deck.nova_carta(lord.Mission('Three is Company',texto))
    deck.nova_carta(lord.Mission('A Shortcut to Mushrooms'))
    deck.nova_carta(lord.Mission('Escape to Buckland'))
    return deck

@pytest.fixture
def deck_de_encontro():
    deck = Baralho()
    deck.nova_carta(lord.Card('A Shadow of the Past'))
    deck.nova_carta(lord.Card('Bag End'))
    deck.nova_carta(lord.Card('Woody End'))
    deck.nova_carta(lord.Card('Stock-Brook'))
    deck.nova_carta(lord.Card('Pathless Country'))
    deck.nova_carta(lord.Card('Buckleberry Ferry'))
    deck.nova_carta(lord.Card('Black Rider'))
    deck.nova_carta(lord.Location('Bag End',0,3,vitoria = 1))
    return deck

@pytest.fixture
def deck_dict():
    d = {
        'id': 20040,
        'name': 'Four Hobbitsesss',
        'date_creation': '2021-03-22T14:47:32-03:00',
        'date_update': '2021-03-22T14:47:32-03:00',
        'description_md': '',
        'user_id': 9451,
        'heroes': {'10001': 1, '17109': 1, '19112': 1, '21002': 1},
        'slots': {
            '01023': 2,
            '01036': 2,
            '01045': 1,
            '01049': 1,
            '01050': 2,
            '01057': 2,
            '01073': 2,
            '02103': 2,
            '06010': 2,
            '06065': 1,
            '08087': 2,
            '09014': 1,
            '10001': 1,
            '10007': 2,
            '10034': 1,
            '12035': 1,
            '17002': 2,
            '17109': 1,
            '19112': 1,
            '19115': 1,
            '19116': 2,
            '19117': 2,
            '19121': 2,
            '21002': 1,
            '22005': 2,
            '22006': 2,
            '22110': 2,
            '22142': 1,
            '22147': 1,
            '141006': 1,
            '141008': 1,
            '142003': 1,
            '142005': 1,
            '144005': 2,
            '146005': 2,
            '146012': 2
        },
        'sideslots': {
            '01034': 2,
            '19149': 2,
            '21001': 1,
            '21009': 2,
            '21010': 2,
            '22033': 1
        },
        'version': '1.0',
        'freeze_comments': None,
        'is_published': True,
        'nb_votes': 0,
        'nb_favorites': 0,
        'nb_comments': 0,
        'starting_threat': 25
    }
    return d


a = ['a','b','c','d']

# delete o eliminar una lista o diccionario
del a[0]
print(a)

del a[:1] # desde el inicio hasta la posicion 1
print(a)

# diccionarios
d = {'uno':1, 'dos':2}
del d['dos']
print(d)

del d


import boto3
import json
import lambda_variables

#Get the dynamodb resource and table
dynamodb = boto3.resource(lambda_variables.database)
table    = dynamodb.Table(lambda_variables.table['Parameter']['Value'])

#Initializes and increments the visitor counter value by 1
def update_visitor_counter():
    response = table.update_item(Key={lambda_variables.hash_key_column['Parameter']['Value']: lambda_variables.hash_key_value['Parameter']['Value']},
    AttributeUpdates={
            lambda_variables.hash_key_count['Parameter']['Value']: {
                'Value': 1,
                'Action': 'ADD'
            }
        }
    )

#This function calls update_visitor_counter() which initializes/increments the visitor counter,
# then returns the current visitor counter value
def get_visitor_counter(event, context):
    update_visitor_counter()

    response = table.get_item(Key={lambda_variables.hash_key_column['Parameter']['Value']: lambda_variables.hash_key_value['Parameter']['Value']})

    apiResponse = {
        "isBase64Encoded": False,
        "statusCode": 200,
        'headers': {
            'Access-Control-Allow-Headers': 'content-type',
            'Access-Control-Allow-Origin': '*',
            'Access-Control-Allow-Methods': 'OPTIONS, POST, GET'
        },
        "body": response['Item'].get(lambda_variables.hash_key_count['Parameter']['Value'])
    }
    return apiResponse

#Calls get_visitor_counter() function to update and return the visitor counter
if __name__ == '__main__':
    get_visitor_counter(event, context)


import click, google, yaml, sys
import hlcloud.config

@click.command(short_help="add or update bucket labels")
@click.argument('url', type=click.STRING)
@click.option('--user', '-u', type=click.STRING, required=False, help="User to use in bucket labels. Default is current logged in username.")
@click.option('--project', '-p', type=click.STRING, required=True, help="Project to use in bucket labels.")
@click.option('--pipeline', '-l', type=click.STRING, required=True, help="Pipeline to use in bucket labels.")
def buckets_update_labels_cmd(url, user, project, pipeline):
    """
    Add or update labels on a bucket according to Hall Lab Cloud Policies
    """
    sys.stderr.write("Update bucket labels...\n")
    if not user:
        user = hlcloud.config.HLCConfig.get_user()

    storage = google.cloud.storage.Client()
    bucket = storage.get_bucket(url)
    if not bucket:
        raise Exception("No bucket for {}".format(url))

    labels = bucket.labels
    if labels is None:
        labels = dict()

    new_labels = {
        "user": user,
        "project": project,
        "pipeline": pipeline,
    }
    if new_labels == labels:
       sys.stderr("New and current bucket labels are the same, not updating.\n")
       return

    labels.update(new_labels)
    bucket.labels = labels
    sys.stderr.write("New labels:\n{}".format(yaml.dump(bucket.labels)))
    bucket.update()
    sys.stderr.write("Update bucket labels...SUCCESS\n")


from openpyxl import load_workbook
from collections import namedtuple
from scripts.loggerhandler import do_logger
from scripts.confighandler import do_config
from scripts.constants import CASE_DATA_DIR
from scripts.ReHandler import ReText

class ExcelHandler(object):
    """
    封装读写excel的操作
    """
    def __init__(self, filename, sheetname='', max_col=do_config('excel', 'max_column')):
        """
        初始化
        :param filename: excel文件路径名
        :param sheetname: 需要定位的表单名
        :param max_col:  读取的最大列数（扣除待写的列）
        """
        self.filename = filename  # 从常量文件中读取excel绝对路径
        self.sheetname = sheetname
        self.data_list = []
        self.wb = load_workbook(self.filename)
        self.ws = self.wb[sheetname] if sheetname != '' else self.wb.active
        self.max_col = max_col
        self.data_header = tuple(self.ws.iter_rows(max_row=1, max_col=self.max_col, values_only=True))[0]
        self.Cases = namedtuple("Cases", self.data_header)

    def write_excel(self, row, *data_tuple):
        """
        写入excel
        :param row: 单元格的行号 int
        :param data_tuple: 待写入的值
        """
        other_wb = load_workbook(self.filename)
        other_ws = other_wb[self.sheetname]
        if isinstance(row, int) and (1 < row <= other_ws.max_row):
            try:
                columns = do_config('excel', 'columns', iseval=True)  # 读取配置文件中单元格的列号，类型可以为int，list，tuple，''
            except SyntaxError:  # columns选项值为空时或者类型不正确时，默认写入列号为max_col的后两个
                if len(data_tuple) != 2:
                    do_logger.error('写入的列数与值的个数不匹配，写入失败！')
                    raise IndexError('tuple index out of range')
                other_ws.cell(row, self.max_col+1, value=data_tuple[0])
                other_ws.cell(row, self.max_col+2, value=data_tuple[1])
                other_wb.save(self.filename)
            else:
                if isinstance(columns, (list, tuple)):
                    sort_columns = list(set(columns))
                    sort_columns.sort()
                    if len(data_tuple) != len(sort_columns):
                        do_logger.error('写入的列数与值的个数不匹配，写入失败！')
                        raise IndexError('tuple index out of range')
                    for i in sort_columns:
                        other_ws.cell(row, i, value=data_tuple[sort_columns.index(i)])
                    other_wb.save(self.filename)
                elif isinstance(columns, int):
                    other_ws.cell(row, columns, value=data_tuple[0])
                    other_wb.save(self.filename)
        else:
            do_logger.error('传入的行号错误，行号必须是大于1的整数！写入失败！')

    def read_excel(self):
        """
        读取excel
        :return:嵌套命名元组的列表
        """
        for tuple_data in self.ws.iter_rows(min_row=2, max_col=self.max_col, values_only=True):
            self.data_list.append(self.Cases(*tuple_data))
        return self.data_list

# sheetnames = do_config('excel', 'sheet_name', iseval=True)    # 处理多个表单的情况下，实例化多个ExcelHandler对象
# register_excel = ExcelHandler(sheetname=sheetnames[0], max_col=do_config('excel', 'max_column'))
# login_excel = ExcelHandler(sheetname=sheetnames[1], max_col=do_config('excel', 'max_column'))

if __name__ == '__main__':
    # a = register_excel.read_excel()
    # b = ReText().register_parametrization(a[0].data)
    # print(eval(b))
    pass


import pygame
from pygame.sprite import Sprite

class Alien(Sprite):
    def __init__(self, ai_settings, screen):
        """Initialise alien and set startng position."""
        super(Alien, self).__init__()
        self.screen = screen
        self.ai_settings = ai_settings

        # Load alien image and set its attribute
        self.image = pygame.image.load('images/alien.bmp')
        self.rect = self.image.get_rect()

        # Start each alien at top left of screen
        self.rect.x = self.rect.width
        self.rect.y = self.rect.height

        # Store the alien's exact position.
        self.x = float(self.rect.x)

    def blitme(self):
        """Draw alien at current location"""
        self.screen.blit(self.image, self.rect)


#!/usr/bin/env python3

# file: ignore.py

"""
This script prints to STDOUT the contents of
    .gitignore  and
    .git/info/exclude.
"""
files2ignore = ['.gitignore', '.git/info/exclude']
for file2ignore in files2ignore:
    print("=======  {}  ==========".format(file2ignore))
    with open(file2ignore, "r") as f:
        for line in f:
            line = line.strip()
            if line and line[0]!='#':
                print('\t' + line)


# Django imports
from django.utils import timezone
from django.shortcuts import render
from django.http import JsonResponse

from apps.backoffice.views.employees import employees
# App imports
from ..services import query_date_range, graph_first_entry, income_per_service, income_per_employee
from ...utils.permissions import admin_member_required

@admin_member_required
def dashboard(request):
    """Render dashboard home page"""
    # Query the database to get data from all appointments from last 30 days
    income, sales, new_customers = query_date_range(request, timezone.now() - timezone.timedelta(days=29), timezone.now())  # This goes to the summary

    # Graph's first entry
    day_summary = graph_first_entry(request)
    graph_months = list(day_summary.keys())
    graph_sales = list(day_summary.values())

    # Services summary
    service_summary = income_per_service(request)
    # Employee summary
    employee_summary = income_per_employee(request)

    context = {
        'user': request.user,
        'income': round(income, 2),
        'sales': sales,
        'new_customers': new_customers,
        'graph_months': graph_months,
        'graph_sales': graph_sales,
        'service_summary': service_summary,
        'employee_summary': employee_summary,
    }
    return render(request, 'admin/dashboard.html', context)

def date_filter_dashboard(request):
    """Returns data filtered by determined date range"""
    # Get date range from the frontend
    start_date = request.GET['startDate']
    end_date = request.GET['endDate']

    # Query the database to get data from all appointments from date range input by user
    income, sales, new_customers = query_date_range(request, start_date, end_date)

    return JsonResponse({
        'income': round(income, 2),
        'sales': sales,
        'new_customers': new_customers
    })

def date_filter_graph(request):
    """returned filtered data for graph any date range"""
    day_summary = graph_first_entry(request)
    return JsonResponse(day_summary)


def mean(sample):
    return sum(sample) / len(sample)

def rsi_calc(close, time_period=6):
    # time_period looks back period to compute gains & losses
    gain_history = []  # history of gains over look back period (0 if no gain, magnitude of gain if gain)
    loss_history = []  # history of losses over look back period (0 if no loss, magnitude of loss if loss)
    avg_gain_values = []  # track avg gains for visualization purposes
    avg_loss_values = []  # track avg losses for visualization purposes
    rsi_values = []  # track computed RSI values
    last_price = 0  # current_price - last_price > 0 => gain. current_price - last_price < 0 => loss.

    for close_price in close:
        if last_price == 0:
            last_price = close_price

        gain_history.append(max(0, close_price - last_price))
        loss_history.append(max(0, last_price - close_price))
        last_price = close_price

        if len(gain_history) > time_period:  # maximum observations is equal to lookback period
            del (gain_history[0])
            del (loss_history[0])

        avg_gain = mean(gain_history)  # average gain over lookback period
        avg_loss = mean(loss_history)  # average loss over lookback period

        avg_gain_values.append(avg_gain)
        avg_loss_values.append(avg_loss)

        rs = 0
        if avg_loss > 0:  # to avoid division by 0, which is undefined
            rs = avg_gain / avg_loss
        elif avg_loss == 0 and avg_gain > 0:
            rs = 100
        else:
            rs = 50

        rsi = 100 - (100 / (1 + rs))
        rsi_values.append(rsi)
        # print(rsi_values)
    rsi_values[:time_period] = [50] * time_period
    return rsi_values

def macd_func(close, short=8, long=22):
    # Get the long EMA of the closing price
    k = close.ewm(span=short, adjust=False, min_periods=short).mean()

    # Get the short EMA of the closing price
    d = close.ewm(span=long, adjust=False, min_periods=long).mean()

    # Subtract the long EMA from the short EMA to get the MACD
    macd = k - d
    macd.fillna(0, inplace=True)

    return macd