from django.conf import settings
import solr

class SolrSearch(object):
    """
        This class is a helper class for translating between query parameters in a GET
        request and the format needed to search in Solr.

        It has three main methods: search, facets, and group_search.

        The search method performs a search. The `parse_request` and `prepare_query` methods
        are automatically called with the request object when the class is initialized. This
        filters all the query keys and translates them to Solr.

        The facets method requests facets from the Solr search server.

        The group_search method performs a search, but can be used to group results on any
        given field.

        The private methods in this class (ones beginning in underscores) are helpers that
        do all the work.

    """
    def __init__(self, request):
        self.server = solr.Solr(settings.SOLR_SERVER)
        self.request = request
        self.parsed_request = {}
        self.prepared_query = u""
        self.solr_params = {}
        self._parse_request()
        self._prepare_query()

    def search(self, **kwargs):
        self.solr_params.update(kwargs)
        res = self._do_query()
        return res

    def facets(self, facet_fields, **kwargs):
        facet_params = {
            'facet': 'true',
            'facet_field': facet_fields,
            'facet_mincount': 1
        }
        self.solr_params.update(facet_params)
        self.solr_params.update(kwargs)

        res = self._do_query()
        return res

    def group_search(self, group_fields, **kwargs):
        group_params = {
            'group': 'true',
            'group_ngroups': 'true',
            'group_field': group_fields
        }
        self.solr_params.update(group_params)
        self.solr_params.update(kwargs)

        res = self._do_query()
        return res

    def _do_query(self):
        return self.server.select(self.prepared_query, **self.solr_params)

    def _parse_request(self):
        qdict = self.request.GET
        for k, v in qdict.lists():
            self.parsed_request[k] = v

    def _prepare_query(self):
        if self.parsed_request:
            arr = []
            for k, v in self.parsed_request.iteritems():
                if not v:
                    continue
                if k == 'q':
                    if v[0] != u"":
                        arr.insert(0, u"{0}".format(v[0]))
                else:
                    arr.append(u"{0}:({1})".format(k, " OR ".join(["\"{0}\"".format(s) for s in v if v is not None])))
            self.prepared_query = u" AND ".join(arr)
        else:
            self.prepared_query = u"*:*"


from flask import Flask, render_template
aiml = Flask(__name__)

list_of_projects = [
    'Mad Libs Generator',
    'Number Guessing',
    'Text-based Adventure Game',
    'Dice Rolling Simulator',
    'Hangman',
    'Contact Book',
    'Email Slicer',
    'Binary search algorithm',
    'Desktop Notifier App',
    'Python Story Generator',
    'YouTube video downloader',
    'Python Website Blocker',
    'Spin a Yarn',
    'What’s the word',
    'Rock, Paper, Scissors',
    'Leap it!',
    'Find out, Fibonacci!',
    'Calculator',
    'Countdown Clock and Timer',
    'Random Password Generator',
    'Random Wikipedia Article',
    'Reddit Bot',
    'Python Command-Line Application',
    'Alarm Clock',
    'Tic-Tac-Toe',
    'Steganography',
    'Currency converter',
    'Post-it Notes',
    'Site Connectivity Checker',
    'Directory Tree Generator',
    'Speed Typing Test',
    'Content Aggregator',
    'Bulk File Rename/ Image Resize Application',
    'Python File Explorer',
    'Plagiarism Checker',
    'Web Crawler',
    'Music Player',
    'Price Comparison Extension',
    'Expense Tracker',
    'Regex Query Tool',
    'Instagram Photo Downloader',
    'Quiz Application'
]

@aiml.route("/")
@aiml.route("/home")
def home():
    return render_template("home.html", t_home="BlueSky_Home")

@aiml.route("/about me")
def about():
    return render_template("about me.html", t_about="about BlueSky")

@aiml.route("/projects")
def projects():
    return render_template("projects.html", t_project="BlueSky Projects", l_pro = list_of_projects, len_list=len(list_of_projects))

@aiml.route("/contact")
def contact():
    return render_template("contact.html")

if __name__ == '__main__':
    aiml.run(debug=True)


perguntas = {
    'Pergunta Atividades - 1': {
        'pergunta': 'Quanto é 2 + 2?',
        'respostas': {
            'a': 'Atividades - 1',
            'b': '4',
            'c': '8'
        },
        'resposta_certa': 'b',
    },
    'Pergunta 2': {
        'pergunta': 'Quanto é 3 * 2?',
        'respostas': {
            'a': '8',
            'b': '12',
            'c': '6'
        },
        'resposta_certa': 'c',
    },
}

respostas_certas = 0
for pk, pv in perguntas.items():
    print(f'{pk}: {pv["pergunta"]}')

    print('Escolha a opção correta abaixo.')
    for rk, rv in pv['respostas'].items():
        print(f'[{rk}]: {rv}')

    resposta_usuario = input('Sua resposta: ')

    if resposta_usuario.lower() == pv['resposta_certa']:
        print(f'EHHHHHHH! Você acerto')
        respostas_certas += 1
    else:
        print('IXIIIIIII! Você errou!')
    print()

qtd_perguntas = len(perguntas)
porcentagem_acerto = respostas_certas / qtd_perguntas * 100

print(f'Voce acertou {respostas_certas} pergunta.')
print(f'Sua porcentagem de acerto foi {porcentagem_acerto}%')


import torch
import torch.nn as nn
import torch.nn.functional as F
from math import log

class mmTensor():
    """
    Object stores a center, an upper offset and a lower offset.
    """
    def __init__(self, input, device=None):
        self.device = device
        self.masks = None
        """
        self.__data is a concatenation of 3 equally sized matrices:
         __min <= __center <= __max
        """
        if isinstance(input, torch.Tensor):
            self.__min = torch.clone(input)
            self.__center = input
            self.__max = torch.clone(input)
            self.__batch_size = input.shape[0]
        elif isinstance(input, tuple) and len(input) == 3:
            x_min, center, x_max = input
            if type(x_min) != torch.Tensor or type(x_max) != torch.Tensor or type(center) != torch.Tensor:
                raise ValueError("Tuple must contain three tensors")
            if x_min.shape != x_max.shape or x_min.shape != center.shape:
                raise ValueError("Tuple must contain three tensors of the same size")
            self.__batch_size = center.shape[0]
            self.__min, self.__center, self.__max = input
        else:
            raise ValueError("\"input\" must be a tensor or a tuple of two (lower, upper) or three (down offset, center, up offset) tensors")

    def lower(self):
        return self.__min

    def center(self):
        return self.__center

    def upper(self):
        return self.__max

    def data(self):
        return self.__min, self.__center, self.__max

    def batch_size(self):
        return self.__batch_size

    def shape(self):
        return (self.__batch_size, ) + self.__center.shape[1:]

    def add_mask(self, lower_mask, upper_mask):
        self.__min = self.__center - lower_mask
        self.__max = self.__center + upper_mask
        return self

    def __iadd__(self, other):
        other_min, other_center, other_max = other.data()
        self.__min += other_min
        self.__center += other_center
        self.__max += other_max
        return self

    def add_rand_range(self, range_width):
        self.__min -= torch.rand(size=self.shape(),device=self.device) * (range_width / 2)
        self.__max += torch.rand(size=self.shape(),device=self.device) * (range_width / 2)
        return self

    def get_range(self):
        return torch.abs(self.lower() - self.upper()).mean().item()

    def scale_range(self, factor):
        if factor != 1:
            self.__min += (self.__center - self.__min) * (1-factor)
            self.__max += (self.__center - self.__max) * (1-factor)

    def cpu(self):
        return mmTensor((self.lower().cpu(), self.center().cpu(), self.upper().cpu()))

def _relu_process(self, input):
    scale = None
    if hasattr(self, 'autoscale_relu'):
        scale = 1
        with torch.no_grad():
            allowed = input.center().numel() * self.autoscale_relu
            L = input.center() / (input.center() - input.lower())
            U = input.center() / (input.center() - input.upper())
            distances = F.relu(L * (L < 1)) + F.relu(U * (U < 1))

            overcrossed = int(torch.sum(distances > 0).item() - allowed)
            if overcrossed > 0:
                scale, index = torch.kthvalue(-distances.flatten(), overcrossed)
                scale = -scale.item()
        input.scale_range(scale)

    if hasattr(self, 'debug_relu'):
        relus_crossed = torch.sum(torch.logical_and(input.lower() <= 0, input.upper() >= 0)).item()
        relus_total = input.center().numel()
        print(type(self).__name__.ljust(16), str(relus_crossed).rjust(16), "/", str(relus_total).ljust(10), "=", "{:.4f}%".format(100 * relus_crossed / relus_total), " - Scale: ", scale)

def _layer_print(self, out, relu=False):
    if hasattr(self, 'debug_range'):
        r = log(out.get_range())
        print(type(self).__name__.ljust(16), r)

def _layer_postprocess(self, out):
    if hasattr(self, 'scale_factor'):
        out.scale_range(self.scale_factor)
    return out

class Conv2d(nn.Conv2d):
    def forward(self, input):
        W_p = F.relu(self.weight)
        W_n = F.relu(-self.weight)

        min_p = self._conv_forward(input.lower(), W_p, self.bias)
        min_n = self._conv_forward(input.lower(), W_n, None)
        max_p = self._conv_forward(input.upper(), W_p, self.bias)
        max_n = self._conv_forward(input.upper(), W_n, None)
        x = self._conv_forward(input.center(), self.weight, self.bias)

        out = mmTensor((min_p - max_n, x, max_p - min_n))
        _layer_print(self, out)
        return _layer_postprocess(self, out)

class Linear(nn.Linear):
    def forward(self, input):
        W_p = F.relu(self.weight)
        W_n = F.relu(-self.weight)

        out_min = F.linear(input.lower(), W_p, self.bias) - F.linear(input.upper(), W_n)
        out_max = F.linear(input.upper(), W_p, self.bias) - F.linear(input.lower(), W_n)
        x = F.linear(input.center(), self.weight, self.bias)

        out = mmTensor((out_min, x, out_max))
        _layer_print(self, out)
        return _layer_postprocess(self, out)

class BatchNorm2d(nn.BatchNorm2d):
    def forward(self, input):
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

class ReLU(nn.ReLU):
    def forward(self, input):
        _relu_process(self, input)
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

class MaxPool2d(nn.MaxPool2d):
    def forward(self, input):
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

class AvgPool2d(nn.AvgPool2d):
    def forward(self, input):
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

class Dropout(nn.Dropout):
    def forward(self, input):
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

class AdaptiveAvgPool2d(nn.AdaptiveAvgPool2d):
    def forward(self, input):
        lower = super().forward(input.lower())
        center = super().forward(input.center())
        upper = super().forward(input.upper())
        out = mmTensor((lower, center, upper))
        _layer_print(self, out)
        return out

def cat(input, *args):
    if type(input) == mmTensor:
        return input
    if type(input) == list:
        lower = torch.cat([i.lower() for i in input], *args)
        center = torch.cat([i.center() for i in input], *args)
        upper = torch.cat([i.upper() for i in input], *args)
        return mmTensor((lower, center, upper))

def flatten(input, *args):
    lower = torch.flatten(input.lower(), *args)
    center = torch.flatten(input.center(), *args)
    upper = torch.flatten(input.upper(), *args)
    return mmTensor((lower, center, upper))

def LogSoftMax(x):
    S_min = x.lower() - (x.upper().exp().sum() - x.upper().exp() + x.lower().exp()).log()
    S_max = x.upper() - (x.lower().exp().sum() - x.lower().exp() + x.upper().exp()).log()
    m = nn.LogSoftmax()
    S = m(x.center())
    return mmTensor((S_min, S, S_max))


class Solution:
    def isAnagram(self, s: str, t: str) -> bool:
        dic1, dic2 = {}, {}
        for c in s:
            dic1[c] = dic1.get(c,0) +1
        for c in t:
            dic2[c] = dic2.get(c,0) +1
        return dic1 == dic2

    def isAnagram2(self, s, t):
        dic1, dic2 = [0]*26, [0]*26
        for item in s:
            dic1[ord(item)-ord('a')] += 1
        for item in t:
            dic2[ord(item)-ord('a')] += 1
        return dic1 == dic2

    def isAnagram3(self, s, t):
        return sorted(s) == sorted(t)


import numpy as np
from sklearn.neural_network import MLPRegressor
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import datetime
from sklearn.model_selection import ShuffleSplit

# this program uses to train models using sklearn
# input is prepared data 1) training data 2) testing data
# output is the predict results

print(datetime.datetime.now(), 'loading data ...')
Xtrain = np.loadtxt('../dat/xtrain.txt')
Ytrain = np.loadtxt('../dat/ytrain.txt')

print('shape:', np.shape(Xtrain))
print(datetime.datetime.now(), 'training data ...')
rs = ShuffleSplit(n_splits = 5, test_size = .1, random_state = 0)
rs.get_n_splits(Xtrain)

#Xtrain, Xtest, Ytrain, Ytest = train_test_split(Xtrain, Ytrain, test_size=0.1, random_state=88)
#rs = ShuffleSplit(n_splits = 5, test_size = .1, random_state = 0)
#rs.get_n_splits(Xtrain)

scaler  = StandardScaler()
scaler_model = scaler.fit(Xtrain)
X_train_trans = scaler_model.transform(Xtrain)

Xtest = np.loadtxt('../dat/xtest20191211.txt')
X_test_trans = scaler_model.transform(Xtest)

clf = MLPClassifier(hidden_layer_sizes=(128,64,), activation = 'relu', solver = 'adam', \
                   alpha = 0.0001, batch_size = 'auto', learning_rate = 'adaptive', \
                   learning_rate_init = 0.001, power_t = 0.5, max_iter=2000, tol = 1e-4)

clf_model = clf.fit(X_train_trans, Ytrain)
pred = clf_model.predict(X_test_trans)

Ytest = np.loadtxt('../dat/ytest20191211.txt')

from sklearn.metrics import mean_squared_error

np.savetxt('../dat/outputv2x.txt', pred)

print(mean_squared_error(Ytest, pred))
print(clf_model.score(X_test_trans, Ytest))

print(datetime.datetime.now(), 'computation is finished ...')


from django.core.paginator import Paginator
from django.shortcuts import render, get_object_or_404
from django.shortcuts import redirect
from .models import Post, Group, User
from .forms import PostForm

def index(request):
    post_list = Post.objects.order_by("-pub_date").all()
    paginator = Paginator(post_list, 10) # показывать по 10 записей на странице.
    page_number = request.GET.get('page') # переменная в URL с номером запрошенной страницы
    page = paginator.get_page(page_number) # получить записи с нужным смещением
    return render(request, 'posts/index.html', {'page': page, 'paginator': paginator})

def group_posts(request, slug):
    group = get_object_or_404(Group, slug=slug)
    posts = Post.objects.filter(group=group).order_by("-pub_date")[:12]
    paginator = Paginator(posts, 10)
    page_number = request.GET.get('page')
    page = paginator.get_page(page_number)
    return render(request, "group.html", {"group": group, 'page': page, 'paginator': paginator})

def new_post(request):
    user = request.user
    form = PostForm(request.POST)
    if not request.user.is_authenticated:
        return redirect(index)
    if request.method == 'POST':
        if form.is_valid():
            n_post = form.save(commit=False)
            n_post.author = user
            n_post.save()
            return redirect('index')
        return render(request, 'posts/new_post.html', {'form': form})
    form = PostForm()
    return render(request, 'posts/new_post.html', {'form': form})

def profile(request, username):
    author = get_object_or_404(User, username = username)
    post_list = Post.objects.filter(author = author).order_by('-pub_date')
    paginator = Paginator(post_list, 10) # показывать по 10 записей на странице.
    page_number = request.GET.get('page') # переменная в URL с номером запрошенной страницы
    page = paginator.get_page(page_number) # получить записи с нужным смещением
    count = post_list.count
    return render(request, "posts/profile.html", {'count':count, 'author':author, 'page': page, 'paginator': paginator})

def post_view(request, username, post_id):
    author = get_object_or_404(User, username = username)
    post = Post.objects.get(pk=post_id)
    posts = Post.objects.filter(author = author).order_by('-pub_date')
    count = posts.count
    return render(request, "posts/post.html", {'post':post, 'count':count, 'author':author})

def post_edit(request, username, post_id):
    post = get_object_or_404(Post, id = post_id)
    if request.user != post.author:
        return redirect("post", username, post_id)
    form = PostForm(request.POST)
    if form.is_valid():
        post.text = form.cleaned_data['text']
        post.group = form.cleaned_data['group']
        post.save()
        return redirect("post", username, post_id)
    form = PostForm()
    return render(request, "posts/post_edit.html", {"form": form, 'post':post})


import torch
import torch.nn as nn
import torch.utils.data as data
import numpy as np
import os

from utils.data import GestureData
from models.res2net import res2net50

def test():
    os.environ["CUDA_VISIBLE_DEVICES"] = "1,0"
    path = "/home/dshah/I6_Gestures"
    saved_model_path = "/home/dshah/2dgesturenet/res2net50.pth"

    test_data = GestureData(path, mode="test")
    test_loader = data.DataLoader(test_data, batch_size=16, shuffle=True, num_workers=4)

    model = nn.DataParallel(res2net50(num_classes=35))
    model = model.cuda()
    model.load_state_dict(torch.load(saved_model_path))

    model.eval()
    test_acc = []
    log_nth = 10
    for i, (inputs, targets) in enumerate(test_loader):
        inputs, targets = inputs.cuda().to(torch.float), targets.cuda().to(torch.long)
        outputs = model(inputs)

        _, preds = torch.max(outputs, 1)

        acc = np.mean((targets == preds).detach().cpu().numpy())
        test_acc.append(acc)

        if i % log_nth == 0:
            avg_acc = sum(test_acc)/len(test_acc)
            print("Average accuracy: %.3f" % avg_acc)

    mean_test_acc = sum(test_acc)/len(test_acc)
    print("Mean accuracy over full test set: %.3f" % mean_test_acc)

if __name__ == "__main__":
    test()


import numpy as np

from typing import Union

from peppermining.utils.enum import EventColumn, ModelColumn
from peppermining.filters.pepper_filter import PepperFilter
from peppermining.peppermining import PepperMining
from peppermining.conformance.violation.pepper_violation import PepperViolation
from peppermining.conformance.process_model import ProcessModel

class UndesiredConnection(PepperViolation):
    """Violation: Undesired Connection.

    Where: activity_x is followed by activity_y.

    Example
    -------
    >>> pm = PepperMining()
    >>> pm.read_event_log_csv("/tests/data/eventlog-example.csv", separator=';', format_date='%d/%m/%Y %H:%M')
    >>> pm.read_cases_csv("/tests/data/case-example.csv", separator=';')
    >>> data = {'activity': ['register request', 'check ticket', 'examine casually', 'decide', 'pay compensation'], 'sorting': [1, 2, 3, 4, 5]}
    >>> df = pd.DataFrame(data, columns=['activity', 'sorting'])
    >>> md = ProcessModel()
    >>> md.set_process_model(df)
    >>> v = UndesiredConnection(pm, md)
    >>> v.get_violation()
    """

    def __init__(self, data: Union[PepperMining, PepperFilter], models: Union[ProcessModel, list]) -> None:
        """Undesired Connection.

        Parameters
        ----------
        data : pd.DataFrame
            PepperMining or PepperFilter object.
        _models : list
            List of ProcessModel objects.
        """
        super().__init__(data, models, "UndesiredConnection")

    def detection(self) -> None:
        """Violation detection.
        """
        # Get eventdata without undesired activity
        event_log = self._component.get_event_log()
        event_log['next_activity'] = event_log.groupby(EventColumn.CASE_ID.value)[EventColumn.ACTIVITY.value].shift(-1)
        violation_log = event_log[event_log[EventColumn.ACTIVITY.value].isin(
            self.get_model_activities()) & event_log['next_activity'].notnull()][[EventColumn.ACTIVITY.value, EventColumn.CASE_ID.value]]
        # Check up new next activity
        violation_log['next_activity'] = violation_log.groupby('case_id')[EventColumn.ACTIVITY.value].shift(-1)
        violation_log = violation_log.replace(np.nan, None)
        # Get process model
        model_log = self.get_model_log()
        model_log['next_activity'] = model_log.groupby(ModelColumn.ID.value)[ModelColumn.ACTIVITY.value].shift(-1)
        model_log = model_log.replace(np.nan, None)
        # Discovery all transitions is not found in process models
        violation_log = violation_log.merge(model_log, on=['activity', 'next_activity'], how='left', indicator=True).query('_merge == "left_only"')
        violation_log = violation_log.drop(columns=['id', '_merge'])
        violation_log = violation_log[violation_log['next_activity'].notnull()]
        violation_log = violation_log.drop_duplicates()
        # Add violation
        for key, row in violation_log.groupby([EventColumn.ACTIVITY.value, 'next_activity'],
                                              group_keys=False)[EventColumn.CASE_ID.value].apply(list).reset_index(name=EventColumn.CASE_ID.value).iterrows():
            self.add_violation(f"{row[EventColumn.ACTIVITY.value]} is followed by {row['next_activity']}",
                               [row[EventColumn.ACTIVITY.value], row['next_activity']], row[EventColumn.CASE_ID.value])


student = int(input())
numbers = list(map(int, input().split()))
line = []

for stand in range(1, student+1):
    num = numbers.pop(0)
    if num:
        line = line[:-num] + [stand] + line[-num:]
    else:
        line += [stand]

print(*line)


# get N, K
n, k = map(int, input().split())
result = 0

# if N >= K, keep dividing
while n >= k:

    # keep -1 until N % K becomes 0
    while n % k != 0:
        n -= 1
        result += 1

    # devide by K
    n //= k
    result += 1

# keep -1 until N becomes 1
result += (n - 1)

print(result)

# from https://github.com/ndb796/python-for-coding-test/blob/master/3/5.py

나동빈_1이 될 때까지.py

나동빈_1이 될 때까지.py


import streamlit as st
import time

st.title('Streamlit 超入門')

st.write('プログレスバーの表示')
'Start!!'

latest_iteration = st.empty()
bar = st.progress(0)

for i in range(100):
    latest_iteration.text(f'Iteration{i+1}')
    bar.progress(i + 1)
    time.sleep(0.1)

'Done!!!'

left_column, right_column = st.beta_columns(2)
button = left_column.button('右カラムに文字を表示')
if button:
    right_column.write('ここは右カラムです')

expander = st.beta_expander('問い合わせ1')
expander.write('問い合わせ1の回答')
expander = st.beta_expander('問い合わせ2')
expander.write('問い合わせ2の回答')
expander = st.beta_expander('問い合わせ3')
expander.write('問い合わせ3の回答')

# text = st.text_input('あなたの趣味を教えてください。')
# 'あなたの趣味: ', text,

# condition = st.slider('あなたの今の調子は?', 0, 100, 50)
# 'コンディション: ', condition

# st.sidebar.write('Interactive Widgets')

# text = st.sidebar.text_input('あなたの趣味を教えてください。')
# 'あなたの趣味: ', text,

# condition = st.sidebar.slider('あなたの今の調子は?', 0, 100, 50)
# 'コンディション: ', condition

# option = st.selectbox(
#     'あなたが好きな数字を教えてください。',
#     list(range(1, 11))
#)

#'あなたの好きな数字は、', option, 'です。'

# if st.checkbox('Show Image'):
#     img = Image.open('IMGP8156.JPG')
#     st.image(img, caption='猫背', use_column_width=True)
# .checkboxにチェックを入れるとTrue、外れているとFalseという仕掛け


"""

--- Main fulltext processing pipeline ---

Be aware: Running the whole pipeline can take 16++ hours, depending on number of processors assigned, setting 5+
processors is beneficial

General prerequisites:
    Python >= 3.5
    NLTK == 3.2.2
    Treetagger == 3.2 Linux
    Treetaggerwrapper == 2.2.2
    NLTK datasets (nltk.download()) and specified TreeTagger directory
    (following instructions at http://www.cis.uni-muenchen.de/~schmid/tools/TreeTagger/)
    NB: Treetagger 3.2 on Windows is currently not supported due to version differences with the Linux version
    of Treetagger 3.2

Processing steps:
    1) Download PMC Open Access Subset: https://www.ncbi.nlm.nih.gov/pmc/tools/openftlist/
    2) Either download abstracts separately for the journals to be analyzed (using get_pubmeddata from
       dataminingfunctions) or extract them from the fulltexts directly
    3) Put fulltext textfiles into subfolders for each journal in ../data/fulltexts/*** (where *** are the ftJournals
       names below) and the abstracts file ("searchresults") into ../data/fulltexts/abstracts/*** with
       subfolders "/id_article/abstracttext_pubdate_year_pmid_articletitle_journal_title_keyword_doi/searchresults"
       (where *** are the abstractsJournals names specified below)
    4) Make sure that provided doi -> pmid matching .csvs are in basePath/ids/ (default ../data/fulltexts/ids/) to
       reproduce our analysis, otherwise ID matching can also be done by calling the Pubmed API, check convert_id.py
    5) Set numProc parameter to number of processors to be assigned to the task (default is 4,
       but 5+ is recommended if possible)

Output:
    .json and .csv dataframes in statsDir (default output path is "../data/fulltexts/stats/")

"""

from multiprocessing import Pool
from functools import partial
import os
import sys
sys.path.append('../')

from readability_fulltexts_vs_abstracts import parse_fulltexts as pf
from readability_fulltexts_vs_abstracts import analyze_fulltexts as af
from readability_fulltexts_vs_abstracts import combine_abstracts_fulltexts as caf

# Initialize variables

# Number of processors for multithreading
numProc = 4

basePath = '../data/fulltexts/'
outPath = basePath + 'output/'
outPathONE = outPath + 'PLoS_ONE/'
abstractsPath = basePath + 'abstracts/'
statsDir = basePath + 'stats/'
idPath = basePath + 'ids/'

ftJournals = ['BMC_Biol', 'eLife', 'Genome_Biol', 'PLoS_Med', 'PLoS_Biol']
abstractJournalsOthers = ['bmc_biol[journal]', 'elife[journal]', 'genome_biol[journal]',
                          'plos_med[journal]', 'plos_biol[journal]']
abstractJournalONE = 'plos_one[journal]'

# For id conversion at Pubmed (not necessary with defaults if doi -> pmid csv mappings exist)
email = 'placeholder'
tool = 'id_converter'

if not os.path.exists(outPath + 'metrics/'):
    os.makedirs(outPath + 'metrics/')

if not os.path.exists(outPath + 'pmid/'):
    os.makedirs(outPath + 'pmid/')

if not os.path.exists(outPathONE + 'pmid/'):
    os.makedirs(outPathONE + 'pmid/')

if not os.path.exists(idPath):
    os.makedirs(idPath)

if not os.path.exists(outPathONE + 'metrics/' + 'PLoS_ONE/'):
    os.makedirs(outPathONE + 'metrics/' + 'PLoS_ONE/')

if not os.path.exists(statsDir):
    os.makedirs(statsDir)

if __name__ == '__main__':

    # Parse raw text files
    pf.parse_bmc(basePath, outPath)
    pf.parse_elife(basePath, outPath)
    pf.parse_genbiol(basePath, outPath)
    pf.parse_plos(basePath, outPath)
    pf.parse_plosone(basePath, outPathONE)

    # Make list of parsed jsons from PLoS ONE
    ftJournalsONE = [t[:-5] for t in os.listdir(outPathONE) if t.endswith('.json')]

    # Other journals convert_id
    af.convert_id_others(outPath, idPath, ftJournals)

    # Other journals abstracts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialAbstracts = partial(af.analyze_abstracts_others, abstractsPath)  # Partial needed for additional arguments
    pool.map(partialAbstracts, abstractJournalsOthers)  # Map jsons to workers
    pool.close()

    # Other journals fulltexts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialFulltexts = partial(af.analyze_ft_others, outPath)  # Partial needed for additional arguments
    pool.map(partialFulltexts, ftJournals)  # Map jsons to workers
    pool.close()

    # PLoS ONE convert_id
    af.convert_id_plosone(outPathONE, idPath, ftJournalsONE)

    # PLoS ONE abstracts analysis
    af.analyze_abstracts_plosone(abstractsPath, abstractJournalONE)

    # PLoS ONE fulltexts analysis
    pool = Pool(processes=numProc)  # Create multiprocessing Pool
    partialFTONE = partial(af.analyze_ft_plosone, outPathONE)  # Partial is needed for additional arguments
    pool.map(partialFTONE, ftJournalsONE)  # Map jsons to workers
    pool.close()

    # Combine abstracts and fulltexts into a single dataframe
    caf.combine_abstracts_fulltexts_one(abstractsPath, outPathONE, statsDir, abstractJournalONE, ftJournalsONE)
    caf.combine_abstracts_fulltexts_others(abstractsPath, outPath, statsDir, abstractJournalsOthers, ftJournals)
    caf.combine_others_one(statsDir)


# Script to take marks from user and print percentage
"""
Take name of the User Take Marks of Maths, Science and English from User Find percentage of given marks, Display,

User name is <NAME>, Scored <PERCENTAGE>  % in exams"""

name=input("Enter your name: ")
mathsmarks=int(input("Enter the Math marks: "))
sciencemarks=int(input("Enter the Science marks: "))
englishmarks=int(input("Enter the English marks: "))
total=mathsmarks+sciencemarks+englishmarks
percentage=(total/3)
print("User name is",name,",Scored",percentage,"% in exams")


from __future__ import annotations
import argparse
import logging
import sys
from typing import List, Callable
from pathlib import Path
import multiprocessing as mp
import pandas as pd
from numpy.typing import ArrayLike, NDArray
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import dual_annealing
from scipy.optimize import OptimizeResult
from mpl_config import latex_preamble, xtick_label_formatter
from num_opt_ga import NumericalOptimizationGA
from kp_model import (
    first_order_ham_factory,
    second_order_ham_factory,
    third_order_ham_factory,
    get_k_k_index,
    get_energies,
    avg_squared_diff,
    get_fitting_region,
    get_plot_domain,
)

PROCESSES = mp.cpu_count()
THRESHOLD = 1e-5

# Genetic algorithm default properties
GENS = 100
POP_SIZE = 500
MUT_PROB = 0.05
REFINEMENT_RATIO = 0.1

# Dual annealing default properties
ITERS = 1000
INIT_TEMP = 5230

# Physical properties of each lattice
CRS2_LATTICE = 3.022302679
CRSE2_LATTICE = 3.167287237
MOS2_LATTICE = 3.165000000
CRS2_ENERGY = 0.3536
CRSE2_ENERGY = 0.8903
MOS2_ENERGY = 0.8535

Y_MARGIN = 0.1
FIT_LIM = 0.375

# Setting up logger
LOG_FORMAT = "%(levelname)s : %(asctime)s : %(message)s"
logging.basicConfig(format=LOG_FORMAT)
logger = logging.getLogger("logger")
logger.setLevel("INFO")

# Setting up argparse
description = """
    A tool to evaluate the energy bands of the k.p model
    for CrS2 and CrSe2 by means of either a genetic algorithm
    or the dual annealing method.
    """
parser = argparse.ArgumentParser(description=description)

# Setting up Matploltib
plt.rcParams.update(
    {
        "text.usetex": True,
        "text.latex.preamble": latex_preamble,
    }
)

def parse_args():
    """
    Parse the command line arguments using argparse
    """

    parser.add_argument(
        "--processes",
        type=int,
        nargs="?",
        default=PROCESSES,
        const=PROCESSES,
        help=(
            "the number of processes to spawn working on the optimizing "
            f"process. Default is {PROCESSES}, the number of logical "
            "processors in the machine"
        ),
    )

    parser.add_argument(
        "--orders",
        type=int,
        choices=(1, 2, 3),
        nargs="*",
        default=(1, 2, 3),
        help=(
            "the order of the k vector in the k.p model expansion "
            "for the hamiltonian of the system to include in the "
            "energy bands plot. Default is (1, 2, 3)."
        ),
    )

    parser.add_argument(
        "--method",
        type=str,
        choices=("genetic_algorithm", "dual_annealing"),
        nargs="?",
        default="genetic_algorithm",
        const="genetic_algorithm",
        help="the optimizer method to use",
    )

    parser.add_argument(
        "--lattices",
        type=str,
        choices=("crs2", "crse2", "mos2"),
        nargs="*",
        default=("crs2", "crse2", "mos2"),
        help="the lattices whose energy bands will be evaluated",
    )

    parser.add_argument(
        "--pop-size",
        type=int,
        nargs="?",
        default=POP_SIZE,
        const=POP_SIZE,
        help=(
            "the number of individuals to compose the populations "
            f"when using genetic algorithms. Default is {POP_SIZE}."
        ),
    )

    parser.add_argument(
        "--gens",
        type=int,
        nargs="?",
        default=GENS,
        const=GENS,
        help=(
            "when using generetic algorithms, the number of generations "
            "to evaluate in the evolution process of each population. "
            f"Default is {GENS}."
        ),
    )

    parser.add_argument(
        "--mut-prob",
        type=float,
        nargs="?",
        default=MUT_PROB,
        const=MUT_PROB,
        help=(
            "when using genetic algorithms, the probability of mutation "
            f"per gene in the recombination process. Default is {MUT_PROB}."
        ),
    )

    parser.add_argument(
        "--refine-ratio",
        type=float,
        nargs="?",
        default=REFINEMENT_RATIO,
        const=REFINEMENT_RATIO,
        help=(
            "when using genetic algorithms, the ratio by which the search region "
            "is shrinked in the second run around the best candidates of minimum. "
            f"If 0.0, no refinement strategy is used. Default is {REFINEMENT_RATIO}."
        ),
    )

    parser.add_argument(
        "--iters",
        type=int,
        nargs="?",
        default=ITERS,
        const=ITERS,
        help=(
            "when using dual annealing method, the max number of iterations "
            f"to perform. Default is {ITERS}."
        ),
    )

    parser.add_argument(
        "--temp",
        type=float,
        nargs="?",
        default=INIT_TEMP,
        const=INIT_TEMP,
        help=(
            "when using dual annealing method, the initial temperature. "
            "use higher values to facilitates a wider search. "
            f"Default is {INIT_TEMP}. Range is (0.01, 5.e4]."
        ),
    )

    parser.add_argument(
        "--no-local-search",
        action="store_true",
        help=(
            "when using dual annealing, perform a traditional generalized "
            "simulated annealing with no local search strategy applied"
        ),
    )

    parser.add_argument(
        "--fix",
        action="store_true",
        help=(
            "fix the non-gamma parameters, replacing them by their "
            "respective expected values, infered from the input data"
        ),
    )

    parser.add_argument(
        "--fit-regions",
        type=float,
        nargs="*",
        default=[FIT_LIM],
        help=(
            "a sequence of floats representing the percentage of points "
            "of the data to consider in the fitting process for each order "
            "specified, in ascending order. Idealy, the lower the order, the "
            "smaller the correspondent value. A sugested sequence for orders "
            "1, 2, 3 is 0.25, 0.375, 0.5. Range of each value is [0.0, 1.0]. "
            f"Default is [{FIT_LIM}]."
        ),
    )

    return parser.parse_args()

def get_refined_search_regions(
    ga_list: List[NumericalOptimizationGA],
) -> List[NumericalOptimizationGA]:
    return [
        [
            (pos - size, pos + size)
            for pos, size in zip(ga.best()[0].pos, refined_interval_sizes)
        ]
        for ga in ga_list
    ]

def get_best_func_value(ga_instance: NumericalOptimizationGA) -> float:
    return abs(ga_instance.function(ga_instance.best()[0].pos))

def evolve_ga(ga_instance: NumericalOptimizationGA) -> NumericalOptimizationGA | None:
    try:
        while ga_instance.gen < gens:
            ga_instance.evolve()
        return ga_instance
    # Ignore KeyboardInterrupt on a child process
    except KeyboardInterrupt:
        return None

def evolve_gas(
    ga_list: List[NumericalOptimizationGA], processes: int = PROCESSES
) -> List[NumericalOptimizationGA]:
    with mp.Pool(processes) as pool:
        try:
            return pool.map(evolve_ga, ga_list)
        except KeyboardInterrupt:
            # Kill the pool when KeyboardInterrupt is raised
            logger.info("Process terminated by the user")
            pool.terminate()
            pool.join()
            sys.exit(1)

def custom_dual_annealing(
    obj_function: Callable[[NDArray], float],
    bounds: ArrayLike,
    maxiter: int,
    initial_temp: float,
    no_local_search: bool,
) -> OptimizeResult:
    return dual_annealing(
        obj_function,
        bounds=bounds,
        maxiter=maxiter,
        initial_temp=initial_temp,
        no_local_search=no_local_search,
    )

if __name__ == "__main__":

    args = parse_args()

    method = args.method
    pop_size = args.pop_size
    gens = args.gens
    mut_prob = args.mut_prob
    refine_ratio = args.refine_ratio
    iters = args.iters
    temp = args.temp
    no_local_search = args.no_local_search
    processes = args.processes
    orders = sorted(args.orders)
    fix = args.fix
    fit_regions = list(map(abs, args.fit_regions))

    # Checking if number of processes is not greater than available cores
    if processes > PROCESSES and method == "genetic_algorithm":
        processes = PROCESSES
        logger.warning(
            "Number of processes greater than available logical processors. "
            f"Using {PROCESSES} processes instead."
        )

    # Checking if fit_region has at least the same length of orders
    # if not, populate fit_region with the default FIT_LIM
    while len(fit_regions) < len(orders):
        fit_regions.append(FIT_LIM)

    csv_dir = Path("data/csv")
    plot_dir = Path("plots")
    results_dir = Path("results")

    lattices = {"crs2": CRS2_LATTICE, "crse2": CRSE2_LATTICE, "mos2": MOS2_LATTICE}
    expected_energies = {
        "crs2": CRS2_ENERGY,
        "crse2": CRSE2_ENERGY,
        "mos2": MOS2_ENERGY,
    }

    titles = {"crs2": r"$\ch{CrS2}$", "crse2": r"$\ch{CrSe2}$", "mos2": r"\ch{MoS2}"}
    files = {key: csv_dir.joinpath(f"{key}_data.csv") for key in lattices.keys()}
    ham_factories = (
        first_order_ham_factory,
        second_order_ham_factory,
        third_order_ham_factory,
    )

    for lattice_name in args.lattices:

        lattice = lattices[lattice_name]
        title = titles[lattice_name]
        file = files[lattice_name]

        # Reading data from files
        logger.info(f"Reading data from file {file}")
        df = pd.read_csv(file)
        ks = df.loc[:, "kx":"kz"].to_numpy()
        energies = df.loc[:, "e1":"e4"].to_numpy()
        sorted_energies = np.sort(energies)

        # Expected values of the parameters
        k_k_index = get_k_k_index(ks)
        k_k_energies = sorted_energies[k_k_index, :]
        expected_energy = expected_energies[lattice_name]
        expected_delta = k_k_energies[2] - k_k_energies[1]
        expected_lambda_c = k_k_energies[3] - k_k_energies[2]
        expected_lambda_v = k_k_energies[1] - k_k_energies[0]
        expected_params = (
            expected_energy,
            expected_delta,
            expected_lambda_c,
            expected_lambda_v,
        )

        sorted_eigen_list = []
        params_list = []
        best_func_value_list = []

        for order, fit_region in zip(orders, fit_regions):
            # Data subset that will be used in the fitting process
            lower_fit_index, upper_fit_index = get_fitting_region(
                ks, lower=-fit_region, upper=fit_region
            )

            fitting_ks = ks[lower_fit_index:upper_fit_index, :]
            fitting_energies = sorted_energies[lower_fit_index:upper_fit_index, :]
            logger.info(
                "Using interval "
                f"{fitting_ks[0, 0]: .3f} < kx < {fitting_ks[-1, 0]: .3f} "
                "as fitting region (equivalent to range "
                f"[{fit_region}, {fit_region}] in the plot domain)"
            )

            # The genetic_algorithm method search for points of
            # maximum, while the dual annealing method seach for
            # points of minimum. This factor flips the objective
            # function according with the method used
            obj_function_factor = -1 if method == "genetic_algorithm" else 1
            ham_factory = ham_factories[order - 1]

            # Objective function to optimize
            def obj_function(params: ArrayLike) -> float:
                return obj_function_factor * avg_squared_diff(
                    ks=fitting_ks,
                    sorted_energies=fitting_energies,
                    ham_factory=ham_factory,
                    params=(lattice, *params),
                )

            # Region where to find a minimum
            if fix is True:
                first_order_search_region = [
                    (expected_param - THRESHOLD, expected_param + THRESHOLD)
                    for expected_param in expected_params
                ] + [(-1.0, 1.0)]
            else:
                first_order_search_region = [
                    (-2.0, 2.0),  # energy
                    (0.5, 2.0),  # delta
                    (0.0, 1.0),  # lambda_c
                    (0.0, 1.0),  # lambda_v
                    (-1.5, 1.5),  # gamma_0
                ]

            higher_order_search_region = [(-1.0, 1.0) for _ in range(3 * order - 3)]
            suggested_search_region = (
                first_order_search_region + higher_order_search_region
            )

            logger.info(f"Starting optimization process for order {order}")
            if method == "genetic_algorithm":

                # First two integers in ELITE must be even
                elite = (
                    int(0.025 * pop_size) * 2,
                    int(0.05 * pop_size) * 2,
                    int(0.1 * pop_size),
                )

                # Evolving gas
                gas = [
                    NumericalOptimizationGA(
                        search_region=suggested_search_region,
                        function=obj_function,
                        pop_size=pop_size,
                        elite=elite,
                        mut_probs=(mut_prob, mut_prob),
                    )
                    for _ in range(processes)
                ]

                logger.info(
                    f"Evolving {processes} populations with {pop_size} "
                    f"individuals for {gens} generations"
                )

                gas = evolve_gas(gas, processes=processes)

                # Evolving gas with refined search regions
                if refine_ratio != 0.0:
                    refined_interval_sizes = tuple(
                        [
                            refine_ratio * (upper - lower) / 2
                            for (lower, upper) in suggested_search_region
                        ]
                    )

                    gas = [
                        NumericalOptimizationGA(
                            search_region=search_region,
                            function=obj_function,
                            pop_size=pop_size,
                            elite=elite,
                            mut_probs=(mut_prob, mut_prob),
                        )
                        for search_region in get_refined_search_regions(gas)
                    ]

                    logger.info(
                        "Restarting process with refined search "
                        "regions arround the best individuals"
                    )

                    gas = evolve_gas(gas, processes=processes)

                func_values = list(map(get_best_func_value, gas))
                best_func_value = min(func_values)
                best_index = func_values.index(best_func_value)
                ga = gas[best_index]

                # Data used in the plot
                label = "GA Fit"
                params = ga.best()[0].pos
                sorted_eigenvalues = get_energies(
                    ks, ham_factory=ham_factory, params=(lattice, *params)
                )

            if method == "dual_annealing":

                logger.info(
                    f"Spawning {processes} dual annealing processes "
                    f"with temp={temp}, maxiter={iters} and "
                    f"no_local_search={no_local_search}"
                )

                dual_annealing_args = (
                    obj_function,
                    np.array(suggested_search_region),
                    iters,
                    temp,
                    no_local_search,
                )

                with mp.Pool(processes) as pool:
                    promises = [
                        pool.apply_async(custom_dual_annealing, dual_annealing_args)
                        for _ in range(processes)
                    ]
                    results = [promise.get() for promise in promises]

                # Data used in the plot
                label = "Dual Annelaing Fit"
                func_values = [result.fun for result in results]
                best_func_value = min(func_values)
                best_index = func_values.index(best_func_value)
                params = results[best_index].x
                sorted_eigenvalues = get_energies(
                    ks, ham_factory=ham_factory, params=[lattice, *params]
                )

            logger.info(f"Best function value reached: {best_func_value: .3e}")
            sorted_eigen_list.append(sorted_eigenvalues)
            params_list.append(params)
            best_func_value_list.append(best_func_value)

        # Creating plots
        logger.info("Creating energy plot")
        fig, ax = plt.subplots()

        # Setting up the axes
        ylim = (
            np.min(sorted_energies[:, 0]) - Y_MARGIN,
            np.max(sorted_energies[:, -1]) + Y_MARGIN,
        )

        ax.set(
            title=title,
            ylabel=r"Energy (\si{\eV})",
            ylim=ylim,
        )

        ax.xaxis.set_major_formatter(plt.FuncFormatter(xtick_label_formatter))

        plot_domain = get_plot_domain(ks)
        ax.plot(
            plot_domain,
            sorted_energies,
            color="black",
            marker="o",
            linestyle="none",
            markersize=1.0,
            label="DFT",
        )

        colors = ("blue", "red", "green")

        for sorted_eigenvalues, color, fit_region, order in zip(
            sorted_eigen_list, colors, fit_regions, orders
        ):
            ax.vlines(
                (-fit_region, fit_region),
                *ylim,
                color=color,
                alpha=0.8,
                linestyles="dashed",
                linewidths=0.7,
                label=f"Fitting Region (Order {order})",
                zorder=-2,
            )

            ax.plot(
                plot_domain,
                sorted_eigenvalues,
                color=color,
                alpha=0.8,
                label=label + f" (Order {order})",
                zorder=-1,
            )

        # Removing repeated entries from legend
        handles, labels = ax.get_legend_handles_labels()
        by_label = dict(zip(labels, handles))
        plt.legend(by_label.values(), by_label.keys(), framealpha=1.0)

        # Basename to save the image and the csv file
        joined_orders = "".join([str(order) for order in orders])
        basename = f"{lattice_name}_{method}_order_{joined_orders}"
        if fix is True:
            basename += "_fix"

        filename = plot_dir.joinpath(f"{basename}.png")
        logger.info(f"Saving figure as {filename}")
        plt.savefig(filename, dpi=300)

        # Creating dataframe with the results
        logger.info("Creating a dataframe with the results")

        # The dataframe will have 7 + 3 * max(orders) - 3 = 4 + 3 * max(orders) rows
        rows = 4 + 3 * max(orders)

        fitted_params_columns = [
            [best_func_value, lattice]
            + list(params)
            + [None for _ in range(rows - len(params) - 2)]
            for params, best_func_value in zip(params_list, best_func_value_list)
        ]

        expected_params_column = [None for _ in range(rows)]
        expected_params_column[2:6] = expected_params
        data_list = [
            *fitted_params_columns,
            expected_params_column,
        ]  # Needs to be transposed

        first_order_index_labels = [
            "obj_func_value",
            "lattice",
            "fermi_energy",
            "delta",
            "lambda_c",
            "lambda_v",
            "gamma_0",
        ]

        higher_order_index_labels = [
            f"gamma_{n + 1}" for n in range(3 * max(orders) - 3)
        ]

        output_df = pd.DataFrame(
            data=np.array(data_list).transpose(),
            index=first_order_index_labels + higher_order_index_labels,
            columns=[f"order_{order}" for order in orders] + ["expected_values"],
        )

        filename = results_dir.joinpath(f"{basename}.csv")
        logger.info(f"Saving output as {filename}")
        output_df.to_csv(filename)


from __future__ import absolute_import
import logging

from irods.models import Zone
from irods.zone import iRODSZone
from irods.manager import Manager
from irods.message import GeneralAdminRequest, iRODSMessage
from irods.api_number import api_number
from irods.exception import ZoneDoesNotExist, NoResultFound

logger = logging.getLogger(__name__)

class ZoneManager(Manager):

    def get(self, zone_name):
        query = self.sess.query(Zone).filter(Zone.name == zone_name)

        try:
            result = query.one()
        except NoResultFound:
            raise ZoneDoesNotExist()
        return iRODSZone(self, result)

    def create(self, zone_name, zone_type):
        message_body = GeneralAdminRequest(
            "add",
            "zone",
            zone_name,
            zone_type,
        )
        request = iRODSMessage("RODS_API_REQ", msg=message_body,
                               int_info=api_number['GENERAL_ADMIN_AN'])
        with self.sess.pool.get_connection() as conn:
            conn.send(request)
            response = conn.recv()
        logger.debug(response.int_info)
        return self.get(zone_name)

    def remove(self, zone_name):
        message_body = GeneralAdminRequest(
            "rm",
            "zone",
            zone_name
        )
        request = iRODSMessage("RODS_API_REQ", msg=message_body,
                               int_info=api_number['GENERAL_ADMIN_AN'])
        with self.sess.pool.get_connection() as conn:
            conn.send(request)
            response = conn.recv()
        logger.debug(response.int_info)


from collections import deque
def solution(progresses, speeds):
    answer = []
    sum = 0
    progresses = deque(progresses)
    # progresses 원소가 빌 때까지
    while progresses:
        # 각 원소에 speed 만큼 더해줌
        for i in range(len(progresses)):
            progresses[i] += speeds[i]
        # 100이 넘는 원소가 끝날때까지 pop:
        while progresses:
            a = progresses.popleft()
            if a >= 100:
                sum += 1
            else :
                progresses.appendleft(a)
                break
        if sum != 0 :
            answer.append(sum)
            sum = 0
    return answer

test1 = [93, 30, 55]
speed1 = [1,30,5 ]
test2 = [95, 90, 99, 99, 80, 99]
speed2 = [1,1,1,1,1,1]
print(solution(test1,speed1))
print(solution(test2,speed2))


from gpiozero import DistanceSensor, Robot
from random import choice
from time import sleep

class Jeeno:

    def __init__(self):
        self.front_sensor = DistanceSensor(echo = 22, trigger = 27)
        #self.back_sensor = DistanceSensor(echo = , trigger = )
        self.robot = Robot(left=(6,12), right=(5,13))
        self.move()

    def move(self):
        self.front_sensor.when_out_of_range = self.robot.forward
        self.front_sensor.when_in_range = self.turn

    def turn(self):
        self.robot.stop()
        if choice(['L','R']) == 'L':
            self.left()
        else:
            self.right()

    def left(self):
        self.robot.forward(curve_left=1)
        sleep(1)
        self.robot.stop()
    def right(self):
        self.robot.forward(curve_right=1)
        sleep(1)
        self.robot.stop()

if __name__ == "__main__":
    j = Jeeno()


'''
Created by Ming Li at 2019-01-30

Feature: Random forest

Description: random forest based on the decision tree

Contact: ming.li2@columbia.edu

Note:
here we use a matrix to store the prediction result for every pair of
tree and data point, we store them in the predicted_matrix
predicted_matrix, each row is each data point, each column is each tree prediction for all data points

predicted_matrix[:, i]
    get access for each column
for each_row in predicted_matrix
    get access for each row

'''

import numpy as np
from decision_tree import ClassificationTree
from utils_RF import get_random_subsets
import pickle
import progressbar

bar_widgets = [
    'Training: ', progressbar.Percentage(), ' ', progressbar.Bar(marker="-", left="[", right="]"),
    ' ', progressbar.ETA()
]
# TODO replace it with tqdm
# from tqdm import tqdm

class RandomForest:

    def __init__(self, n_estimators=100, max_features=None, min_samples_split=2,
                 min_gain=0, max_depth=float("inf")):
        '''

        :param n_estimators: number of trees
        :param max_features: number of features for each tree generation, maxmium number of features
        :param min_samples_split: used for decision tree split, param for decision tree
        :param min_gain: Minimum information gain req. to continue, param for decision tree
        :param max_depth: Maximum depth for tree, param for decision tree
        '''
        self.n_estimators = n_estimators
        self.max_features = max_features
        self.min_samples_split = min_samples_split
        self.min_gain = min_gain
        self.max_depth = max_depth
        self.progressbar = progressbar.ProgressBar(widgets=bar_widgets)
        self.trees_feature_idx = []

        self.trees = [] # initial the list of trees
        for _ in self.n_estimators:
            self.trees.append(
                ClassificationTree(
                    min_samples_split=self.min_samples_split,
                    min_impurity=min_gain,
                    max_depth=self.max_depth
                )
            )

    def fit(self, X, y):
        '''
        fit the dataset using X, y
        #### process:
        1. choose a subset of dataset
        2. for each dataset, choose a subset of features
        3. using the features of subset data for fitting the decision tree classifier
        :param X:
        :param y:
        :return:
        '''
        n_features = X.shape[1]
        # default, using the sqrt(n_features) for each tree generation
        n_subset_features = np.sqrt(n_features)
        '''get the subset of data points for each tree'''
        subsets = get_random_subsets(X, y, self.n_estimators)
        '''fit each tree'''
        for i in self.progressbar(range(self.n_estimators)):
            '''for each subset of the dataset'''
            X_subset, y_subset = subsets[i]
            '''get a subset of features number n_subset_features'''
            feature_idx = np.random.randint(range(n_features), size=n_subset_features, replace=False)
            '''store the subset of features for each tree'''
            self.trees_feature_idx[i] = feature_idx
            '''fit the single tree classifier'''
            self.trees[i].fit(X_subset[:, feature_idx], y_subset)

        '''store the filled trees'''
        TREES_STORE = open('fitted_trees.pickle', 'w')
        pickle.dump(self.trees, TREES_STORE)
        TREES_STORE.close()

    def predict(self, X_new):
        '''
        predict based on the fitted trees, using the X_new
        :param X_new: X_new
        :return: predicted class

        Note: here we use a matrix to store the prediction result for every pair of
        tree and data point, we store them in the predicted_matrix
        predicted_matrix, each row is each data point, each column is each tree prediction for all data points

        predicted_matrix[:, i] get access for each column
        for each_row in predicted_matrix get access for each row
        '''
        # assert X_new.shape[1] == X.shape[1]
        predicted_matrix = np.zeros(shape=(X_new.shape[0], self.n_estimators))
        for i, tree in enumerate(self.trees):
            feature_idx = self.trees_feature_idx[i]
            # get the prediction for the whole dataset using one tree
            prediction = tree.predict(X_new[:, feature_idx])
            predicted_matrix[:, i] = prediction

        y_pred = []
        for predicted_sample_result in predicted_matrix:
            y_pred.append(np.bincount(predicted_sample_result.astype('int')).argmax())
        return y_pred

from sklearn import datasets
from sklearn.model_selection import train_test_split

class Test:
    def __init__(self):
        data = datasets.load_digits()
        self.X = data.data
        self.y = data.target

    def test_fit(self, n_estimators):
        X_train, X_test, y_train, y_test = train_test_split(self.X, self.y, test_size=0.3, seed=2)
        clf = RandomForest(n_estimators=n_estimators)
        clf.fit(X_train, y_train)

    def test_predict(self, n_estimators):
        X_train, X_test, y_train, y_test = train_test_split(self.X, self.y, test_size=0.3)
        clf = RandomForest(n_estimators=n_estimators)
        clf.fit(X_train, y_train)
        y_pred = clf.predict(X_test)
        print(y_pred)

def predict_demo(X_new, n_estimators):
    '''
    predict demo for RF, from storing the output to the predicted class
    :param X_new: X_new
    :param n_estimators: n_estimator
    :return: predicted class
    '''
    # assert X_new.shape[1] == X.shape[1]
    predicted_matrix = np.zeros(shape=(X_new.shape[0], n_estimators))
    for i, tree in enumerate(range(n_estimators)):
        prediction = np.random.randint(i+1, size=X_new.shape[0])
        predicted_matrix[:, i] = prediction
        # print(i, predicted_matrix[:, i])

    y_pred = []
    for predicted_sample_result in predicted_matrix:
        # for each_row in predicted_matrix
        # y_pred.append(np.bincount(predicted_sample_result.astype('int')).argmax())
        predicted_for_sample_bincount = np.bincount(predicted_sample_result.astype(np.int))
        y_pred.append(np.argmax(predicted_for_sample_bincount))
    return predicted_matrix, y_pred

def main():
    #test = Test()
    #test.test_predict(n_estimators=10)
    X_new = np.random.rand(10, 5)
    predicted_matrix = predict_demo(X_new, n_estimators=7)
    print(predicted_matrix)

if __name__ == '__main__':
    main()


from __future__ import annotations
from dataclasses import dataclass
from decimal import Decimal

@dataclass
class EnvelopeData:
    top:     Decimal
    average: Decimal
    bottom:  Decimal

@dataclass
class Statistics:
    values: List[Decimal]

    def sma(self, term:int) -> List[Decimal]:
        result = list()

        if len(self.values) < term:
            return result

        for i in range(len(self.values)-term+1):
            value = Decimal('0')
            for j in range(term):
                value = value + self.values[i + j]/term
            result.append(value)

        return result

    def wma(self, term:int) -> List[Decimal]:
        result = list()

        if len(self.values) < term:
            return result

        for i in range(len(self.values)-term+1):
            value = Decimal('0')
            for j in range(term):
                value = value + self.values[i + j]*(j + 1)
            result.append(value/sum(range(1, term+1)))

        return result

    def ema(self, term:int) -> List[Decimal]:
        result = list()

        if len(self.values) < term:
            return result

        last_value = sum(self.values[0:term])/term
        result.append(last_value)
        for value in self.values[term:]:
            last_value = last_value + (Decimal('2')/(term + 1))*(value - last_value)
            result.append(last_value)

        return result

    def envelope_sma(self, term:int, percent:Decimal) -> [EnvelopeData]:
        result = list()
        sma = self.sma(term = term)

        for value in sma:
            result.append(EnvelopeData(top = value*(1 + percent/100), average = value, bottom = value*(1 - percent/100)))

        return result

    def mean(self) -> Decimal:
        return sum(self.values)/Decimal(len(self.values))

    def variance(self) -> Decimal:
        mean = self.mean()
        return sum([(value - mean)**2 for value in self.values])/Decimal(len(self.values))

    def standard_deviation(self) -> Decimal:
        return self.variance()**Decimal('0.5')


# 기초 문법

a = 0.47
print(type(a))
#float #실수

b = 46
print(type(b))
#int #정수

c = "박우성"
print(type(c))
#str #문자

d = True
print(type(d))
#bool #참과 거짓

e = ["Mon", "Tue", "Wed"]
print(type(e))
#list
#list = mutable = 변할 수 있는

print(e[0])

f = ("Mon", "Tue", "Wed")
print(type(f))
#tuple
#tuple = immutable ()=만 사용

print(e[0])

weekday = ["Mon", "Tue", "Wed", "Thu", "Fri"]
weekend = ["Sat", "Sun"]
week = [] #대괄호
week.append(weekday)
print(week)
week.append(weekend)
print(week)
week.remove(weekend)
print(week)

######
'''
[] = list - 안의 있는 요소들 수정 가능
{} = dict #dictionary (key, value) 단어, 정의
() = tuple - 안의 있는 요소들 수정 불가능

소문자로만 하고 싶을 땐 .lower()
대문자로만 하고 싶을 땐 .upper()

.split() -> 특정 문자로 요소를 나누고 싶을 때
.strip() -> 양 끝의 공백을 없애고 싶을 때
.append() -> 추가
'''
#####

#####
'''
for 문
for 변수 in 리스트
    수행할 문장 1 #tap 필수
    수행할 문장 2
'''
#example
days = [" Mon ", " Tue ", " Wed ", " Thu ", " Fri "]
print(days)

for day in days:
    print(day) #공백 있이 출력
    print(day.strip()) #공백없이 출력
    days.append("Sat") #("Sat", "Sun") X
    days.append("Sun")
    print(days) #Sat, Sun 추가됨
#####
'''
for in range 함수

range(10) -> 1, 2, 3, 4, 5, 6, 7, 8, 9

range(start, stop)
range(1,10) -> 1, 2, 3, 4, 5, 6, 7, 8, 9

range(start, stop, step)
range(0, 20, 2) -> 0, 2, 4, 6, 8, 10, 12, 14, 16, 18
'''
#####

#####
'''
def 함수 #def = definition
def 함수이름(매개변수):
    수행할 문장 1
    수행할 문장 2
'''
#example
def abc(a, b, c, d):
    return (a + b + c + d) #보통 print 대신 return 씀

def park(a, b):
    return (a - b)

print(park(3, abs(2 + 4 + 5 + 5)))
# -13
#####

#####
'''
조건문 - if 문 and swich 문
if 문
'''
#example
user_input = input("Enter you age: ") #이름엔 띄어쓰기 대신 _(언더바)
print(user_input) #age는 터미널에 입력

def check_soju(age):
    if age > 21:
        return("You can drint soju")
    elif age == 20 or age == 21:  # elif -> Python  , else if -> Java
        return("You are new to drink soju")
    else:
        return("You cannot drink soju")

print(check_soju(user_input))
#####

##### example
# how many credit you take? # if player say no then stop
def restart():
  answer = str(input("Do you want to start over? y/n ")).lower()
  if answer == "y" or answer =="n":
    if answer == "n":
        print("k. bye!")
        return
    elif answer == "y":
      main()
  else:
    print("That's not a valid answer")
    restart()

def main():
  credit = int(input("TYPE HOW MANY CREDITS YOU TAKE: "))
  if credit < 20:
    print("You are freshman")
  elif credit >= 20 and credit < 40:
    print("You are sophomore")
  elif credit >= 40 and credit < 60:
    print("You are junior")
  else:
    print("You are senior")
  restart()

main()
#####

#####
def add_byuh_class(dictionary={}, course="", description=""):
    dictionary[course] = description

byuh_class = {
    #key      #value
    "MUSC 101" : "Introduction of Music",
    "CIS 101" : "Inctro to CS"
}
print(byuh_class)
print(byuh_class["CIS 101"])

add_byuh_class(byuh_class, "MATH 121", "Principle of Statistic")

print(byuh_class)
#####

#####
#example
def delete_exercise(type={}, name=""):
    del type[name]

types_of_exercise = {
#tab
    "lat pull down" : "back",
    "bench press" : "chest",
    "push up" : "back"
}

print(type(types_of_exercise))
print(types_of_exercise)

delete_exercise(types_of_exercise, "lat pull down")
print(types_of_exercise)

for type_of_exercise in types_of_exercise:
    print(type_of_exercise)
#####

#####
#example
animals = input("Enter animal: ").split(",")
print(animals)
stripped_animals = []
for animal in animals:                       #항목별로 공백을 없애고 싶어서 for 문 삽입
    stripped_animals.append(animal.strip())  #추가하기 때문에 .append
print(stripped_animals)

#Dog, Cat, Pig
#####


from pwn import *

r = process("./pwntools")

r.recvuntil('er :)\n')

r.send(p32(0xdeadbeef))
r.recvline()

for _ in range(1000):
    q = r.recvuntil(' = ?').decode().replace(' = ?', '')
    ans = eval(q)
    print(ans)
    r.sendline(str(ans))
    print("ok")

r.interactive()


import logging, os, urllib, sys, hashlib, threading, operator
import simplejson as json
from config.config import *
from lib.typecode.util import *
from nltk import *
import voxpop
import vp.vpStats
import vp.itemManager as itemManager
import vp.item as item
from controllers.controller import *

class Readmodal(Controller):

	def GET(self, _1="", _2=""):
		logging.info("#### Readmodal.get -> " + _1 + " - " + _2)
		if _1 == 'keys' and _2 != "": return self.get_rm_children(_2)
		if _1 == 'ids' and _2 != "": return self.get_rm_children_by_ids(_2)
		if _1 == 'articleid' and _2 != "": return self.get_rm_children_by_article_id(_2)
		return self.json_not_found()

	def get_rm_children(self,_id):
		logging.error("#### Readmodal.get_rm_children["+repr(_id)+"]")
		with voxpop.VPE.memcache_lock:
			_output = voxpop.VPE.get_memcache().get('readmodalChildren_'+_id.encode('utf-8'))
		if not _output:
			_ids = [_id]
			if ',' in _id:
				_ids = _id.split(',')
			comments = None
			with voxpop.VPE.db_lock:
				comments = json.loads(voxpop.VPE.get_db().open_document('_design/readmodal/_view/children',key='"'+_ids[len(_ids)-1]+'"'))['rows']
			articles = []
			_myComments = []
			_aggregateSentiment = {}
			_aggregateSentiment['n_positive'] = 0
			_aggregateSentiment['n_negative'] = 0
			_aggregateSentiment['ratio'] = 0.5
			for i in comments:
				if u'caches' in i['value']:
					if len(_ids) > 1:
						_matches = True
						for j in _ids:
							if j not in i['value']['caches']:
								_matches = False
						if not _matches:
							continue
					for j in i['value']['caches']:
						if j[0:8] == 'article_':
							i['article_id'] = j
							if j not in articles:
								articles.append(j)
				_aggregateSentiment['n_positive'] = _aggregateSentiment['n_positive'] + len(i['value']['lasswell']['positive'])
				_aggregateSentiment['n_negative'] = _aggregateSentiment['n_negative'] + len(i['value']['lasswell']['negative'])
				_myComments.append(i)
			_myArticles = {}
			for i in articles:
				with voxpop.VPE.db_lock:
					article = json.loads(voxpop.VPE.get_db().open_document(i))
				_myArticles[i] = article
			if (float(_aggregateSentiment['n_positive']) + float(_aggregateSentiment['n_negative'])) > 0:
				_aggregateSentiment['ratio'] = float(_aggregateSentiment['n_positive']) / (float(_aggregateSentiment['n_positive']) + float(_aggregateSentiment['n_negative']))
			else:
				_aggregateSentiment['ratio'] = 0.5
			_output = {'comments':_myComments,'articles':_myArticles, 'aggregateSentiment': _aggregateSentiment}
			with voxpop.VPE.memcache_lock:
					voxpop.VPE.get_memcache().set('readmodalChildren_'+_id.encode('utf-8'), _output, 600)
		return self.json(_output)

	def get_rm_children_by_ids(self,qid):
		logging.error("#### Readmodal.get_rm_children_by_ids["+str(qid)+"]")
		with voxpop.VPE.memcache_lock:
			_output = voxpop.VPE.get_memcache().get('readmodalChildrenById_'+str(qid).encode('utf-8'))
		if not _output:
			_ids = [web.input()['ids']]
			if ',' in web.input()['ids']:
				_ids = web.input()['ids'].split(',')
			_myArticleIds = []
			_myComments = []
			_aggregateSentiment = {}
			_aggregateSentiment['n_positive'] = 0
			_aggregateSentiment['n_negative'] = 0
			_aggregateSentiment['ratio'] = 0.5
			for i in _ids:
				_myComment = json.loads(voxpop.VPE.get_db().open_document(i))
				if u'caches' in _myComment:
					_myReducedComment = {}
					for j in _myComment[u'caches']:
						if j[0:8] == 'article_':
							_myReducedComment['article_id'] = j
							if _myArticleIds.count(j) == 0:
								_myArticleIds.append(j)
					_myReducedComment['value'] = {}
					_myReducedComment['value']['id'] = _myComment['_id']
					_myReducedComment['value']['approveDate'] = _myComment['approveDate']
					_myReducedComment['value']['commentBody'] = _myComment['commentBody']
					if 'geo' in _myComment and 'name' in _myComment['geo']:
						_myReducedComment['value']['geo_name'] = _myComment['geo']['name']
					_myReducedComment['value']['display_name'] = _myComment['display_name']
					_myReducedComment['value']['caches'] = _myComment['caches']
					_myReducedComment['value']['lasswell'] = {}
					_myReducedComment['value']['lasswell']['positive'] = _myComment['NLP']['lasswell_words']['positive']
					_aggregateSentiment['n_positive'] = _aggregateSentiment['n_positive'] + len(_myComment['NLP']['lasswell_words']['positive'])
					_myReducedComment['value']['lasswell']['negative'] = _myComment['NLP']['lasswell_words']['negative']
					_aggregateSentiment['n_negative'] = _aggregateSentiment['n_negative'] + len(_myComment['NLP']['lasswell_words']['negative'])
					_myComments.append(_myReducedComment)
			_myArticles = {}
			for i in _myArticleIds:
				with voxpop.VPE.db_lock:
					article = json.loads(voxpop.VPE.get_db().open_document(i))
				_myArticles[i] = article

			if (float(_aggregateSentiment['n_positive']) + float(_aggregateSentiment['n_negative'])) > 0:
				_aggregateSentiment['ratio'] = float(_aggregateSentiment['n_positive']) / (float(_aggregateSentiment['n_positive']) + float(_aggregateSentiment['n_negative']))
			else:
				_aggregateSentiment['ratio'] = 0.5
			_output = {'comments':_myComments,'articles':_myArticles, 'aggregateSentiment': _aggregateSentiment}
			with voxpop.VPE.memcache_lock:
					voxpop.VPE.get_memcache().set('readmodalChildrenById_'+str(qid).encode('utf-8'), _output, 600)
		return self.json(_output)

		def get_rm_children_by_article_id(self,aid):
			logging.error("#### Readmodal.get_rm_children_by_article_id["+str(aid)+"]")
			with voxpop.VPE.memcache_lock:
				_output = voxpop.VPE.get_memcache().get('readmodalChildrenByArticleId_'+str(aid).encode('utf-8'))
			if not _output:
				article = voxpop.VPE.get_items().get(_id=aid)
				for i in article.children:
					logging.error(i)
				_output = {}
				with voxpop.VPE.memcache_lock:
					pass
					#voxpop.VPE.get_memcache().set('readmodalChildrenById_'+str(qid).encode('utf-8'), _output, 600)
			return self.json(_output)


from __future__ import division
import math
import os

from nvidia.dali.pipeline import Pipeline
import nvidia.dali.ops as ops
import nvidia.dali.types as types
from nvidia.dali.plugin.paddle import DALIGenericIterator

import paddle
from paddle import fluid

def convert_data_layout(data_layout):
    if data_layout == 'NCHW':
        return types.NCHW
    elif data_layout == 'NHWC':
        return types.NHWC
    else:
        assert False, "not supported data_layout:{}".format(data_layout)

class HybridTrainPipe(Pipeline):
    """
    Create train pipe line.  You can find document here:
    https://docs.nvidia.com/deeplearning/sdk/dali-master-branch-user-guide/docs/plugins/paddle_tutorials.html
    Note: You may need to find the newest DALI version.
    """

    def __init__(self,
                 rec_path,
                 idx_path,
                 batch_size,
                 crop,
                 min_area,
                 lower,
                 upper,
                 mean,
                 std,
                 device_id,
                 shard_id=0,
                 num_shards=1,
                 random_shuffle=True,
                 num_threads=4,
                 seed=42,
                 data_layout="NCHW",
                 output_dtype='float16',
                 pad_output=False,
                 prefetch_queue_depth=5,
                 nvjpeg_padding=0,
                 decoder_buffer_hint=0,
                 normalize_buffer_hint=0):
        super(HybridTrainPipe, self).__init__(
            batch_size, num_threads, device_id, seed=seed)

        self.input = ops.MXNetReader(
            path=[rec_path],
            index_path=[idx_path],
            lazy_init=True,
            dont_use_mmap=0,
            random_shuffle=random_shuffle,
            shard_id=shard_id,
            num_shards=num_shards)

        self.decode = ops.ImageDecoderRandomCrop(
            device="mixed",
            output_type=types.RGB,
            device_memory_padding=nvjpeg_padding,
            host_memory_padding=nvjpeg_padding,
            random_area=[min_area, 1.0],
            random_aspect_ratio=[lower, upper],
            bytes_per_sample_hint=decoder_buffer_hint,
            affine=False)
        self.rrc = ops.Resize(
            device="gpu",
            resize_x=crop,
            resize_y=crop,
            bytes_per_sample_hint=decoder_buffer_hint)
        dtype = types.FLOAT if output_dtype == 'float32' else types.FLOAT16
        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            output_dtype=dtype,
            output_layout=convert_data_layout(data_layout),
            crop=(crop, crop),
            image_type=types.RGB,
            mean=mean,
            std=std,
            pad_output=pad_output,
            bytes_per_sample_hint=normalize_buffer_hint)
        self.coin = ops.CoinFlip(probability=0.5)
        self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")

    def define_graph(self):
        rng = self.coin()
        jpegs, labels = self.input(name="Reader")
        images = self.decode(jpegs)
        images = self.rrc(images)
        output = self.cmnp(images.gpu(), mirror=rng)
        return [output, self.to_int64(labels.gpu())]

    def __len__(self):
        return self.epoch_size("Reader")

class HybridValPipe(Pipeline):
    """
    Create validate pipe line.
    """

    def __init__(self,
                 rec_path,
                 idx_path,
                 batch_size,
                 resize_shorter,
                 crop,
                 mean,
                 std,
                 device_id,
                 shard_id=0,
                 num_shards=1,
                 random_shuffle=False,
                 num_threads=3,
                 seed=42,
                 data_layout='NCHW',
                 output_dtype='float32',
                 pad_output=False,
                 prefetch_queue_depth=5,
                 nvjpeg_padding=0):
        super(HybridValPipe, self).__init__(
            batch_size,
            num_threads=num_threads,
            device_id=device_id,
            seed=seed,
            prefetch_queue_depth=5)
        self.input = ops.MXNetReader(
            path=[rec_path],
            index_path=[idx_path],
            lazy_init=True,
            dont_use_mmap=0,
            random_shuffle=random_shuffle,
            shard_id=shard_id,
            num_shards=num_shards)

        self.decode = ops.ImageDecoder(
            device="mixed",
            output_type=types.RGB,
            device_memory_padding=nvjpeg_padding,
            host_memory_padding=nvjpeg_padding,
            affine=False)

        self.resize = ops.Resize(device="gpu", resize_shorter=resize_shorter)

        dtype = types.FLOAT if output_dtype == 'float32' else types.FLOAT16
        self.cmnp = ops.CropMirrorNormalize(
            device="gpu",
            output_dtype=dtype,
            output_layout=convert_data_layout(data_layout),
            crop=(crop, crop),
            image_type=types.RGB,
            mean=mean,
            std=std,
            pad_output=pad_output)

        self.to_int64 = ops.Cast(dtype=types.INT64, device="gpu")

    def define_graph(self):
        jpegs, labels = self.input(name="Reader")
        images = self.decode(jpegs)
        images = self.resize(images)
        output = self.cmnp(images)
        return [output, self.to_int64(labels.gpu())]

    def __len__(self):
        return self.epoch_size("Reader")

def build(args,
          trainer_id,
          num_trainers,
          mode='train',
          device_id=0,
          output_dtype='float16'):
    assert mode in ['train', 'val']
    assert trainer_id is not None
    assert num_trainers is not None

    file_root = args.data_dir
    batch_size_train = args.batch_size_train
    batch_size_test = args.batch_size_test

    mean = [v * 255 for v in args.image_mean]
    std = [v * 255 for v in args.image_std]
    image_shape = [int(m) for m in args.image_shape.split(",")]
    crop = image_shape[1]
    pad_output = True if image_shape[0] == 4 else False
    min_area = args.lower_scale
    lower = args.lower_ratio
    upper = args.upper_ratio
    seed = args.random_seed

    if mode == 'val':
        rec_path = os.path.join(file_root, 'val.rec')
        idx_path = os.path.join(file_root, 'val.idx')

        pipe = HybridValPipe(
            rec_path=rec_path,
            idx_path=idx_path,
            batch_size=batch_size_test,
            resize_shorter=args.resize_short_size,
            crop=crop,
            mean=mean,
            std=std,
            device_id=device_id,
            shard_id=trainer_id,
            num_shards=num_trainers,
            seed=seed + trainer_id,
            data_layout=args.data_format,
            num_threads=args.dali_num_threads,
            output_dtype=output_dtype,
            pad_output=pad_output,
            nvjpeg_padding=args.dali_nvjpeg_memory_padding)
        pipe.build()
        real_size = int(
            math.floor(1.0 * (trainer_id + 1) * len(pipe) / num_trainers) -
            math.floor(1.0 * trainer_id * len(pipe) / num_trainers))

        return DALIGenericIterator(
            pipe, ['feed_image_test', 'feed_label_test'],
            size=real_size,
            dynamic_shape=True,
            fill_last_batch=True,
            last_batch_padded=True)

    rec_path = os.path.join(file_root, 'train.rec')
    idx_path = os.path.join(file_root, 'train.idx')

    print("dali device_id:", device_id, "shard_id:", trainer_id, "num_shard:",
          num_trainers)
    pipe = HybridTrainPipe(
        rec_path=rec_path,
        idx_path=idx_path,
        batch_size=args.batch_size_train,
        crop=crop,
        min_area=args.lower_scale,
        lower=args.lower_ratio,
        upper=args.upper_ratio,
        mean=mean,
        std=std,
        device_id=device_id,
        shard_id=trainer_id,
        num_shards=num_trainers,
        random_shuffle=False,
        seed=seed + trainer_id,
        data_layout=args.data_format,
        num_threads=args.dali_num_threads,
        output_dtype=output_dtype,
        pad_output=pad_output,
        nvjpeg_padding=args.dali_nvjpeg_memory_padding,
        decoder_buffer_hint=args.dali_decoder_buffer_hint,
        normalize_buffer_hint=args.dali_normalize_buffer_hint)
    pipe.build()
    pipelines = [pipe]
    sample_per_shard = len(pipe) // num_trainers
    return DALIGenericIterator(
        pipelines, ['feed_image', 'feed_label'], size=sample_per_shard)


# coding=utf8
from base.models import Log
from mongoengine import *
import datetime

class WebLogPrev(EmbeddedDocument):
    page = StringField(required=True)
    appendix = DictField(required=False)

class WebLog(Log):
    page = StringField(required=True)
    prev = EmbeddedDocumentField(WebLogPrev, required=False)

def _gen_prev(prev_str):
    _tokens = prev_str.split('_')
    _page = _tokens[0].upper()
    _appendix = None
    if _page == 'SEARCH':
        _appendix = { 'query' : _tokens[1] }

    if _page in ['ENTITY']:
        _appendix = { 'id' : Entity.get_entity_id_by_hash(_tokens[1]) }

    if _page in ['POPULAR']:
        _appendix = { 'scale' : _tokens[1] }

    if _page == 'TAG':
        _appendix = { 'tag' : _tokens[1] }

    if _page in ['USER']:
        _appendix = {
            'id' : _tokens[1],
            'block' : _tokens[2]
        }

    if _page == 'EXTERNAL':
        _appendix = { 'source' : _tokens[1] }

    _prev_doc = WebLogPrev(
        page = _page,
    )
    if _appendix != None:
        _prev_doc.appendix = _appendix
    return _prev_doc

def log(duration, page, ip, log_time, entry='web', request_user_id=None, prev_str=None, appendix=None):
    _doc = WebLog(
        entry=entry,
        duration=duration,
        ip=ip,
        page=page.upper(),
        log_time=log_time,
    )
    if request_user_id == None:
        _doc.user_id = -1
    else:
        _doc.user_id = int(request_user_id)
    if _doc.page in ['ENTITY', 'CATEGORY_ENTITY'] and prev_str != None:
        _doc.prev = _gen_prev(prev_str)
    if appendix != None:
        _doc.appendix = appendix
    _doc.save()


from airflow.decorators import task
from airflow.providers.postgres.hooks.postgres import PostgresHook
import csv
import pandas as pd
import logging

@task(task_id="load_csv_to_staging", weight_rule="upstream")
def load_csv_to_staging(data):
    csv_filename = data["filename"]
    store_id = data["store_id"]

    logging.info(f"Loading to Staging {csv_filename}")

    df = pd.read_csv(
        f"./dags/stage/etl_openmeteo/csv/transformed/{csv_filename}",
        sep=";",
        quoting=csv.QUOTE_NONNUMERIC,
        quotechar='"',
        encoding="utf-8",
        dtype=str,
    )
    postgres_hook = PostgresHook(postgres_conn_id="postgres", schema="airflow")
    extract_table = f"e_openmeteo_data_{store_id}"
    df.to_sql(
        name=extract_table,
        con=postgres_hook.get_sqlalchemy_engine(),
        schema="staging",
        index=False,
        if_exists="replace",
        chunksize=1000,
    )

    return {"extract_table": extract_table, "store_id": store_id}


class Node:
    def __init__(self, data):
        self.data = data
        self.nextnode = None

def insertNodeAtEnd(headnode, data):
    NewNode = Node(data)
    if headnode.nextnode is None:
        headnode.nextnode = NewNode
        return headnode
    laste = headnode
    while laste.nextnode:
        laste = laste.nextnode
    laste.nextnode = NewNode
    return headnode

def printList(head):
    printdata = head
    while True:
        print(printdata.data)
        if printdata.nextnode is None:
            break
        printdata = printdata.nextnode

e1 = Node(16)
e2 = Node(13)
e3 = Node(7)
e2.nextnode = e3
e1.nextnode = e2

printList(e1)

insertNodeAtEnd(e1, 1)

printList(e1)


__author__ = 'Arun'

def getHPUXIPName(con):
    stdin, stdout, stderr = con.exec_command(r"cat /etc/rc.config.d/netconf | grep -F INTERFACE_NAME[")
    ipname= ""
    for i in stdout.readlines():
        ipname = ipname + i.split('\"', 1)[1].split('\"')[0] + "\n"
    return ipname

def getHPUXIPAddr(con):
    stdin, stdout, stderr = con.exec_command(r"cat /etc/rc.config.d/netconf | grep -F IP_ADDRESS[")
    ip= ''
    for i in stdout.readlines():
        if '\"' in i:
         ip = ip + i.split('\"', 1)[1].split('\"')[0] + "\n"
        else:
         ip= ip + i.split('=')[1].rstrip() +"\n"
    return ip

def getHPUXMACAddr(con):
    stdin, stdout, stderr = con.exec_command(r"cat /etc/rc.config.d/netconf | grep -F INTERFACE_NAME[")
    ip= ''
    mac=''
    for i in stdout.readlines():
        #ip = ip + i.split('\"', 1)[1].split('\"')[0] + "\n"

        stdin1, stdout1, stderr1 = con.exec_command(r"lanscan")
        ip= ''
        for j in stdout1.readlines():
            if '\"' in i:            # "lan0" format
                if  j.split()[4] == i.split('\"', 1)[1].split('\"')[0]:
                    mac=mac+j.split()[1][2:]+"\n"
            else:                    # lan0 format
                if  j.split()[4] == i.split("=")[1].rstrip():
                    mac=mac+j.split()[1][2:]+"\n"

    return mac


# gives u a list of string and u have to return the longest string chain
# time - O( N * M^2 + N(log(N)) )
# space - O( N * M)

def fun(strings):
    stringChains = {}
    for string in strings:
        stringChains[string] = {"nextString":"","maxChainLength":1}

    sortedStrings = sorted(strings,key=len)
    for string in sortedStrings:
        findLongestStringChain(string,stringChains)

    return buildLongestStringChain(strings,stringChains)

def findLongestStringChain(string,stringChains):
    for i in range(len(string)):
        smallerString = getSmallerString(string,i)
        if smallerString not in stringChains:
            continue
        tryUpdateLongestStringChain(string,smallerString,stringChains)

def getSmallerString(string,index):
    return string[0:index] + string[index + 1 :]

def tryUpdateLongestStringChain(currentString,smallerString,stringChains):

    smallerStringChainLength = stringChains[smallerString]["maxChainLength"]
    currentStringChainLength = stringChains[currentString]["maxChainLength"]

    if smallerStringChainLength + 1 > currentStringChainLength:
        stringChains[currentString]["maxChainLength"] = smallerStringChainLength+1
        stringChains[currentString]["nextString"] = smallerString

def buildLongestStringChain(strings,stringChains):
    maxChainLength = 0
    chainStartingString = ""
    for string in strings:
        if stringChains[string]["maxChainLength"] > maxChainLength:
            maxChainLength = stringChains[string]["maxChainLength"]

    ourLongestStringChain = []
    currentString = chainStartingString
    while currentString != "":
        ourLongestStringChain.append(currentString)
        currentString = stringChains[currentString]["nextString"]

    return [] if len(ourLongestStringChain) == 1 else ourLongestStringChain


#!/usr/bin/python3
"""
Python script that takes in a URL and an email, sends a POST
request to the passed URL with the email as a parameter, and
displays the body of the response
"""
from urllib import request, parse
import sys

if __name__ == "__main__":
    """create a dictionary 'values' with email as a key"""
    values = {'email': sys.argv[2]}
    """encode the dic into a url-encoded string"""
    data = parse.urlencode(values)
    """encode the url-encoded string as ascii bytes"""
    data = data.encode('ascii')
    """create a http post request with the url and data"""
    req = request.Request(sys.argv[1], data)
    """use urllib.request.urlopen() to send the post request"""
    with request.urlopen(req) as response:
        """read and store the body of the http response as
           'body'"""
        body = response.read()
        print(body.decode('utf-8'))


from pyspark import SparkConf, SparkContext
import collections

#Deals with Key/Value rdd's and average friends by age

conf = SparkConf().setMaster("local").setAppName("FriendsByAge")
sc = SparkContext(conf = conf)

def parseLine(line):
    fields = line.split(',')
    age = int(fields[2])
    numFriends = int(fields[3])
    return (age, numFriends)

lines = sc.textFile("file:////Users/abdulahad/Documents/data-engineering/sparkCourse/fakefriends.csv")
rdd = lines.map(parseLine)
#mapValues --> we add a 1 so we can later add up times up occurence  (age, (#friends,1))
#reduceByKey --> groupby age group, add up #friends for that age group and times of occurence
#now we have for each age group --> (age, (total #friends,total occurences))
totalsByAge = rdd.mapValues(lambda x: (x, 1)).reduceByKey(lambda x, y: (x[0] + y[0], x[1] + y[1]))
#now divide total #friends/total occurences, we get (age, avg # of friends for that age)
averagesByAge = totalsByAge.mapValues(lambda x: x[0] / x[1])
results = averagesByAge.collect()
for result in results:
    print(result)

#for sorted results do this
sortedResults = collections.OrderedDict(sorted(results))
for key, value in sortedResults.items():
    print("%s %i" % (key, value))

Apache Spark - PySpark Code/friends-by-age.py


#!/usr/bin/python
import sys
import argparse
import multiprocessing
from common import *
import termcolor as T
from expt import Expt
from iperf import Iperf
from time import sleep
from host import *
import os

parser = argparse.ArgumentParser(description="Perfiso consistent TX/RX overhead.")

parser.add_argument('--rate',
                    dest="rate",
                    action="store",
                    help="Aggregate rate of VQ/TXC.",
                    required=True)

parser.add_argument('--dir',
                    dest="dir",
                    action="store",
                    help="Directory to store outputs.",
                    required=True)

parser.add_argument('-n',
                    dest="n",
                    action="store",
                    type=int,
                    help="Number of VMs/tenants.",
                    default=1)

parser.add_argument('-t',
                    dest="t",
                    action="store",
                    type=int,
                    help="Time to run expt in seconds.",
                    default=120)

parser.add_argument('--enable',
                    dest="enable",
                    action="store_true",
                    default=False,
                    help="Enable eyeq")

parser.add_argument('--mtu',
                    dest="mtu",
                    help="Set MTU on hosts",
                    default="1500")

parser.add_argument('--vqupdate',
                    dest="vqupdate",
                    help="vqupdate interval",
                    default="25")

parser.add_argument('--datapath',
                    dest="datapath",
                    help="Measure overhead of this datapath",
                    choices=["rx", "tx"],
                    default="rx")

parser.add_argument("--profile",
                    dest="profile",
                    help="Directory to store profile data.  Omit if you don't want to profile",
                    default=None)

args = parser.parse_args()

class RxOverhead(Expt):
    def start(self):
        dir = self.opts("dir")
        if not os.path.exists(dir):
            os.makedirs(dir)

        h1 = Host("10.0.1.1")
        h2 = Host("10.0.1.2")
        hlist = HostList(h1, h2)
        self.hlist = hlist
        n = self.opts("n")

        hlist.rmmod()
        hlist.remove_qdiscs()
        if self.opts("enable"):
            hlist.insmod()
        else:
            # Rate limit second host to just the aggregate rate
            h2.insmod()
            h2.perfiso_set("ISO_MAX_TX_RATE", self.opts("rate"))
            h2.perfiso_set("ISO_RFAIR_INITIAL", self.opts("rate"))

        h1.perfiso_set("ISO_VQ_DRAIN_RATE_MBPS", self.opts('rate'))
        h1.perfiso_set("ISO_VQ_UPDATE_INTERVAL_US", self.opts("vqupdate"))
        if True:
            # Only for QCN tests
            perrate = int(self.opts("rate")) / n
            dt_qcn = min(1200, 100 * 1500 * 8 / perrate)
            hlist.perfiso_set("ISO_RFAIR_INCREASE_INTERVAL_US", dt_qcn)
            hlist.perfiso_set("ISO_RFAIR_DECREASE_INTERVAL_US", dt_qcn)
        # Create all IP routes etc.
        for i in xrange(n):
            hlist.create_ip_tenant(i+1)
        hlist.setup_tenant_routes(self.opts("n"))
        h1.start_monitors(self.opts("dir"))

        # Start iperf server
        iperf = Iperf({'-p': 5001})
        self.procs.append(iperf.start_server(h1))
        sleep(1)

        # Start all iperf clients
        for i in xrange(n):
            parallel = 4
            iperf = Iperf({'-p': 5001,
                           '-P': parallel,
                           '-i': '1',
                           '-c': h1.get_tenant_ip(i+1),
                           '-B': h2.get_tenant_ip(i+1),
                           'dir': os.path.join(self.opts("dir"), "iperf"),
                           '-t': self.opts('t')})
            self.procs.append(iperf.start_client(h2))

        if self.opts("profile"):
            self.h1 = h1
            h1.start_profile(dir=self.opts("profile"))

    def stop(self):
        if self.opts("profile"):
            self.h1.stop_profile(dir=self.opts("profile"))

        self.hlist.remove_tenants()
        self.hlist.killall()

class TxOverhead(Expt):
    def start(self):
        h1 = Host("10.0.1.1")
        h2 = Host("10.0.1.2")
        hlist = HostList(h1, h2)
        self.hlist = hlist
        n = self.opts('n')

        hlist.rmmod()
        hlist.remove_qdiscs()
        hlist.configure_rps()
        n = self.opts('n')

        if self.opts("enable"):
            h1.insmod()
            h1.perfiso_set("ISO_MAX_TX_RATE", self.opts("rate"))
            h1.perfiso_set("ISO_RFAIR_INITIAL", self.opts("rate"))
        else:
            # insert htb qdiscs in hierarchy
            dev = h1.get_10g_dev()
            ceil = '%sGbit' % (int(self.opts('rate')) / 1000)
            cmd = "tc qdisc add dev %s root handle 1: htb default 1000" % dev
            h1.cmd(cmd)

            for i in xrange(n):
                classid = 10+i
                tip = h1.get_tenant_ip(i+1)
                rate = "%.3f" % (float(self.opts("rate")) / n)
                cmd = "tc class add dev %s parent 1: classid 1:%s " % (dev, classid)
                cmd += "htb rate %sMbit ceil %s mtu 64000"  % (rate, ceil)
                h1.cmd(cmd)

                cmd = 'tc filter add dev %s protocol ip parent 1: prio 1 ' % dev
                cmd += " u32 match ip src %s flowid 1:%s " % (tip, classid)
                h1.cmd(cmd)

        for i in xrange(n):
            #hlist.create_ip_tenant(i+1)
            ip = h1.get_tenant_ip(i+1)
            h1.perfiso_create_txc(ip)
            h1.tenants.append(i+1)
            h1.cmd("ifconfig %s:%d %s" % (h1.get_10g_dev(), i+1, ip))
            h2.cmd("ifconfig %s:%d %s" % (h2.get_10g_dev(), i+1, h2.get_tenant_ip(i+1)))
        hlist.setup_tenant_routes(n)
        h1.start_monitors(self.opts("dir"))

        # Start iperf server
        iperf = Iperf({'-p': 5001})
        self.procs.append(iperf.start_server(h2))
        sleep(1)

        # Start all iperf clients
        for i in xrange(n):
            parallel = 4
            iperf = Iperf({'-p': 5001,
                           '-P': parallel,
                           '-i': '1',
                           '-c': h2.get_tenant_ip(i+1),
                           'dir': os.path.join(self.opts("dir"), "iperf"),
                           '-t': self.opts('t')})
            self.procs.append(iperf.start_client(h1))
        if self.opts("profile"):
            self.h1 = h1
            h1.start_profile(dir=self.opts("profile"))
        return

    def stop(self):
        if self.opts("profile"):
            self.h1.stop_profile(dir=self.opts("profile"))

        self.hlist.remove_tenants()
        self.hlist.killall()
        self.hlist.remove_qdiscs()

if __name__ == "__main__":
    if args.datapath == "rx":
        RxOverhead(vars(args)).run()
    else:
        TxOverhead(vars(args)).run()