# coding: utf-8
import logging
from django.conf import settings
from django.contrib import messages
from django.http import HttpResponse, Http404
from django.template import loader, Context
from django.core.urlresolvers import reverse
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.shortcuts import render_to_response
from django.utils.translation import ugettext as _
from django.template.context import RequestContext
from django.template.defaultfilters import slugify
from django.forms.models import inlineformset_factory
from django.forms.formsets import formset_factory
from django.contrib.auth.decorators import login_required, permission_required, user_passes_test
from django.db.models import Max

from journalmanager.models import Journal, JournalMission, Issue
from journalmanager.forms import RestrictedJournalForm, JournalMissionForm

from editorialmanager import notifications
from scielomanager.tools import get_paginated
from audit_log import helpers

from . import forms
from . import models

logger = logging.getLogger(__name__)

def _get_order_for_new_members(board, role_pk, new_member_pk):
    """
    return the value for new members order field.
    - if does not exist other members in board, return 1 (first board member)
    - if does not exist other members with the same role of the new member, return the max order of board + 1 (member with a new role on board)
    - if exist members with the same role, return the order field value of those members.
    """
    # collect all board members except the new one
    board_members = board.editorialmember_set.all().exclude(pk=new_member_pk)
    if board_members.count() > 0:  # already exist others members on this board
        # collect all others board members with the same role as the new members
        members_with_role = board_members.filter(role__pk=role_pk)
        if members_with_role.count() > 0:
            # already exist others members with the same role as the new member
            # the result will be the same order field that those members found
            return members_with_role[0].order
        else:
            # no members with this role, so return the maximum order found + 1
            max_order_of_board = board_members.aggregate(Max('order'))
            return max_order_of_board['order__max'] + 1
    else:
        # no members with this board, this should be the first one
        return 1

def _update_member_order(member, old_role):
    """
    update the @param ``member``'s order attribute, depending on this situations:
    - (A) more than one user with role == ``old_role``, ``member`` is moved to a new role that DOES NOT have any members yet.
    - (B) more than one user with role == ``old_role``, ``member`` is moved to a new role that ALREADY have other members.
    - (C) ``member`` is the last one with role == ``old_role``, ``member`` is moved to a new role that DOES NOT have any members yet.
    - (D) ``member`` is the last one with role == ``old_role``, ``member`` is moved to a new role that ALREADY have other members.
    """
    board = member.board
    board_members = board.editorialmember_set.all()
    members_of_old_role = board_members.filter(role__pk=old_role.pk).exclude(pk=member.pk)
    members_of_new_role = board_members.filter(role__pk=member.role.pk).exclude(pk=member.pk)
    new_order_value = None

    if members_of_old_role.count() == 0:
        # ``member`` WAS the only member with role == old_role
        previous_order = member.order
        if members_of_new_role.count() == 0:
            # ``member`` IS the FIRST member with role == new_role -----> (C)
            max_order_of_board = board_members.aggregate(Max('order'))
            new_order_value = max_order_of_board['order__max'] + 1
        else:
            # there are more members with role == new_role -----> (D)
            new_order_value = members_of_new_role[0].order

        # update member.order to: new_order_value
        member.order = new_order_value
        member.save()

        # decrease order by one, to avoid the empty "bucket" of old_role let by the change
        for m in board.editorialmember_set.filter(order__gte=previous_order):
            m.order = m.order - 1
            m.save()
    else:
        # there are more members with role == old_role
        if members_of_new_role.count() == 0:
            # ``member`` IS the FIRST member with role == new_role -----> (A)
            max_order_of_board = board_members.aggregate(Max('order'))
            new_order_value = max_order_of_board['order__max'] + 1
        else:
            # there are more members with role == new_role  -----> (B)
            new_order_value = members_of_new_role[0].order

        # update member.order to: new_order_value
        member.order = new_order_value
        member.save()

def _update_members_order_when_delete(member_deleted):
    """
    Checks if ``member_deleted`` is the last one of this board member with this role,
    and if any members with an order bigger than ``member_deleted.order`` then update it,
    to decrease the order value by one
    """
    board = member_deleted.board
    other_board_members = board.editorialmember_set.filter(role=member_deleted.role).exclude(pk=member_deleted.pk)
    # after member delete, still more members with this role ?
    if other_board_members.count() == 0:
        # the ``member_deleted`` was the last one of this role, so
        # update all members other greater than ``member_deleted.order`` to decrease 1
        for member in board.editorialmember_set.filter(order__gt=member_deleted.order):
            member.order -= 1
            member.save()

def _do_move_board_block(board_pk, position, direction):
    """
    moves de members the board with pk == board_pk), and with order == position, to direction ``direction`` (up, or down).
    * when direction is up, and have more members above, make a swap, updating member's order attribute.
    * when direction is down, and have more members below, make a swap, updating member's order attribute.
    """
    board = models.EditorialBoard.objects.get(pk=board_pk)

    # creating a list of members to reallocate, becasue simply querysets are lazy
    target_members = [m for m in board.editorialmember_set.filter(order=position)]

    if direction.upper() == "UP":
        position_above = int(position) - 1
        if position_above > 0:
            # move members above to target_members's current position
            board.editorialmember_set.filter(order=position_above).update(order=position)
            # move target_members's position to a higher one
            for m in target_members:
                m.order = position_above
                m.save()

    elif direction.upper() == "DOWN":
        lowest_position_possible = board.editorialmember_set.all().order_by('role__pk').distinct('role__pk').count()
        position_below = int(position) + 1
        if position_below <= lowest_position_possible:
            # move members above to target_members's current position
            board.editorialmember_set.filter(order=position_below).update(order=position)
            # move target_members's position to a lower one
            for m in target_members:
                m.order = position_below
                m.save()
    else:
        # direction is not UP nor DOWN, so, ignore it, do nothing, skip it
        logger.error(u"Trying to move a board (pk: %s) block (position: %s) in this direction: %s is not possible, so doing nothing!" %  (board_pk, position, direction))

def _user_has_access(user):
    return user.is_superuser or user.get_profile().is_editor or user.get_profile().is_librarian or user.get_profile().is_trainee

def _get_journals_by_user_access(user):
    user_profile = user.get_profile()
    if user_profile.is_editor:
        journals = Journal.userobjects.active().filter(editor=user)
    elif user_profile.is_librarian or user.is_superuser or user_profile.is_trainee:
        journals = Journal.userobjects.active()
    else:
        journals = []
    return journals

def _get_journal_or_404_by_user_access(user, journal_id):
    journals = _get_journals_by_user_access(user)
    return get_object_or_404(journals, id=journal_id)

@login_required
@user_passes_test(_user_has_access, login_url=settings.AUTHZ_REDIRECT_URL)
def index(request):
    journals = _get_journals_by_user_access(request.user)

    objects = get_paginated(journals, request.GET.get('page', 1))
    context = {
        'objects_journal': objects,
    }
    return render_to_response('journal/journal_list.html', context, context_instance=RequestContext(request))

@login_required
@user_passes_test(_user_has_access, login_url=settings.AUTHZ_REDIRECT_URL)
def journal_detail(request, journal_id):
    journal = _get_journal_or_404_by_user_access(request.user, journal_id)
    context = {
        'journal': journal,
    }
    return render_to_response('journal/journal_detail.html', context, context_instance=RequestContext(request))

@login_required
@user_passes_test(_user_has_access, login_url=settings.AUTHZ_REDIRECT_URL)
def edit_journal(request, journal_id):
    """
    Handles only existing journals
    """
    user_profile = request.user.get_profile()
    if request.user.is_superuser or user_profile.is_librarian:
        # redirect to full edit view:
        return HttpResponseRedirect(reverse('journal.edit', args=[journal_id, ]))

    journal = _get_journal_or_404_by_user_access(request.user, journal_id)

    JournalMissionFormSet = inlineformset_factory(Journal, JournalMission, form=JournalMissionForm, extra=1, can_delete=True)

    if request.method == "POST":
        journalform = RestrictedJournalForm(request.POST, request.FILES, instance=journal, prefix='journal')
        missionformset = JournalMissionFormSet(request.POST, instance=journal, prefix='mission')

        # this view only handle existing journals, so always exist previous values.
        audit_old_values = helpers.collect_old_values(journal, journalform, [missionformset, ])

        if journalform.is_valid() and missionformset.is_valid():
            journal = journalform.save()
            missionformset.save()

            audit_data = {
                'user': request.user,
                'obj': journal,
                'message': helpers.construct_change_message(journalform, [missionformset, ]),
                'old_values': audit_old_values,
                'new_values': helpers.collect_new_values(journalform, [missionformset, ]),
            }
            # this view only handle existing journals, so always log changes.
            helpers.log_change(**audit_data)

            messages.success(request, _(u'Journal updated successfully.'))

            if request.user.get_profile().is_editor:
                return HttpResponseRedirect(reverse('editorial.index'))
            else:
                return HttpResponseRedirect(reverse('journal.dash', args=[journal.id]))
        else:
            messages.error(request, _(u'Check mandatory fields.'))

    else:
        journalform = RestrictedJournalForm(instance=journal, prefix='journal')
        missionformset = JournalMissionFormSet(instance=journal, prefix='mission')

    return render_to_response('journal/edit_journal.html', {
                              'add_form': journalform,
                              'missionformset': missionformset,
                              }, context_instance=RequestContext(request))

@login_required
@user_passes_test(_user_has_access, login_url=settings.AUTHZ_REDIRECT_URL)
def board(request, journal_id):
    journal = _get_journal_or_404_by_user_access(request.user, journal_id)
    issues = journal.issue_set.all().order_by('-publication_year', '-volume', '-number')
    context = {
        'journal': journal,
        'issues': issues,
    }
    return render_to_response('board/board_list.html', context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.change_editorialmember', login_url=settings.AUTHZ_REDIRECT_URL)
def edit_board_member(request, journal_id, member_id):
    """
    Handles only existing editorial board member
    """

    if request.is_ajax():
        template_name = 'board/board_member_edit_form.html'
    else:
        template_name = 'board/board_member_edit.html'

    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    board_member = get_object_or_404(models.EditorialMember, id=member_id)
    post_url = reverse('editorial.board.edit', args=[journal_id, member_id, ])
    board_url = reverse('editorial.board', args=[journal_id, ])
    context = {
        'board_member': board_member,
        'post_url': post_url,
        'board_url': board_url,
    }

    if request.method == "POST":
        old_role = board_member.role
        form = forms.EditorialMemberForm(request.POST, instance=board_member)
        # this view only handle existing editorial board member, so always exist previous values.
        audit_old_values = helpers.collect_old_values(board_member, form)

        if form.is_valid():
            board_member = form.save()
            if 'role' in form.changed_data:  # change role -> change order
                _update_member_order(board_member, old_role)

            audit_data = {
                'user': request.user,
                'obj': board_member,
                'message': helpers.construct_change_message(form),
                'old_values': audit_old_values,
                'new_values': helpers.collect_new_values(form),
            }
            # this view only handle existing editorial board member, so always log changes.
            helpers.log_change(**audit_data)
            # notify librarians
            notifications.board_members_send_email_by_action(board_member, request.user, audit_data['message'], 'board_edit_member')

            messages.success(request, _(u'Board Member updated successfully.'))
            return HttpResponseRedirect(board_url)
        else:
            messages.error(request, _(u'Check mandatory fields.'))
            context['form'] = form
            return render_to_response(template_name, context, context_instance=RequestContext(request))
    else:
        form = forms.EditorialMemberForm(instance=board_member)

    context['form'] = form
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.add_editorialmember', login_url=settings.AUTHZ_REDIRECT_URL)
def add_board_member(request, journal_id, issue_id):
    """
    Handles only NEW editorial board member
    """
    if request.is_ajax():
        template_name = 'board/board_member_edit_form.html'
    else:
        template_name = 'board/board_member_edit.html'

    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    # get the issue, then the board if exist
    issue = Issue.userobjects.active().filter(pk=issue_id)
    if len(issue) != 1:
        messages.error(request, _(u'The issue is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    issue = issue[0]
    # get the related issue::board, if none: create one
    if not hasattr(issue, 'editorialboard'):
        board = models.EditorialBoard(issue=issue)
        board.save()

    board = issue.editorialboard

    board_url = reverse('editorial.board', args=[journal_id, ])
    post_url = reverse('editorial.board.add', args=[journal_id, issue_id, ])

    context = {
        'post_url': post_url,
        'board_url': board_url,
    }
    if request.method == "POST":
        form = forms.EditorialMemberForm(request.POST)
        if form.is_valid():
            new_member = form.save(commit=False)
            new_member.board = board
            new_member.save()
            new_member.order = _get_order_for_new_members(new_member.board, new_member.role.pk, new_member.pk)
            new_member.save()

            audit_data = {
                'user': request.user,
                'obj': new_member,
                'message': helpers.construct_create_message(form),
                'old_values': None,
                'new_values': helpers.collect_new_values(form),
            }
            # this view only handle NEW editorial board member, so always log create.
            helpers.log_create(**audit_data)
            # notify librarians
            notifications.board_members_send_email_by_action(new_member, request.user, audit_data['message'], 'board_add_member')

            messages.success(request, _(u'Board Member created successfully.'))
            return HttpResponseRedirect(board_url)
        else:
            messages.error(request, _(u'Check mandatory fields.'))
            context['form'] = form
            return render_to_response(template_name, context, context_instance=RequestContext(request))
    else:
        form = forms.EditorialMemberForm()

    context['form'] = form
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.delete_editorialmember', login_url=settings.AUTHZ_REDIRECT_URL)
def delete_board_member(request, journal_id, member_id):
    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    if request.is_ajax():
        template_name = 'board/board_member_delete_data.html'
    else:
        template_name = 'board/board_member_delete.html'

    board_member = get_object_or_404(models.EditorialMember, id=member_id)
    board_url = reverse('editorial.board', args=[journal_id, ])
    post_url = reverse('editorial.board.delete', args=[journal_id, member_id, ])
    context = {
        'board_member': board_member,
        'post_url': post_url,
        'board_url': board_url,
    }
    if request.method == "POST":
        # save the audit log
        audit_message = helpers.construct_delete_message(board_member)
        helpers.log_delete(request.user, board_member, audit_message)
        _update_members_order_when_delete(board_member)
        # notify librarians
        notifications.board_members_send_email_by_action(board_member, request.user, audit_message, 'board_delete_member')
        # delete member!
        board_member.delete()
        messages.success(request, _(u'Board Member DELETED successfully.'))
        return HttpResponseRedirect(board_url)

    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.change_editorialmember', login_url=settings.AUTHZ_REDIRECT_URL)
def board_move_block(request, journal_id):
    board_url = reverse('editorial.board', args=[journal_id, ])

    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(board_url)

    if request.method == "POST":
        # gather data and validate with a form
        data = {
            'journal_pk': request.POST.get("journal_pk", None),
            'issue_pk': request.POST.get("issue_pk", None),
            'board_pk': request.POST.get("board_pk", None),
            'role_name': request.POST.get("role_name", None),
            'role_position': request.POST.get("role_position", None),
            'direction': request.POST.get("direction", None),
        }
        form = forms.BoardMoveForm(data)

        if form.is_valid():
            _do_move_board_block(
                board_pk=data['board_pk'],
                position=data['role_position'],
                direction=data['direction'])

            messages.success(request, _(u'Board block moved successfully.'))
        else:
            messages.error(request, _(u'Board block can not be moved'))
            logger.error(u"Board block can not be moved. form is not valid. Errors: %s" % form.errors)

    return HttpResponseRedirect(board_url)

@login_required
@permission_required('editorialmanager.add_roletype', login_url=settings.AUTHZ_REDIRECT_URL)
def add_role_type(request, journal_id):
    """
    Handles only NEW editorial Role Types
    """
    if request.is_ajax():
        template_name = 'board/role_type_edit_form.html'
    else:
        template_name = 'board/role_type_edit.html'

    board_url = reverse('editorial.board', args=[journal_id, ])
    post_url = reverse('editorial.role.add', args=[journal_id, ])
    context = {
        'post_url': post_url,
        'board_url': board_url,
    }

    if request.method == "POST":
        form = forms.RoleTypeForm(request.POST)
        if form.is_valid():
            new_role = form.save()
            audit_data = {
                'user': request.user,
                'obj': new_role,
                'message': helpers.construct_create_message(form),
                'old_values': None,
                'new_values': helpers.collect_new_values(form),
            }
            # this view only handle NEW editorial board member, so always log create.
            helpers.log_create(**audit_data)
            messages.success(request, _(u'Role created successfully.'))
            return HttpResponseRedirect(board_url)
        else:
            messages.error(request, _(u'Check mandatory fields.'))
            context['form'] = form
            return render_to_response(template_name, context, context_instance=RequestContext(request))
    else:
        form = forms.RoleTypeForm()

    context['form'] = form
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.change_roletype', login_url=settings.AUTHZ_REDIRECT_URL)
def edit_role_type(request, journal_id, role_id):
    """
    Handle only existing editorial Role Types
    """
    if request.is_ajax():
        template_name = 'board/role_type_edit_form.html'
    else:
        template_name = 'board/role_type_edit.html'

    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    role_type = get_object_or_404(models.RoleType, id=role_id)
    post_url = reverse('editorial.role.edit', args=[journal_id, role_id, ])
    board_url = reverse('editorial.board', args=[journal_id, ])
    context = {
        'role_type': role_type,
        'post_url': post_url,
        'board_url': board_url,
    }

    if request.method == "POST":
        form = forms.RoleTypeForm(request.POST, instance=role_type)
        audit_old_values = helpers.collect_old_values(role_type, form)

        if form.is_valid():
            role_type = form.save()
            # audit
            audit_data = {
                'user': request.user,
                'obj': role_type,
                'message': helpers.construct_change_message(form),
                'old_values': audit_old_values,
                'new_values': helpers.collect_new_values(form),
            }
            # this view only handle existing roles, so always log changes.
            helpers.log_change(**audit_data)
            messages.success(request, _(u'Role updated successfully.'))
            return HttpResponseRedirect(board_url)
        else:
            messages.error(request, _(u'Check mandatory fields.'))
            context['form'] = form
            return render_to_response(template_name, context, context_instance=RequestContext(request))
    else:
        form = forms.RoleTypeForm(instance=role_type)

    context['form'] = form
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
def list_role_type(request, journal_id):
    if request.is_ajax():
        template_name = 'board/role_type_list_data.html'
    else:
        template_name = 'board/role_type_list.html'

    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    board_url = reverse('editorial.board', args=[journal_id, ])
    roles = models.RoleType.objects.all().order_by('name')
    context = {
        'board_url': board_url,
        'journal_id': journal_id,
        'roles': roles,
    }
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
@permission_required('editorialmanager.change_roletype', login_url=settings.AUTHZ_REDIRECT_URL)
def translate_role_type(request, journal_id, role_id):
    template_name = 'board/role_type_translate.html'
    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    role_type = get_object_or_404(models.RoleType, id=role_id)
    post_url = reverse('editorial.role.translate', args=[journal_id, role_id, ])
    board_url = reverse('editorial.board', args=[journal_id, ])

    context = {
        'journal_id': journal_id,
        'role_type': role_type,
        'post_url': post_url,
        'board_url': board_url,
    }

    RoleTypeTranslationFormSet = inlineformset_factory(
                                    models.RoleType, models.RoleTypeTranslation,
                                    form=forms.RoleTypeForm, formset=forms.RoleTypeTranslationFormSet,
                                    extra=3, can_delete=True)

    if request.method == "POST":
        formset = RoleTypeTranslationFormSet(request.POST, instance=role_type, prefix="role-translations-formset")
        audit_old_values = helpers.collect_old_values(role_type, form=None, formsets=[formset, ])

        if formset.is_valid():
            formset.save()

            # audit
            audit_data = {
                'user': request.user,
                'obj': role_type,
                'message': helpers.construct_change_message(form=None, formsets=[formset, ]),
                'old_values': audit_old_values,
                'new_values': helpers.collect_new_values(form=None, formsets=[formset, ]),
            }
            # this view only handle existing roles, so always log changes.
            helpers.log_change(**audit_data)

            messages.success(request, _(u'Role updated successfully.'))
            return HttpResponseRedirect(board_url)
        else:
            messages.error(request, _(u'Check mandatory fields.'))
    else:
        formset = RoleTypeTranslationFormSet(instance=role_type, prefix="role-translations-formset")

    context['formset'] = formset
    return render_to_response(template_name, context, context_instance=RequestContext(request))

@login_required
def export_csv(request, journal_id, issue_id=None):
    '''
    Export a list of members following this format:

    journal, issn_print, issn_eletronic, issue_year, issue_volume, issue_number,
    role_name, first_name, last_name, full_name, email, institution, link_cv,
    state, country, country_code, country_code_alpha3, research_id, orcid

    19 fields separeted by comma.

    '''

    # check if user have correct access to view the journal:
    if not Journal.userobjects.active().filter(pk=journal_id).exists():
        messages.error(request, _(u'The journal is not available for you.'))
        return HttpResponseRedirect(reverse('editorial.index'))

    filters = {}
    journal = _get_journal_or_404_by_user_access(request.user, journal_id)
    journal_slug = slugify(journal.title)

    if issue_id:
        filters['pk'] = issue_id
        try:
            issue_year, issue_volume, issue_number = models.Issue.objects.filter(pk=issue_id).values_list(
                'publication_year', 'volume', 'number')[0]
        except models.Issue.DoesNotExist:
            raise Http404("Requested Issue does not exist")
        filename = u'board_%s_%s_v%s_n%s' % (journal_slug, issue_year, issue_volume, issue_number)
    else:
        filename = u'full_board_%s' % journal_slug

    issues = journal.issue_set.filter(**filters).order_by('-publication_year', '-volume', '-number')

    response = HttpResponse(mimetype='text/csv')
    response['Content-Disposition'] = u'attachment; filename="%s.csv"' % filename

    template = loader.get_template('board/export_member_csv.txt')
    context = Context({'journal': journal, 'issues': issues})

    def gen_csv_data_windows_line():
        csv_data = template.render(context)
        for line in csv_data.splitlines():
            yield line.rstrip() + u'\r\n'

    response.write(u''.join(gen_csv_data_windows_line()))

    return response


import numpy as np
import matplotlib.pyplot as plt
import torch
import vit_pytorch
import torchvision
import torch
import pandas as pd
from PIL import Image, ImageFile
from modules.encoder import get_network  # encoder
from modules.TSFE_model import TSFE_model
from modules.dataset_loader import colorspaces, ResizeCrop
from sklearn.linear_model import Ridge, Lasso
from sklearn.model_selection import train_test_split

def srocc_score(lst1, lst2):
    lst1_sort, lst2_sort = list(lst1).copy(), list(lst2).copy()
    N = len(lst1)
    lst1_sort.sort()
    lst2_sort.sort()

    sum = 0
    for i in range(N):
        vi = lst1_sort.index(lst1[i])
        pi = lst2_sort.index(lst2[i])
        sum += (vi - pi) ** 2

    return 1 - (6 * sum / (N * (N**2 - 1)))

def plcc_score(lst1, lst2):
    sum1, sum2, sum3 = 0, 0, 0
    m1, m2 = np.mean(lst1), np.mean(lst2)
    for i in range(len(lst1)):
        sum1 += (lst1[i] - m1) * (lst2[i] - m2)
        sum2 += (lst1[i] - m1) ** 2
        sum3 += (lst2[i] - m2) ** 2

    return sum1 / (np.sqrt(sum2 * sum3))

if __name__ == "__main__":
    mos = np.load('./data/mos.npy')
    feat_cl = np.load('./data/feat_cl(VGG16).npy')  # res50, res101, res152, VGG16, VGG16bn, googlenet, (VGG16)_ImageNet
    feat_vit = np.load('./data/feat_vit(ssim).npy')  # ssim, fsim, gmsd, sf, sg, fg, sfg, sfg2
    feat_all = np.concatenate((feat_cl, feat_vit), 1)
    seed = 42

    X_train, X_test, y_train, y_test = train_test_split(feat_all, mos, test_size=0.2, random_state=seed)
    X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=1/8, random_state=seed)

    max_val_srocc = -1
    for alpha in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 ,1]:
        clf = Ridge(alpha=alpha)
        clf.fit(X_train, y_train)
        y_val_pred = clf.predict(X_val)
        srocc_val = srocc_score(y_val_pred, y_val)
        if srocc_val > max_val_srocc:
            max_val_srocc = srocc_val
            best_alpha = alpha

            y_test_pred = clf.predict(X_test)
            cur_test_srocc = srocc_score(y_test_pred, y_test)
            cur_test_plcc = plcc_score(y_test_pred, y_test)

    print(cur_test_srocc, cur_test_plcc)


import pandas as pd
from bs4 import BeautifulSoup as bs
from splinter import Browser

def scrap_all():

    executable_path = {'executable_path': 'chromedriver.exe'}
    browser = Browser('chrome', **executable_path, headless=False)
    news_title, news_p = mars_news(browser)

    data = {
        "news_title": news_title,
        "news_paragraph": news_p,
        "featured_image": f_link_url(browser),
        "table": tables(),
        "hemisphere_img": hemisphere_image_urls()
    }

    browser.quit()
    return data
def mars_news(browser):

    url = 'https://mars.nasa.gov/news/'
    browser.visit(url)

    html = browser.html
    soup = bs(html, 'html.parser')

    news_title_div = soup.find('div', class_='list_text').find('div', class_='content_title')
    news_title = news_title_div.get_text()
    news_title

    news_p_div = soup.find('div', class_='article_teaser_body')
    news_p_div
    news_p = news_p_div.get_text()
    news_p
def f_link_url(browser):
#JPL Mars Space Images - Featured Image
#visit url
    url_jpl = 'https://www.jpl.nasa.gov/spaceimages//?search=&category=Mars'
    browser.visit(url_jpl)

#find image button
    image = browser.find_by_id('full_image')
    image.click()

#find more info button
    browser.is_element_present_by_text('more info', wait_time=1)
    find_button = browser.links.find_by_partial_text('more info')
    find_button.click()

    html = browser.html
    soup = bs(html, 'html.parser')

    image_url_a = soup.find('figure', class_='lede').find('img', class_='main_image').get('src')
    image_url_a

    base = 'https://www.jpl.nasa.gov'

    f_link_url = base + image_url_a
    browser.visit(f_link_url)
    f_link_url
def tables():
#Mars Facts
    table_url = 'https://space-facts.com/mars/'
    browser.visit(table_url)

    tables = pd.read_html(table_url)
    type(tables)
    list
    tables[0]
    df=tables[0]
    df.columns = ['Description', 'Mars']
    # df.head()
    df.set_index('Description', inplace=True)
    # df.head()
    return df.to_html('table.html')
def hemisphere_image_urls():
#Mars Hemispheres
    hem_url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'
    browser.visit(hem_url)

    base = 'https://astrogeology.usgs.gov/'

#Cerberus Hemisphere Enhanced
    cer_hem = browser.links.find_by_partial_text('Cerberus Hemisphere Enhanced')
    cer_hem.click()

    html = browser.html
    soup = bs(html, 'html.parser')

    cer_hem_url = soup.find('img', class_='wide-image').get('src')

    mars_cer = base + cer_hem_url
    mars_cer

#Schiaparelli Hemisphere Enhanced
    sch_hem = browser.links.find_by_partial_text('Schiaparelli Hemisphere Enhanced')
    sch_hem.click()

    html = browser.html
    soup = bs(html, 'html.parser')

    sch_hem_url = soup.find('img', class_='wide-image').get('src')

    mars_sch = base + cer_hem_url
    mars_sch

#Syrtis Major Hemisphere Enhanced
    syr_hem = browser.links.find_by_partial_text('Syrtis Major Hemisphere Enhanced')
    syr_hem.click()

    html = browser.html
    soup = bs(html, 'html.parser')

    syr_hem_url = soup.find('img', class_='wide-image').get('src')

    mars_syr = base + cer_hem_url
    mars_syr

#Valles Marineris Hemisphere Enhanced
    val_hem = browser.links.find_by_partial_text('Valles Marineris Hemisphere Enhanced')
    val_hem.click()

    html = browser.html
    soup = bs(html, 'html.parser')

    val_hem_url = soup.find('img', class_='wide-image').get('src')

    mars_val = base + val_hem_url
    mars_val

    hemisphere_image_urls = [
        {"title": "Cerberus Hemisphere Enhanced", "img_url": mars_cer},
        {"title": "Schiaparelli Hemisphere Enhanced", "img_url": mars_sch},
        {"title": "Syrtis Major Hemisphere Enhanced", "img_url": mars_syr},
        {"title": "Valles Marineris Hemisphere Enhanced", "img_url": mars_val},
    ]

    hemisphere_image_urls


score = 0
q = int(input('what is 2x2: '))
if q == 4:
    print('correct answer, two points')
    score = score + 2
else:
    print('incorrect')
q2 = input('What is the capital city of England: ')
if q2 == 'london':
    print('correct, 3 points')
    score = score + 3
else:
    print('incorrect')

name = input('enter your name: ')

file = open('sc.txt', 'a')
e = file.write(str(score) + ',' + name + '\n')

file.close()

display = str(score)
print('Score: ' + display)

print('''
            __________________
            |                |
            |  @        @    |
            |       !        |
            |   \_______/    |
            |________________|''')


## Seat name gen(A and C)
def train_seat():
    rows = ['A', 'C']
    arr = []

    for i in range(12):
        arr.append([])

        for j in range(2):
            arr[i].append(str(i+1)+rows[j])

    return arr

## Seat name gen(B and D)
def train_seat2():
    rows = ['B', 'D']
    arr = []

    for i in range(12):
        arr.append([])

        for j in range(2):
            arr[i].append(str(i+1)+rows[j])

    return arr

## Calling Function
seat1 = train_seat()
seat2 = train_seat2()

## TrainList
TrainList = []

for i in range(len(seat1)):
    for j in range(len(seat1[i])):
        if(i < 9):
            TrainList.append('0' + seat1[i][j])
            TrainList.append('0' + seat2[i][j])
        else:
            TrainList.append(seat1[i][j])
            TrainList.append(seat2[i][j])

## Loop to make pattern of seats arrangement

def train():
    print()

    for i in range(9):
        for j in range(2):
            print('0'+seat1[i][j], end = ' ')
            print('0'+seat2[i][j], end = ' ')

            if(j == 0):
                print("  -  ", end = ' ')

        print()

    for i in range(9, 12):
        for j in range(2):
            print(seat1[i][j], end = ' ')
            print(seat2[i][j], end = ' ')

            if(j == 0):
                print("  -  ", end = ' ')

        print()

def checkTrain_ID(x):
    check_bool = False
    for i in range(len(TrainList)):
        if TrainList[i] == x:
            return True


import pandas as pd
import requests
from io import StringIO
import numpy as np

orig_url='https://drive.google.com/file/d/1FW9JcMoovgV3pEq4TJiqQ4DqSWC0JbfE/view'

file_id = orig_url.split('/')[-2]
dwn_url='https://drive.google.com/uc?export=download&id=' + file_id

class calcData:
    def __init__(self):
        pass
    def GloBal_Total(self):
        url = requests.get(dwn_url).text
        csv_raw = StringIO(url)
        dfs = pd.read_csv(csv_raw)
        max_date = max(dfs["Date"])
        total_cases = np.sum(dfs[dfs["Date"]==max_date]["Confirmed"])
        new_cases = np.sum(dfs[dfs["Date"]==max_date]["newConfirmed"])
        total_deaths = np.sum(dfs[dfs["Date"]==max_date]["Death"])
        new_deaths = np.sum(dfs[dfs["Date"]==max_date]["newDeath"])
        total_recovered = np.sum(dfs[dfs["Date"]==max_date]["Recovered"])
        new_recovered = np.sum(dfs[dfs["Date"]==max_date]["newRecovered"])
        total_active = (total_cases-total_deaths) - total_recovered
        return [total_cases, new_cases, total_deaths, new_deaths, total_recovered,new_recovered, total_active]

    def country_data(self, country_name):
        url = requests.get(dwn_url).text
        csv_raw = StringIO(url)
        dfs = pd.read_csv(csv_raw)
        dfs["Country"] =  dfs["Country"].apply(lambda x: x.lower())
        country_q = country_name.lower()
        max_date = max(dfs[dfs["Country"]==country_q]["Date"])
        cond1 = dfs["Country"]==country_q
        cond2 = dfs["Date"]==max_date
        total_cases = np.sum(dfs[cond1 & cond2]["Confirmed"])
        new_cases = np.sum(dfs[cond1 & cond2]["newConfirmed"])
        total_deaths = np.sum(dfs[cond1 & cond2]["Death"])
        new_deaths = np.sum(dfs[cond1 & cond2]["newDeath"])
        total_recovered = np.sum(dfs[cond1 & cond2]["Recovered"])
        new_recovered = np.sum(dfs[cond1 & cond2]["newRecovered"])
        total_active = (total_cases-total_deaths) - total_recovered
        return [total_cases, new_cases, total_deaths, new_deaths, total_recovered, new_recovered, total_active, country_q]

    def country_list(self):
        url = requests.get(dwn_url).text
        csv_raw = StringIO(url)
        dfs = pd.read_csv(csv_raw)
        list_cntry = dfs["Country"].unique()
        text_cntry = ["Countries: \n"]
        for index, item in enumerate(list_cntry):
            text_cntry.append(str(index + 1)+ ": "+ item +"\n")
        message_cntry = " ".join(text_cntry)
        return message_cntry


from __future__ import annotations

from wekan.base import WekanBase

class Swimlane(WekanBase):
    def __init__(self, parent_board, swimlane_id: str) -> None:
        """ Reference to a Wekan Swimlane """
        super().__init__()
        self.board = parent_board
        self.id = swimlane_id

        data = self.board.client.fetch_json(f'/api/boards/{self.board.id}/swimlanes/{self.id}')
        self.title = data['title']
        self.archived = data['archived']
        self.created_at = self.board.client.parse_iso_date(data['createdAt'])
        self.updated_at = self.board.client.parse_iso_date(data.get('updatedAt', data['createdAt']))
        self.sort = data.get('sort')
        self.color = data.get('color', '')
        self.type = data['type']

    def __repr__(self) -> str:
        return f"<Swimlane (id: {self.id}, title: {self.title})>"

    @classmethod
    def from_dict(cls, parent_board, data: dict) -> Swimlane:
        """
        Creates an instance of class Swimlane by using the API-Response of Swimlane GET.
        :param parent_board: Instance of Class Board pointing to the current Board
        :param data: Response of Swimlane GET.
        :return: Instance of class Swimlane
        """
        return cls(parent_board=parent_board, swimlane_id=data['_id'])

    @classmethod
    def from_list(cls, parent_board, data: list) -> list:
        """
        Wrapper around function from_dict to process multiple objects within one function call.
        :param parent_board: Instance of Class Board pointing to the current Board
        :param data: Responses of Swimlane GET.
        :return: Instances of class Swimlane
        """
        instances = []
        for swimlane in data:
            instances.append(cls(parent_board=parent_board, swimlane_id=swimlane['_id']))
        return instances

    def delete(self) -> None:
        """
        Delete the Swimlane instance according to https://wekan.github.io/api/v6.22/#get_swimlane
        :return: None
        """
        self.board.client.fetch_json(f'/api/boards/{self.board.id}/swimlanes/{self.id}', http_method="DELETE")


import numpy as np
import pandas as pd
import math
import matplotlib.pyplot as plt

def calculate_garman_klass_volatility(open_price, close_price, high_price, low_price, num_trading_days):

    try:
        ln_ratio_open_close = math.log(close_price / open_price)
        ln_ratio_high_low = math.log(high_price / low_price)

        first_component = (0.5 * ln_ratio_open_close) ** 2
        second_component = (2 * ln_ratio_high_low - ln_ratio_open_close) ** 2

        average_daily_volatility = (first_component - second_component) / (num_trading_days * math.log(2))
        #print(f"average_daily_volatility: {average_daily_volatility}")
        realized_volatility = math.sqrt(abs(average_daily_volatility))

        return realized_volatility
    except:
        pass

df = pd.read_csv("optionsdata.csv")
#print(df.head())
#print(df.info())
#print(df.iloc[1])
#print(df.get(["OPEN","CLOSE","HIGH","LOW"]))
test = pd.DataFrame(df.get(["OPEN","CLOSE","HIGH","LOW"]))
l = test.values.tolist()
#l = test.iloc[1:3].values.tolist()
#print(l)
gks = []

for i in l:
    #print(f"i: {i}")
    garm = calculate_garman_klass_volatility(i[0],i[1],i[2],i[3],20)
    if garm != None:
        gks.append(garm)
    #print("Garman-Klass Vol: ", garm)

#time = np.arange(0, len(gks)*200, 200)

# Calculate the average of every 200 values
agks = [np.mean(gks[i:i+110]) for i in range(0, len(gks), 110)]

# Plotting the data
plt.plot(agks)

# Adding labels and title
plt.xlabel('Time')
plt.ylabel('Data')
plt.title('Data vs. Time')

# Display the plot
plt.show()

#Fourier Analysis on the signal

from numpy.fft import rfft, irfft, rfftfreq
from scipy import pi, signal, fftpack

gks_ft = np.abs(rfft(agks))

gks_freq = rfftfreq(len(agks))

plt.plot(gks_freq, gks_ft)

plt.xlabel('frequency (1/day)')
plt.show()

# Apply FFT
fft_values = np.fft.fft(agks)

# Calculate magnitude spectrum
magnitude_spectrum = np.abs(fft_values)

# Generate frequency axis
N = len(agks)
T = 1  # Assuming unit time interval between data points
frequencies = np.fft.fftfreq(N, T)

# Define the threshold value
threshold = 30  # Adjust as needed

# Filter out high-frequency components
fft_values_filtered = fft_values.copy()
fft_values_filtered[magnitude_spectrum < threshold] = 0
denoised_data = np.fft.ifft(fft_values_filtered).real
plt.plot(range(N), agks, label='Original Data')
plt.plot(range(N), denoised_data, label='Denoised Data')
plt.xlabel('Time')
plt.ylabel('Value')
plt.title('Original Data vs Denoised Data')
plt.legend()

#ARIMA
#
#import pandas as pd
#import matplotlib.pyplot as plt
#from statsmodels.tsa.arima.model import ARIMA
#
## Example time series data
#
#data = agks
## Create the ARIMA model
#model = ARIMA(agks, order=(1, 0, 10))
#
## Fit the ARIMA model
#model_fit = model.fit()
#
## Generate forecasts
#forecast_values = model_fit.forecast(steps=5)
#
#
## Plot the original data and the forecasts
#plt.plot(range(len(data)), data, label='Original Data')
#plt.plot(range(len(data), len(data) + len(forecast_values)), forecast_values, label='Forecasted Data')
#plt.xlabel('Time')
#plt.ylabel('Value')
#plt.title('ARIMA Forecast')
#plt.legend()
#plt.show()


from modules.helper.redis_client import RedisClient

class TestRedisClient:
    def test_redis_client_connect_to_server(self, mocker):
        mock_redis = mocker.Mock()
        mocker.patch('redis.Redis', return_value=mock_redis)

        with RedisClient() as redis_client:
            assert redis_client == mock_redis


import asyncio
import logging

import requests

from chitanda import BotError
from chitanda.config import config
from chitanda.util import args, register

logger = logging.getLogger(__name__)

@register('wolframalpha')
@args(r'(.+)')
async def call(*, bot, listener, target, author, args, private):
    """Queries the Wolfram|Alpha API and relays the response."""
    future = asyncio.get_event_loop().run_in_executor(
        None,
        lambda: requests.get(
            'https://api.wolframalpha.com/v1/result',
            headers={'User-Agent': config['user_agent']},
            params={'appid': config['wolframalpha']['appid'], 'i': args[0]},
            timeout=15,
        ),
    )

    try:
        response = (await future).text
    except requests.RequestException as e:
        logger.error(f'Failed to query Wolfram|Alpha: {e}')
        raise BotError('Failed to query Wolfram|Alpha.')

    return f'{response[:497]}...' if len(response) >= 500 else response


# P6.py
#
# Write a program containing a pair of nested while loops that displays the
# integer values 1–100, ten numbers per row, with the columns aligned as
# shown below:
#    1   2   3   4   5   6   7   8   9  10
#   11  12  13  14  15  16  17  18  19  20
#   21  22  23  24  25  26  27  28  29  30
#   ...
#   91  92  93  94  95  96  97  98  99 100
#
# date:    10/02/2016
# author:  Chiayo Lin
# license: GPL 3.0

# i can't write a pair of while loops for this problem. a
# _potential_ O(n^2) solution to this problem is just a sin.
# but if i really have to nest it, here is the solution:
counter = 0
while counter != 100:
    while counter != 100:
        counter += 1
        print(format(counter, "4d"), end = '')
        not counter % 10 and print() or None
# and yes, the outer while loop does nothing :)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import re
from setuptools import setup, find_packages

def get_version(*file_paths):
    filename = os.path.join(os.path.dirname(__file__), *file_paths)
    version_file = open(filename).read()
    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M)
    if version_match:
        return version_match.group(1)
    raise RuntimeError('Unable to find version string.')

version = get_version("django_secrets", "__init__.py")
readme = open('README.rst').read()
url = 'https://github.com/kakulukia/django-secrets'

setup(
    name='django-secrets',
    version=version,
    description='The easy way of handling Django secrets.',
    long_description=readme,
    author='Andy Grabow',
    author_email='andy@freilandkiwis.de',
    license='MIT',
    keywords=['keys', 'key', 'secret', 'secrets','Django', 'deployment'],
    url=url,
    download_url=url + '/tarball/' + version,
    install_requires=[
        'Django>=2.0',
        'future',
        'six',
    ],
    python_requires='>=3.6',
    packages=find_packages(),
    include_package_data=True,
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Framework :: Django',
        'Framework :: Django :: 2.0',
        'Framework :: Django :: 3.0',
        'Framework :: Django :: 4.0',
        'Topic :: Software Development :: Libraries :: Python Modules',
        'Topic :: Security',
    ],
)


from PIL import Image
from cassandra.auth import PlainTextAuthProvider
from getpass import getpass
from tqdm import tqdm
import io
import numpy as np
import os
import uuid
import pandas as pd
from cassandradl import CassandraWriter

def get_data(path):
    with open(path, "rb") as f:
        data = f.read()
    return data

def save_images(cassandra_ip, cass_user, cass_pass, suffix):
    auth_prov = PlainTextAuthProvider(cass_user, cass_pass)

    def ret(jobs):
        cols = [
            "image_id",
            "top_label_name",
            "type_name",
            "type",
            "grade_name",
            "grade",
            "wsi",
            "roi",
            "mpp",
            "x",
            "y",
            "w",
            "h",
            "or_split",
        ]
        cw = CassandraWriter(
            auth_prov,
            [cassandra_ip],
            table_ids=f"unito.ids_{suffix}",
            table_data=f"unito.data_{suffix}",
            table_metadata=f"unito.metadata_{suffix}",
            id_col="patch_id",
            label_col="top_label",
            data_col="data",
            cols=cols,
            get_data=get_data,
        )
        for path, label, partition_items in tqdm(jobs):
            cw.save_image(path, label, partition_items)

    return ret

def get_jobs(src_dir):
    jobs = []
    labels = dict()
    ln = 0  # next-label number
    for or_split in ["train", "test"]:
        csv = os.path.join(src_dir, or_split + ".csv")
        tab = pd.read_csv(csv, keep_default_na=False)
        for _, r in tab.iterrows():
            cur_dir = os.path.join(src_dir, r["top_label_name"])
            path = os.path.join(cur_dir, r["image_id"])
            partition_items = r.to_list() + [or_split]
            label = partition_items.pop(2)
            jobs.append((path, label, partition_items))
    return jobs


import datetime as dt

import pytest
import pytz

from analytix.groups import Group, GroupItem, GroupItemList, GroupList

@pytest.fixture()
def group_data():
    return {
        "kind": "youtube#group",
        "etag": "f6g7h8i9j0",
        "id": "a1b2c3d4e5",
        "snippet": {
            "publishedAt": "2022-11-30T12:34:56.789Z",
            "title": "Barney the Dinosaur",
        },
        "contentDetails": {
            "itemCount": "69",
            "itemType": "youtube#video",
        },
    }

@pytest.fixture()
def group():
    return Group(
        kind="youtube#group",
        etag="f6g7h8i9j0",
        id="a1b2c3d4e5",
        published_at=dt.datetime(2022, 11, 30, 12, 34, 56, 789000, tzinfo=pytz.utc),
        title="Barney the Dinosaur",
        item_count=69,
        item_type="youtube#video",
    )

def test_create_group_from_json(group_data, group):
    assert Group.from_json(group_data) == group

def test_group_data_property(group_data, group):
    assert group.data == group_data

@pytest.fixture()
def group_list_data(group_data):
    return {
        "kind": "youtube#groupListResponse",
        "etag": "f6g7h8i9j0",
        "items": [group_data],
        "nextPageToken": "a1b2c3d4e5",
    }

@pytest.fixture()
def group_list(group):
    return GroupList(
        kind="youtube#groupListResponse",
        etag="f6g7h8i9j0",
        items=[group],
        next_page_token="a1b2c3d4e5",
    )

def test_create_group_list_from_json(group_list_data, group_list):
    assert GroupList.from_json(group_list_data) == group_list

def test_group_list_data_property(group_list_data, group_list):
    assert group_list.data == group_list_data

def test_group_list_get_item(group_list, group):
    assert group_list[0] == group_list.items[0] == group

@pytest.fixture()
def group_item_data():
    return {
        "kind": "youtube#groupItem",
        "etag": "f6g7h8i9j0",
        "id": "e5d4c3b2a1",
        "groupId": "a1b2c3d4e5",
        "resource": {
            "kind": "youtube#video",
            "id": "j0i9h8g7f6",
        },
    }

@pytest.mark.dependency()
def test_create_group_item_from_json(group_item_data):
    group = GroupItem.from_json(group_item_data)
    assert group.kind == "youtube#groupItem"
    assert group.etag == "f6g7h8i9j0"
    assert group.id == "e5d4c3b2a1"
    assert group.group_id == "a1b2c3d4e5"
    assert group.resource.kind == "youtube#video"
    assert group.resource.id == "j0i9h8g7f6"

@pytest.mark.dependency(depends=["test_create_group_item_from_json"])
@pytest.fixture()
def group_item(group_item_data):
    return GroupItem.from_json(group_item_data)

@pytest.mark.dependency(depends=["test_create_group_item_from_json"])
def test_group_item_data_property(group_item_data, group_item):
    assert group_item.data == group_item_data

@pytest.fixture()
def group_item_list_data(group_item_data):
    return {
        "kind": "youtube#groupItemListResponse",
        "etag": "a1b2c3d4e5",
        "items": [group_item_data],
    }

@pytest.fixture()
def group_item_list(group_item):
    return GroupItemList(
        kind="youtube#groupItemListResponse",
        etag="a1b2c3d4e5",
        items=[group_item],
    )

@pytest.mark.dependency(depends=["test_create_group_item_from_json"])
def test_create_group_item_list_from_json(group_item_list_data, group_item_list):
    assert GroupItemList.from_json(group_item_list_data) == group_item_list

@pytest.mark.dependency(depends=["test_create_group_item_from_json"])
def test_group_item_list_data_property(group_item_list_data, group_item_list):
    assert group_item_list.data == group_item_list_data

@pytest.mark.dependency(depends=["test_create_group_item_from_json"])
def test_group_item_list_get_item(group_item_list, group_item):
    assert group_item_list[0] == group_item_list.items[0] == group_item


from django.shortcuts import render

# Create your views here.

def product(request, product="BananaChips"):
    price = request.GET.get('price', 'N/A')
    nutrition = request.GET.get('nutrition', 'N/A')
    origin = request.GET.get('origin', 'N/A')
    # Create an information dictionary
    info = {'product': product, 'price': price,
            'nutrition': nutrition, 'origin': origin}
    return render(request, 'product/details.html', {'info': info})


'''
Created on Jul 7, 2015

@author: Yohan
'''
import csv
import re
from collections import defaultdict

class Quiz:

    def __init__(self, params):
        pass

    @staticmethod
    def quizzes(db_path):
        fields = ['id','parent_id','open_time','soft_close_time','hard_close_time','maximum_submissions','title','duration','quiz_type','proctoring_requirement','authentication_required','deleted','last_updated']
        for line in open(db_path):
            if not line.startswith("INSERT INTO `quiz_metadata` VALUES "): continue
            for item in csv.DictReader( map(lambda s: re.sub('^\\(','', re.sub('\\);?$','',s)), re.split('(?<=\\)),(?=\\()', line.replace("INSERT INTO `quiz_metadata` VALUES ",""))), quotechar="'", fieldnames=fields ):
                yield item

    @staticmethod
    def quiz_submissions(db_path):
        fields = ['id','item_id','session_user_id','submission_time','submission_number','raw_score','grading_error','authenticated_submission_id']
        for line in open(db_path):
            if not line.startswith("INSERT INTO `quiz_submission_metadata` VALUES "): continue
            for item in csv.DictReader( map(lambda s: re.sub('^\\(','', re.sub('\\);?$','',s)), re.split('(?<=\\)),(?=\\()', line.replace("INSERT INTO `quiz_submission_metadata` VALUES ",""))), quotechar="'", fieldnames=fields ):
                yield item

    @staticmethod
    def valid_quizzes(db_path, threshold):
        quiz_users = defaultdict(set)
        for submission in Quiz.quiz_submissions(db_path):
            quiz_users[submission['item_id']].add(submission['session_user_id'])
        return set( [quiz_id for quiz_id,users in quiz_users.iteritems() if len(users) >= threshold] )


"""
Coursera: Introduction to Bioinformatics
Module implements requirements of lesson 1.3 from week 1.
25 December 2021 - Bahadir Arslan
"""

from pathlib import Path
import sys

PROJECT_DIRECTORY = Path(__file__).parent.parent
sys.path.append(str(PROJECT_DIRECTORY))

from week1.lesson_1_2 import pattern_count

def frequency_map(string: str, length: int) -> dict:

    freq_map = {}
    for i in range(len(string) - length + 1):
        pattern = string[i: i + length]
        freq_map[pattern] = pattern_count(string, pattern)
    return freq_map

def frequent_words(string: str, length: str) -> list:

    words = []
    freq_map = frequency_map(string, length)
    max_val = max(freq_map.values())
    for word, count in freq_map.items():
        if count == max_val:
            words.append(word)
    return sorted(words)


from PIL import Image, ImageGrab
import time
from pynput.mouse import Button, Controller

STACK_COUNT = 6
INITIAL_CARDS = 6

GAME_UPPER_LEFT_X = 800
GAME_UPPER_LEFT_Y = 470
GAME_BOTTOM_RIGHT_X = 1762
GAME_BOTTOM_RIGHT_Y = 820

STARTING_X = 2
STARTING_Y = 2
CARD_WIDTH = 164
CARD_HEIGHT = 32
REF_WIDTH = 34
REF_HEIGHT = 24

class CardImageMatcher:
    def __init__(self):
        self.ref6 = list(Image.open("ref/6.png").getdata())
        self.ref7 = list(Image.open("ref/7.png").getdata())
        self.ref8 = list(Image.open("ref/8.png").getdata())
        self.ref9 = list(Image.open("ref/9.png").getdata())
        self.ref10 = list(Image.open("ref/10.png").getdata())
        self.refv = list(Image.open("ref/v.png").getdata())
        self.refd = list(Image.open("ref/d.png").getdata())
        self.refk = list(Image.open("ref/k.png").getdata())
        self.reft = list(Image.open("ref/t.png").getdata())

    def match(self, image):
        image_data = list(image.getdata())

        if image_data == self.ref6:
            return "6"
        if image_data == self.ref7:
            return "7"
        if image_data == self.ref8:
            return "8"
        if image_data == self.ref9:
            return "9"
        if image_data == self.ref10:
            return "10"
        if image_data == self.refv:
            return "v"
        if image_data == self.refd:
            return "d"
        if image_data == self.refk:
            return "k"
        if image_data == self.reft:
            return "t"
        raise Exception("Can't determine card value")

class GameApi:
    def __init__(self):
        self.stacks = []
        self.mouse = Controller()
        # Get window focus first
        self.__focus()
        self.__new_game()
        self.__populate_game_state()

    def __populate_game_state(self):
        game_image = ImageGrab.grab().crop((GAME_UPPER_LEFT_X, GAME_UPPER_LEFT_Y, GAME_BOTTOM_RIGHT_X, GAME_BOTTOM_RIGHT_Y))
        matcher = CardImageMatcher()

        for x in range(STACK_COUNT):
            stack = []
            for y in range(INITIAL_CARDS):
                current_x = STARTING_X + x * CARD_WIDTH
                current_y = STARTING_Y + y * CARD_HEIGHT
                current_card = game_image.crop((current_x, current_y, current_x + REF_WIDTH, current_y + REF_HEIGHT)).convert('L')
                stack.append(matcher.match(current_card))
            self.stacks.append(stack)

        print("Found card stacks:")
        print(self.stacks)

    def __click_on(self, position_tuple):
        self.mouse.position = position_tuple
        time.sleep(0.05)
        self.mouse.press(Button.left)
        time.sleep(0.05)
        self.mouse.release(Button.left)
        time.sleep(0.05)

    def __focus(self):
        self.__click_on((1280, 1015))

    def __new_game(self):
        self.__click_on((1280, 1215))
        time.sleep(5)

    '''
    Card tuple consists of (stack index, card index).
    '''
    def click_card(self, card_tuple):
        stack_num = card_tuple[0]
        card_num = card_tuple[1]
        x_pos = GAME_UPPER_LEFT_X + STARTING_X + (stack_num * CARD_WIDTH)
        y_pos = GAME_UPPER_LEFT_Y + STARTING_Y + (card_num * CARD_HEIGHT)
        self.__click_on((x_pos, y_pos))


""" Scrape NCBI for missing gene lengths
"""
import requests
from bs4 import BeautifulSoup

missing_genes = [_.strip() for _ in open("DATA/missingGenesAll.txt", 'r').readlines()]

with open("DATA/missedGenes.scraped.tsv", 'a') as ofile:
    for gene in missing_genes:
        print(gene)
        url = "https://www.ncbi.nlm.nih.gov/gene/{}".format(gene)
        print(url)
        page = requests.get(url)
        soup = BeautifulSoup(page.text, 'html.parser')

        # parse for gene length
        gContext = soup.find("table", class_="jig-ncbigrid")
        if gContext:
            gContext = gContext.find_all("td")
            geneCoord = gContext[4].prettify().partition('(')[2].partition(')')[0].split('..')
        else:
            gContext = soup.find("dd", class_="rs-range").find_all("span")
            geneCoord = str(gContext[0]).partition('>')[2].partition('<')[0].split('..')

        print(geneCoord)

        ofile.write('\t'.join([gene] + geneCoord + ['\n']))


# Generator of wordlists

# Author: @andvsilva
# date 2021-11-08

import itertools

string = input("String to be permutated: ")

results = itertools.permutations(string, len(string))

j=1
for i in results:
    print(f'# {j} -> ' + ''.join(i))
    j += 1


from PIL import Image
import numpy as np

import os

src_dir = "celegans"
dst_dir = "celegans_multi_cond"
time_interval = 8

# assume the file is read in order

dst_dir += "_{}".format(time_interval)
for subdir, dirs, files in os.walk(src_dir):
    for fn in files:
        if len(subdir) < 9:
            continue
        src_fn = os.path.join(subdir, fn)

        slice_beg = fn.find("_t") + 2
        slice_end = fn.find(".jpg")
        time_slice = int(fn[slice_beg:slice_end])
        next_fn = fn[0:slice_beg] + \
                "{}".format(time_slice + time_interval) + \
                fn[slice_end:]
        next_src_fn = os.path.join(subdir, next_fn)

        if os.path.isfile(next_src_fn):
            images_list = [src_fn, next_src_fn]
            images = [Image.open(i) for i in images_list]

            merged_image_array = \
                    np.hstack((np.asarray(image) for image in images))
            merged_image = Image.fromarray(merged_image_array)
            new_fn = subdir[-2:] + "_" + fn
            fn_time_slice = (time_slice - 1) / 10
            dst_fn = "{}/{}/{}".format(dst_dir, fn_time_slice, new_fn)

            merged_image.save(dst_fn)


from faker.providers import BaseProvider
from collections import OrderedDict

class CustomPartProvider(BaseProvider):
    """Custom Ethnic Provider"""

    PARTS: OrderedDict[str, float] = OrderedDict([
    ("Head", 0.15),
    ("Shoulders", 0.15),
    ("Chest", 0.15),
    ("Back", 0.15),
    ("Stomach", 0.05),
    ("Waist", 0.05),
    ("Hands", 0.05),
    ("Wrists", 0.05),
    ("Thighs", 0.15),
    ("Feet", 0.10)
    ])

    def part(self) -> str:
        return self.random_elements(self.PARTS, length=1, use_weighting=True)[0]


# -*- coding: utf-8 -*-

import logging
import os
import torch

from TransXL.metrics import classification_report
from TransXL.metrics import f1_score

class InputFeatures(object):
  """
  Input feature for BERT.
  """
  def __init__(self, input_id, label_id, input_mask):
    """
    Args:
      input_id(int): Id of character.
      label_id(str): Id of label for classify.
      input_mask(int): Mask flag, 0 or 1.
    """
    self.input_id = input_id
    self.label_id = label_id
    self.input_mask = input_mask

def load_vocab(vocab_file):
  """
  Loads a vocabulary file into a dictionary.

  Args:
    vocab_file(str): file path.

  Returns:
    vocab(dict): Character and index map.
  """
  vocab = {}
  index = 0
  with open(vocab_file, "r", encoding="utf-8") as reader:
    while True:
      token = reader.readline()
      if not token:
          break
      token = token.strip()
      vocab[token] = index
      index += 1
  return vocab

def get_text_line_feature(sentense, vocab, max_length=512):
  """
  Gets text feature.

  Args:
    sentense(str): Input string.
    vocab(dict): Character and index map.
    max_length(int): Input content max length.

  Returns:
    feature(InputFeatures): Feature of sentense.
  """
  tokens = list(sentense.replace(" ", ""))
  tokens_f = ['[CLS]'] + tokens + ['[SEP]']
  input_ids = [
    int(vocab[i]) if i in vocab else int(vocab['[UNK]']) for i in tokens_f]
  input_mask = [1] * len(input_ids)
  while len(input_ids) < max_length:
    input_ids.append(0)
    input_mask.append(0)
  feature = InputFeatures(
    input_id=input_ids, input_mask=input_mask, label_id=None)
  return feature

def id2tag(tag_id, ids):
  """
  Index to tag.

  Args:
    tag_id(dict): Label and index map.
    ids(list): List of tag.

  Returns:
    tags(list): Tags of ids.
  """
  id_tag = dict([val, key] for key, val in tag_id.items())
  tags = [id_tag.get(id) for id in ids]
  return tags

def decode_tags_biose(text, tags):
  """
  Decodes BIOSE to string with tag.

  Args:
    text(str): Origin string.
    tags(list): Tag of string.

  Returns:
    item(dict): Decoded result of current string.
  """
  item = {"string": text, "entities": []}
  entity_name = ""
  entity_start = 0
  idx = 0
  for char, tag in zip(text, tags):
    if tag[0] == "S":
      item["entities"].append(
        {"word": char, "start": idx, "end": idx+1, "type": tag[2:]})
    elif tag[0] == "B":
      entity_name += char
      entity_start = idx
    elif tag[0] == "I":
      if idx - 1 >= 0:
        if tags[idx-1] == "O":
          entity_start = idx
      entity_name += char
    elif tag[0] == "E":
      if entity_name:
        entity_name += char
        if len(set([t[2:] for t in tags[entity_start:idx +1]])) == 1:
          item["entities"].append(
            {"word": entity_name, "start": entity_start, "end": idx + 1, "type": tag[2:]})
        entity_name = ""
    else:
      entity_name = ""
      entity_start = idx
    idx += 1
  return item

def decode_tags_io(text, tags):
  """
  Decodes IO to string with tag.

  Args:
    text(str): Origin string.
    tags(list): Tag of string.

  Returns:
    item(dict): Decoded result of current string.
  """
  item = {"string": text, "entities": []}
  entity_name = ""
  entity_start = 0
  last_tag = ""
  for idx, (char, tag) in enumerate(zip(text, tags)):
    if tag not in ["O", "<pad>", "<start>", "<eos>"]:
      entity_tag = tag[2:]
    else:
      entity_tag = "O"
    if entity_tag != "O":
      # entity_tag = tag[2:]
      if last_tag != entity_tag:
        entity_name = char
        entity_start = idx
      else:
        entity_name += char
      if idx == len(tags)-1 or tags[idx+1][2:] != entity_tag:
        if len(set([t[2:] for t in tags[entity_start:idx + 1]])) == 1:
          if text[entity_start:idx + 1] != entity_name:
            print("tttttttttttt")
          item["entities"].append(
            {"word": entity_name, "start": entity_start, "end": idx + 1, "type": tag[2:]})
        entity_name = ""
    else:
      entity_name = ""
      entity_start = idx
    last_tag = entity_tag
  return item

def compute(pre, groud_true):
  """
  Computes correct.

  Args:
    pre(list): Predicted tags.
    groud_true(list): Tags of groud true.

  Returns:
    correct_num(int): Num of correct.
  """
  correct_num = 0
  use_num = 0
  for p, t in zip(pre, groud_true):
    if p == "o":
      continue
    use_num += 1
    if p == t:
      correct_num += 1
  return correct_num

def evaluate(true_tags, pred_tags, verbose=False):
  """
  Evaluate method.

  Args:
    true_tags(list): Tags of groud true.
    pred_tags(list): Tags of predict.
    verbose(bool): Whether need verbose f1 report.

  Returns:
    metrics(dict): Metrics of evaluation.
  """
  metrics_all = f1_score(true_tags, pred_tags)
  # logger.debug("Metrics: {}".format(metrics_all))
  metrics = metrics_all.get("avg")

  metrics_str = "; ".join("{}: {:05.2f}".format(k, v)
    for k, v in metrics.items())
  logger.debug("- {} metrics: ".format(metrics_str))
  print("- {} metrics: ".format(metrics_str))
  if verbose:
    report = classification_report(true_tags, pred_tags)
    logger.debug(report)
  return metrics

def decode_tags_bio(text, tags):
  """
  Decodes BIO to string with tag.

  Args:
    string(str): Origin string.
    tags(list): Tag of string.

  Returns:
    item(dict): Decoded result of current string.
  """
  item = {"text": text, "entities": []}
  entity_name = ""
  entity_start = 0
  count = 0
  entity_tag = ""
  for c_idx in range(len(text)):
    c, tag = text[c_idx], tags[c_idx]
    if c_idx < len(tags)-1:
      tag_next = tags[c_idx+1]
    else:
      tag_next = ''
    if tag[0] == 'B':
      entity_tag = tag[2:]
      entity_name = c
      entity_start = count
      if tag_next[2:] != entity_tag:
        item["entities"].append({"word": c, "start": count, "end": count + 1, "type": tag[2:]})
    elif tag[0] == "I":
      if tag[2:] != tags[c_idx-1][2:] or tags[c_idx-1][2:] == 'O':
        tags[c_idx] = 'O'
        pass
      else:
        entity_name = entity_name + c
        if tag_next[2:] != entity_tag:
          item["entities"].append(
            {"word": entity_name, "start": entity_start, "end": iCount + 1, "type": entity_tag})
          entity_name = ''
    count += 1
  return item

def save_model(model, epoch, path='result', **kwargs):
  """
  Saves model.

  Args:
    model(BERT_LSTM_CRF): Model of BERT_LSTM_CRF.
    epoch(int): Number of epoch.
    path(str): Model saved path.
  """
  if not os.path.exists(path):
    os.mkdir(path)
  if kwargs.get('name', None) is None:
    f1 = 0.0
    if kwargs.get("f1"):
      f1 = kwargs.get("f1")
    name = ('epoch_' + str(epoch) +
      "_loss_" + str('%.3f' % kwargs.get("loss")) + "f1_" + str(f1) + ".pkl")
    full_name = os.path.join(path, name)
    if isinstance(model, torch.nn.DataParallel):
      model = model.module
    torch.save(model, full_name)
    print('Saved model at epoch {} successfully'.format(epoch))
    with open('{}/checkpoint'.format(path), 'w') as file:
      file.write(name)
      print('Write to checkpoint')
    file_list = os.listdir(path)
    file_list = sorted(
      file_list, key=lambda x: os.path.getmtime(os.path.join(path, x)))
    surplus = file_list[:-6]
    for file_name in surplus:
      os.remove(os.path.join(path, file_name))

def save_train_model(model, epoch, path='result', **kwargs):
  """
  Saves model.

  Args:
    model(BERT_LSTM_CRF): Model of BERT_LSTM_CRF.
    epoch(int): Number of epoch.
    path(str): Model saved path.
  """
  if not os.path.exists(path):
    os.mkdir(path)
  if kwargs.get('name', None) is None:
    f1 = 0.0
    if kwargs.get("f1"):
      f1 = kwargs.get("f1")
    name = ('epoch_' + str(epoch) +
      "_loss_" + str('%.3f' % kwargs.get("loss")) + "f1_" + str(f1) + ".pth")
    full_name = os.path.join(path, name)
    if isinstance(model, torch.nn.DataParallel):
      model = model.module

    print('Saved model at epoch {} successfully'.format(epoch))
    with open('{}/checkpoint'.format(path), 'w') as file:
      file.write(name)
      print('Write to checkpoint')
    file_list = os.listdir(path)
    file_list = sorted(
      file_list, key=lambda x: os.path.getmtime(os.path.join(path, x)))
    surplus = file_list[:-1]
    for file_name in surplus:
      os.remove(os.path.join(path, file_name))
    torch.save(model.state_dict(), full_name)

def load_model(path='result', **kwargs):
  """
  Loads model.

  Args:
    model(BERT_LSTM_CRF): Model of BERT_LSTM_CRF.
    path(str): Model saved path.

  Returns:
    model(BERT_LSTM_CRF): Model of BERT_LSTM_CRF after added weights.
  """
  if kwargs.get('name', None) is None:
    with open('{}/checkpoint'.format(path)) as file:
      content = file.read().strip()
      name = os.path.join(path, content)
  else:
    name = kwargs['name']
    name = os.path.join(path, name)
  model = torch.load(name, map_location=lambda storage, loc: storage)
  print('load model {} successfully'.format(name))
  return model

def load_train_model(model, path='result', **kwargs):
  if kwargs.get('name', None) is None:
    with open('{}/checkpoint'.format(path)) as file:
      content = file.read().strip()
      name = os.path.join(path, content)
  else:
    name = kwargs['name']
    name = os.path.join(path, name)
  model.load_state_dict(torch.load(name, map_location=lambda storage, loc: storage))
  print('load model {} successfully'.format(name))
  return model

def iob_iobes(tags):
  """
  BIO mode to BIOSE.

  Args:
    tags(list): BIO tags.

  Returns:
    new_tags(list): BIOSE tags.
  """
  new_tags = []
  for i, tag in enumerate(tags):
    if tag == 'O':
      new_tags.append(tag)
    elif tag.split('-')[0] == 'B':
      if i + 1 != len(tags) and tags[i + 1].split('-')[0] == 'I':
        new_tags.append(tag)
      else:
        new_tags.append(tag.replace('B-', 'S-'))
    elif tag.split('-')[0] == 'I':
      if i + 1 < len(tags) and tags[i + 1].split('-')[0] == 'I':
        new_tags.append(tag)
      else:
        new_tags.append(tag.replace('I-', 'E-'))
    else:
      raise Exception('Invalid IOB format!')
  return new_tags

def iobes_iob(tags):
  """
  BIOSE mode to BIO.

  Args:
    tags(list): BIOSE tags.

  Returns:
    new_tags(list): BIO tags.
  """
  new_tags = []
  for i, tag in enumerate(tags):
    if tag.split('-')[0] == 'B':
      new_tags.append(tag)
    elif tag.split('-')[0] == 'I':
      new_tags.append(tag)
    elif tag.split('-')[0] == 'S':
      new_tags.append(tag.replace('S-', 'B-'))
    elif tag.split('-')[0] == 'E':
      new_tags.append(tag.replace('E-', 'I-'))
    elif tag.split('-')[0] == 'O':
      new_tags.append(tag)
    else:
      raise Exception('Invalid format!')
  return new_tags

def get_logger():
  """
  Gets logger obj.

  Returns:
    logger(loggerClass): Logger util.
  """
  logger = logging.getLogger("train")
  logger.setLevel(logging.DEBUG)
  # Builds filehandler write log, above debug level.
  fh = logging.FileHandler("train.log")
  fh.setLevel(logging.DEBUG)
  # Builds streamhandler CMD window，above error.
  ch = logging.StreamHandler()
  ch.setLevel(logging.ERROR)
  # Sets log format.
  formatter = logging.Formatter(
    "%(asctime)s - %(name)s - %(levelname)s - %(message)s")
  ch.setFormatter(formatter)
  fh.setFormatter(formatter)
  # Adds handler to logger obj.
  logger.addHandler(ch)
  logger.addHandler(fh)
  return logger

logger = get_logger()


import Constants as C

class TreeNode():
    leftPointer = 0
    rightPointer = 0
    data = ""
    def __init__(self,LeftPointer=0,RightPointer=0,Data = ""):
        self.data = Data
        self.leftPointer = LeftPointer
        self.rightPointer = RightPointer

class BinaryTree():

    rootPointer = 0
    freePointer = 0
    def __init__(self):
        self.tree  = [TreeNode() for i in range(7)]
        self.rootPointer = C.nullPointer
        self.freePointer = 0
        for i in range(6):
            self.tree[i].leftPointer = i+1
        self.tree[6].leftPointer = C.nullPointer
    def InsertNode(self,newItem):
        if self.freePointer != C.nullPointer:
            self.newNodePointer = self.freePointer
            self.freePointer = self.tree[self.freePointer].leftPointer
            self.tree[self.newNodePointer].data = newItem
            self.tree[self.newNodePointer].leftPointer = C.nullPointer
            self.tree[self.newNodePointer].rightPointer = C.nullPointer
            if self.rootPointer == C.nullPointer:
                self.rootPointer = self.newNodePointer
            else:
                self.thisNodePointer = self.rootPointer
                while self.thisNodePointer != C.nullPointer:
                    self.previousNodePointer  = self.thisNodePointer
                    if self.tree[self.thisNodePointer].data > newItem:
                        self.TurnedLeft = True
                        self.thisNodePointer = self.tree[self.thisNodePointer].leftPointer
                    else:
                        self.TurnedLeft = False
                        self.thisNodePointer = self.tree[self.thisNodePointer].rightPointer
                if self.TurnedLeft == True:
                    self.tree[self.previousNodePointer].leftPointer = self.newNodePointer
                else:
                    self.tree[self.previousNodePointer].rightPointer = self.newNodePointer
    def FindNode(self,searchItem):
        self.thisNodePointer = self.rootPointer
        while self.thisNodePointer != C.nullPointer and self.tree[self.thisNodePointer].data != searchItem:
            if self.tree[self.thisNodePointer].data > searchItem:
                self.thisNodePointer = self.tree[self.thisNodePointer].leftPointer
            else:
                self.thisNodePointer = self.tree[self.thisNodePointer].rightPointer
        return self.thisNodePointer

SamsTree = BinaryTree()
SamsTree.InsertNode("F")
SamsTree.InsertNode("C")
SamsTree.InsertNode("S")
SamsTree.InsertNode("B")
SamsTree.InsertNode("E")
SamsTree.InsertNode("R")
SamsTree.InsertNode("D")
print(SamsTree.FindNode("F"))
print(SamsTree.FindNode("D"))


# coding: utf-8

# In[1]:

import numpy as np
def read_data(infile):
    stats = np.loadtxt('infile',delimiter=',',unpack=True,dtype='int')
    number = stats[0]
    data = stats[1]
    return data

# In[ ]:


import boto3
from botocore.exceptions import *
import logging

def get_ec2_info():
    try:
        ec2 = boto3.resource('ec2')
        print_instances = ec2.instances.all()

        print('Outputting EC2 Info...\n')

        for ec2instance in print_instances:
            print(
                "ID: {}\nType: {} \nPublic IPv4: {}\nAMI: {}\nState: {}\nTags: {}\nSecurityGroups: {}\n".format(
                    ec2instance.id, ec2instance.instance_type, ec2instance.public_ip_address, ec2instance.image.id,
                    ec2instance.state, ec2instance.tags, ec2instance.security_groups
                )
            )

    except ClientError as e:
        logging.error(e)

    except Exception as e:
        logging.debug('Checking resource')
        logging.warning(e)
        print(e)

if __name__ == '__main__':
    get_ec2_info()
else:
    print('Running As Imported...')
    get_ec2_info()

Python-For-DevOps/Interacting with Python in AWS/List EC2 Instances/listec2.py


import datetime
import math
import sys
import time
from functools import partial
from typing import List

import torch
from torch.nn.parallel import DistributedDataParallel
from torch.utils.data import DataLoader

import dist
import encoder
from decoder import LightDecoder
from models import build_sparse_encoder
from sampler import DistInfiniteBatchSampler, worker_init_fn
from spark import SparK
from utils import arg_util, misc, lamb
from utils.imagenet import build_dataset_to_pretrain
from utils.lr_control import lr_wd_annealing, get_param_groups

class LocalDDP(torch.nn.Module):
    def __init__(self, module):
        super(LocalDDP, self).__init__()
        self.module = module

    def forward(self, *args, **kwargs):
        return self.module(*args, **kwargs)

def main_pt():
    args: arg_util.Args = arg_util.init_dist_and_get_args()
    print(f'initial args:\n{str(args)}')
    args.log_epoch()

    # build data
    print(f'[build data for pre-training] ...\n')
    dataset_train = build_dataset_to_pretrain(args.data_path, args.input_size)
    data_loader_train = DataLoader(
        dataset=dataset_train, num_workers=args.dataloader_workers, pin_memory=True,
        batch_sampler=DistInfiniteBatchSampler(
            dataset_len=len(dataset_train), glb_batch_size=args.glb_batch_size,
            shuffle=True, filling=True, rank=dist.get_rank(), world_size=dist.get_world_size(),
        ), worker_init_fn=worker_init_fn
    )
    itrt_train, iters_train = iter(data_loader_train), len(data_loader_train)
    print(f'[dataloader] gbs={args.glb_batch_size}, lbs={args.batch_size_per_gpu}, iters_train={iters_train}')

    # build encoder and decoder
    enc: encoder.SparseEncoder = build_sparse_encoder(args.model, input_size=args.input_size, sbn=args.sbn, drop_path_rate=args.dp, verbose=False)
    dec = LightDecoder(enc.downsample_raito, sbn=args.sbn)
    model_without_ddp = SparK(
        sparse_encoder=enc, dense_decoder=dec, mask_ratio=args.mask,
        densify_norm=args.densify_norm, sbn=args.sbn,
    ).to(args.device)
    print(f'[PT model] model = {model_without_ddp}\n')

    # the model has been randomly initialized in their construction time
    # now try to load some checkpoint as model weight initialization; this ONLY loads the model weights
    misc.initialize_weight(args.init_weight, model_without_ddp)

    if dist.initialized():
        model: DistributedDataParallel = DistributedDataParallel(model_without_ddp, device_ids=[dist.get_local_rank()], find_unused_parameters=False, broadcast_buffers=False)
    else:
        model = LocalDDP(model_without_ddp)

    # build optimizer and lr_scheduler
    param_groups: List[dict] = get_param_groups(model_without_ddp, nowd_keys={'cls_token', 'pos_embed', 'mask_token', 'gamma'})
    opt_clz = {
        'sgd': partial(torch.optim.SGD, momentum=0.9, nesterov=True),
        'adamw': partial(torch.optim.AdamW, betas=(0.9, args.ada)),
        'lamb': partial(lamb.TheSameAsTimmLAMB, betas=(0.9, args.ada), max_grad_norm=5.0),
    }[args.opt]
    optimizer = opt_clz(params=param_groups, lr=args.lr, weight_decay=0.0)
    print(f'[optimizer] optimizer({opt_clz}) ={optimizer}\n')

    # try to resume the experiment from some checkpoint.pth; this will load model weights, optimizer states, and last epoch (ep_start)
    # if loaded, ep_start will be greater than 0
    ep_start, performance_desc = misc.load_checkpoint(args.resume_from, model_without_ddp, optimizer)
    if ep_start >= args.ep: # load from a complete checkpoint file
        print(f'  [*] [PT already done]    Min/Last Recon Loss: {performance_desc}')
    else:   # perform pre-training
        tb_lg = misc.TensorboardLogger(args.tb_lg_dir, is_master=dist.is_master(), prefix='pt')
        min_loss = 1e9
        print(f'[PT start] from ep{ep_start}')

        pt_start_time = time.time()
        for ep in range(ep_start, args.ep):
            ep_start_time = time.time()
            tb_lg.set_step(ep * iters_train)
            if hasattr(itrt_train, 'set_epoch'):
                itrt_train.set_epoch(ep)

            stats = pre_train_one_ep(ep, args, tb_lg, itrt_train, iters_train, model, optimizer)
            last_loss = stats['last_loss']
            min_loss = min(min_loss, last_loss)
            performance_desc = f'{min_loss:.4f} {last_loss:.4f}'
            misc.save_checkpoint_with_meta_info_and_opt_state(f'{args.model}_withdecoder_1kpretrained_spark_style.pth', args, ep, performance_desc, model_without_ddp.state_dict(with_config=True), optimizer.state_dict())
            misc.save_checkpoint_model_weights_only(f'{args.model}_1kpretrained_timm_style.pth', args, model_without_ddp.sparse_encoder.sp_cnn.state_dict())

            ep_cost = round(time.time() - ep_start_time, 2) + 1    # +1s: approximate the following logging cost
            remain_secs = (args.ep-1 - ep) * ep_cost
            remain_time = datetime.timedelta(seconds=round(remain_secs))
            finish_time = time.strftime("%m-%d %H:%M", time.localtime(time.time() + remain_secs))
            print(f'  [*] [ep{ep}/{args.ep}]    Min/Last Recon Loss: {performance_desc},    Cost: {ep_cost}s,    Remain: {remain_time},    Finish @ {finish_time}')

            args.cur_ep = f'{ep + 1}/{args.ep}'
            args.remain_time, args.finish_time = str(remain_time), str(finish_time)
            args.last_loss = last_loss
            args.log_epoch()

            tb_lg.update(min_loss=min_loss, head='train', step=ep)
            tb_lg.update(rest_hours=round(remain_secs/60/60, 2), head='z_burnout', step=ep)
            tb_lg.flush()

        # finish pre-training
        tb_lg.update(min_loss=min_loss, head='result', step=ep_start)
        tb_lg.update(min_loss=min_loss, head='result', step=args.ep)
        tb_lg.flush()
        print(f'final args:\n{str(args)}')
        print('\n\n')
        print(f'  [*] [PT finished]    Min/Last Recon Loss: {performance_desc},    Total Cost: {(time.time() - pt_start_time) / 60 / 60:.1f}h\n')
        print('\n\n')
        tb_lg.close()
        time.sleep(10)

    args.remain_time, args.finish_time = '-', time.strftime("%m-%d %H:%M", time.localtime(time.time()))
    args.log_epoch()

def pre_train_one_ep(ep, args: arg_util.Args, tb_lg: misc.TensorboardLogger, itrt_train, iters_train, model: DistributedDataParallel, optimizer):
    model.train()
    me = misc.MetricLogger(delimiter='  ')
    me.add_meter('max_lr', misc.SmoothedValue(window_size=1, fmt='{value:.5f}'))
    header = f'[PT] Epoch {ep}:'

    optimizer.zero_grad()
    early_clipping = args.clip > 0 and not hasattr(optimizer, 'global_grad_norm')
    late_clipping = hasattr(optimizer, 'global_grad_norm')
    if early_clipping:
        params_req_grad = [p for p in model.parameters() if p.requires_grad]

    for it, inp in enumerate(me.log_every(iters_train, itrt_train, 3, header)):
        # adjust lr and wd
        min_lr, max_lr, min_wd, max_wd = lr_wd_annealing(optimizer, args.lr, args.wd, args.wde, it + ep * iters_train, args.wp_ep * iters_train, args.ep * iters_train)

        # forward and backward
        inp = inp.to(args.device, non_blocking=True)
        SparK.forward
        loss = model(inp, active_b1ff=None, vis=False)
        optimizer.zero_grad()
        loss.backward()
        loss = loss.item()
        if not math.isfinite(loss):
            print(f'[rk{dist.get_rank():02d}] Loss is {loss}, stopping training!', force=True, flush=True)
            sys.exit(-1)

        # optimize
        grad_norm = None
        if early_clipping: grad_norm = torch.nn.utils.clip_grad_norm_(params_req_grad, args.clip).item()
        optimizer.step()
        if late_clipping: grad_norm = optimizer.global_grad_norm
        torch.cuda.synchronize()

        # log
        me.update(last_loss=loss)
        me.update(max_lr=max_lr)
        tb_lg.update(loss=me.meters['last_loss'].global_avg, head='train_loss')
        tb_lg.update(sche_lr=max_lr, head='train_hp/lr_max')
        tb_lg.update(sche_lr=min_lr, head='train_hp/lr_min')
        tb_lg.update(sche_wd=max_wd, head='train_hp/wd_max')
        tb_lg.update(sche_wd=min_wd, head='train_hp/wd_min')

        if grad_norm is not None:
            me.update(orig_norm=grad_norm)
            tb_lg.update(orig_norm=grad_norm, head='train_hp')
        tb_lg.set_step()

    me.synchronize_between_processes()
    return {k: meter.global_avg for k, meter in me.meters.items()}

if __name__ == '__main__':
    main_pt()


"""
Models
"""
from sqlalchemy import Column, Integer, String, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql.sqltypes import DateTime
from sqlalchemy.orm import relationship

Base = declarative_base()

class Person(Base):
    """Information about Person"""

    __tablename__ = "person"
    id = Column(Integer, primary_key=True)
    name = Column(String(50), nullable=False)
    phones = relationship("Phones", cascade="all, delete", backref="person")
    birthday = Column(DateTime, nullable=True)

class Phones(Base):
    """Information about Person phones"""

    __tablename__ = "phones"
    id = Column(Integer, primary_key=True)
    phone = Column(Integer)
    person_id = Column(Integer, ForeignKey("person.id", ondelete="CASCADE"))