"""
Handlers related to "reports" about students' activity - badge evidence pages
and badge issuing routines.
"""
from controllers.utils import BaseHandler, ReflectiveRequestHandler, XsrfTokenManager
from common import prefetch
import pprint
from models.roles import Roles
import re
from collections import defaultdict
from models import transforms
from modules.badges.badge_models import Badge, BadgeAssertion
from report import UnitReport, PartReport, ExpertBadgeReport
from report import _parts as part_config
from models.models import Student
from models.models import EventEntity
from wiki_models import Annotation
from jinja2 import Markup
import urllib
import wtforms as wtf
from google.appengine.api import users
from google.appengine.ext import db
from google.appengine.ext import deferred
import page_templates
from common.querymapper import LoggingMapper
import logging

import wtforms as wtf

def exam_display_choices(exam_info):
    "Decide which display choices are available to a student, depending on their score."
    choices = [
            ('blank', '(Blank)'),
            ]
    default = 'blank'

    if exam_info['completed']:
        choices.append( ('completed', 'Submitted the exam') )

    if exam_info['did_pass']:
        choices.append( ('passed', 'Passed the exam, with at least (passing score) out of 100%') )
        choices.append( ('scored', 'Passed the exam, scoring (your score) out of 100%') )

        default = 'passed'

    return locals()

class EvidenceHandler(BaseHandler, ReflectiveRequestHandler):
    """
    Handler to show badge evidence - both the 'top' page and the text of each unit's wikifolio.

    Also the display settings for each badge - whether to show the text of the wikifolio
    and how to display the exam.
    """
    get_actions = ['view', 'settings']
    default_action = 'view'
    post_actions = ['save_settings']

    class SettingsForm(wtf.Form):
        report_id = wtf.HiddenField()
        units_are_public = wtf.BooleanField(
                "Show my Wikifolio entries for this badge on the evidence page?")

        # Will set choices dynamically.
        exam_display = wtf.SelectField(
                "How to display exam scores on the evidence page?")

        review_is_public = wtf.BooleanField(
                """Show the instructor's review of the paper on the evidence page?  Only
                relevant for the term paper.""")

    def get_settings(self):
        user = self.personalize_page_and_get_enrolled()
        if not user:
            return

        try:
            report = PartReport.get_by_id(int(self.request.GET.get('id', -1)))
        except ValueError:
            report = None
        if not report:
            self.abort(404, "That evidence report was not found.")

        if not self.can_edit(user, report):
            self.abort(403)

        form = self.SettingsForm(
                report_id=report.key().id(),
                units_are_public=report.units_are_public,
                exam_display=report.exam_display,
                review_is_public=report.review_is_public)

        #if report.part != 4:
            #del form.review_is_public

        display_field_params = {
                'choices': [('blank', '(Blank)')],
                'default': 'blank'
                }

        if report.assessment_scores:
            if len(report.assessment_scores) > 1:
                logging.warning("Evidence page settings assuming there's just one exam per part, but there is more than one")
            display_field_params = exam_display_choices(
                    report.assessment_scores[0])

        form.exam_display.choices = display_field_params['choices']
        form.exam_display.default = display_field_params['default']

        self.template_value['report'] = report
        self.template_value['form'] = form
        self.template_value['xsrf_token'] = XsrfTokenManager.create_xsrf_token('save_settings')
        self.template_value['navbar'] = {}
        self.template_value['action_url'] = '/badges/evidence?action=save_settings'
        self.template_value['badge_name'] = report._config['name']
        self.render('wf_evidence_settings.html')

    def can_edit(self, user, report):
        #Roles.is_course_admin(self.app_context)
        return report.student_email == user.key().name()

    def post_save_settings(self):
        user = self.personalize_page_and_get_enrolled()
        if not user:
            return

        form = self.SettingsForm(self.request.POST)

        try:
            report = PartReport.get_by_id(int(form.report_id.data))
        except ValueError:
            report = None
        if not report:
            self.abort(404, "That evidence report was not found.")

        if not self.can_edit(user, report):
            self.abort(403, "You can't edit that user's report.")

        if report.assessment_scores:
            display_field_params = exam_display_choices(
                    report.assessment_scores[0])
            form.exam_display.choices = display_field_params['choices']
            form.exam_display.default = display_field_params['default']
        else:
            del form.exam_display

        if not form.validate():
            self.redirect('/')
            return

        report.units_are_public = form.units_are_public.data
        report.review_is_public = form.review_is_public.data
        if report.assessment_scores:
            report.exam_display = form.exam_display.data
        report.put()

        EventEntity.record(
                'set-evidence-settings',
                users.get_current_user(),
                transforms.dumps({
                    'part': report.part,
                    'slug': report.slug,
                    'review_is_public': report.review_is_public,
                    'public': report.units_are_public,
                    'exam_display': report.exam_display,
                    'email': user.key().name()
                    }))

        self.template_value['navbar'] = {}
        self.template_value['content'] = '''
OK, saved settings.

        Back to your account page...
'''
        self.render('bare.html')

    def head(self):
        try:
            report = PartReport.get_by_id(int(self.request.GET.get('id', -1)))
        except ValueError:
            report = None
        if not report:
            self.abort(404)

        self.abort(200)

    def get_view(self):
        try:
            report = PartReport.get_by_id(int(self.request.GET.get('id', -1)))
        except ValueError:
            report = None
        if not report:
            self.abort(404)

        if not report.exam_display:
            if report.assessment_scores:
                display_info = exam_display_choices(report.assessment_scores[0])
                report.exam_display = display_info['default']
            else:
                report.exam_display = 'blank'

        self.report = report
        self.template_value['inline_save'] = lambda: ''
        self.template_value['navbar'] = {}
        self.template_value['author'] = self.report.student
        self.template_value['review_is_public'] = self.report.review_is_public
        if report.units_are_public:
            self.template_value['unit_link'] = self._unit_link
        else:
            self.template_value['unit_link'] = None
            self.template_value['no_unit_links'] = True
        self.template_value['unit_title'] = self._unit_title

        self.unit_num = None
        try:
            if report.units_are_public:
                self.unit_num = int(self.request.GET.get('unit', ''))
        except ValueError:
            self.unit_num = None

        if self.unit_num:
            self.unit = self.report.get_unit(self.unit_num)
            if self.unit:
                self.render_unit()
                return
            else:
                logging.warning('Could not find the right unit %d for PartReport %s',
                        self.unit_num, self.report.key())
        self.render_top()

    def _unit_link(self, unit):
        return self.request.path + "?" + urllib.urlencode({
            'id': self.request.GET['id'],
            'unit': unit,
            })

    def _unit_title(self, unit):
        unit_obj = self.find_unit_by_id(unit)
        title = unit_obj.title
        title = re.sub(r'\(.*?\)', '', title)
        return title.strip()

    def render_unit(self):
        fields = {
                k: Markup(v) for k,v in self.unit.wiki_fields.iteritems()}
        fields['reflection'] = Markup('
Removed from this view for peers\' privacy.
--BOOC Instructors and Tech Staff
')
        self.template_value['fields'] = fields
        self.template_value['unit'] = self.find_unit_by_id(self.unit_num)
        self.template_value['report'] = self.unit
        self.template_value['badge_slug'] = self.report.badge.key().name()
        # THIS is kinda magic - we render a page like wf_temp_u1.html, but have
        # it inherit from wf_evidence.html rather than wf_page.html.  This
        # removes the comment section, for instance.
        self.template_value['layout_template'] = 'wf_evidence.html'
        self.template_value['review'] = Annotation.reviews(whose=self.report.student, unit=self.unit_num).get()
        self.render(page_templates.templates[self.unit_num])

    def render_top(self):
        self.template_value['part'] = self.report
        self.render('wf_evidence_top.html')

class ExpertEvidenceHandler(BaseHandler, ReflectiveRequestHandler):
    """
    Handler for the "expert" badge - issued at the end of the course.
    This badge has links to all the previous badges, rather than links
    to the internal units.
    """
    get_actions = ['view', 'settings']
    default_action = 'view'
    post_actions = ['save_settings']

    def can_edit(self, user, report):
        return report.student_email == user.key().name()

    class SettingsForm(wtf.Form):
        report_id = wtf.HiddenField()

        # Will set choices dynamically.
        exam_display = wtf.SelectField(
                "How to display exam scores on the evidence page?")

    def get_settings(self):
        user = self.personalize_page_and_get_enrolled()
        if not user:
            return

        try:
            report = ExpertBadgeReport.get_by_id(int(self.request.GET.get('id', -1)))
        except ValueError:
            report = None
        if not report:
            self.abort(404, "That evidence report was not found.")

        if not self.can_edit(user, report):
            self.abort(403)

        form = self.SettingsForm(
                report_id=report.key().id(),
                exam_display=report.exam_display)

        display_field_params = {
                'choices': [('blank', '(Blank)')],
                'default': 'blank'
                }

        display_field_params = exam_display_choices(
                report.final_exam_score)

        form.exam_display.choices = display_field_params['choices']
        form.exam_display.default = display_field_params['default']

        self.template_value['report'] = report
        self.template_value['form'] = form
        self.template_value['xsrf_token'] = XsrfTokenManager.create_xsrf_token('save_settings')
        self.template_value['navbar'] = {}
        self.template_value['badge_name'] = "Assessment Expert Badge"
        self.template_value['action_url'] = '/badges/expert_evidence?action=save_settings'
        self.render('wf_evidence_settings.html')

    def post_save_settings(self):
        user = self.personalize_page_and_get_enrolled()
        if not user:
            return

        form = self.SettingsForm(self.request.POST)

        try:
            report = ExpertBadgeReport.get_by_id(int(form.report_id.data))
        except ValueError:
            report = None
        if not report:
            self.abort(404, "That evidence report was not found.")

        if not self.can_edit(user, report):
            self.abort(403, "You can't edit that user's report.")

        display_field_params = exam_display_choices(
                report.final_exam_score)
        form.exam_display.choices = display_field_params['choices']
        form.exam_display.default = display_field_params['default']

        if not form.validate():
            self.redirect('/')
            return

        report.exam_display = form.exam_display.data
        report.put()

        EventEntity.record(
                'set-evidence-settings',
                users.get_current_user(),
                transforms.dumps({
                    'slug': report.slug,
                    'exam_display': report.exam_display,
                    'email': user.key().name()
                    }))

        self.template_value['navbar'] = {}
        self.template_value['content'] = '''
OK, saved settings.

        Back to your account page...
'''
        self.render('bare.html')

    def get_view(self):
        try:
            report = ExpertBadgeReport.get_by_id(int(self.request.GET.get('id', -1)))
        except ValueError:
            report = None
        if not report:
            self.abort(404)

        all_assertions_q = BadgeAssertion.all()
        all_assertions_q.filter('recipient', report.student_key)
        all_assertions_q.filter('revoked', False)
        all_assertions = all_assertions_q.run(limit=10)

        if not report.exam_display:
            if report.final_exam_score:
                display_info = exam_display_choices(report.final_exam_score)
                report.exam_display = display_info['default']
            else:
                report.exam_display = 'blank'

        self.template_value['report'] = report
        self.template_value['navbar'] = {}
        self.template_value['author'] = report.student
        # TODO: links to the other badges

        all_assertions = prefetch.prefetch_refprops(
                list(all_assertions), BadgeAssertion.badge)
        course_parts = {'practices': None, 'principles': None, 'policies': None}
        for ass in all_assertions:
            name_parts = ass.badge_name.split('.')
            if name_parts[0] in course_parts:
                if (not course_parts[name_parts[0]]) or (len(name_parts) > 1):
                    course_parts[name_parts[0]] = ass
        self.template_value['part_assertions'] = course_parts

        self.render('wf_expert_evidence.html')

def combine_badge_slug_parts(*parts):
    return '.'.join(parts)

def choose_badge_version(slug, completion):
    # Choose the base badge version - normal vs expertise.
    # Doesn't choose Leader vs. normal - that is later.

    if not completion['units']:
        return None
    elif not completion['assessments']:
        return slug
    else:
        return combine_badge_slug_parts(slug, 'expertise')

class SingleIssueHandler(BaseHandler):
    """
    Decide whether to issue a badge to one student, and optionally really issue it.
    """
    class Form(wtf.Form):
        part = wtf.IntegerField('Which part of the course to issue a badge for? (1,2,3)')
        really_save = wtf.BooleanField('Really issue the badge and freeze the scores?', default=False)
        re_run = wtf.BooleanField('Re-run all unit and part reports? Will delete old ones if you also choose Really freeze above.', default=False)
        email = wtf.StringField('The email of the student to reconsider')

    def get(self):
        if not users.is_current_user_admin():
            self.abort(403)
        form = self.Form()
        self.template_value['form'] = form
        self.template_value['xsrf_token'] = XsrfTokenManager.create_xsrf_token('post')
        self.template_value['action_url'] = self.request.url
        self.template_value['title'] = 'Reconsider a single participant'
        self.render('badge_bulk_issue.html')

    def post(self):
        if not users.is_current_user_admin():
            self.abort(403)
        if not XsrfTokenManager.is_xsrf_token_valid(self.request.POST.get('xsrf_token', ''), 'post'):
            self.abort(403, 'XSRF token failed.')
        form = self.Form(self.request.POST)
        if not form.validate():
            self.response.write('
'.join(form.errors))
            return

        student = Student.get_by_key_name(form.email.data)
        report = PartReport.on(student, course=self.get_course(),
                part=form.part.data,
                force_re_run=form.re_run.data,
                put=form.really_save.data)

        badge_version = choose_badge_version(part_config[form.part.data]['slug'], report.completion())
        if badge_version:
            badge = Badge.get_by_key_name(badge_version)
            if not badge:
                self.response.write(' There is no badge with key_name %s (so I cannot issue a badge)' % badge_version)
            if form.really_save.data and badge:
                b = Badge.issue(badge, student, put=False)
                b.evidence = self.request.host_url + '/badges/evidence?id=%d' % report.key().id()
                b.put()
                self.response.write('Issued badge %s!' % badge_version)
            else:
                self.response.write('Would have issued badge %s!' % badge_version)
        else:
            self.response.write('Not issuing because at least one of: %s' % (', '.join(report.incomplete_reasons)))

class BulkIssueMapper(LoggingMapper):
    """
    Issue completion badges to many folks!  This includes 'expertise' and 'knowledge' badges.
    """
    KIND = Student
    FILTERS = [('is_participant', True)]

    def __init__(self, really, course, part, host_url, re_run):
        super(BulkIssueMapper, self).__init__()
        self.really = really
        self.course = course
        self.part = part
        self.host_url = host_url
        self.re_run = re_run
        self.num_issued = 0

    def map(self, student):
        self.log.append('########## Student %s ##########' % student.key().name())
        report = PartReport.on(student, course=self.course, part=self.part,
                force_re_run=self.re_run, put=self.really)

        completion = report.completion()
        self.log.append(' Passed? %s.' % str(completion))

        badge_version = choose_badge_version(report.slug, completion)

        if badge_version:
            badge = Badge.get_by_key_name(badge_version)
            if not badge:
                self.log.append(' There is no badge with key_name %s (so I cannot issue a badge)' % badge_version)

            self.num_issued += 1
            if self.really and badge:
                b = Badge.issue(badge, student, put=False) # need to include evidence URL here somehow
                b.evidence = self.host_url + '/badges/evidence?id=%d' % report.key().id()
                b.put()
                self.log.append(' Issued badge, name=%s, assertion id=%d' % (
                    badge_version, b.key().id()))
                return ([b], [])
            else:
                self.log.append(' WOULD issue badge.')
        else:
            self.log.append('Not issuing because at least one of: %s' % (', '.join(report.incomplete_reasons)))

            ##TODO: this is not comprehensive: they could still have .expertise or .leader versions.
            #if self.really and badge:
                #Badge.ensure_not_issued(badge, student)
        return ([], [])

    def finish(self):
        self.log.append('DONE.  Issued %d badges total.' % self.num_issued)
        self._batch_write()

def choose_expert_badge_version(completion):
    badge_version = None
    if completion['badges']:
        badge_version = 'expert'
    if badge_version and completion['assessments'] \
            and all('expertise' in slug for slug in completion['badge_slugs']):
        badge_version = 'expert.expertise' # lol
    return badge_version

class BulkExpertBadgeIssueMapper(LoggingMapper):
    """
    Issue end-of-course badges to many folks.
    """
    KIND = Student
    FILTERS = [('is_participant', True)]

    def __init__(self, really, course, unused_part_num, host_url, force_re_run):
        LoggingMapper.__init__(self)
        self.really = really
        self.course = course
        self.host_url = host_url
        self.force_re_run = force_re_run

        self.num_issued = 0

    def map(self, student):
        self.log.append('--------------- Student %s' % student.key().name())

        report = ExpertBadgeReport.on(student, self.course,
                force_re_run=self.force_re_run, put=self.really)

        completion = report.completion()
        self.log.append(' Passed? %s.' % str(completion))

        badge_version = choose_expert_badge_version(completion)

        if badge_version:
            badge = Badge.get_by_key_name(badge_version)
            if not badge:
                self.log.append('no such badge! %s' % badge_version)
            self.num_issued += 1
            if self.really and badge:
                b = Badge.issue(badge, student, put=False) # need to include evidence URL here somehow
                b.evidence = self.host_url + '/badges/expert_evidence?id=%d' % report.key().id()
                b.put()
                self.log.append(' Issued badge, name=%s, assertion id=%d' % (
                    badge.key().name(), b.key().id()))
                return ([b], [])
            else:
                self.log.append(' WOULD issue badge %s' % badge_version)
        else:
            self.log.append('Incomplete, we are missing: %s' % (', '.join(report.incomplete_reasons())))

        return ([], [])

    def finish(self):
        self.log.append('DONE.  Issued %d badges total.' % self.num_issued)
        self._batch_write()

NOBODY = object()
def default_dict_entry():
    return ([NOBODY], -1)

class BulkLeaderIssueMapper(LoggingMapper):
    """
    Issue leader badges.  In this one, the badges aren't issued during the main phase
    of the 'mapper' loop over all the students - instead, the badges are issued
    in the "finish" phase.
    """
    KIND = Student
    FILTERS = [('is_participant', True)]

    def __init__(self, really, course, part, host_url, re_run):
        super(BulkLeaderIssueMapper, self).__init__()
        self.really = really
        self.course = course
        self.part = part
        self.host_url = host_url
        self.re_run = re_run
        self.best_by_group = defaultdict(default_dict_entry)
        self.leader_badge_keys = (
                part_config[part]['slug'] + '.expertise.leader',
                part_config[part]['slug'] + '.leader',
                )

        leader_badge_e = Badge.get_by_key_name(self.leader_badge_keys[0])
        leader_badge = Badge.get_by_key_name(self.leader_badge_keys[1])
        if not leader_badge and leader_badge_e:
            logging.warning('Missing a badge with one of: %s', str(self.leader_badge_keys))
            self.log.append('Missing a badge with one of: %s'% str(self.leader_badge_keys))
            if self.really:
                raise ValueError('Missing a badge with one of: %s'% str(self.leader_badge_keys))

    def map(self, student):
        self.log.append('######### Student %s ##########' % student.key().name())

        part_report = PartReport.on(student, course=self.course, part=self.part,
                force_re_run=self.re_run)
        completion = part_report.completion()
        base_badge_version = choose_badge_version(part_report.slug, completion)
        if not base_badge_version:
            self.log.append(' Skipping, since not complete.')
            return ([], [])

        self.log.append(' Part is complete, considering units.')

        unit_reports = part_report.unit_reports
        promotions = 0

        for ur in unit_reports:
            promotions += ur.promotions

        best_so_far = self.best_by_group[student.group_id][1]
        if promotions > best_so_far:
            self.best_by_group[student.group_id] = ([student.key().name()], promotions)

            self.log.append(' They have current best for group %s, with %d.' % (
                student.group_id, promotions))

        elif promotions == best_so_far:
            self.best_by_group[student.group_id][0].append(student.key().name())

            self.log.append(' They ARE TIED FOR CURRENT BEST for group %s, with %d.' % (
                student.group_id, promotions))
        return ([], [])

    def finish(self):
        if self.really:
            leader_badges = dict((key, Badge.get_by_key_name(key)) for key in self.leader_badge_keys)

        for group_id, (emails, count) in self.best_by_group.iteritems():
            self.log.append('Considering group %s, best score is %d' % (
                str(group_id), count))
            if count < 1:
                self.log.append('... Best score is too low, skipping.')
                continue
            if self.really:
                for email in emails:
                    report = PartReport.on(
                            db.Key.from_path(Student.kind(), email),
                            course=self.course, part=self.part,
                            force_re_run=self.re_run)
                    base_badge_version = choose_badge_version(report.slug, report.completion())
                    leader_badge_version = combine_badge_slug_parts(base_badge_version, 'leader')
                    leader_badge = leader_badges[leader_badge_version]
                    assert leader_badge, "Should be a badge called %s" % leader_badge_version

                    b = Badge.issue(leader_badge,
                            db.Key.from_path(Student.kind(), email), put=False)
                    b.evidence = self.host_url + '/badges/evidence?id=%d' % report.key().id()
                    b.put()
                    self.log.append('... ISSUED leader badge %s to %s, id=%d' % (
                        leader_badge_version, email, b.key().id()))
            else:
                self.log.append('... WOULD ISSUE leader badge to %s' % ' '.join(emails))
        self._batch_write()

class BulkExpertLeaderIssueMapper(LoggingMapper):
    """
    Expert badges go to people who pass all the parts of the course,
    and do the final exam, and do the survey.  Leaders are calculated
    based on who gets the most Exemplaries in the entire course.
    """
    KIND = ExpertBadgeReport

    def __init__(self, really, course, part, host_url, re_run):
        super(BulkExpertLeaderIssueMapper, self).__init__()
        self.really = really
        self.course = course
        self.host_url = host_url
        self.re_run = re_run
        self.best_by_group = defaultdict(default_dict_entry)

    def map(self, report):
        student = report.student
        self.log.append('---------------- Student %s' % student.key().name())

        if self.re_run:
            report._run(self.course)

        best_so_far = self.best_by_group[student.group_id][1]
        promotions = report.exemplary_count

        completion = report.completion()

        if promotions > best_so_far:
            self.log.append('New best for group %s, %d promotions' % (
                student.group_id, promotions))

            if completion['badges']:
                self.best_by_group[student.group_id] = ([student.key().name()], promotions)
            else:
                self.log.append('BUT, Skipping, not complete.')

        elif promotions == best_so_far:
            self.log.append('TIED best for group %s, %d promotions' % (
                student.group_id, promotions))

            if completion['badges']:
                self.best_by_group[student.group_id][0].append(student.key().name())
            else:
                self.log.append('BUT, Skipping, not complete.')

        return ([], [])

    def finish(self):
        leader_badge_slugs = ('expert.leader', 'expert.expertise.leader')
        leader_badges = dict((k, Badge.get_by_key_name(k)) for k in leader_badge_slugs)

        for group_id, (emails, count) in self.best_by_group.iteritems():
            self.log.append('Considering group %s, best score is %d' % (
                str(group_id), count))
            if count < 1:
                self.log.append('... Best score is too low, skipping.')
                continue
            for email in emails:
                report = ExpertBadgeReport.on(
                        db.Key.from_path(Student.kind(), email),
                        course=self.course,
                        force_re_run=self.re_run)
                base_badge_version = choose_expert_badge_version(report.completion())
                if not base_badge_version:
                    raise AssertionError('They should have passed, wat?')
                leader_badge_slug = base_badge_version + '.leader'
                if self.really:
                    leader_badge = leader_badges[leader_badge_slug]
                    b = Badge.issue(leader_badge,
                            db.Key.from_path(Student.kind(), email), put=False)
                    b.evidence = self.host_url + '/badges/expert_evidence?id=%d' % report.key().id()
                    b.put()
                    self.log.append('... ISSUED %s to %s, id=%d' % (
                        leader_badge_slug, email, b.key().id()))
                else:
                    self.log.append('... WOULD ISSUE %s to %s' % (
                        leader_badge_slug, email))
        self._batch_write()

issuer_mappers = {
        'completion': BulkIssueMapper,
        'leader': BulkLeaderIssueMapper,
        'expert': BulkExpertBadgeIssueMapper,
        'expert-leader': BulkExpertLeaderIssueMapper,
        }

class BulkIssuanceHandler(BaseHandler, ReflectiveRequestHandler):
    """
    Request handler for kicking off all these different kinds of badge issuing mappers.
    """
    default_action = 'prep'
    get_actions = ['prep', 'watch']
    post_actions = ['start']

    TITLE = 'Bulk Issue Badges'

    class IssueForm(wtf.Form):
        part = wtf.IntegerField('Which part of the course to issue a badge for? (1,2,3)')
        really_save = wtf.BooleanField('Really issue the badges and freeze the scores?', default=False)
        leader_or_completion = wtf.RadioField('Do you want to issue completion badges, or leader badges?',
                choices=[(k, k) for k in issuer_mappers.keys()])
        force_re_run_reports = wtf.BooleanField('Re-run all unit and part reports? Will delete old ones if you also choose Really freeze above.', default=False)
        one_email = wtf.TextField('Only consider one student? Enter their e-mail here.',
                validators=[wtf.validators.Optional()])

    def _action_url(self, action, **kwargs):
        params = dict(kwargs)
        params['action'] = action
        return '?'.join((
            self.request.path,
            urllib.urlencode(params)))

    def get_prep(self):
        if not users.is_current_user_admin():
            self.abort(403)
        self.render_form(self.IssueForm())

    def render_form(self, form):
        self.template_value['form'] = form
        self.template_value['xsrf_token'] = self.create_xsrf_token('start')
        self.template_value['action_url'] = self._action_url('start')
        self.template_value['title'] = self.TITLE
        self.render('badge_bulk_issue.html')

    def post_start(self):
        if not users.is_current_user_admin():
            self.abort(403)

        form = self.IssueForm(self.request.POST)
        if not form.validate():
            self.render_form(form)
            return

        REALLY = form.really_save.data
        part_num = form.part.data

        problems = set()
        student_infos = []

        issuer = issuer_mappers[form.leader_or_completion.data]

        job = issuer(REALLY, self.get_course(), part_num, self.request.host_url, form.force_re_run_reports.data)
        if form.one_email.data:
            job.FILTERS.append(
                    ('__key__', db.Key.from_path('Student', form.one_email.data)))
            logging.debug('Filters for issuing: %s', repr(job.FILTERS))
        job_id = job.job_id
        deferred.defer(job.run, batch_size=50)
        self.redirect(self._action_url('watch', job_id=job_id))

    def get_watch(self):
        if not users.is_current_user_admin():
            self.abort(403)

        job_id = self.request.GET.get('job_id', None)
        if not job_id:
            self.abort(404)

        messages = BulkIssueMapper.logs_for_job(job_id)

        self.template_value['title'] = self.TITLE
        self.template_value['problems'] = []
        self.template_value['log'] = messages
        self.render('badge_bulk_issue_done.html')

class DammitMapper(LoggingMapper):
    """Goes through all PartReports and re-runs their assessment grades."""
    KIND = PartReport

    def __init__(self, course):
        super(DammitMapper, self).__init__()
        self.course = course

    def map(self, report):
        self.log.append("Working on report for %s, part %d" % (report.student_email, report.part))
        had_before = bool(report.assessment_scores)
        report._run_assessments(self.course)
        has_after = bool(report.assessment_scores)
        self.log.append("had before? %s.  has after? %s" % (had_before, has_after))

        return ([report], [])

class DammitHandler(BaseHandler, ReflectiveRequestHandler):
    get_actions = ['start', 'watch']
    default_action = 'watch'

    def _action_url(self, action, **kwargs):
        params = dict(kwargs)
        params['action'] = action
        return '?'.join((
            self.request.path,
            urllib.urlencode(params)))

    def get_start(self):
        if not users.is_current_user_admin():
            self.abort(403)

        course = self.get_course()
        job = DammitMapper(course)
        job_id = job.job_id
        deferred.defer(job.run, batch_size=50)
        self.redirect(self._action_url('watch', job_id=job_id))

    def get_watch(self):
        if not users.is_current_user_admin():
            self.abort(403)

        job_id = self.request.GET.get('job_id', None)
        if not job_id:
            self.abort(404)

        messages = BulkIssueMapper.logs_for_job(job_id)

        self.template_value['title'] = "GRRRRRR"
        self.template_value['problems'] = []
        self.template_value['log'] = messages
        self.render('badge_bulk_issue_done.html')


# -*- coding:utf-8 -*-

import pandas as pd
import requests
import time
import json
import os

def get_month_data(month):
    page = 1
    num = 1
    video_list = []

    while(True):
        print('*****' * 10, f'page[{page}]', '*****' * 10)

        # 处理月份格式
        if month in ['01', '03', '05', '07', '08', '10', '12']:
            url = f'https://s.search.bilibili.com/cate/search?main_ver=v3&search_type=video&view_type=hot_rank&pic_size=160x100&order=click&copy_right=-1&cate_id=22&page={page}&pagesize=20&time_from=2018{month}01&time_to=2018{month}31'
        elif month in ['04', '06', '09', '11']:
            url = f'https://s.search.bilibili.com/cate/search?main_ver=v3&search_type=video&view_type=hot_rank&pic_size=160x100&order=click&copy_right=-1&cate_id=22&page={page}&pagesize=20&time_from=2018{month}01&time_to=2018{month}30'
        else:
            url = f'https://s.search.bilibili.com/cate/search?main_ver=v3&search_type=video&view_type=hot_rank&pic_size=160x100&order=click&copy_right=-1&cate_id=22&page={page}&pagesize=20&time_from=2018{month}01&time_to=2018{month}28'

        headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063'}
        html_code = requests.get(url, headers=headers).text

        # 获取当前网页最大页数
        max_page = json.loads(html_code)['numPages']
        # 获取视频信息
        results = json.loads(html_code)['result']

        for result in results:
            print(result['title'], result['arcurl'], result['pubdate'].split(' ')[0], result['play'], result['video_review'], result['favorites'], result['author'])
            video_list.append([result['title'],
                               result['arcurl'],
                               result['pubdate'].split(' ')[0],
                               result['play'],
                               result['video_review'],
                               result['favorites'],
                               result['author']])
            num += 1
        page += 1
        time.sleep(0.5)
        if page > max_page:
            if not os.path.isdir('csv_files'):
                os.mkdir('csv_files')
            result_df = pd.DataFrame(video_list, columns=['title', 'link', 'date', 'play_count', 'danmu_num', 'favorites', 'author'])
            result_df.to_csv(f'csv_files/month_{month}.csv', sep=',', na_rep='NA')
            break

for month in range(1,13):
    if month < 10:
        month = '0' + str(month)
    else:
        month = str(month)
    get_month_data(month)


import os
import io
import imageio
import logging

import cv2
import numpy as np
from PIL import Image
import torch
from ts.torch_handler.base_handler import BaseHandler

logger = logging.getLogger(__name__)

def test_resize(img, size=640, pad=False):
    h, w, c = img.shape
    scale_w = size / w
    scale_h = size / h
    scale = min(scale_w, scale_h)
    h = int(h * scale)
    w = int(w * scale)

    new_img = None
    if pad:
        new_img = np.zeros((size, size, c), img.dtype)
        new_img[:h, :w] = cv2.resize(img, (w, h))
    else:
        new_img = cv2.resize(img, (w, h))

    return new_img

def test_preprocess(img,
                    mean=[103.939, 116.779, 123.68],
                    to_tensor=True,
                    pad=False):
    img = test_resize(img, size=640, pad=pad)

    img = img.astype(np.float32)
    img[..., 0] -= mean[0]
    img[..., 1] -= mean[1]
    img[..., 2] -= mean[2]
    img = np.expand_dims(img, axis=0)

    if to_tensor:
        img = torch.Tensor(img.transpose(0, 3, 1, 2))

    return img

class DBTextDetectionHandler(BaseHandler):
    def __init__(self):
        self.model = None
        self.device = None
        self.initialized = False

    def initialize(self, ctx):
        self.manifest = ctx.manifest
        properties = ctx.system_properties
        self.device = torch.device('cpu')
        model_dir = properties.get("model_dir")
        serialized_file = self.manifest['model']['serializedFile']
        model_pt_path = os.path.join(model_dir, serialized_file)
        assert os.path.exists(model_pt_path)
        self.model = torch.jit.load(model_pt_path)
        self.model.to(self.device)
        self.model.eval()

        logger.debug(
            'Model file {0} loaded successfully'.format(model_pt_path))
        self.initialized = True

    def preprocess(self, request):

        tensor_imgs = []
        for _, data in enumerate(request):
            image = data.get("data")
            if image is None:
                image = data.get("body")

            input_image = Image.open(io.BytesIO(image))
            input_image = np.array(input_image)
            tensor_img = test_preprocess(input_image, pad=False)
            tensor_imgs.append(tensor_img)

        tensor_imgs = torch.cat(tensor_imgs)
        return tensor_imgs

    def inference(self, img):
        return self.model(img)

    def postprocess(self, data):

        res = []
        data = data.detach().cpu().numpy()
        for pred in data:
            prob_mask = (pred[0] * 255).astype(np.uint8)
            thresh_mask = (pred[1] * 255).astype(np.uint8)
            prob_mask = prob_mask.tolist()
            thresh_mask = thresh_mask.tolist()
            res.append({"prob_mask": prob_mask, "thresh_mask": thresh_mask})

        return res

_service = DBTextDetectionHandler()

def handle(data, context):
    if not _service.initialized:
        _service.initialize(context)

    if data is None:
        return None

    data = _service.preprocess(data)
    data = _service.inference(data)
    data = _service.postprocess(data)

    return data


#!/usr/bin/env python3

# AWS Lambda function for creating an AMI image from a given instance.
# By Michael Ludvig - https://aws.nz

# Trigger this function from CloudWatch Scheduler (cron-like)
# Pass the Instance ID in 'instance_id' environment variable.

import os
import boto3
from datetime import datetime, timedelta
import time

ec2 = boto3.client('ec2')

def create_image(instance_id, reboot):
    def _print_log(message):
        print('%s @ %s: %s' % (instance_id, snapshot_timestamp, message))

    snapshot_timestamp = datetime.strftime(datetime.now(), '%s')
    _print_log('Snapshotting instance')
    instance = ec2.describe_instances(InstanceIds=[instance_id])
    description = ''
    tags = {}

    try:
        tags = {item['Key']:item['Value'] for item in instance['Reservations'][0]['Instances'][0]['Tags']}
    except:
        pass

    if 'Name' in tags:
        description = tags['Name']
    elif 'aws:cloudformation:stack-name' in tags:
        description = tags['aws:cloudformation:stack-name']
    else:
        description = instance_id

    name = instance_id+'_'+snapshot_timestamp
    description = description + ' ' + datetime.strftime(datetime.now(), '%Y-%m-%d %H-%M-%S')
    r = ec2.create_image(InstanceId = instance_id, Name = name, Description = description, NoReboot = not reboot)
    image_id = r['ImageId']
    _print_log('Created image: id=%s name=%s' % (image_id, name))
    image_tags = [
        {'Key': 'SnapshotTimestamp', 'Value': snapshot_timestamp },
        {'Key': 'InstanceId', 'Value': instance_id },
    ]
    if 'Name' in tags:
        image_tags.append({ 'Key': 'Name', 'Value': tags['Name'] })
    if 'aws:cloudformation:stack-name' in tags:
        image_tags.append({ 'Key': 'StackName', 'Value': tags['aws:cloudformation:stack-name'] })

    ec2.create_tags(Resources = [image_id], Tags = image_tags)
    image_tags_string = ' '.join(map(lambda x: '%(Key)s=%(Value)s' % x, image_tags))
    _print_log('Created tags: %s' % (image_tags_string))

    return (image_id, snapshot_timestamp)

def lambda_handler(event, context):
    try:
        instance_id = os.environ['instance_id']
    except:
        print('ERROR: Environment variables must be set: instance_id')
        raise

    try:
        reboot = event.get('reboot', True)
        assert(type(event['reboot']) in [ bool, int ])
    except:
        print('ERROR: Event JSON expected: { "reboot": true / false }')
        raise

    image_id, snapshot_timestamp = create_image(instance_id, reboot)

    return image_id


import pandas as pd
import time
import sys
import numpy as np
import os
import torch
import torchvision
from numpy import sqrt
import math
from torch import nn
import latticeglass
from args import args
from resmade import MADE
from utils import (
    clear_checkpoint,
    clear_log,
    get_last_checkpoint_step,
    ignore_param,
    init_out_dir,
    my_log,
    print_args,
    EarlyStopping,
    mysorting,
)
import itertools
import plotly.express as px
import math
from transformer import Transformer
from torch.nn import functional as F

def main():
    start_time = time.time()

    init_out_dir()
    if args.clear_checkpoint:
        clear_checkpoint()
    last_step = get_last_checkpoint_step()
    if last_step >= 0:
        my_log('\nCheckpoint found: {}\n'.format(last_step))
    else:
        clear_log()
    print_args()
    with open('{}.myargs'.format(args.out_filename), 'w') as f:
        print_args(f.write)

    args.L=int(round(np.power(args.n,1/3)))
    args.patch_size = int(args.L*0.5)
    boundary_size = 5**3 - 3**3

    ntokens = 5  # size of vocabulary
    max_output_length = 29 #extra end of word token
    net = Transformer(ntokens, max_output_length, 2**9).to(args.device)
    net.to(args.device)
    my_log('{}\n'.format(net))

    params = list(net.parameters())
    params = list(filter(lambda p: p.requires_grad, params))
    nparams = int(sum([np.prod(p.shape) for p in params]))
    my_log('Total number of trainable parameters: {}'.format(nparams))
    named_params = list(net.named_parameters())

    if args.optimizer == 'sgd':
        optimizer = torch.optim.SGD(params, lr=args.lr)
    elif args.optimizer == 'sgdm':
        optimizer = torch.optim.SGD(params, lr=args.lr, momentum=0.9)
    elif args.optimizer == 'rmsprop':
        optimizer = torch.optim.RMSprop(params, lr=args.lr, alpha=0.99)
    elif args.optimizer == 'adam':
        optimizer = torch.optim.Adam(params, lr=args.lr, betas=(0.9, 0.999))
    elif args.optimizer == 'adam0.5':
        optimizer = torch.optim.Adam(params, lr=args.lr, betas=(0.5, 0.999))
    else:
        raise ValueError('Unknown optimizer: {}'.format(args.optimizer))

    if args.lr_schedule:
        #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
        #    #optimizer, factor=0.5, patience=500, threshold=1e-4, min_lr=1e-5)
        #    optimizer, factor=0.92, patience=500, threshold=1e-4, min_lr=1e-5)
        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, 10000, 0)

    if args.early_stopping:
        my_log('INFO: Initializing early stopping')
        early_stopping = EarlyStopping(patience=5000)

    if last_step >= 0:
        state = torch.load('{}_save/{}.state'.format(args.out_filename, last_step),map_location=torch.device(args.device))
        ignore_param(state['net'], net)
        net.load_state_dict(state['net'])
        if state.get('optimizer'):
            optimizer.load_state_dict(state['optimizer'])
        if args.lr_schedule and state.get('scheduler'):
            scheduler.load_state_dict(state['scheduler'])

    init_time = time.time() - start_time
    my_log('init_time = {:.3f}'.format(init_time))

    my_log('\n*** Patch size {} from a box of size {}'.format(args.patch_size,args.L))
    nn_list_full  = latticeglass.make_nn(args.L)
    nn_list_patch = latticeglass.make_nn(args.patch_size)

    #***********************************************************************************************************
    #******************************************   IMPORT DATA FILES ********************************************
    with torch.no_grad():
        my_log('%%%% Loading the full sample')

        rawfilename = 'LG_eq_configs_T_{}.txt'.format(args.T)
        full_sample = latticeglass.read_sample(rawfilename)

        # Now I measure the Energy of the full sample
        samplefilename = 'Eequi_T{}_L{}_ps{}.npy'.format(args.T,args.L,args.patch_size)
        if os.path.isfile(samplefilename):
            E_equi = np.load(samplefilename)
        else:
            my_log('Calculating the energy')
            full_sample_energy = torch.Tensor(np.asarray(latticeglass.energy(full_sample, nn_list_full, args.q)))
            E_equi =full_sample_energy.mean()/(args.L**3)
            np.save(samplefilename,E_equi)
        my_log('*** Equilibrium E/N={}'.format(E_equi))

        # pass to numpy which is faster to operate in parallel
        full_sample = full_sample.to(dtype=int).detach().cpu().numpy()

        # and then the energy of the patches
        # ***[!]*** The patches are NOT defined in PBC, so to get their bulk energy I have to exclude the boundaries
        # to do so I define a mask which is 0 on the boundaries
        mask_patch= torch.ones(size=(1,args.patch_size,args.patch_size,args.patch_size))
        mask_patch[:,0,:,:]=0
        mask_patch[:,args.patch_size-1,:,:]=0
        mask_patch[:,:,0,:]=0
        mask_patch[:,:,args.patch_size-1,:]=0
        mask_patch[:,:,:,0]=0
        mask_patch[:,:,:,args.patch_size-1]=0
        onedmask = torch.zeros(size=(1,args.patch_size**3))
        for x in range(args.patch_size):
            for y in range(args.patch_size):
                for z in range(args.patch_size):
                    s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                    onedmask[:, s] = mask_patch[:, x, y, z]
        # save the mask
        samplefilename = 'onedmask_T{}_L{}_ps{}.npy'.format(args.T,args.L,args.patch_size)
        np.save(samplefilename,onedmask)

        # I want to impose a different ordering for the variables in the patches that has the following requisites:
        #     1) first I put the fixed boundaries
        #     2) the first (mobile) spin is the center of the patch
        #     3) then we have the center of the faces (1boundary connection)
        #     4) then the edges (touching 2 boundaries)
        #     5) finally the corners
        # I can achieve this similarly to what I did in the coloring for the ordering based on the graph
        # So I create an array of all the IDs in the order that I want
        order_of_sites=[]
        # (1)
        for x in [0,args.patch_size-1]:
            for y in range(args.patch_size):
                for z in range(args.patch_size):
                    s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                    order_of_sites.append(s)
        for x in range(args.patch_size):
            for y in [0,args.patch_size-1]:
                for z in range(args.patch_size):
                    s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                    order_of_sites.append(s)
        for x in range(args.patch_size):
            for y in range(args.patch_size):
                for z in [0,args.patch_size-1]:
                    s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                    order_of_sites.append(s)
        # (2)
        x = 2
        y = 2
        z = 2
        s = x + args.patch_size*y + args.patch_size*args.patch_size*z
        order_of_sites.append(s)
        # (3)
        y = 2
        z = 2
        for x in [1,3]:
            s = x + args.patch_size*y + args.patch_size*args.patch_size*z
            order_of_sites.append(s)
        x = 2
        z = 2
        for y in [1,3]:
            s = x + args.patch_size*y + args.patch_size*args.patch_size*z
            order_of_sites.append(s)
        x = 2
        y = 2
        for z in [1,3]:
            s = x + args.patch_size*y + args.patch_size*args.patch_size*z
            order_of_sites.append(s)
        # (4)
        x = 2
        for y in [1,3]:
            for z in [1,3]:
                s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                order_of_sites.append(s)
        y = 2
        for x in [1,3]:
            for z in [1,3]:
                s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                order_of_sites.append(s)
        z = 2
        for x in [1,3]:
            for y in [1,3]:
                s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                order_of_sites.append(s)
        # (5)
        for x in [1,3]:
            for y in [1,3]:
                for z in [1,3]:
                    s = x + args.patch_size*y + args.patch_size*args.patch_size*z
                    order_of_sites.append(s)
        # remove repetitions
        order_of_sites = list(dict.fromkeys(order_of_sites))
        my_log('## I ordered the sites in the following way:\n{}'.format(order_of_sites))

        # I also need the inverse order which will be used to revert back the generated samples and compute their energy
        reverse_order = np.arange(args.patch_size**3)
        reverse_order = [x for _,x in sorted(zip(order_of_sites,reverse_order), key=lambda pair: pair[0])]

        my_log('storing info')
        samplefilename = 'ordering_T{}_L{}_ps{}.npy'.format(args.T,args.L,args.patch_size)
        np.save(samplefilename,order_of_sites)

    train_sample = full_sample[args.batch_size:]
    val_sample = full_sample[:args.batch_size]#.to(args.device)

    my_log('Preparing validation sample...')
    # prepare the validation sample once
    val_patches = torch.zeros(size=(args.batch_size, args.patch_size**3))
    # ** (1) select patch center
    centers = np.random.choice(np.arange(len(val_sample[0])),size=(args.batch_size,1)).astype(int)
    # ** (2) select how to transform the patch
    know_transformations = 13
    transformation = np.random.choice(np.arange(know_transformations),size=(args.batch_size,1)).astype(int)
    csamp = np.concatenate((centers,transformation,val_sample),axis=-1)
    # ** (3) extract the patch and transform it
    f = lambda x: latticeglass.patch_from_config_not_ordered(x[2:],x[0],args.L,False, True, x[1])
    val_patches = torch.Tensor(np.apply_along_axis(f, 1, csamp).squeeze(axis=1))
    patch_energy = torch.Tensor(np.asarray(latticeglass.patch_energy(val_patches, onedmask.cpu().detach().numpy(), nn_list_patch, args.q)))
    my_log('The validation patches have E/N={}'.format(patch_energy.mean()/3**3))
    # ** (4) sort the validation sample
    for sample_i in range(val_patches.shape[0]):
        val_patches[sample_i] =  mysorting(order_of_sites,val_patches[sample_i])
    # ** (5) measure the density (that I am planning to conserve)
    zeros= (val_patches == 0).sum(dim=-1).unsqueeze(dim=-1)
    ones = (val_patches == 1).sum(dim=-1).unsqueeze(dim=-1)
    twos = (val_patches == 2).sum(dim=-1).unsqueeze(dim=-1)
    Npatch = 5**3
    Nall = args.batch_size*Npatch
    #val_sample_wrho = torch.concat([zeros/Npatch,ones/Npatch,twos/Npatch,torch.Tensor(val_patches)],dim=-1)
    val_sample_wrho = torch.concat([zeros/Npatch,ones/Npatch,twos/Npatch,val_patches],dim=-1)
    # ** (6) pass the sample to deice for faster processing
    val_sample_wrho = val_sample_wrho.to(device=args.device)

    # create start and end tokens
    start_token = torch.zeros(size=zeros.shape).to(device=args.device)
    end_token = start_token + 1

    criterion = nn.CrossEntropyLoss()

    my_log('Training...')
    newsample_resorted = torch.zeros(size=(args.batch_size,5**3))
    oldsample_resorted = torch.zeros(size=(args.batch_size,5**3))
    sample_time = 0
    train_time = 0
    start_time = time.time()

    # **********************************************
    # **************** TRAINING LOOP
    # **********************************************
    for step in range(last_step + 1, args.max_step + 1):

        # Use the annealing rate same as Wu et al. to decrease T
        beta = 1/args.T * (1 - args.beta_anneal**step)

        #  **** Maximum likelihood
        if args.ARtype == 'maxlike':
            sample_start_time = time.time()

            # **** Batch preparation
            # * (1) extract random configs
            indices = np.random.choice(np.arange(train_sample.shape[0]),size=(args.batch_size))
            sample = train_sample[indices][:]
            # * (2) select patch center
            centers = np.random.choice(np.arange(len(sample[0])),size=(args.batch_size,1)).astype(int)
            # * (3) select how to transform the patch
            transformation = np.random.choice(np.arange(know_transformations),size=(args.batch_size,1)).astype(int)
            csamp = np.concatenate((centers,transformation,sample),axis=-1)
            # * (4) extract the patch and transform it
            f = lambda x: latticeglass.patch_from_config_not_ordered(x[2:],x[0],args.L,False, True, x[1])
            patches = torch.Tensor(np.apply_along_axis(f, 1, csamp).squeeze(axis=1))
            # * (5) sort the training sample
            for sample_i in range(args.batch_size):
                patches[sample_i] =  mysorting(order_of_sites,patches[sample_i])
            # * (6) measure the density (that I am planning to conserve)
            zeros= (patches == 0).sum(dim=-1).unsqueeze(dim=-1).to(device=args.device)
            ones = (patches == 1).sum(dim=-1).unsqueeze(dim=-1).to(device=args.device)
            twos = (patches == 2).sum(dim=-1).unsqueeze(dim=-1).to(device=args.device)

            patches = patches.to(device=args.device)

            sample_wrho = torch.concat([start_token,zeros/Npatch,ones/Npatch,twos/Npatch,patches],dim=-1).to(device=args.device)

            # Teacher forcing: use end and start tokens
            data = patches[:,:98]
            #data = torch.concat([data+2,end_token],dim=-1).to(device=args.device,dtype=torch.long)
            data = data.to(device=args.device,dtype=torch.long)+2
            target = patches[:,98:]
            target = torch.concat([start_token,target+2,end_token],dim=-1).to(device=args.device,dtype=torch.long)

            if args.print_step and step % args.print_step == 0 :
                # generate a new sample to evaluate the energy that the network can generate and evalaute the acceptance
                with torch.no_grad():
                    net.eval()
                    new_sample = net.predict(data)[:,1:-1]
#                    print('\n')
#                    print('\n')
#                    print(data)
#                    print(data.shape)
#                    print(target)
#                    print(target.shape)
#                    print(new_sample)
#                    print(new_sample.shape)
                    logits = net(data, target[:,:-1])
                    p = F.softmax(logits, dim = 1)
                    this_sample_p = torch.gather(p, 1, new_sample.unsqueeze(dim=1)).squeeze(dim=1)
                    entropy = -torch.log(this_sample_p).sum(dim=-1)
                    # * fix the start of word token
                    new_sample = new_sample-2
                    new_sample[new_sample<0] = 0

                    # check how many samples conserved the density
                    ok_samples=0
                    for i in range(args.batch_size):
                        if (target[i]==2).sum()==(new_sample[i]==0).sum():
                            if (target[i]==3).sum()==(new_sample[i]==1).sum():
                                if (target[i]==4).sum()==(new_sample[i]==2).sum():
                                    ok_samples+=1

                    # I have to resort the samples to compute energy
                    for s_i in range(args.batch_size):
                        newsample_resorted[s_i] = mysorting(reverse_order, torch.concat([patches[s_i,:98],new_sample[s_i]],dim=-1))
                        oldsample_resorted[s_i] = mysorting(reverse_order, patches[s_i])

                    new_energy = torch.Tensor(np.asarray(latticeglass.patch_energy(newsample_resorted, onedmask.cpu().detach().numpy(), nn_list_patch, args.q)))
                    old_energy = torch.Tensor(np.asarray(latticeglass.patch_energy(oldsample_resorted, onedmask.cpu().detach().numpy(), nn_list_patch, args.q)))
                    zeros = (new_sample == 0).sum()
                    ones = (new_sample == 1).sum()
                    twos = (new_sample == 2).sum()
                    N = args.batch_size*(3**3)
                    my_log('\naverage density of zeros={:.3g}\tones={:.3g}\ttwos={:.3g}\n{}/{} samples conserved the density'.format(zeros/N,ones/N,twos/N,ok_samples,args.batch_size))
                    net.train()

            logits = net(data, target[:,:-1])
            loss = criterion(logits, target[:,1:]) #exclude from loss the start of word token

            # and measure the validation loss
            #val_loss = -net.log_prob(val_sample_wrho).mean()
            val_loss = 0*loss

            sample_time += time.time() - sample_start_time
            train_start_time = time.time()

        #  ************
        # Zero the gradient
        optimizer.zero_grad()

        # If required add regularization
        if args.regularization == 'l1':
            l1_norm = sum(p.sum() for p in net.parameters())
            loss_reinforce += args.lambdal1 * l1_norm
        if args.regularization == 'l2':
            l2_norm = sum(p.pow(2.0).sum() for p in net.parameters())
            loss_reinforce += args.lambdal2 * l2_norm

        # Backpropagate
        loss.backward()

        if args.clip_grad:
            nn.utils.clip_grad_norm_(params, args.clip_grad)

        optimizer.step()

        # If I am doing maxlike, I will start scheduler and early stop from epoch=50
        # while if I am doing var, I will start only when beta~equi

        #if args.lr_schedule and args.ARtype == 'maxlike' and step>50 :
        #    scheduler.step(loss.mean())
        #elif args.lr_schedule and round(beta,3) == args.beta:
        #    scheduler.step(loss.mean())
        if args.lr_schedule:
            scheduler.step()

        # Notice that early stopping is done over the validation loss
        if args.early_stopping and args.ARtype == 'maxlike' and step>50 :
             early_stopping(val_loss)
             if early_stopping.early_stop:
                 my_log('Exiting for early stopping trigger')
                 break
        elif args.early_stopping and round(beta,3) == args.beta:
             early_stopping(val_loss)
             if early_stopping.early_stop:
                 my_log('Exiting for early stopping trigger')
                 break

        train_time += time.time() - train_start_time

        if args.print_step and step % args.print_step == 0:
            entropy_mean = float(entropy.mean())/((args.patch_size-2)**3)
            energy_mean = float(new_energy.mean())/((args.patch_size-2)**3)
            old_energy_mean = float(old_energy.mean())/((args.patch_size-2)**3)
            free_energy = energy_mean - entropy_mean/ args.beta
            if step > 0:
                sample_time /= args.print_step
                train_time /= args.print_step
            used_time = time.time() - start_time
            my_log(
                'step = {}, F = {:.4g}, S/N = {:.4g}, E(generated)/N = {:.5g}, E(ref)/N = {:.5g}, E(equi)/N = {:.5g}, lr = {:.3g}, beta = {:.3g}, T={:.3g}, sample_time = {:.3f}, loss = {:.3g}, val_loss = {:.4g}, train_time = {:.3f}, used_time = {:.3f}'
                .format(
                    step,
                    free_energy,
                    entropy_mean,
                    energy_mean,
                    old_energy_mean,
                    E_equi,
                    optimizer.param_groups[0]['lr'],
                    beta,
                    1./max(0.000001,beta),
                    sample_time,
                    loss,
                    val_loss,
                    train_time,
                    used_time,
                ))
            sample_time = 0
            train_time = 0

            if args.save_sample:
                state = {
                    'sample': sample,
                    'log_prob': log_prob,
                    'energy': energy,
                    'loss': loss,
                }
                torch.save(state, '{}_save/{}.sample'.format(
                    args.out_filename, step))

        if (args.out_filename and args.save_step
                and step % args.save_step == 0):
            state = {
                'net': net.state_dict(),
                'optimizer': optimizer.state_dict(),
            }
            if args.lr_schedule:
                state['scheduler'] = scheduler.state_dict()
            torch.save(state, '{}_save/{}.state'.format(
                args.out_filename, step))

        if (args.out_filename and args.visual_step
                and step % args.visual_step == 0):

            if args.print_sample and 0:
                energy_np = np.asarray(new_energy)
                energy_count = np.stack(
                    np.unique(energy_np, return_counts=True)).T
                my_log(
                    '\nsample\n{}\nlog_prob\n{}\nenergy\n{}\nloss\n{}\nenergy_count\n{}\n'
                    .format(
                        sample[:args.print_sample, :],
                        log_prob[:args.print_sample],
                        energy_np,
                        loss[:args.print_sample],
                        energy_count,
                    ))

            if args.print_grad:
                my_log('grad max_abs min_abs mean std')
                for name, param in named_params:
                    if param.grad is not None:
                        grad = param.grad
                        grad_abs = torch.abs(grad)
                        my_log('{} {:.3g} {:.3g} {:.3g} {:.3g}'.format(
                            name,
                            torch.max(grad_abs).item(),
                            torch.min(grad_abs).item(),
                            torch.mean(grad).item(),
                            torch.std(grad).item(),
                        ))
                    else:
                        my_log('{} None'.format(name))
                my_log('')

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
environments = (u'en la web de Renfe',
                u'en la base de datos corporativa',
                u'en la web del congreso',
                u'en la web de la ESA',
                u'en la base de datos del FBI',
                u'en la web de Twitter',
                u'en Facebook',
                u'en la NASA'
                u'en el patio de mi casa',
                u'en mi movil')


#Lista de Exercício 2 - Questão 10
#Dupla: 2020314273 - Cauã Alexandre Torres de Holanda e 2021327294 - Kallyne Ferro Veiga
#Disciplina: Programação Web
#Professor: Ítalo Arruda

#10.Faça um Programa que pergunte em que turno você estuda. Peça para digitar M-matutino ou V-Vespertino ou N- Noturno. Imprima a mensagem "Bom Dia!", "Boa Tarde!" ou "Boa Noite!" ou "Valor Inválido!", conforme o caso.

class TurnoEstudo:
    def __init__(self):
        self.turno = ''
    def ler_turno(self):
        while True:
            self.turno = input("Em qual turno você estuda? (M-matutino, V-vespertino, N-noturno): ")
            if self.turno in ['M', 'm', 'V', 'v', 'N', 'n']:
                break
            else:
                print("Valor inválido. Digite M, V ou N.")
    def exibir_mensagem(self):
        if self.turno in ['M', 'm']:
            print("Bom Dia!")
        elif self.turno in ['V', 'v']:
            print("Boa Tarde!")
        elif self.turno in ['N', 'n']:
            print("Boa Noite!")
        else:
            print("Valor inválido!")
turno_estudo = TurnoEstudo()
turno_estudo.ler_turno()
turno_estudo.exibir_mensagem()


import pandas as pd
import sqlite3

def load_data(messages_filepath, categories_filepath):
    '''
    Create dataframes for messages and categories data.
    '''
    messages_df = pd.read_csv(messages_filepath)
    categories_df = pd.read_csv(categories_filepath)

    return messages_df, categories_df

def clean_data(messages_df, categories_df):
    '''
    Create a clean, combined dataframe of messages and category dummy variables.

    Args:
        messages_df: DataFrame. Contains 'id' column for joining.
        categories_df: DataFrame. Contains 'id' column for joining and
        'categories' column with strings of categories separated by ;.
    '''
    # Merge datasets
    df = pd.merge(messages_df, categories_df, on='id')

    # Drop duplicates
    df.drop_duplicates(inplace=True)

    # Create categories columns
    new_cat_df = df.categories.str.split(';', expand=True)

    col_names = df.iloc[0].str[:-2]
    new_cat_df.columns = col_names

    for col in col_names:
        new_cat_df[col] = new_cat_df[col].str[-1].astype(int)

    df = pd.concat([df.drop('categories', axis=1), new_cat_df], axis=1)

    # Clean 'related' values
    df.loc[(df.related == 2), 'related'] = 1

    return df

def save_data(df, database_filepath):
    '''
    Save dataframe to database in 'messages' table. Replace any existing data.
    '''
    conn = sqlite3.connect(database_filepath)
    df.to_sql('messages', con=conn, if_exists='replace', index=False)
    conn.commit()
    conn.close()

def main(messages_filepath, categories_filepath, database_filepath):
    '''
    Extract messages and categories data from csv files, cleans the data and
    saves merged data into database. Checks if data already saved.
    '''
    # Load data
    print('Loading data...\n    MESSAGES: {}\n    CATEGORIES: {}'
              .format(messages_filepath, categories_filepath))
    messages_df, categories_df = load_data(messages_filepath,
                                           categories_filepath)

    conn = sqlite3.connect(database_filepath)
    # Check if data already in db
    try:
        row_count = pd.read_sql('SELECT COUNT(*) FROM messages', conn).iloc[0][0]
        if categories_df.drop_duplicates().shape[0] == row_count:
            print('Database is up to date')
            conn.close()
        else:
            print('Cleaning data...')
            df = clean_data(messages_df, categories_df)

            print('Saving data...')
            save_data(df, database_filepath)
            conn.close()

    except sqlite3.OperationalError:
        print('Cleaning data...')
        df = clean_data(messages_df, categories_df)

        print('Saving data...')
        save_data(df, database_filepath)
        conn.close()

if __name__ == '__main__':
    # Create argparser
    import argparse
    parser = argparse.ArgumentParser(description='Categorize disaster messages')
    parser.add_argument("messages_filepath", help="File path for messages csv")
    parser.add_argument("categories_filepath", help="File path for categories csv")
    parser.add_argument("database_filepath", help="File path for database")
    args = parser.parse_args()

    main(messages_filepath=args.messages_filepath,
         categories_filepath=args.categories_filepath,
         database_filepath=args.database_filepath)


# Given an array of strings strs, group the anagrams together. You can return the answer in any order.

# An Anagram is a word or phrase formed by rearranging the letters of a different word or phrase,
# typically using all the original letters exactly once.

# Example

# Input: strs = ["eat","tea","tan","ate","nat","bat"]
# Output: [["bat"],["nat","tan"],["ate","eat","tea"]]
def compare_maps(map1, map2):
    keys_1 = map1.keys()
    keys_2 = map2.key()
    if len(keys_1) != len(keys_2):
        return False
    for x in keys_1:
        try:
            if map1[x] != map2[x]:
                return False
        except KeyError:
            return False
    return True

def getMapOfWord(word):
    map1 = {}
    for ch in word:
        if map1.get(ch, False):
            map1[ch] +=1
        else:
            map1[ch] = 1
    return map1

def group_anagrams(string_array):
    groups = {}
    for words in string_array:
        map_word = getMapOfWord(words)
        if len(groups.keys()) == 0:
            groups[words]= [words]
            continue
        for key in groups.keys():
            map_key = getMapOfWord(key)
            if compare_maps(map_key, map_word):
                groups[key].append(words)
    output = []
    for x in groups:
        output.append(groups[x])
    return output


import requests
import json ,os,pprint

while True:
	if os.path.exists("s_courses.json")==False:
		a=requests.get('http://saral.navgurukul.org/api/courses')
		pprint.pprint(a.text)
		b=a.text
		c=open('s_courses.json','w')
		json.dump(b,c)
		c.close()

	c=open('s_courses.json','r')
	jsl1=json.load(c)
	jsl2=json.loads(jsl1)
	print(type(jsl2))
	c.close()

	for i in jsl2:
		for j in range(len(jsl2[i])):
			if j<9:
				x='0'+str(j+1)
				print(x,'-',jsl2[i][j]['id'],' ','name -',jsl2[i][j]['name'])
			else:
				print(1+j,'-',jsl2[i][j]['id'],' ','name -',jsl2[i][j]['name'])

	j=int(input('enter no.'))
	print(j,'-',jsl2[i][j-1]['id'],' ','name -',jsl2[i][j-1]['name'],'\n')
	x=jsl2[i][j-1]['id']

	while True:

		a1=requests.get(' http://saral.navgurukul.org/api/courses/'+str(x)+'/exercises')
		b1=a1.text
		if os.path.exists("'exercises_'+str(x)+'.json'")==False:
			c1=open('exercises_'+str(x)+'.json','w+')
			json.dump(b1,c1)
			c1.close()
		c1=open('exercises_'+str	(x)+'.json','r')
		js1=json.load(c1)
		js2=json.loads(js1)
		c.close()
		for i in js2:
			for j in range(len(js2[i])):
				print('\t',j+1,'-',js2[i][j]['name'])
		inp=input('''
			*** YOU WANT TO
			PRIVIOUS PAGE - THEN PRESS ("P")
			NEXT PAGE     - TNEN PRESS ("N")
			STARTING PAGE - THEN PRESS ("S")''').lower()
		if inp=='p':
			x-=1
		elif(inp=='n'):
			x+=1
		else:
			break
		if x<14:
			x=14
			print('have no PRIVIOUS PAGE ')
		elif(x>92):
			x=92
			print('have no NEXT PAGE')
		print('Id-',x)
	inp=input('**you want to exit then press "exit"').lower()
	if 'e' in inp:
		break


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 24 13:39:12 2016

Multi-Layer-Perceptron as Learner

@author: Mats Richter
"""

import numpy as np
import sknn.mlp as mlp

class Learner:

    #@input w_size: size of the sliding window used for batch learning
    #@input size:   numbers of parameters fed in the input layer
    #@input mode:   switch between processing returns or prices
    #
    #@param learner: A MLP-Regressor for learning
    #@param data:    The data fed into the system
    def __init__(self,w_size=100,input_size=12,layers=1,n_itr=50,learn=0.05,AutoEncoder=False,adaption = 0.1):
        self.layers = list()
        #setup layers HIGHLY TENTETIVE AND SUBJECT TO CHANGE
        for i in range(layers):
            layer = mlp.Layer('Rectifier',units=input_size)
            self.layers.append(layer)
        self.layers.append(mlp.Layer('Softmax'))
        self.learner = mlp.Classifier(self.layers,learning_rate=learn,n_iter=n_itr)
        self.input_size = input_size
        self.w_size = self.input_size * w_size
        self.data = list()
        self.returns = list()
        self.labels = list()
        self.tstep = 0

        self.sharpeA = 1
        self.sharpeB = 1
        self.adaption = adaption

        self.std = 1
        return

    def predict(self,price1,price2):
        result = 0
        #add data to the data-vector
        self.returns.append(price2-price1)
        if(len(self.data)> self.w_size):
            self.returns.pop(0)
        #build input-vector if enought data is available
        if(self.tstep > self.input_size):
            #label returns
            self.labels.append(self.label_returns(price2-price1))
            #init data vector
            x = list()
            #setup input vector:
            x = self.returns[len(self.returns)-self.input_size-1:len(self.returns)-1]
            x = np.array(x)
            x = x.reshape((len(x),1))
            self.data.append(x)
            x = np.transpose(x)
            if(self.learner.is_initialized):
                self.update_sharpe(price2-price1)
                #predict the probability of each class
                pred = self.learner.predict_proba(x)
                #prepare output for return
                if(pred[0][0] < pred[0][1]):
                    result = pred[0][0]
                else:
                   result = pred[0][1] * (-1)

            else:
                result = 0
        #if window is full, start training
        if(self.tstep%self.w_size == 0 and self.tstep != 0):
            self.train()
            self.is_initialized = True
            #clear window
        self.tstep += 1
        return np.tanh(result * np.fabs(self.sharpeA/self.sharpeB))

    def train(self):
        #arrange data for processing
        data = np.array(self.data)
        train_dat = np.zeros((len(self.labels),self.input_size))
        for i in range(len(train_dat)):
            train_dat[i][:] = np.transpose(self.data[i])

        self.learner.fit(train_dat,np.array(self.labels))
        self.std = np.std(self.data)
        #print("training done")
        #clear data and labels
        self.data = list()
        self.labels = list()
        return

    def label_returns(self,next_return):
        if next_return > 0:
            return 1
        else:
            return -1

    def update_sharpe(self,latest_return):
        self.sharpeA += self.adaption*(latest_return - self.sharpeA)
        self.sharpeB += self.adaption*(latest_return**2 - self.sharpeB)


"""
aimall_utils.py v0.1
F. Falcioni, L. J. Duarte, P. L. A. Popelier

Library with function to submit job to AIMAll and get properties values from output
AIMAll version: 19.10.12
Check for updates at github.com/FabioFalcioni
For details about the method, please see XXXXXXX

"""

import numpy as np
from typing import List

def distance_A_B(xyz_file: str, atom_A: int, atom_B: int) -> float:
    """distance_A_B gets the distance between atom A and B in an XYZ type file

    :return: distance value
    :rtype: float
    """
    # INTERNAL VARIABLES:
    all_coordinates = []

    # WORKING IN THE FILE:
    with open(xyz_file) as f:
        coordinates_list = f.readlines()[3:]  # remove the first 2 lines of xyz file
        for i in range(0, len(coordinates_list)):
            coordinates_of_atom = [float(c) for c in coordinates_list[i].split()[1:]]
            all_coordinates.append(coordinates_of_atom)

    coord_atom_A = all_coordinates[atom_A - 1]
    coord_atom_B = all_coordinates[atom_B - 1]
    x = 0
    y = 1
    z = 2
    # GET DISTANCE
    r_AB = np.sqrt(
        (coord_atom_B[x] - coord_atom_A[x]) ** 2
        + (coord_atom_B[y] - coord_atom_A[y]) ** 2
        + (coord_atom_B[z] - coord_atom_A[z]) ** 2
    )
    return r_AB

def get_atom_list_wfn(wfn_file: str) -> List[str]:
    """get_atom_list_wfn is a function that gets atom labels from a wavefunction file (.wfn format)

    :raises ValueError: Atomic labels not found
    :return: list of atom labels
    :rtype: List
    """
    # INTERNAL VARIABLES:
    atom_list = []

    # OPEN FILE:
    file = open(wfn_file, "r")
    lines = file.readlines()  # Convert file into a array of lines
    file.close()  # Close file

    # ERRORS:
    if "(CENTRE " not in lines[2]:
        raise ValueError(
            "Atomic labels not found"
        )  # Checks if atomic list exist inside file

    # GET ATOM LIST:
    for i in range(len(lines)):
        if "(CENTRE" in lines[i]:
            split_line = lines[i].split()
            atom_list.append(
                split_line[0].lower() + str(split_line[1])
            )  # uppercase to lowercase

    return atom_list

def get_atom_list_wfx(wfx_file: str) -> List[str]:
    """get_atom_list_wfx is a function that gets atom labels from a wavefunction file (.wfx format)

    :raises ValueError: Atomic labels not found
    :return: list of atom labels
    :rtype: List
    """
    # INTERNAL VARIABLES:
    atom_list = []

    # OPEN FILE:
    file = open(wfx_file, "r")
    lines = file.readlines()  # Convert file into a array of lines
    file.close()  # Close file

    # ERRORS:
    if "<Nuclear Names>" not in lines[33]:
        raise ValueError(
            "Atomic labels not found"
        )  # Checks if atomic list exist inside file

    # GET ATOM LIST:
    for i in range(len(lines)):
        if "<Nuclear Names>" in lines[i]:
            i += 1
            while "</Nuclear Names>" not in lines[i]:
                split_line = lines[i].split()
                atom_list.append(split_line[0].lower())  # uppercase to lowercase
                i += 1

    return atom_list

def get_aimall_wfn_energies(wfn_files: List[str]) -> List[float]:
    """get_aimall_wfn_energies gets the total energy for each of the wavefunction (.wfn) files in a list

    :raises ValueError: Energy values not found in wavefunction file
    :return: list of energies
    :rtype: List
    """
    # INTERNAL VARIABLES:
    wfn_energy = []  # List of wfn files

    # READ FILES
    for path in wfn_files:
        file = open(path, "r")
        lines = file.readlines()
        # ERRORS:
        if (
            "TOTAL ENERGY " not in lines[-1]
        ):  # Checks if there is an energy value at the end of wfn file.
            raise ValueError("Energy values not found in file: ", path)
        wfn_energy.append(float(lines[-1].split()[3]))
        file.close()

    return wfn_energy

def get_aimall_wfx_energies(wfx_files: List[str]) -> List[float]:
    """get_aimall_wfx_energies gets the total energy for each of the wavefunction (.wfx type) files in a list

    :raises ValueError: Energy values not found in wavefunction file
    :return: list of energies
    :rtype: List
    """
    # INTERNAL VARIABLES:
    wfx_energy = []  # List of wfn files

    # READ FILES
    for path in wfx_files:
        file = open(path, "r")
        lines = file.readlines()
        # ERRORS:
        if (
            "<Energy = T + Vne + Vee + Vnn>" not in lines[-6]
        ):  # Checks if there is an energy value at the end of wfn file.
            raise ValueError("Energy values not found in file: ", path)
        wfx_energy.append(float(lines[-5].split()[0]))
        file.close()

    return wfx_energy

def intra_property_from_int_file(
    folders: List[str], prop: List[str], atom_list: List[str]
) -> List:
    """intra_property_from_int_file gets IQA intra-atomic energy values from .int files output from AIMAll

    :raises ValueError: File is empty or does not exist
    :return: Lists of intra-atomic energies with the corresponding intra-atomic labeling
    :rtype: List of floats and strings
    """
    # INTERNAL VARIABLES:
    temp1 = []  # Temporary array
    temp2 = []  # Temporary array
    temp3 = []  # Temporary array
    intra_properties = []  # Output
    contributions_list = []  # Output

    # READ PROPERTIES FROM .INT FILES
    for folder in folders:
        for atom in atom_list:
            file = open(folder + "/" + atom + ".int", "r")
            lines = file.readlines()

            file.close()
            for i in lines:
                if 'IQA Energy Components (see "2EDM Note")' in i:
                    start = lines.index(i)
                elif "2EDM Note:" in i:
                    end = lines.index(i)

            if end >= len(lines):  # Checks the .int file.
                raise ValueError(
                    "File is empty or does not exist: " + folder + "/" + atom + ".int"
                )

            lines = [lines[i] for i in range(start + 1, end)]
            for term in prop:
                for i in lines:
                    if (term + "           ") in i:
                        temp1.append(float(i.split()[-1]))

    # ORGANIZE ARRAY ORDER
    for j in range(len(prop)):
        for i in range(j, len(temp1), len(prop)):
            temp2.append(temp1[i])
    for j in range(len(atom_list)):
        temp3.append([temp2[i] for i in range(j, len(temp2), len(atom_list))])

    start = 0
    for j in range(len(prop)):
        for atom_prop in temp3:
            intra_properties.append(
                [atom_prop[i] for i in range(start, (j + 1) * len(folders))]
            )
        start = (j + 1) * len(folders)

    # CREATE CONTRIBUTIONS LIST ARRAY:
    for a in prop:
        for b in atom_list:
            contributions_list.append(a + "-" + b)

    return intra_properties, contributions_list

def inter_property_from_int_file(
    folders: List[str], prop: List[str], atom_list: List[str]
) -> List:
    """inter_property_from_int_file gets IQA inter-atomic energy values from .int files output from AIMAll

    :raises ValueError: File is empty or does not exist
    :return: Lists of inter-atomic energies with the corresponding intra-atomic labeling
    :rtype: List of floats and strings
    """
    # INTERNAL VARIABLES:
    temp1 = []  # Temporary array
    temp2 = []  # Temporary array
    temp3 = []  # Temporary array
    inter_properties = []  # Output
    contributions_list = []  # Output

    for path in folders:
        for i in range(len(atom_list)):
            atom1 = atom_list[i]
            for j in range(i + 1, len(atom_list)):
                atom2 = atom_list[j]
                file = open(path + "/" + atom1 + "_" + atom2 + ".int", "r")
                lines = file.readlines()
                file.close()
                for i in lines:
                    if ' Energy Components (See "2EDM Note"):' in i:
                        start = lines.index(i)
                    elif "2EDM Note:" in i:
                        end = lines.index(i)

                if end >= len(lines):  # Checks the .int file.
                    raise ValueError(
                        "File is empty or does not exist: "
                        + path
                        + "/"
                        + atom1
                        + "_"
                        + atom2
                        + ".int"
                    )

                lines = [lines[i] for i in range(start + 1, end)]
                for term in prop:
                    for i in lines:
                        if (term + "  ") in i:
                            temp1.append(float(i.split()[-1]))
    # ORGANIZE ARRAY ORDER
    for j in range(len(prop)):
        for i in range(j, len(temp1), len(prop)):
            temp2.append(temp1[i])
    for j in range(int(len(atom_list) * (len(atom_list) - 1) / 2)):
        temp3.append(
            [
                temp2[i]
                for i in range(
                    j, len(temp2), int(len(atom_list) * (len(atom_list) - 1) / 2)
                )
            ]
        )
    start = 0
    for j in range(len(prop)):
        for atom_prop in temp3:
            inter_properties.append(
                [atom_prop[i] for i in range(start, (j + 1) * len(folders))]
            )
        start = (j + 1) * len(folders)
        # CREATE CONTRIBUTIONS LIST ARRAY:
    for a in prop:
        for i in range(len(atom_list)):
            for j in range(i + 1, len(atom_list)):
                contributions_list.append(a + "-" + atom_list[i] + "_" + atom_list[j])

    return inter_properties, contributions_list

def charge_transfer_and_polarisation_from_int_file(
    folders: List[str],
    atom_list: List[str],
    inter_properties: List[str],
    xyz_files: List[str],
) -> List:
    """charge_transfer_and_polarisation_from_int_file gets IQA polarisation and charge-transfer energies
    values by computing them as Vct = qAqB/rAB and Vpl = Vcl - Vct

    :raises ValueError: File is empty or does not exist
    :return: Returns lists of energies and labels for both Vct and Vpl
    :rtype: List of floats and strings
    """
    # INTERNAL VARIABLES:
    n = len(atom_list)
    f = len(folders)
    temp1 = []  # Temporary array
    temp2 = []  # Temporary array
    temp3 = []  # Temporary array
    net_charges = []
    charge_transfer_properties = []  # Output
    polarisation_properties = []  # Output
    contributions_list_CT = []  # Output
    contributions_list_PL = []  # Output

    # CREATE CONTRIBTUIONS LIST ARRAY
    for i in range(len(atom_list)):
        for j in range(i + 1, len(atom_list)):
            contributions_list_PL.append(
                "Vpl_IQA(A,B)-" + atom_list[i] + "_" + atom_list[j]
            )
            contributions_list_CT.append(
                "Vct_IQA(A,B)-" + atom_list[i] + "_" + atom_list[j]
            )

    # READ NET-CHARGE PROPERTIES FROM .INT FILES
    for folder in folders:
        net_charge_group = []
        for i in range(0, len(atom_list)):
            file = open(folder + "/" + atom_list[i] + ".int", "r")
            lines = file.readlines()
            file.close()

            for line in lines:
                if "Results of the basin integration:" in line:
                    start = lines.index(line)
                elif "|Dipole|" in line:
                    end = lines.index(line)
            if end >= len(lines):  # Checks the .int file.
                raise ValueError(
                    "File is empty or does not exist: "
                    + folder
                    + "/"
                    + atom_list[i]
                    + ".int"
                )

            lines = [lines[j] for j in range(start + 1, end)]
            Q = float(lines[0].split()[-4])
            net_charge_group.append(Q)
        net_charges.append(net_charge_group)

    # GET CHARGE TRANSFER TERMS
    for k in range(len(net_charges)):
        for i in range(len(atom_list)):
            for j in range(i + 1, len(atom_list)):
                temp1.append(
                    (net_charges[k][i] * net_charges[k][j])
                    / ((distance_A_B(xyz_files[k], i + 1, j + 1)) * 1.8897259886)
                )

    # ORGANIZE CHARGE TRANSFER ARRAY ORDER
    for j in range(int(n * (n - 1) / 2)):
        temp2.append([temp1[i] for i in range(j, len(temp1), int(n * (n - 1) / 2))])
    start = 0
    for atom_prop in temp2:
        charge_transfer_properties.append([atom_prop[i] for i in range(start, f)])

    # Isolate Vcl terms
    classical_properties = inter_properties[: len(charge_transfer_properties)]

    # OBTAIN POLARISATION TERMS AS Vpl = Vcl - Vct
    for i in range(len(classical_properties)):
        for j in range(len(classical_properties[i])):
            temp3.append(classical_properties[i][j] - charge_transfer_properties[i][j])

    # ORGANIZE POLARISATION ARRAY ORDER
    polarisation_properties = [
        temp3[i * f : (i + 1) * f] for i in range((len(temp3) + f - 1) // f)
    ]

    return (
        charge_transfer_properties,
        contributions_list_CT,
        polarisation_properties,
        contributions_list_PL,
    )


"""added profile pict

Revision ID: d03756b7815b
Revises: 845e6def5277
Create Date: 2022-03-02 16:17:18.286475

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd03756b7815b'
down_revision = '845e6def5277'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('u_pict', sa.String(), nullable=True))
    op.create_index(op.f('ix_user_u_pict'), 'user', ['u_pict'], unique=True)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_index(op.f('ix_user_u_pict'), table_name='user')
    op.drop_column('user', 'u_pict')
    # ### end Alembic commands ###


# create venv
# python3 -m venv env

import markdown
from datetime import datetime

md = markdown.Markdown(extensions=['markdown.extensions.fenced_code'])

file_data = None
with open('raw.md') as f:
	file_data = f.read()
	#print(file_data)

data = md.convert(file_data)

header_list = []

# link
i = len(data)
target = '<a '
while i != -1:
	i = data.rfind(target, 0, i)
	if i == -1:
		break
	#print(i)
	j = i + len(target)
	data = data[:j] + 'target="_blank" ' + data[j:]
	data = '{}target="_blank" {}'.format(data[:j], data[j:])

# header
i = len(data)
target = '
'
while i != -1:
	i = data.rfind(target, 0, i)
	if i == -1:
		break
	j = i + len(target)
	k = data[i:].find('
')
	#print(j, i+k)
	head_text = data[j:i+k]
	header_id = head_text.lower().replace(' ', '-')
	header_list.insert(0, (head_text, header_id))
	data = '{} id="{}"{}'.format(data[:j-1], header_id, data[j-1:])

# code block
i = len(data)
target = 'class="code-block"'
prefix = '><code '
while i != -1:
	i = data.rfind(target, 0, i)
	if i == -1:
		break
	#print(i)
	j = i + len(target)
	data = '{} {}{}{}'.format(data[:i-len(prefix)], target, prefix[:-1], data[j:])

# runtime embedded box
i = len(data)
target = 'runtime-embedded-box'
prefix = '
<code class="'
postfix = '
'
box_extra_class = {
	0: '',
	1: 'runtime-show-canvas',
	2: 'runtime-hide-console',
	3: 'runtime-show-canvas runtime-hide-console',
}
while i != -1:
	i = data.rfind(target, 0, i) # class start
	if i == -1:
		break
	#print(i)
	j = i + len(target) # class params start
	k = j + data[j:].find('"') # class end
	l = k + data[k:].find(postfix) # code block end
	box_style = data[j+1:k].split('-') # [type, height]
	data = '{}
{}
{}'.format(
		data[:i-len(prefix)],
		box_extra_class[int(box_style[0])],
		box_style[1],
		data[k+2:l],
		data[l+len(postfix):]
		)

template = ''
with open('template.html') as f:
	template = f.read()

toc = '\n'.join(['
{}
'.format(header[1], header[0]) for header in header_list])

out_data = template.replace('{{toc}}', toc)
now = datetime.now()
out_data = out_data.replace('{{date}}', now.strftime("%d %b, %Y"))
out_data = out_data.replace('{{year}}', now.strftime("%Y"))
out_data = out_data.replace('{{contents}}', data)

with open('index.html', 'w') as f:
	f.write(out_data)


# A Python program that simulates the "Rock, Paper, Scissors" game.

import random

options = ("rock", "paper", "scissors")

comp = options[random.randint(0, 2)]

print("===== Welcome =====")

player = input("Please enter rock, paper or scissors: ")

if player.lower() == comp:
    print("It's a tie, please try again")
elif player.lower() == "rock":
    if comp == "paper":
        print("You Lose!")
    else:
        print("You win")
elif player.lower() == "paper":
    if comp == "scissors":
        print("You lose!")
    else:
        print("You win")
elif player.lower() == "scissors":
    if comp == "rock":
        print("You lose!")
    else:
        print("You win")
else:
    print("Please enter a valid option")


# coding: utf-8

from __future__ import unicode_literals

import datetime
import unittest

from uwsgi_log_plugin import import_from_uwsgi_log

class UwsgiLogPluginTestCase(unittest.TestCase):
    def test_import_from_uwsgi_log(self):
        filename = "uwsgi.log"
        table = import_from_uwsgi_log(filename, "utf-8")
        self.assertEqual(len(table), 2)
        first = table.Row(
            pid=879,
            ip="127.0.0.1",
            datetime=datetime.datetime(2015, 6, 1, 11, 23, 16),
            generation_time=0.17378,
            http_path="/something",
            http_verb="GET",
            http_version=1.1,
            http_status=404,
        )
        second = table.Row(
            pid=31460,
            ip="127.0.1.1",
            datetime=datetime.datetime(2015, 7, 15, 23, 49, 20),
            generation_time=0.000466,
            http_path="/about",
            http_verb="OPTIONS",
            http_version=1.1,
            http_status=200,
        )
        self.assertEqual(table[0], first)
        self.assertEqual(table[1], second)


#cog by @maxy_dev (maxy#2866)
import asyncio
import sys
import disnake as discord
import random
import os
from main import bot
from enum import Enum
import datetime, time
from disnake.ext import commands
from utils import db

if "debug" not in db:
  db["debug"] = {}

class Required1(str, Enum):
  true = "True"
  false = ""

class Debug(commands.Cog):
  def __init__(self, bot):
    self.bot = bot

  #when slash error
  @commands.Cog.listener()
  async def on_slash_command_error(self, inter, error):
    if isinstance(error, commands.CommandNotFound):
      return
    if not isinstance(error, commands.CommandOnCooldown):
      if not "Command raised an exception:" in str(error):
        e = discord.Embed(title = "Error", description = f"```{str(error)}```", color = random.randint(0, 16777215))
      else:
        e = discord.Embed(title = "Error", description = f"```{str(error)[29:]}```", color = random.randint(0, 16777215))
    else:
      e = discord.Embed(title = "Error", description = f"{str(error)[:31]} <t:{int(time.time() + error.retry_after)}:R>", color = random.randint(0, 16777215))
    await inter.send(embed = e, ephemeral = True)

  #debug group
  @commands.slash_command()
  async def debug(self, inter):
    pass

  @debug.sub_command()
  @commands.is_owner()
  async def toggle(self, inter, toggler: Required1 = Required1.true):
    '''
    debug,,

    Parameters
    ----------
    text: None
    '''
    if str(inter.author.id) not in db["debug"] and toggler:
      db["debug"][str(inter.author.id)] = "True"
      e = discord.Embed(title = "Success", description = "Debug mode enabled", color = random.randint(0, 16777215))
      await inter.send(embed = e, ephemeral = True)
      return
    if str(inter.author.id) in db["debug"] and not toggler:
      del db["debug"][str(inter.author.id)]
      e = discord.Embed(title = "Success", description = "Debug mode disabled", color = random.randint(0, 16777215))
      await inter.send(embed = e, ephemeral = True)

  #load extension command
  @debug.sub_command()
  @commands.is_owner()
  async def load(self, inter, extension):
    '''
    Loads an extension

    Parameters
    ----------
    extension: Cog name
    '''
    bot.load_extension(f"cogs.{extension}")
    await inter.send(f"cogs.{extension} is loaded", ephemeral = True)

  #reload extension command
  @debug.sub_command()
  @commands.is_owner()
  async def reload(self, inter, extension):
    '''
    Loads an extension

    Parameters
    ----------
    extension: Cog name
    '''
    bot.unload_extension(f"cogs.{extension}")
    bot.load_extension(f"cogs.{extension}")
    await inter.send(f"cogs.{extension} is reloaded", ephemeral = True)

  #unload extension command
  @debug.sub_command()
  @commands.is_owner()
  async def unload(self, inter, extension):
    '''
    Loads an extension

    Parameters
    ----------
    extension: Cog name
    '''
    bot.unload_extension(f"cogs.{extension}")
    await inter.send(f"cogs.{extension} is unloaded", ephemeral = True)

  #restart bot
  # @debug.sub_command()
  # @commands.is_owner()
  # async def restart(self, inter):
  #   '''
  #   Restarts the bot
  #   '''
  #   await inter.send("Restarting", ephemeral = True)
  #   before = time.perf_counter()
  #   await bot.close()
  #   await bot.login(os.environ["DISCORD_TOKEN"])
  #   after = time.perf_counter()
  #   await inter.edit_original_message(f"Restarted, took `{round((after - before) * 1000)}`ms")

  @debug.sub_command()
  @commands.is_owner()
  async def shutdown(self, inter):
    '''
    Shutdowns the bot
    '''
    await inter.send("Shutdown", ephemeral = True)
    try:
      await bot.close()
    except Exception:
      print("something went wrong")
    sys.exit()

def setup(bot):
  bot.add_cog(Debug(bot))