# -*- coding: utf-8 -*-

import logging

from latex import LatexDocument

from django.utils.translation import ugettext as _
from django.template import Template as DjTemplate
from django.template import Context as DjContext

from django.core.files.uploadedfile import InMemoryUploadedFile
from django.core.files.base import ContentFile

from storage.models import Template, GeneratedFile, KYCFile

logger = logging.getLogger("api.latex")

# https://pypi.python.org/pypi/django-latex/0.1
def _generatePDF(template, context={}):
    """
    Compute a pdf from a LaTeX @template, with some dynamic data in @context
    return a @GeneratedFile instance
    """
    t = DjTemplate(template.content)
    c = DjContext(context)

    latex_str = t.render(c)
    latex_str += '' # to django.utils.safestring.SafeText to unicode

    latex_doc = LatexDocument(latex_str)

    try:
        gen_file = GeneratedFile()
        gen_file.template = template
        result = latex_doc.as_pdf(debug=True) # returns a StringIO instance
    except LatexDocument.LatexPdfGenerationError as e:
        gen_file.errors = "Error during pdf generation: %s" % str(e)
        logger.error(str(e))
    else:
        gen_file.file.save("%s-%d" % (template.name, template.id,), ContentFile(result.read()))

    gen_file.save()

    return gen_file

def generateInvestorBook(ba):
    template = Template.objects.get(type='INVESTOR_BOOK')

    context = {
    	'business_angel': ba,
    }

    return _generatePDF(template, context)


import pyttsx3
import datetime
import speech_recognition as sr
import wikipedia
import webbrowser
import os
import smtplib

engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')

engine.setProperty('voice',voices[0].id)

def speak(audio):
	engine.say(audio)
	engine.runAndWait()

def wishMe():
	hour = int(datetime.datetime.now().hour)
	if hour >=0 and hour <12:
		speak("Good Morning!")
	elif hour >=12 and hour <18:
		speak("Good Afternoon!")
	else:
		speak("Good Evening!")
	speak("I am Leonardo sir, how may I help you?")

def takeCommand():
	'''
	takes microphone command and returns input
	'''

	r=sr.Recognizer()
	with sr.Microphone() as source:
		print("Listening...")
		r.pause_threshold = 1
		audio = r.listen(source)

	try:
		print("Recognizing...")
		query = r.recognize_google(audio, language = 'en-in')
		print(f"You said: {query}\n")
	except Exception as e:
		print(e)
		print("Say that again please...")
		return "None"
	return query

def sendEmail(to, content):
	server = smtplib.SMTP('smtp.gmail.com',587)
	server.ehlo()
	server.starttls()
	server.login('theconscienceofficial@gmail.com', 'conscience@google')
	server.sendmail('theconscienceofficial@gmail.com',to,content)
	server.close()

if __name__ == '__main__':
	chrome_path = 'C:/Program Files (x86)/Google/Chrome/Application/chrome.exe %s'
	wishMe()
	while True:
	    query = takeCommand().lower()
	    if 'wikipedia' in query:
	    	speak("searching wikipedia...")
	    	query = query.replace("wikipedia","")
	    	results = wikipedia.summary(query,sentences=2)
	    	speak("According to wikipedia")
	    	speak(results)
	    elif 'exit' in query:
	    	exit()

	    elif 'shutdown' == query:
	    	speak("Do you really want to shut down your Computer boss?")
	    	query2 = takeCommand().lower()
	    	if(query == "yes shutdown"):
	    	    os.system("shutdown /s /t 1");

	    elif 'who are you' in query:
	    	speak("I am Leonardo, your virtual assistant, boss")

	    elif 'who am i' in query:
	    	speak("You are Tarang, boss. You are my creator")
	    elif 'hey leonardo' in query or 'hey leo' in query:
	    	speak("Hello boss")

	    elif 'open youtube' in query or 'youtube open' in query:
	    	webbrowser.get(chrome_path).open("youtube.com")

	    elif 'search youtube for' in query:
	    	query = query.replace("search youtube for ","")
	    	query=query.replace(" ","+")
	    	webbrowser.get(chrome_path).open("youtube.com/results?search_query="+query)

	    elif 'open facebook' in query:
	    	webbrowser.get(chrome_path).open("facebook.com")

	    elif 'open instagram' in query:
	    	webbrowser.get(chrome_path).open("instagram.com")

	    elif 'open linkedin' in query:
	    	webbrowser.get(chrome_path).open("linkedin.com")

	    elif 'open google' in query:
	    	webbrowser.get(chrome_path).open("google.com")

	    elif 'open udemy' in query:
	    	webbrowser.get(chrome_path).open("udemy.com")

	    elif 'open github' in query:
	    	webbrowser.get(chrome_path).open("github.com")

	    elif 'open heroku' in query:
	    	webbrowser.get(chrome_path).open("heroku.com")

	    elif 'open the conscience' in query:
	    	webbrowser.get(chrome_path).open("theconscience.cf")

	    elif 'search google for' in query:
	    	query = query.replace("search google for ","")
	    	query=query.replace(" ","+")
	    	webbrowser.get(chrome_path).open("google.com/search?q="+query)

	    elif 'open stackoverflow' in query:
	    	webbrowser.get(chrome_path).open("stackoverflow.com")

	    elif 'play music' in query:
	    	music_dir = "D:\\Alan Walker"
	    	songs = os.listdir(music_dir)
	    	print(songs)
	    	os.startfile(os.path.join(music_dir, songs[0]))

	    elif 'the time' in query:
	    	strTime = datetime.datetime.now().strftime("%H:%M:%S")
	    	speak(f"Sir, the time is {strTime}")

	    elif 'open sublime' in query:
	    	sublimePath = "C:\\Program Files\\Sublime Text 3\\sublime_text.exe"
	    	os.startfile(sublimePath)
	    elif 'send email' in query:
	    	try:
	    		speak("What should I say?")
	    		content = takeCommand()
	    		to = "taranggarlapally@gmail.com"
	    		sendEmail(to, content)
	    		speak("Sir, your Email has been sent")
	    	except Exception as e:
	    		print(e)
	    		speak("Sorry sir, I am unable to send an Email")


import random

#1. feladat---------------------------------------------------------------------

random_erme_dobas = random.choice(['I','F'])
print("1. feladat")
print("A pénzfeldobás eredménye:", random_erme_dobas)

#2. feladat---------------------------------------------------------------------

print("2. feladat")
tipp = input("Tippeljen! (F/I)= ")
random_erme_dobas = random.choice(['I','F'])
print(f'A tipp {tipp}, a dobás eredménye {random_erme_dobas} volt')

if tipp == random_erme_dobas:
    print("Ön eltalálta!")

if tipp != random_erme_dobas:
    print("Ön nem találta el!")

#3. feladat---------------------------------------------------------------------

txt = open(r'..\fej_vagy_iras_feladat\kiserlet.txt', 'r')
sor = txt.readline()
sorok_szama = 0

while sor != "":
    sor = txt.readline()
    sorok_szama += 1

print("3. feladat")
print(f"A kísérlet {sorok_szama} dobásból állt.")
txt.close()

#4. feladat---------------------------------------------------------------------

txt = open(r'..\fej_vagy_iras_feladat\kiserlet.txt', 'r')
sor = txt.readline()
fejek_szama = 0

while sor != "":
    if sor == "F\n":
        fejek_szama += 1
    sor = txt.readline()

print("4. feladat")
print(f'A kísérlet során a fej relatív gyakorisága {round(fejek_szama/sorok_szama*100,2)}% volt.')
txt.close()

#5. feladat---------------------------------------------------------------------

txt = open(r'..\fej_vagy_iras_feladat\kiserlet.txt', 'r')
sor = txt.readline()
ket_fej = 0
jo = True

while sor != "":
    if sor == "F\n" and jo == True:
        sor = txt.readline()
        if sor == "F\n":
            sor = txt.readline()
            if sor != "F\n":
                ket_fej += 1
            else:
                jo = False
        else:
            jo = False
    if sor == "I\n":
        jo = True
    sor = txt.readline()

print("5. feladat")
print(f'A kísérlet során {ket_fej} alkalommal dobtak pontosan két fejet egymás után')
txt.close()

#6. feladat---------------------------------------------------------------------

txt = open(r'..\fej_vagy_iras_feladat\kiserlet.txt', 'r')
sor = txt.readline()
sorok_szama = 1
fejek_sorozata = 0
max_lista = []
maximum = 0
kezdet = 0

while sor != "":
    sor = txt.readline()
    sorok_szama += 1

    if sor == "I\n":
        fejek_sorozata = 0
    if sor == "F\n":
        fejek_sorozata += 1
    if maximum < fejek_sorozata:
        kezdet = sorok_szama-maximum
    max_lista.append(fejek_sorozata)
    maximum = max(max_lista)

print("6. feladat")
print(f'A leghosszabb tisztafej sorozat {maximum} tagból áll, kezdete a(z) {kezdet}. dobás.')
txt.close()

#7. feladat---------------------------------------------------------------------

ezres_lista = []
fej_kovette = 0
iras_kovette = 0

for i in range(250):

    random_erme_dobas_1 = random.choice(['I', 'F'])
    random_erme_dobas_2 = random.choice(['I', 'F'])
    random_erme_dobas_3 = random.choice(['I', 'F'])
    random_erme_dobas_4 = random.choice(['I', 'F'])

    ezres_lista.append([random_erme_dobas_1, random_erme_dobas_2, random_erme_dobas_3,random_erme_dobas_4])

for j in range(250):
    if ezres_lista[j][0] == 'F' and ezres_lista[j][1] == 'F' and ezres_lista[j][2] == 'F' and ezres_lista[j][3] == 'F':
        fej_kovette += 1
    if ezres_lista[j][0] == 'F' and ezres_lista[j][1] == 'F' and ezres_lista[j][2] == 'F' and ezres_lista[j][3] == 'I':
        iras_kovette += 1

dobasok = open(r'..\fej_vagy_iras_feladat\dobasok.txt', 'w')
dobasok.write(f'FFFF: {fej_kovette},FFFI: {iras_kovette}\n')

for k in range(250):
    dobasok.write(f'{ezres_lista[k][0]}{ezres_lista[k][1]}{ezres_lista[k][2]}{ezres_lista[k][3]} ')

dobasok.close()

input("Enter megnyomására bezáródik a program...")


from Tkinter import *
import ttk
import functools
import helpers
import chapter1.section1.problem1.problem1

def load_problem_one(tree, frame):
  tree.insert("ch1.1", "end", "ch1.1.1", text="Problem 1", tags=["ch1.1.1"])
  tree.tag_bind("ch1.1.1", '<1>',
    functools.partial(problem_one_view, tree, frame))

def problem_one_view(tree, frame, event):
  helpers.clear_frame(frame)

  prompt = Label(frame, text=u"Given a number \u03B1, find rational number " +
    u"p/q such that |\u03B1 - p/q| \u2264 1/q^2")
  prompt.grid(row=0, columnspan=3)

  entryLabel = Label(frame, text=u"Enter an \u03B1:")
  entryLabel.grid(row=1, column=0)
  alpha = Entry(frame)
  alpha.grid(row=1, column=1)
  submitButton = Button(frame,
    text="submit",
    command=functools.partial(problem_one_display_answer, tree, frame, alpha))
  submitButton.grid(row=1, column=2)

def problem_one_display_answer(tree, frame, alpha):
  helpers.clear_row(frame, 2)

  answer = chapter1.section1.problem1.problem1.problem(int(alpha.get()))

  answerText = Label(frame, text=u"With an \u03B1 of " +
    unicode(int(alpha.get())) + u" we have the inequality " +
    unicode(answer[0][2]) + u" \u2264 " + unicode(answer[0][3]) +
    u" when p=" + unicode(answer[0][0]) + u" and q=" +
    unicode(answer[0][1]))

  answerText.grid(row=2, columnspan=3)


import base64
import ast

import json
import tweepy
import re
from collections import Counter
import time
from decouple import config
import sys
import os

from google.cloud import pubsub_v1

import argparse
import time

from google.cloud import firestore

import datetime

def get_user_interactions(list_dict):
    """Crawls the targeted user's timeline and returns interactions.
    Args:
        `list_dict`, dict: A dictionary containing the following variables:
            `search`, string: The name of the user who's timeline to search.
            `output`, list of tuples: paired list of search & interaction targets.
            `next_query`, list: a list of names to search on the next run.
            `limit`, int: A flag to indicate how many results should be returned.
            'level'

    Functionality:
        Initialize search for the specific user.
        Get the user's tweets from their timeline.
        Cycle through all the tweets' text and join it into a mega-string.
        Do some standardizing and replacing.
        Strip away everything except usernames, into a string.
        Make a list of the counts, and take the top (X) most common people.
        Creates a list of the `top` people. No duplicates
        Tuples the results of the search & output together.
        Make search inputs for the next level of crawling.

    Returns:
        This function is async and has no return statement, rather it
        instead updates the values of `output` and `next_query` extending
        the lists that were passed to it as args.
    """
    search, output, found, search_depth, level, errors = list_dict

    if level > 1:

        project_id = os.environ['GCP_PROJECT']
        db = firestore.Client()
        get_user = db.collection(u'interactions').document(search).get()
        user_info = get_user.to_dict()

        if user_info is not None:

            time = user_info['timestamp']

            d = time.strftime("%Y-%m-%d")
            d = datetime.datetime.strptime(d, "%Y-%m-%d")
            now = datetime.datetime.now()

            delta = now - d
            if delta.days < 7:
                print("User has already been checked this week.")

                print('TEST CHECK DB: ', time)
                print('since: ', delta)
                return

    try:
        twitter_user = TWITTER.get_user(search)
        print('grabbing timeline for user: ', search)
        print('The search depth is:', search_depth)
        tweets = twitter_user.timeline(
                    count=200,
                    exclude_replies=False,
                    include_rts=True,
                    tweet_mode='extended'
        )
        full_tweets = [i.full_text for i in tweets]
        long_string = " ".join(full_tweets)
        lowercase = long_string.lower()
        remove_orgin_user = lowercase.replace(search, "")
        out = re.findall(r'(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)',remove_orgin_user)

        if search_depth > 0:

            top = Counter(out).most_common(search_depth)
            if len(top) == 0:
                errors.append(search)
                return
            interactions = []
            for interaction_count in top:
                interactions += ([interaction_count[0]] * interaction_count[1])

            tweet_data = [(search, i) for i in interactions]
            output.extend(tweet_data)

            found_users = [person[0] for person in top]
            found.extend(found_users)

        elif search_depth == -1:
            top = Counter(out).most_common()
            tweet_data = [(search, i) for i in out]
            output = output.extend(tweet_data)

            found_users = [person[0] for person in top]
            found.extend(found_users)

    except tweepy.TweepError:
        errors.append(search)
        print("tweepy.TweepError")

    except:
        e = sys.exc_info()[0]
        errors.append(search)
        print("Error: %s" % e)

def interaction_chain(origin_user, first_search_depth, second_search_depth):
    from concurrent.futures import ThreadPoolExecutor as PoolExecutor
    """Calls our async function & loops through it for each target user."""

    # init function variables
    data = []
    level = 0

    start_time = time.time()
    last_time = time.time()

    users_to_search = []# Next search targets

    blacklist = []

    found_users = []
    found_users.extend(origin_user)

    past_users = []  # Current users deposited after llvl, cleans out f users.
    #past_users += origin_user
    print('test',found_users)

    # Updates the time for each loop.
    def funct_time(last, d, l):
        now = time.time()
        check_time = now - last
        print("\n\nLevel %s Completed\nLevel %s time to complete: %s." % (l, l, check_time))
        print("Level %s total cumulative interactions found:", (l, len(d)))
        return now

    # Dedupe & Check user list for previous runs (no duplicated work).
    def update_lists(searched_users, found_users, past_users, level):
        found_users = list(dict.fromkeys(found_users)) # Dedupe f_users.
        past_users.extend([x for x in searched_users if x not in blacklist]) # Deposit last search(ed)_users into past_users. (input to last run)

        found_users = [x for x in found_users if x not in past_users]

        searched_users = found_users #Overwrite ss_users list with cleaned ff_users loop output.
        found_users = [] #Reset ff_users for next run.
        level += 1
        if level == 1:
        	print("----------Entering %sst level. Searching the following users:----------\n" % level, searched_users)
        else: # on other runs...
            print("Level %s new connections found:" % len(searched_users), "\n\n")
            print("Total %s connections discovered so far." % (len(past_users)+len(searched_users)) )
            print("---------Beginning Level %s - Searching the following users:\n" % level, searched_users)
        return searched_users, found_users, past_users, level

    # Pre-run print.
    users_to_search, found_users, past_users, level = update_lists(users_to_search, found_users, past_users, level)

    # Level 1 Run.
    loop_num = 1
    with PoolExecutor(max_workers=20) as executor:
        for _ in executor.map(get_user_interactions, [(x, data, found_users, first_search_depth, level, blacklist) for x in users_to_search]):
            print("Loop # ", loop_num, " . Time so far:", time.time() - start_time)
            loop_num +=1
            pass

    last_time = funct_time(last_time, data, level)
    users_to_search, found_users, past_users, level = update_lists(users_to_search, found_users, past_users, level)

    # Level 2 Run.
    loop_num = 1
    with PoolExecutor(max_workers=20) as executor:
        for _ in executor.map(get_user_interactions, [(x, data, found_users, second_search_depth, level, blacklist) for x in users_to_search]):
            print("Loop # ", loop_num, " . Time so far:", time.time() - start_time)
            loop_num +=1
            pass

    last_time = funct_time(last_time, data, level)

    users_to_search, found_users, past_users, level = update_lists(users_to_search, found_users, past_users, level)

    print("Total time:", time.time() - start_time)
    print("Total Connections found:", (len(past_users)-1))
    print("Total Overall Interactions found:", len(data))
    num_total_interactions = len(data)
    print(past_users)
    return data, past_users

def get_callback(api_future, data):
    """Wrap message data in the context of the callback function."""

    def callback(api_future):
        try:
            print("Published message {} now has message ID {}".format(
                data, api_future.result()))
        except Exception:
            print("A problem occurred when publishing {}: {}\n".format(
                data, api_future.exception()))
            raise
    return callback

def publish(interactions, topic):
    """Publishes a message to a Pub/Sub topic."""
    # [START pubsub_quickstart_pub_client]
    # Initialize a Publisher client
    client = pubsub_v1.PublisherClient()
    # [END pubsub_quickstart_pub_client]
    # Create a fully qualified identifier in the form of
    # `projects/{project_id}/topics/{topic_name}`
    topic_path = client.topic_path('bert-optimization-testing', topic)

    # Data sent to Cloud Pub/Sub must be a bytestring

    data = json.dumps(interactions, ensure_ascii=False).encode('utf8')

    api_future = client.publish(topic_path, data=data)
    api_future.add_done_callback(get_callback(api_future, data))

    # Keep the main thread from exiting until background message
    # is processed.
    while api_future.running():
        time.sleep(0.1)

    print("data published")

def generate_recommendation_response(original_user, TWITTER_ACCESS_TOKEN, TWITTER_ACCESS_TOKEN_SECRET, message_id, depth):
    """
    Takes in all the established variables and passes them to the correct functions.
    """
    # Create Twitter Connection
    first_search_depth = depth
    second_search_depth = depth

    twitter_auth = tweepy.OAuthHandler(config('TWITTER_CONSUMER_KEY'),config('TWITTER_CONSUMER_SECRET'))

    print('Twitter Consumer Key: ', config('TWITTER_CONSUMER_KEY'))
    print('Twitter Consumer Secret: ', config('TWITTER_CONSUMER_SECRET'))

    access_token = str(TWITTER_ACCESS_TOKEN)
    access_token_secret = str(TWITTER_ACCESS_TOKEN_SECRET)

    #access_token = config('ACCESS_TOKEN')
    #access_token_secret = config('ACCESS_SECRET')

    #print("Access Token: ", access_token)
    #print("Access Token Secret: ", access_token_secret)

    twitter_auth.set_access_token(access_token, access_token_secret)
    global TWITTER
    TWITTER = tweepy.API(twitter_auth)

    #original_user = str(original_user)
    output_data = [message_id]

    # Call the Multithreaded function
    response_data, users_found = interaction_chain(original_user, first_search_depth, second_search_depth)
    print("Called Multitreading Function")
    output_data.append(users_found)
    output_data.append(response_data)

    print(response_data)
    print(users_found)
    #post_to_firestore(response_data, interactions_found, original_user)
    publish(output_data, 'listen_for_interactions')

def hello_pubsub(event, context):
    """Triggered from a message on a Cloud Pub/Sub topic.
    Args:
         event (dict): Event payload.
         context (google.cloud.functions.Context): Metadata for the event.
    """
    pubsub_message = base64.b64decode(event['data']).decode('utf-8')
    print(pubsub_message)
    parsed = ast.literal_eval(pubsub_message)

    print(parsed[0])
    job_id = parsed[3]

    to_search = []
    search_users = []
    print(type(parsed[0]))
    search_users.extend([x for x in parsed[0]])
    print(search_users)
    print(search_users[0])
    num_of_searches = int(parsed[4]/5)

    if (parsed[4]%5) > 0:
        num_of_searches = num_of_searches + 1

    print('num of searches: ', num_of_searches)
    for i in range(num_of_searches):

        index = ((i+1)*5)-5
        print(index)
        to_search.append(str(search_users[index]))

    depth = 50 // (num_of_searches * 5)

    if depth < 3:
        depth = 3

    to_publish = []
    to_publish.append(job_id)
    to_publish.append(parsed[4])
    publish(to_publish, 'post_to_db')

    print(to_search)
    generate_recommendation_response(to_search, parsed[1], parsed[2], job_id, depth)


#!/usr/bin/python
# Example of calling SecureNet Charge API from Python
import requests

secureNetId = '8008966'  # Replace with your own ID
secureKey = 'MgvfuwWMGUJl'  # Replace with your own Key
url = 'https://gwapi.demo.securenet.com/api/Payments/Charge'
data = {
    'amount': 1.23,
    'check': {
        'firstName': 'Fred',
        'lastName': 'Flintstone',
        'routingNumber': 222371863,
        'accountNumber': 123456
    },
    'developerApplication': {
        'developerId': 12345678,
        'version': '1.2'
    }
}

print("Sending charge request")
response = requests.post(
    url,
    headers={"content-type": "application/json"},
    auth=(secureNetId, secureKey),
    json=data,
    verify=True)

print("HTTP Status Code: {0}".format(response.status_code))
if (response.ok):
    json = response.json()
    print("success: {0}".format(json["success"]))
    print("result: {0}".format(json["result"]))
    print("message: {0}".format(json["message"]))
    print("transactionId: {0}".format(json["transaction"]["transactionId"]))


from django.db import models
from management.models import Environment, _TEAMS
from .validators import is_valid_hostname, is_unprivileged_port
from .utils import timedgethostbyname
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
from django.core.cache import cache
from .services import all_jobs

_SERVERROLES = (
    ('app', 'Strata App Server'),
    ('ass', 'Assentis Server'),
    ('orb', 'ORBS Server'),
    ('lb', 'HAProxy Load Balancer'),
    ('ovm', 'OVM Host'),
    ('vm', 'VMWare Host'),
)

_OS = (
    ('el6', 'CentOS 6'),
    ('el7', 'CentOS 7'),
    ('win2k8', 'Windows Server 2008'),
    ('win2k12', 'Windows Server 2012'),
    ('vmware', 'VMWare ESXi'),
    ('ovm', 'Oracle Virtual Machine'),
)

_WEBLOGIC_EDITIONS = (
    ('developer', 'Developer Edition'),
    ('standard', 'Standard Edition'),
    ('enterprise', 'Enterprise Edition'),
)

_TOMCAT_VERSIONS = (
    ('7', 'Tomcat 7'),
    ('8', 'Tomcat 8'),
)

_SERVER_STATUSES = (
    ('blank', 'Needs Provisioning'),
    ('installed', 'OS Installed'),
    ('unassigned', 'Unassigned'),
    ('live', 'Live System'),
    ('maintenance', 'Out for Maintenance'),
    ('broken', 'Needs Fixing'),
    ('burn', 'Can be Humanely Destroyed'),
    ('decom', 'Decommisioned'),
)

_APPLICATION_STATUSES = (
    ('develop', 'Under Development'),
    ('production', 'In Production'),
    ('retired', 'Retired'),
)

_TASK_STATUSES = (
    ('success', 'SUCCESS'),
    ('running', 'IN PROGRESS'),
    ('failed', 'FAILED'),
    ('rollback', 'ROLLBACK'),
)

class Server(models.Model):
    """
    A server, this is used as a class to extend application specific servers from such as a WebLogic server.
    """
    name = models.CharField(max_length=255, db_index=True, unique=True, validators=[is_valid_hostname],
                            help_text='The FQDN of the server.')
    role = models.CharField(max_length=8, choices=_SERVERROLES)
    owner = models.CharField(max_length=8, choices=_TEAMS, default='server')
    primary_ip = models.GenericIPAddressField(verbose_name='Primary IP', blank=True, null=True, unique=True,
                                              help_text='Usually assigned to eth0 or ens160. Leave blank to try and resolve.')
    environment = models.ForeignKey(Environment)
    status = models.CharField(max_length=32, choices=_SERVER_STATUSES, default='live')
    override_puppet_role = models.CharField(max_length=64, blank=True, null=True)
    override_puppet_environment = models.CharField(max_length=64, blank=True, null=True)
    is_virtual = models.BooleanField(default=False)
    virtual_host = models.ForeignKey('HypervisorHost', null=True, blank=True)
    os = models.CharField(verbose_name='OS', max_length=8, choices=_OS, null=True, blank=True)
    cpus = models.PositiveSmallIntegerField(default=1)
    cores = models.PositiveSmallIntegerField(default=0)
    ram = models.PositiveSmallIntegerField(default=0)

    class Meta(object):
        ordering = ('name',)

    def clean(self, *args, **kwargs):
        try:
            if self.stratacustomerstack_set.get()[0].environment is not self.environment:
                raise ValidationError(
                    _('Server is in use by: %(customer_stack)s. Cannot move environment.'),
                    params={'customer_stack': self.stratacustomerstack_set.get()},
                )
        except:
            pass

        try:
            if self.stratacustomerstack_set.get()[0].stack.valid_target is not self.role:
                raise ValidationError(
                    _('Server is in use by: %(customer_stack)s. Cannot move environment.'),
                    params={'customer_stack': self.stratacustomerstack_set.get()},
                )
        except:
            pass

        super(Server, self).clean(*args, **kwargs)

    def save(self, *args, **kwargs):
        """If the primary ip address wasn't known, we try to resolve it on save"""
        if self.primary_ip is '':
            self.primary_ip = timedgethostbyname(self.name)

        super(Server, self).save(*args, **kwargs)

    def puppet_role(self):
        """
        Return the Puppet role assigned to this server
        """
        try:
            if self.override_puppet_role is not None:
                return self.override_puppet_role
            else:
                return PuppetRole.objects.all().filter(role=self.role)[0].puppet_role
        except:
            return None

    def puppet_environment(self):
        """
        Return the Puppet environment assigned to this server
        """
        try:
            if self.override_puppet_environment is not None:
                return self.override_puppet_environment
            else:
                return self.environment.short_name
        except:
            return None

    def connect_to(self):
        """
        Returns the primary ip address or the hostname to connect to
        """
        if self.primary_ip is not None:
            return self.name
        else:
            return self.name

    def total_cores(self):
        """
        Total number of cores on this server
        """
        try:
            return self.cpus * self.cores
        except:
            return None

    def __str__(self):
        return self.name

class HypervisorHost(Server):
    """
    A hypervisor host is used for the relationship between host / vm
    and reporting things like WebLogic licences etc...
    """
    def guest_servers(self):
        return self.server_set.all()

    def __str__(self):
        return self.name

class WeblogicServer(Server):
    """
    Weblogic servers have added WebLogic stuff
    """
    edition = models.CharField(max_length=16, choices=_WEBLOGIC_EDITIONS, default='standard')
    nodemanager_port = models.PositiveSmallIntegerField(validators=[is_unprivileged_port], default=5666)
    nodemanager_ssl = models.BooleanField(default=False)
    adminserver_port = models.PositiveSmallIntegerField(validators=[is_unprivileged_port], default=7001)
    adminserver_ssl = models.BooleanField(default=False)
    adminserver_user = models.CharField(max_length=256, default='weblogic')
    adminserver_pass = models.CharField(max_length=256, default='letmein1')

    def __str__(self):
        return self.name

class WindowsServer(Server):
    """
    Windows servers have Windows stuff
    """

    def __str__(self):
        return self.name

class TomcatServer(Server):
    """
    Tomcat servers have added Tomcat stuff
    """
    version = models.CharField(max_length=8, choices=_TOMCAT_VERSIONS, default='8')

    def __str__(self):
        return self.name

class PuppetRole(models.Model):
    """
    A server's role can be linked to a Puppet role.
    """
    role = models.CharField(max_length=8, choices=_SERVERROLES, unique=True)
    description = models.TextField(max_length=512)
    # Auto-generated on save
    puppet_role = models.CharField(max_length=32, help_text='Autogenerated Puppet role applied to this server')

    def save(self, *args, **kwargs):
        self.puppet_role = '::role::%s' % self.role
        super(PuppetRole, self).save(*args, **kwargs)

    class Meta(object):
        ordering = ('role',)

    def __str__(self):
        return self.get_role_display()

class Application(models.Model):
    """
    An application is an asset too. This base class is used to extend upon
    in team specific classes.
    """
    name = models.CharField(max_length=32, unique=True)
    short_name =  models.CharField(max_length=16, unique=True)
    description = models.TextField(max_length=512)
    status = models.CharField(max_length=32, choices=_APPLICATION_STATUSES, default='production')
    # This should be set in team specific Save method
    owner = models.CharField(max_length=8, choices=_TEAMS)

    default_port = models.PositiveSmallIntegerField(validators=[is_unprivileged_port], null=True, blank=True)
    default_ssl_port = models.PositiveSmallIntegerField(validators=[is_unprivileged_port], null=True, blank=True)
    default_jmx_port = models.PositiveSmallIntegerField(validators=[is_unprivileged_port], null=True, blank=True)
    healthcheck_url = models.CharField(max_length=64, default='/healthcheck')
    has_healthcheck = models.BooleanField(default=False)

    nexus_url = models.URLField(null=True, blank=True,
                                help_text='URL to Nexus assets')
    pre_deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                  help_text='The name of the Jenkins job to run before deploying this application')
    deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                      help_text='The name of the Jenkins job to deploy this application')
    post_deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                      help_text='The name of the Jenkins job to run after deploying this application')

    class Meta(object):
        ordering = ('name',)

    def __str__(self):
        return self.name

class ApplicationStack(models.Model):
    """
    An application stack is a collection (or 1) of applications that are deloyed together for something to run.
    """
    name = models.CharField(max_length=32)
    short_name =  models.CharField(max_length=16, unique=True)
    # This should be set in team specific Save method
    owner = models.CharField(max_length=8, choices=_TEAMS)
    description = models.TextField(max_length=512)
    valid_target = models.CharField(max_length=32, choices=_SERVERROLES, null=True, blank=True,
                                    help_text='What server role can this stack be deployed onto?')
    pre_deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                  help_text='The name of the Jenkins job to run before deploying this application')
    deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                      help_text='The name of the Jenkins job to deploy this application')
    post_deploy_job = models.CharField(max_length=32, null=True, blank=True, choices=all_jobs(),
                                      help_text='The name of the Jenkins job to run after deploying this application')

    class Meta(object):
        ordering = ('name',)

class ServerTask(models.Model):
    """
    basic class to achieve multiple choice in admin page
    """
    name = models.CharField(max_length=32, unique=True, choices=all_jobs())

    def __str__(self):
        return self.name

class ServerRoleTask(models.Model):
    """
    Stores the allowed tasks by server type. Used to limit what tasks can
    execute on a server type and limit selection in UI
    """
    role = models.CharField(max_length=8, choices=_SERVERROLES)
    tasks = models.ManyToManyField("ServerTask")

    def __str__(self):
        return self.role

class TaskHistory(models.Model):
    """
    Essentially a high-level, long lived log of task information
    We will add data programmatically and avoid foreign key relationships here
    """
    task_name = models.CharField(max_length=64, db_index=True)
    customer_environment = models.CharField(max_length=64, db_index=True)
    servers = models.TextField(null=True, blank=True)
    start_time = models.DateTimeField()
    end_time = models.DateTimeField(null=True, blank=True)
    duration = models.DurationField(null=True, blank=True)
    status = models.CharField(max_length=16, choices=_TASK_STATUSES, db_index=True)
    task_log = models.URLField()
    ran_by = models.CharField(max_length=64, default='Admin User')
    reason = models.CharField(max_length=128, default='-')
    level = models.CharField(max_length=32, default='primary')

    class Meta:
        ordering = ('-start_time',)
        verbose_name_plural = 'Task history'

    def save(self, *args, **kwargs):
        if bool(self.end_time) and bool(self.start_time):
            self.duration = self.end_time - self.start_time
        super(TaskHistory, self).save(*args, **kwargs)

    def __str__(self):
        return self.task_name


def FormEndSetData(uideflist, auiname, apobjconst):
  config = uideflist.Config
  user_id = config.SecurityContext.UserID

  if user_id == 'root':
    branch_code = 'root'

  oUserApp = config.CreatePObjImplProxy('UserApp')
  oUserApp.Key = user_id
  branch_code = oUserApp.branch_code

  isBranchFiltered = 0
  NoLimitLocation = oUserApp.NoLimitLocation
  if NoLimitLocation == 'F':
    isBranchFiltered = 1

  uipNoData = uideflist.uipNoData
  recNoData = uipNoData.Dataset.AddRecord()
  recNoData.bc_filter = branch_code
  recNoData.isBranchFiltered = isBranchFiltered

  uipJournalItem = uideflist.uipJournalItem

  dsJournalItem = uipJournalItem.Dataset
  for i in range(dsJournalItem.RecordCount):
    recJournalItem = dsJournalItem.GetRecord(i)

    Account_Code = recJournalItem.GetFieldByName('ln_accountinstance.ln_account.Account_Code')
    recJournalItem.Account_Code = Account_Code


import media
import fresh_tomatoes

toy_story = media.Movie("Toy Story",
                        "Toys come to life",
                        "https://upload.wikimedia.org/wikipedia/en/1/13/Toy_Story.jpg",
                        "https://www.youtube.com/watch?v=KYz2wyBy3kc")
#print(toy_story.storyline)
#toy_story.show_trailer()

avatar = media.Movie("Avatar",
                     "The story of unobtanium",
                     "https://upload.wikimedia.org/wikipedia/en/b/b0/Avatar-Teaser-Poster.jpg",
                     "https://www.youtube.com/watch?v=d1_JBMrrYw8")
#print(avatar.storyline)
#avatar.show_trailer()

school_of_rock = media.Movie("School_of_Rock",
                             "The story of a school rockstar",
                             "https://upload.wikimedia.org/wikipedia/en/1/11/School_of_Rock_Poster.jpg",
                             "https://www.youtube.com/watch?v=3PsUJFEBC74")
ratatouille = media.Movie("Ratatouille",
                          "The story of a rat who turns into a chef",
                          "https://upload.wikimedia.org/wikipedia/en/5/50/RatatouillePoster.jpg",
                          "https://www.youtube.com/watch?v=uVeNEbh3A4U")
midnight_in_paris = media.Movie("Midnight in Paris",
                                "Sounds like a romantic movie with a message",
                                "https://upload.wikimedia.org/wikipedia/en/9/9f/Midnight_in_Paris_Poster.jpg",
                                "https://www.youtube.com/watch?v=BYRWfS2s2v4")

hunger_games = media.Movie("The Hunger Games",
                           "The story of some hungry people playing silly games",
                           "https://upload.wikimedia.org/wikipedia/en/4/42/HungerGamesPoster.jpg",
                           "https://www.youtube.com/watch?v=mfmrPu43DF8")

movies = [toy_story, avatar, school_of_rock, ratatouille, midnight_in_paris, hunger_games]
fresh_tomatoes.open_movies_page(movies)
print(media.Movie.VALID_RATINGS)


def show_menu():
    print('*' * 10 + '员工管理系统' + '*' * 10)
    print('1：添加员工信息')
    print('2：删除员工信息')
    print('3：修改员工信息')
    print('4：显示所有员工信息')
    print('5：退出管理系统')
    print('*' * 30)

employees = {}

def show_employees():
    if len(employees) == 0:
        print('当前没有员工信息！')
        return

    print('显示所有员工信息：')
    for emp_id, emp in employees.items():
        print(emp_id, emp.get('name'), emp.get('salary'))

def delete_employee():
    emp_id = input('请输入要删除的员工编号：')
    if employees.get(emp_id) is not None:
        del employees[emp_id]
        print('员工', emp_id, '删除成功！')
    else:
        print('员工编号不存在！')

def add_employee():
    print('开始添加员工信息！')
    employee_id = input('请输入员工编号：')
    if employees.get(employee_id) is not None:
        print('员工编号已存在，无法添加！')
        return

    name = input('请输入员工姓名：')
    salary = input('请输入员工工资：')
    employees[employee_id] = {'name': name, 'salary': salary}
    print('员工信息添加成功！')
    print('编号：', employee_id, '姓名：', name, '工资：', salary)

def edit_employee():
    employee_id = input('请输入要修改的员工编号：')
    if employees.get(employee_id) is None:
        print('员工编号不存在，无法修改！')
        return
    employee = employees[employee_id]
    print('员工编号：', employee_id, '姓名：', employee.get('name'), '工资：', employee.get('salary'))

    name = input('修改员工姓名为：')
    salary = input('修改员工工资为：')

    # 如果用户直接回车，表示不修改
    if name == '':
        name = employee.get('name')
    if salary == '':
        salary = employee.get('salary')

    employees[employee_id] = {'name': name, 'salary': salary}
    print('员工信息修改成功！')
    print('编号：', employee_id, '姓名：', name, '工资：', salary)

while True:
    show_menu()
    operation = input('请输入操作：')
    if operation == '1':
        add_employee()
    elif operation == '2':
        delete_employee()
    elif operation == '3':
        edit_employee()
    elif operation == '4':
        show_employees()
    elif operation == '5':
        print('退出管理系统!')
        break
    else:
        print('输入有误！')

12_员工管理系统.py

12_员工管理系统.py


# -*- coding: utf-8 -*-
# @Time    : 2020/4/29 10:33 PM
# @Author  : Yinghao Qin
# @Email   : y.qin@hss18.qmul.ac.uk
# @File    : utils.py
# @Software: PyCharm
# Reference:
#   https://github.com/WZMIAOMIAO/deep-learning-for-image-processing/tree/master/pytorch_learning
#   https://github.com/39239580/googlenet-pytorch/blob/master/Inception_v1_mnist.py
#   https://www.kaggle.com/grfiv4/plot-a-confusion-matrix
import os
import itertools
import numpy as np
import matplotlib.pyplot as plt

def plot_accuracy_curve(train_list, test_list, show=False, save=True, path='results/accuracy_epoch.png'):
    """
    It is used to plot the accuracy curve. Note that train_list and test_list have the same size.

    :param train_list: the accuracy list containing accuracies per epoch for train data
    :param test_list: the accuracy list containing accuracies per epoch for test data
    :param show: whether to show the plot
    :param save: whether to save the plot
    :param path: if the plot is saved, the path that will be saved
    :return: no
    """
    num = len(train_list)
    x_axis = np.linspace(1, num, num, endpoint=True)
    plt.plot(x_axis, train_list, color='r', label='Train accuracy')
    plt.plot(x_axis, test_list, color='b', label='Test accuracy')
    plt.legend()
    plt.title('Accuracy against epoch in the train and test sets')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy(%)')
    if save:
        if not os.path.exists('results'):
            os.mkdir('results')
        else:
            pass
        plt.savefig(path)

    if show:
        plt.show()
    else:
        plt.close()
    plt.close()

def plot_loss_curve(train_list, test_list, show=False, save=True, path='results/loss_epoch.png'):
    """
    It is used to plot the accuracy curve. Note that train_list and test_list have the same size.

    :param train_list: the loss list containing losses per epoch for train data
    :param test_list: the loss list containing losses per epoch for train data
    :param show: whether to show the plot
    :param save: whether to save the plot
    :param path: if the plot is saved, the path that will be saved
    :return: no
    """
    num = len(train_list)
    x_axis = np.linspace(1, num, num, endpoint=True)
    plt.plot(x_axis, train_list, color='r', label='Train loss')
    plt.plot(x_axis, test_list, color='b', label='Test loss')
    plt.legend()
    plt.title('Loss against epoch in the train and test sets')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    if save:
        if not os.path.exists('results'):
            os.mkdir('results')
        else:
            pass
        plt.savefig(path)

    if show:
        plt.show()
    else:
        plt.close()
    plt.close()

def get_confusion_matrix(classes, predicts_list, labels_list):
    """
    Given predicted results and ground-truth labels, the function return the corresponding confusion matrix.

    :param classes: the classes of labels, it is used to compute the length of the classes
    :param predicts_list: the list of predicted results
    :param labels_list: the list of label results
    :return: confusion matrix
    """
    size = len(classes)
    length = len(predicts_list)
    confusion_matrix = np.zeros((size, size))
    for i in range(size):
        for j in range(size):
            confusion_matrix[i][j] = np.sum([(predicts_list[k] == i) and (labels_list[k] == j) for k in range(length)])
    return confusion_matrix

def plot_confusion_matrix(cm, target_names, title='Confusion matrix', cmap=None, normalize=True):
    """
    Given a sklearn confusion matrix (cm), make a nice plot

    :param cm: confusion matrix from sklearn.metrics.confusion_matrix
    :param target_names: given classification classes such as [0, 1, 2], the class names,
                         for example: ['high', 'medium', 'low']
    :param title: the text to display at the top of the matrix
    :param cmap: the gradient of the values displayed from matplotlib.pyplot.cm
                  see http://matplotlib.org/examples/color/colormaps_reference.html
                  plt.get_cmap('jet') or plt.cm.Blues
    :param normalize: If False, plot the raw numbers
                  If True, plot the proportions
    :return: no
    Usage
    -----
        plot_confusion_matrix(cm  = cm,                  # confusion matrix created by sklearn.metrics.confusion_matrix
                          normalize    = True,                # show proportions
                          target_names = y_labels_vals,       # list of names of the classes
                          title        = best_estimator_name) # title of graph
    Citiation
    ---------
    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html
    """

    accuracy = np.trace(cm) / float(np.sum(cm))
    misclass = 1 - accuracy

    if cmap is None:
        cmap = plt.get_cmap('Blues')

    plt.figure(figsize=(8, 6))
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()

    if target_names is not None:
        tick_marks = np.arange(len(target_names))
        plt.xticks(tick_marks, target_names, rotation=45)
        plt.yticks(tick_marks, target_names)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 1.5 if normalize else cm.max() / 2
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        if normalize:
            plt.text(j, i, "{:0.4f}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")
        else:
            plt.text(j, i, "{:,}".format(cm[i, j]),
                     horizontalalignment="center",
                     color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label\naccuracy={:0.4f}; misclass={:0.4f}'.format(accuracy, misclass))

    # the result will be stored at
    if not os.path.exists('results'):
        os.mkdir('results')
    else:
        pass
    plt.savefig('results/confusion_matrix.png')
    plt.show()


from django.conf.urls import patterns, include, url

# Uncomment the next two lines to enable the admin:
from django.contrib import admin
admin.autodiscover()

urlpatterns = patterns('',
    # Examples:
    # url(r'^$', 'pyDnsAdmin.views.home', name='home'),
    # url(r'^pyDnsAdmin/', include('pyDnsAdmin.foo.urls')),
    url(r'^$', 'dashboard.views.index', name='dashboard'),

    url(r'^customer/', include('customer.urls')),
    url(r'^user/', include('useraccount.urls')),
    url(r'^recordtype/', include('recordtype.urls')),
    url(r'^nameserver/', include('nameserver.urls')),
    url(r'^domain/', include('domain.urls')),
    url(r'^revision/', include('revision.urls')),
#    url(r'^record/', include('record.urls')),

    # Uncomment the admin/doc line below to enable admin documentation:
    # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),

    # Uncomment the next line to enable the admin:
    url(r'^admin/', include(admin.site.urls)),
)


import numpy as np
import tensorflow as tf
import pdb

'''
A Multilayer Perceptron implementation example using TensorFlow library.
This example is using the MNIST database of handwritten digits
(http://yann.lecun.com/exdb/mnist/)
Author: Aymeric Damien
Project: https://github.com/aymericdamien/TensorFlow-Examples/
'''

# Parameters
learning_rate = 0.001
training_epochs = 4100
batch_size = 7
display_step = 50

# Network Parameters
n_hidden_1 = 256 # 1st layer number of features
n_hidden_2 = 256 # 2nd layer number of features
n_input = 6 * 12 # MNIST data input (img shape: 28*28)
n_classes = 7 # MNIST total classes (0-9 digits)

# tf Graph input
x = tf.placeholder("float", [None, n_input])
y = tf.placeholder("float", [None, n_classes])

# Create model
def multilayer_perceptron(x, weights, biases):
    # Hidden layer with RELU activation
    layer_1 = tf.add(tf.matmul(x, weights['h1']), biases['b1'])
    layer_1 = tf.nn.sigmoid(layer_1)
    # Hidden layer with RELU activation
    layer_2 = tf.add(tf.matmul(layer_1, weights['h2']), biases['b2'])
    layer_2 = tf.nn.sigmoid(layer_2)
    # Output layer with linear activation
    out_layer = tf.matmul(layer_2, weights['out']) + biases['out']
    return out_layer

# Store layers weight & bias
weights = {
    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes]))
}
biases = {
    'b1': tf.Variable(tf.random_normal([n_hidden_1])),
    'b2': tf.Variable(tf.random_normal([n_hidden_2])),
    'out': tf.Variable(tf.random_normal([n_classes]))
}

# Construct model
pred = multilayer_perceptron(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(pred, y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Initializing the variables
init = tf.initialize_all_variables()

def input_data():
    puzzle_states = []
    classes = []
    for i in range(1536):
        filename = "ground_truth/string_of_puzzle_states_" + str(i) + ".csv"
        puzzle_state = np.genfromtxt(filename, delimiter=",")
        if puzzle_state.shape[0] == 48:
            puzzle_states.append(puzzle_state.reshape((8,6,12)))

            filename = "ground_truth/list_of_moves_" + str(i) + ".csv"
            puzzle_class = np.genfromtxt(filename, delimiter=",")
            classes.append(puzzle_class)

    return np.array(puzzle_states), np.array(classes)

def get_batch(puzzle_states, classes, index):
    if index > puzzle_states.shape[0] - batch_size:
        index = 0

    puzzle_batch = puzzle_states[index,1:8,:,:]
    puzzle_batch = puzzle_batch.reshape((7,6*12))
    classes =  np.rot90(np.identity(7))

    return puzzle_batch, classes

saver = tf.train.Saver()
index = 0
# Launch the graph
with tf.Session() as sess:
    sess.run(init)

    puzzle_states, classes = input_data()

    # Training cycle
    for epoch in range(training_epochs):
        avg_cost = 0.
        # Loop over all batches
        for i in range(batch_size):
            # pdb.set_trace()
            batch_x, batch_y = get_batch(puzzle_states, classes, index)
            index = index + 1
            # Run optimization op (backprop) and cost op (to get loss value)
            _, c = sess.run([optimizer, cost], feed_dict={x: batch_x,
                                                          y: batch_y})
            # Compute average loss
            avg_cost += c / puzzle_states.shape[0]
        # Display logs per epoch step
        if epoch % display_step == 0:
            print("Epoch:", '%04d' % (epoch+1), "cost=", \
                "{:.9f}".format(avg_cost))
    print("Optimization Finished!")

    save_model = "heuristic.ckpt"
    saver.save(sess,save_model)

    # pdb.set_trace()
    temp = batch_x[6].reshape((1,72))
    prediction = sess.run(tf.nn.sigmoid(pred), feed_dict={x: temp})

    # Test model
    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
    # Calculate accuracy
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
    print("Accuracy:", accuracy.eval({x: batch_x, y: batch_y}))


#!/usr/bin/env python3
"""This module contains a function that sets up Adam optimization for a keras
model with categorical crossentropy loss and accuracy metrics"""

import tensorflow.keras as K

def optimize_model(network, alpha, beta1, beta2):
    """This function sets up Adam optimization for a keras model with
categorical crossentropy loss and accuracy metrics"""

    network.compile(optimizer=K.optimizers.Adam(alpha, beta1, beta2),
                    loss="categorical_crossentropy", metrics=["accuracy"])

    return None


from django.conf.urls import url

from . import views

urlpatterns = [
    # 判断用户名是否重复
    url(r'^usernames/(?P<username>\w{5,20})/count/$', views.UsernameCountView.as_view()),
    # 判断手机号是否重复
    url(r'^mobiles/(?P<mobile>1[3-9]\d{9})/count/$', views.MobileCountView.as_view()),
    # 注册
    url(r'^users/$', views.UserView.as_view()),
]


"""transformer devmap"""

########### Make your changes below the line ######################

# subfolder under classifyapp_logs/
subfolder = 'devmap_logs/transformer_supervised_amd'
# choose 3 letters to identify your experiment
jobname = 'sta'

model = 'transformer_devmap'  # 'ggnn_SET' or 'transformer_SET' or 'x_pretraining' (dataset without split!)
dataset = 'devmap_amd'  # poj104, ncc, devmap_amd, _nvidia, threadcoarsening_Cypress, _Tahiti, _Fermi, _Kepler
kfold = '--kfold' # '' = no kfold, '--kfold' for kfold.
transfer = ''  # '--transfer=transformer_devmap' # '' for no transfer, '--transfer=transformer_devmap'
restore = ''  # "--restore=deeplearning/ml4pl/poj104/logs/ncc_logs/transformer/2020-03-02_04:12:31_000_31632_model_best.pickle"

# 4h runtime per submission
resubmit_times_per_job = 3

# a set of hyperparameters to grid search over
choices_dict = {
    'lr': [0.00025],
    'output_dropout': [0.0], #, 0.5],
    'aux_use_better': [False, True],
    'num_epochs': [200],
    # 'vocab_size': [8569],
    }

################ Make your changes above the line ################

template = """#!/bin/bash
#SBATCH --job-name={jobname}{i:03d}
#SBATCH --time={timelimit}
#SBATCH --partition=total
#SBATCH --gres=gpu:1
#SBATCH --mail-type=ALL
#SBATCH --cpus-per-task=8
#SBATCH --mail-user=zacharias.vf@gmail.com
#SBATCH --exclude=ault07,ault08,ault20
#SBATCH --dependency=singleton

source /users/zfisches/.bash_profile;
cd /users/zfisches/ProGraML;

srun python deeplearning/ml4pl/models/ggnn/run.py \
--dataset={dataset} \
--model={model} \
{kfold} \
{transfer} \
--log_dir=deeplearning/ml4pl/poj104/logs/{subfolder}/ \
{restore_by_pattern} \
--config_json="{config_str}" \
{restore} \
"""  # NO WHITESPACE!!!

# make this file executable from anywhere
import os, sys
from pathlib import Path
full_path = os.path.realpath(__file__)
print(full_path)
repo_root = full_path.rsplit('ProGraML', maxsplit=1)[0] + 'ProGraML'
print(repo_root)
#insert at 1, 0 is the script path (or '' in REPL)
sys.path.insert(1, repo_root)
repo_root = Path(repo_root)

from deeplearning.ml4pl.poj104.scripts.AULT_write_runscripts import write_runscripts

if __name__ == '__main__':
    write_runscripts(subfolder=subfolder, template=template, choices_dict=choices_dict, jobname=jobname, steps=resubmit_times_per_job, model=model, dataset=dataset, kfold=kfold, transfer=transfer, restore=restore)


# Paradigms Final Project: Multiplayer Pong Game
# Authors: Matt Reilly & Nick Palutsis
# Date: May 10, 2017

import random
import pygame, sys
from pygame.locals import *

# Global variables
screenWidth = 700
screenHeight = 500
paddleWidth = 24
paddleHeight = 80
ballRadius = 10
positionPlayer1 = 0
positionPlayer2 = 0
positionBall = [0, 0]
velocityPlayer1 = 0
velocityPlayer2 = 0
velocityBall = [0, 0]
scorePlayer1 = 0
scorePlayer2 = 0
borderThickness = 4
play = False

# Colors
whiteColor = (255, 255, 255)
blackColor = (0, 0, 0)
redColor = (255, 0, 0)
navyColor = (17, 36, 76)
goldColor = (194, 152, 50)

class GameSpace:
    def main(self):
        global positionPlayer1, velocityPlayer1, scorePlayer1, positionPlayer2, velocityPlayer2, scorePlayer2
        self.isPlayer1 = False
        # if playerNum == 1:
        #     self.isPlayer1 = True
        pygame.init()
        fps = pygame.time.Clock()

        #canvas declaration
        window = pygame.display.set_mode((screenWidth, screenHeight), 0, 32)
        pygame.display.set_caption("Let's Play Pong!")

        self.background = pygame.Surface(window.get_size())
        self.background = self.background.convert()
        self.background.fill((blackColor))

        self.field = Gameboard(self)
        self.sprites = pygame.sprite.Group()
        self.board = pygame.sprite.RenderPlain(self.field)
        self.sprites.add(self.board)

        positionPlayer1 = [(paddleWidth / 2) - 1, screenHeight / 2]
        positionPlayer2 = [screenWidth + 1 - (paddleWidth / 2), screenHeight / 2]
        scorePlayer1 = 0
        scorePlayer2 = 0
        ball_init(True)

        window.blit(self.background, (0,0))
        pygame.display.flip()

        fps.tick(60)
        window.fill(blackColor)
        self.sprites.update()
        self.sprites.draw(window)
        draw(window)
        pygame.display.flip()

        # Loop to play the game
        while True:

            for event in pygame.event.get():

                if event.type == KEYDOWN:
                    keydown(event)
                    # self.sendData("DOWN")
                elif event.type == KEYUP:
                    keyup(event)
                    # self.sendData("UP")
                elif event.type == QUIT:
                    # reactor.stop()
                    pygame.quit()
                    sys.exit()

            if play:    # Wait until user presses spacebar to start
                fps.tick(60)

                window.fill(blackColor)
                # self.sprites.update()
                self.sprites.draw(window)
                draw(window)
                pygame.display.flip()
            else:       # Display pause screen
                window.fill(blackColor)
                # self.sprites.update()
                self.sprites.draw(window)

                font3Background = pygame.font.SysFont("Arial", 48, bold = True)
                pausedBackground = font3Background.render("Press Space to Play", 1, blackColor)
                window.blit(pausedBackground, (120, 225))

                font3Foreground = pygame.font.SysFont("Arial", 48 , bold = True)
                pausedForeground = font3Foreground.render("Press Space to Play", 1, whiteColor)
                window.blit(pausedForeground, (125, 220))

                pygame.display.flip()

    # def sendData(self, eventKey):
    #     if self.isPlayer1:
    #         self.outgoingConn.transport.write("1:" + eventKey)
    #     else:
    #         self.outgoingConn.transport.write("2:" + eventKey)

    # def transferConnectionObject(self, obj):
    #     self.outgoingConn = obj

    # def handleData(self, data):
    #     positionPlayer1 = int(data['posPlayer1'])
    #     positionPlayer2 = int(data['posPlayer2'])
    #     positionBall = int(data['posBall'])
    #     velocityPlayer1 = int(data['velPlayer1'])
    #     velocityPlayer1 = int(data['velPlayer2'])
    #     velocityBall = int(data['velBall'])
    #     scorePlayer1 = int(data['scorePlayer1'])
    #     scorePlayer2 = int(data['scorePlayer2'])

def ball_init(right):
    global positionBall, velocityBall
    positionBall = [screenWidth / 2, screenHeight / 2]
    horz = random.randrange(5, 10)
    vert = random.randrange(3, 7)

    if right == False:
        horz = - horz

    velocityBall = [horz, -vert]

# Draw the shapes (paddles, ball, etc.)
def draw(canvas):
    global positionPlayer1, positionPlayer2, positionBall, velocityBall, scorePlayer1, scorePlayer2, play

    # Move the paddles if they are within the bounds of the screen
    # Left paddle
    if positionPlayer1[1] > (paddleHeight / 2) + borderThickness and positionPlayer1[1] < screenHeight - (paddleHeight / 2) - borderThickness:      # paddle is between top and bottom
        positionPlayer1[1] += velocityPlayer1
    elif positionPlayer1[1] <= (paddleHeight / 2) + borderThickness and velocityPlayer1 > 0:      # paddle is at top & moving down
        positionPlayer1[1] += velocityPlayer1
    elif positionPlayer1[1] >= screenHeight - (paddleHeight / 2) - borderThickness and velocityPlayer1 < 0:       # paddle is at bottom & moving up
        positionPlayer1[1] += velocityPlayer1

    # Right paddle
    if positionPlayer2[1] > (paddleHeight / 2) + borderThickness and positionPlayer2[1] < screenHeight - (paddleHeight / 2) - borderThickness:      # paddle is between top and bottom
        positionPlayer2[1] += velocityPlayer2
    elif positionPlayer2[1] <= (paddleHeight / 2) + borderThickness and velocityPlayer2 > 0:      # paddle is at top & moving down
        positionPlayer2[1] += velocityPlayer2
    elif positionPlayer2[1] >= screenHeight - (paddleHeight / 2) - borderThickness and velocityPlayer2 < 0:       # paddle is at bottom & moving up
        positionPlayer2[1] += velocityPlayer2

    # Update the ball's position
    positionBall[0] += int(velocityBall[0])
    positionBall[1] += int(velocityBall[1])

    # Draw ball
    pygame.draw.circle(canvas,
        redColor,
        positionBall,
        ballRadius,
        0)

    # Draw ball border
    pygame.draw.circle(canvas,
        whiteColor,
        positionBall,
        ballRadius + borderThickness,
        borderThickness)

    # Draw left paddle
    pygame.draw.polygon(canvas,
        goldColor,
        [[positionPlayer1[0] - (paddleWidth / 2), positionPlayer1[1] - (paddleHeight / 2)],
        [positionPlayer1[0] - (paddleWidth / 2), positionPlayer1[1] + (paddleHeight / 2)],
        [positionPlayer1[0] + (paddleWidth / 2), positionPlayer1[1] + (paddleHeight / 2)],
        [positionPlayer1[0] + (paddleWidth / 2), positionPlayer1[1] - (paddleHeight / 2)]],
        0)

    # Draw left paddle border
    pygame.draw.polygon(canvas,
        navyColor,
        [[positionPlayer1[0] - (paddleWidth / 2) + borderThickness/2, positionPlayer1[1] - (paddleHeight / 2)],
        [positionPlayer1[0] - (paddleWidth / 2) + borderThickness/2, positionPlayer1[1] + (paddleHeight / 2)],
        [positionPlayer1[0] + (paddleWidth / 2), positionPlayer1[1] + (paddleHeight / 2)],
        [positionPlayer1[0] + (paddleWidth / 2), positionPlayer1[1] - (paddleHeight / 2)]],
        borderThickness)

    # Draw right paddle
    pygame.draw.polygon(canvas,
        goldColor,
        [[positionPlayer2[0] - (paddleWidth / 2), positionPlayer2[1] - (paddleHeight / 2)],
        [positionPlayer2[0] - (paddleWidth / 2), positionPlayer2[1] + (paddleHeight / 2)],
        [positionPlayer2[0] + (paddleWidth / 2), positionPlayer2[1] + (paddleHeight / 2)],
        [positionPlayer2[0] + (paddleWidth / 2), positionPlayer2[1] - (paddleHeight / 2)]],
        0)

    # Draw right paddle border
    pygame.draw.polygon(canvas,
        navyColor,
        [[positionPlayer2[0] - (paddleWidth / 2), positionPlayer2[1] - (paddleHeight / 2)],
        [positionPlayer2[0] - (paddleWidth / 2), positionPlayer2[1] + (paddleHeight / 2)],
        [positionPlayer2[0] + (paddleWidth / 2) - borderThickness, positionPlayer2[1] + (paddleHeight / 2)],
        [positionPlayer2[0] + (paddleWidth / 2) - borderThickness, positionPlayer2[1] - (paddleHeight / 2)]],
        borderThickness)

    # Check if ball hit left paddle; reverse x direction and increase ball speed
    if int(positionBall[0]) <= ballRadius + paddleWidth and int(positionBall[1]) in range(positionPlayer1[1] - (paddleHeight / 2) - (borderThickness * 4), positionPlayer1[1] + (paddleHeight / 2) + (borderThickness * 4), 1):
        velocityBall[0] = -velocityBall[0]
        velocityBall[0] *= 1.1
        velocityBall[1] *= 1.1
    # Ball missed left paddle; increase player 2 score and reset
    elif int(positionBall[0]) <= ballRadius + paddleWidth:
        scorePlayer2 += 1
        ball_init(True)

    # Check if ball hit right paddle; reverse x direction and increase ball speed
    if int(positionBall[0]) >= screenWidth + 1 - ballRadius - paddleWidth and int(positionBall[1]) in range(positionPlayer2[1] - (paddleHeight / 2) - (borderThickness * 4), positionPlayer2[1] + (paddleHeight / 2) + (borderThickness * 4), 1):
        velocityBall[0] = -velocityBall[0]
        velocityBall[0] *= 1.1
        velocityBall[1] *= 1.1
    # Ball missed right paddle; increase player 1 score and reset
    elif int(positionBall[0]) >= screenWidth + 1 - ballRadius - paddleWidth:
        scorePlayer1 += 1
        ball_init(False)

    # Check if ball hit top or bottom wall
    if int(positionBall[1]) <= ballRadius:
        velocityBall[1] = - velocityBall[1]
    if int(positionBall[1]) >= screenHeight + 1 - ballRadius:
        velocityBall[1] = -velocityBall[1]

    # Display scores
    font1Background = pygame.font.SysFont("Arial", 20, bold = True)
    homeBackground = font1Background.render("Home: " + str(scorePlayer1), 1, blackColor)
    canvas.blit(homeBackground, (63, 22))

    font1Foreground = pygame.font.SysFont("Arial", 20, bold = True)
    homeForeground = font1Foreground.render("Home: " + str(scorePlayer1), 1, whiteColor)
    canvas.blit(homeForeground, (65, 20))

    font2Background = pygame.font.SysFont("Arial", 20, bold = True)
    awayBackground = font2Background.render("Away: " + str(scorePlayer2), 1, blackColor)
    canvas.blit(awayBackground, (563, 22))

    font2Foreground = pygame.font.SysFont("Arial", 20, bold = True)
    awayForeground = font2Foreground.render("Away: " + str(scorePlayer2), 1, whiteColor)
    canvas.blit(awayForeground, (565, 20))

# User pressed key down
def keydown(event):
    global velocityPlayer1, velocityPlayer2, play

    if event.key == K_SPACE:
        if play:
            play = False
        else:
            play = True
    elif event.key == K_UP:
        velocityPlayer2 = -16
    elif event.key == K_DOWN:
        velocityPlayer2 = 16
    elif event.key == K_w:
        velocityPlayer1 = -16
    elif event.key == K_s:
        velocityPlayer1 = 16

# User released key
def keyup(event):
    global velocityPlayer1, velocityPlayer2

    if event.key in (K_w, K_s):
        velocityPlayer1 = 0
    elif event.key in (K_UP, K_DOWN):
        velocityPlayer2 = 0

# Setup field
class Gameboard(pygame.sprite.Sprite):
    def __init__(self, environment):
        pygame.sprite.Sprite.__init__(self)
        self.image, self.rect = loadImage("./football_field.jpg")
        self.image = pygame.transform.scale(self.image, (screenWidth, screenHeight))
        self.rect.topleft = (0, 0)

    def update(self):
        self.rect.x = self.rect.x
        self.rect.y = self.rect.y

def loadImage(name, colorkey = None):
    try:
        image = pygame.image.load(name)
    except pygame.error:
        print ("Failed to load image: ", fullname)
        raise SystemExit(str(geterror()))

    if image.get_alpha is None:
        image = image.convert()
    else:
        image = image.convert_alpha()

    return image, image.get_rect()

#####################################################################
############################ Play Game ##############################
#####################################################################

if __name__ == "__main__":
    gs = GameSpace()
    gs.main()


from pymavlink import mavutil
from dronekit import connect, VehicleMode, LocationGlobalRelative
import time

def send_body_ned_velocity(velocity_x, velocity_y, velocity_z, duration=0):
    msg = vehicle.message_factory.set_position_target_local_ned_encode(
        0,       # time_boot_ms (not used)
        0, 0,    # target system, target component
        mavutil.mavlink.MAV_FRAME_BODY_NED, # frame Needs to be MAV_FRAME_BODY_NED for forward/back left/right control.
        0b0000111111000111, # type_mask
        0, 0, 0, # x, y, z positions (not used)
        velocity_x, velocity_y, velocity_z, # m/s
        0, 0, 0, # x, y, z acceleration
        0, 0)
    for x in range(0,duration):
        vehicle.send_mavlink(msg)
        time.sleep(1)

takeoff_alt = 0.015
vehicle = connect('/dev/serial0', wait_ready= True,baud=57600)
while not vehicle.is_armable:
    time.sleep(1)
vehicle.mode = VehicleMode("GUIDED")
vehicle.armed = True
while not vehicle.armed:
    print('Waiting for arming...')
    time.sleep(1)
vehicle.simple_takeoff(takeoff_alt) # Take off to target altitude
while True:
    print('Altitude: %d' %  vehicle.location.global_relative_frame.alt)
    if vehicle.location.global_relative_frame.alt >= takeoff_alt * 0.95:
        print('REACHED TARGET ALTITUDE')
        break
    time.sleep(1)

# This is the command to move the copter 5 m/s forward for 10 sec.
print("move right")
velocity_x = -5
velocity_y = 0
velocity_z = 0
duration = 10
send_body_ned_velocity(velocity_x, velocity_y, velocity_z, duration)

# left at 5 m/s for 10 sec.
print("move left")
velocity_x = 5
velocity_y = 0
velocity_z = 0
duration = 10
send_body_ned_velocity(velocity_x, velocity_y, velocity_z, duration)

vehicle.mode = VehicleMode("LAND")


from django import template
from realestate.models import RentOffer, SellOffer

register = template.Library()

@register.simple_tag
def dispaly_offer_label(offer):
    if isinstance(offer, RentOffer):
        label = "For Rent"
    elif isinstance(offer, SellOffer):
        label = "For Sale"
    else:
        label = "Offer Type not known"
    return label

# @register.simple_tag
# def url_replace(request, field, value):
#     """
#     Usage:
#      <a href="?{% url_replace request 'page' paginator.next_page_number %}">
#     """
#     dict_ = request.GET.copy()
#
#     dict_[field] = value
#
#     return dict_.urlencode()

from urllib.parse import urlencode

@register.simple_tag(takes_context=True)
def url_replace(context, **kwargs):
    query = context['request'].GET.dict()
    query.update(kwargs)
    return urlencode(query)


import pandas as pd
import numpy as np
import os
from .visualize import feature_plot, eva_plot
from . import feature_evaluation
from .feature_explore import *
import simplejson
from itertools import chain
from collections import defaultdict
from ..monitor import data_monitor as dm
import pickle
import traceback
from ..base_utils import *
# from ...service.base_utils import *
from ..data_service import get_data
from ..monitor.psi import cmpt_psi
from .feature_explore import DescStats

# TODO: report各函数df_test应该作为可选参数
# TODO: 计算psi时目前只考虑了两个数据集、其中之一为base的情况，对于多数据集情况没有考虑和适应。

def _update_dict(s, k1, k2):
    """
    json to dict，dict值转成频率，外部套上对应数据集名称
    :param s:
    :param k1:
    :param k2:
    :return:
    """
    dic = json.loads(s[k2])
    total = sum(dic.values())
    dic = {k: v / total for k, v in dic.items()}
    return {s[k1]: dic}

def _merge_dict(s):
    """
    不同字典key值对齐，不存在的key填0
    :param s:
    :return:
    """
    dic = {}
    keys_list = []
    for i in s:
        for k, dic_i in i.items():
            keys_list += list(dic_i.keys())
    keys_list = set(keys_list)

    for i in s:
        for outk, dic_i in i.items():
            ii = {k: dic_i.get(k, 0) for k in keys_list}
            i[outk] = ii
        dic.update(i)
    return json.dumps(dic)

def _get_single_plot_data(s, k1, k2):
    return json.dumps(json.loads(s[k2])[s[k1]])

def _get_distribution_psi(s):
    return cmpt_psi(list(json.loads(s[0]).values()), list(json.loads(s[1]).values()))

def _gen_plot_data(df, k, set_index=False):
    tmp = df.reset_index()

    tmp[k] = tmp.apply(lambda s: _update_dict(s, df.index.names[-1], k), axis=1)
    tmp2 = tmp.groupby(df.index.names[0])[k].agg(_merge_dict)
    tmp2 = pd.DataFrame(tmp2)
    tmp2.columns = ['plot']
    tmp = df.join(tmp2, how='outer')
    tmp = tmp.reset_index()
    tmp['distribution'] = tmp.apply(lambda s: _get_single_plot_data(s, df.index.names[-1], 'plot'), axis=1)
    tmp = tmp.set_index(df.index.names)

    t = tmp.groupby(level=0)['distribution'].agg(_get_distribution_psi)
    t.name = 'H-PSI'
    tmp = tmp.join(t, how='outer')
    col_list = list(tmp.columns)
    col_list = col_list[-1:] + col_list[:-3] + col_list[-3:-1]
    tmp = tmp[col_list]

    if set_index:
        tmp = tmp.set_index('plot', append=True).swaplevel(-2, -1)
    return tmp

def _gen_desc_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # data_stats
    enc = DescStats()
    enc.fit(df_train)
    c_tr = enc.transform(df_train)
    c_te = enc.transform(df_test)
    df = merge_multi_df({'train': c_tr, 'test': c_te}, axis=0).round(decimals)
    df = _gen_plot_data(df, 'distribution')
    # TODO: plot列应该放到索引中以图片形式呈现，因为暂时没有解决excel画图插入cell问题，先删除
    del df['plot']

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'data_stats.report'))
        dic = {'train': table_to_dict(c_tr), 'test': table_to_dict(c_te)}
        simplejson.dump(dic, open(os.path.join(report_dst, 'data_stats.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
    return df

def _gen_woe_report(df_train_origin, df_test_origin, label_col, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # woe_report
    wr = feature_evaluation.WOEReport()
    d_tr = wr.fit_transform(df_train, label_col)
    d_te = wr.transform(df_test, label_col)
    df = merge_multi_df({'train': d_tr, 'test': d_te}, axis=1)

    df['group_sample_ratio'] = df['group_sample_ratio'].fillna(0).astype(float)
    df['total_sample_num'] = df['total_sample_num'].fillna(0).astype(float)
    df['overdue_sample_num'] = df['overdue_sample_num'].fillna(0).astype(float)
    df['overdue_ratio'] = df['overdue_ratio'].fillna(0).astype(float)

    t = df.iloc[:, [i for i, v in enumerate(list(df.columns.labels[0])) if v == list(df.columns.levels[0]).index('IV')]]
    del df['IV']
    df = t.join(df, how='outer')
    df.insert(0, 'L-PSI', value=np.nan)
    df.insert(0, 'V-PSI', value=np.nan)
    for i in df.index.levels[0]:
        # TODO： 以下写法在train和test数据集index有重复时会报错,why?
        # df.loc[i, 'V-PSI'] = cmpt_psi(df.loc[i, 'group_sample_ratio']['train'],
        #                               df.loc[i, 'group_sample_ratio']['test'])
        # df.loc[i, 'L-PSI'] = cmpt_psi(df.loc[i, 'overdue_ratio']['train'],
        #                               df.loc[i, 'overdue_ratio']['test'])
        df.loc[i, 'V-PSI'] = cmpt_psi(df.loc[i]['group_sample_ratio']['train'],
                                      df.loc[i]['group_sample_ratio']['test'])
        df.loc[i, 'L-PSI'] = cmpt_psi(df.loc[i]['overdue_ratio']['train'],
                                      df.loc[i]['overdue_ratio']['test'])
    df.round(decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'woe_stats.report'))
    return df

def _gen_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=False):
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()
    # feature_indice
    train_data = feature_evaluation.feature_evaluation(df_train, [label])
    test_data = feature_evaluation.feature_evaluation(df_test, [label])

    df = merge_multi_df({'train': train_data, 'test': test_data}, axis=1).round(decimals=decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'feature_indices.report'))

        text = {'train': table_to_dict(train_data), 'test': table_to_dict(test_data)}
        simplejson.dump(text, open(os.path.join(report_dst, 'feature_indices.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
    return df

def _gen_woe_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True):
    df1 = _gen_woe_report(df_train_origin, df_test_origin, label, report_dst)
    df2 = _gen_eva_report(df_train_origin, df_test_origin, label, report_dst)
    df2.columns = df2.columns.droplevel(1)
    del df2['IV']
    df2.index.name = 'feature_name'
    df = df2.join(df1, how='outer').round(decimals)

    df = df.reset_index()
    tmp = df[['feature_name', 'IV']]
    tmp.columns = ['_'.join(col).strip() for col in tmp.columns.values]
    tmp = tmp[['feature_name_', 'IV_test']].drop_duplicates().sort_values('IV_test', ascending=False)
    columns = tmp.feature_name_

    df_res = pd.DataFrame()
    for c in columns:
        tmp = df[df['feature_name'] == c]
        try:
            tmp.level = tmp.level.replace('missing', np.nan).astype(float)
        except:
            pass
        tmp = tmp.sort_values([('IV', 'train'), 'feature_name', 'level'], ascending=[False, True, True])
        tmp.level = tmp.level.replace(np.nan, 'missing')
        tmp = tmp.set_index(['feature_name', 'level', 'bin'])
        df_res = pd.concat([df_res, tmp], axis=0)

    if gen_report:
        df_res.to_pickle(os.path.join(report_dst, 'woe_eva_report.report'))
    return df_res

def zeros_deal(test_value, train_value, smooth=0.01):
    difference_rate = 1 - (test_value + smooth) / (train_value + smooth)
    return difference_rate

def stable_different(df, process_name, threshold):
    df_diff = df[df['psi'] >= threshold]
    df_result = pd.DataFrame()
    if df_diff.empty == False:

        for feat_name in df_diff.index.tolist():
            df_result_m = pd.DataFrame()
            df_result_m['feature_name'] = [feat_name]
            df_result_m['process_name'] = process_name
            df_result_m['warning_indice'] = 'PSI'
            df_result_m['warning_reason'] = "数据集的PSI为" + str(df_diff[df_diff.index == feat_name].psi.values[0])
            df_result_m['detailed_data'] = "PSI : " + str(df_diff[df_diff.index == feat_name].psi.values[0])
            df_result = pd.concat([df_result_m, df_result], axis=0)
    return df_result

def woe_different(df, process_name, threshold, woe_columns=['KS', 'AUC', 'IV']):
    df_result_fin = pd.DataFrame()
    for woe_column in woe_columns:
        df[woe_column + '_difference_rate'] = zeros_deal(df[woe_column]['test'], df[woe_column]['train'], smooth=0.01)
        df_diff = df[
            (df[woe_column + '_difference_rate'] >= threshold) | (df[woe_column + '_difference_rate'] <= -threshold)]
        df_result = pd.DataFrame()
        if df_diff.empty == False:
            for feat_name, feat_name_df in df_diff.reset_index().groupby('feature_name'):
                df_result_m = pd.DataFrame()
                df_result_m['feature_name'] = [feat_name]
                df_result_m['process_name'] = process_name
                df_result_m['warning_indice'] = woe_column
                df_result_m['warning_reason'] = "train数据集和test数据集" + woe_column + "差异率为" + format(
                    feat_name_df[woe_column + '_difference_rate'].values[0], '.2%')
                df_result_m['detailed_data'] = woe_column + " train : " + str(
                    feat_name_df[woe_column]['train'].values[0]) + " " + woe_column + " test : " + str(
                    feat_name_df[woe_column]['test'].values[0])
                df_result = pd.concat([df_result_m, df_result], axis=0)
        df_result_fin = pd.concat([df_result, df_result_fin], axis=0)
        df_result_fin
    return df_result_fin

def desc_different(df, process_name, threshold, desc_columns=['cover', '50%']):
    df = df.sort_index(level='feature_name')
    df_result_fin = pd.DataFrame()
    for desc_column in desc_columns:
        diff_rate_m = pd.DataFrame(index=df.index)
        train_i = pd.Series(df.loc[(df.index.levels[0].tolist(), slice(None), ['train']), desc_column])
        test_i = pd.Series(df.loc[(df.index.levels[0].tolist(), slice(None), ['test']), desc_column])
        diff_rate_m['test_difference_rate'] = test_i
        diff_rate_m = diff_rate_m.fillna(method='ffill')
        diff_rate_m['train_difference_rate'] = train_i
        diff_rate_m = diff_rate_m.fillna(method='bfill')
        diff_rate_m[desc_column + '_difference_rate'] = zeros_deal(diff_rate_m['test_difference_rate'],
                                                                   diff_rate_m['train_difference_rate'])
        df_m = df.join(diff_rate_m, how='inner')

        df_diff = df_m[(df_m[desc_column + '_difference_rate'] >= threshold) | (
            df_m[desc_column + '_difference_rate'] <= -threshold)]
        df_result = pd.DataFrame()
        if df_diff.empty == False:
            for feat_name, feat_name_df in df_diff.groupby('feature_name'):
                df_result_m = pd.DataFrame()
                df_result_m['feature_name'] = [feat_name]
                df_result_m['process_name'] = process_name
                df_result_m['warning_indice'] = desc_column
                df_result_m['warning_reason'] = "train数据集和test数据集" + desc_column + "差异率为" + format(
                    feat_name_df[desc_column + '_difference_rate'][0], '.2%')

                df_result_m['detailed_data'] = desc_column + " train : " + str(
                    feat_name_df.loc[(feat_name_df.index.levels[0].tolist(), slice(None), ['train']), desc_column][
                        0]) + " " + desc_column + " test : " + str(
                    feat_name_df.loc[(feat_name_df.index.levels[0].tolist(), slice(None), ['test']), desc_column][0])
                df_result = pd.concat([df_result_m, df_result], axis=0)
        df_result_fin = pd.concat([df_result, df_result_fin], axis=0)

    return df_result_fin

def _gen_warning_report(report_src, report_dst, gen_report=True):
    if os.path.exists(report_src) == False:
        raise ValueError('%s is not exists!' % report_src)
    else:
        df_result = pd.DataFrame()
        for root, dirs, files in os.walk(report_src):
            if root != report_src:
                for file in files:
                    if file.endswith('report'):
                        path = os.path.join(root, file)
                        sheetname = os.path.basename(os.path.dirname(root)) + '_' + file[:-7]
                        if 'woe_eva' in file:
                            df_woe = pd.read_pickle(path)
                            df_diff_m = woe_different(df_woe, sheetname, 0.2)
                        elif 'stable_test' in file:
                            df_stable = pd.read_pickle(path)
                            df_diff_m = stable_different(df_stable, sheetname, 0.2)
                        elif 'data_stats' in file:
                            df_desc = pd.read_pickle(path)
                            df_diff_m = desc_different(df_desc, sheetname, 0.2)
                        else:
                            df_diff_m = pd.DataFrame()
                        df_result = pd.concat([df_diff_m, df_result], axis=0)
        df_result = df_result.sort_values('feature_name')
        df_result = df_result.reset_index(drop=True)
    if gen_report:
        if not os.path.exists(os.path.join(report_dst, 'reporter')):
            os.makedirs(os.path.join(report_dst, 'reporter'))
        df_result.to_pickle(os.path.join(report_dst, 'reporter', 'warning_report.report'))
    return df_result

def _gen_stable_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True,
                       covariate_shift_eva=True):
    df_train_cp = df_train_origin.copy()
    df_test_cp = df_test_origin.copy()
    # stable_test
    dic, mcc, weights = dm.consistent_dataset_test(df_train_cp, df_test_cp, label, report_dst=None,
                                                   covariate_shift_eva=covariate_shift_eva)

    df = pd.DataFrame(dic).T
    df = pd.concat([df, pd.DataFrame({'mcc': {'All_Dataset': mcc}})], axis=0).round(decimals)

    if gen_report:
        df.to_pickle(os.path.join(report_dst, 'stable_test.report'))
        stable_test = {'single_feature_test': dic, 'mcc': mcc}
        simplejson.dump(stable_test, open(os.path.join(report_dst, 'stable_test.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
        pickle.dump(weights, open(os.path.join(report_dst, 'weights.pkl'), 'wb'))
    return df

def _gen_eda_report(df_train_origin, df_test_origin, label, report_dst, decimals=4, gen_report=True,
                    covariate_shift_eva=True):
    tmp = _gen_woe_eva_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals,
                              gen_report=gen_report)
    tmp = _gen_desc_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals, gen_report=gen_report)
    tmp = _gen_stable_report(df_train_origin, df_test_origin, label, report_dst, decimals=decimals,
                             gen_report=gen_report,
                             covariate_shift_eva=covariate_shift_eva)
    return

def reporter_report(report_src, report_dst=None):
    """

    :param report_dst: 工作流的根目录
    :return:
    """
    if report_dst is None:
        report_dst = os.path.join(report_src, 'reporter')
        if not os.path.exists(report_dst):
            os.makedirs(report_dst)
    tmp = _gen_warning_report(report_src, report_dst)
    merge_excel(report_src, os.path.join(report_dst, 'all_report.xlsx'))
    return

def sampler_report(sample_num, dic, group_ratio, report_dst):
    """
    报告分组基准，分组前后样本分布对比
    :param sample_num:
    :param dic:
    :param group_ratio:
    :param report_dst:
    :return:
    """
    simplejson.dump(sample_num, open(os.path.join(report_dst, 'sample_num.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)
    simplejson.dump(dic, open(os.path.join(report_dst, 'dic.json'), 'w'), ignore_nan=True, cls=simplejsonEncoder)
    simplejson.dump(group_ratio, open(os.path.join(report_dst, 'group_ratio.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def spliter_report(X_train, X_test, time_col, label_col, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    覆盖率、饱和度统计，数据描述性统计和直方图，目标分布统计
    :param df_train:
    :param df_test:
    :param time_col:
    :param label_col:
    :param report_dst:
    :return:
    """
    df_train = X_train.copy()
    df_test = X_test.copy()

    # # TODO: 等待删除，暂时前端需要该输出
    # c_tr = cover_stats(df_train)
    # c_te = cover_stats(df_test)
    # c_tr.to_pickle(os.path.join(report_dst, 'c_tr.pkl'))
    # c_te.to_pickle(os.path.join(report_dst, 'c_te.pkl'))
    #
    # dic = {'train': table_to_dict(c_tr), 'test': table_to_dict(c_te)}
    # simplejson.dump(dic, open(os.path.join(report_dst, 'cover_stats.json'), 'w'), ignore_nan=True,
    #                 cls=simplejsonEncoder)

    d_tr = target_stats(df_train, label_col, time_col)
    d_te = target_stats(df_test, label_col, time_col)
    dic = {'train': d_tr, 'test': d_te}
    df = pd.DataFrame(dic).T
    df.to_pickle(os.path.join(report_dst, 'target_stats.report'))
    simplejson.dump(dic, open(os.path.join(report_dst, 'target_stats.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

    del df_train[time_col], df_test[time_col]

    _gen_eda_report(df_train, df_test, label_col, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

def transformer_report(df_train_origin, df_test_origin, label, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    单变量分析、直方排序图分析、train/test特征一致性分布分析
    :param df_train:
    :param df_test:
    :param label:
    :param report_dst:
    :return:
    """
    df_train = df_train_origin.copy()
    df_test = df_test_origin.copy()

    _gen_eda_report(df_train_origin, df_test_origin, label, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

    # feature_plot
    df_train.fillna(-999, inplace=True)
    train_plot_data = feature_plot(df_train, label, path=os.path.join(report_dst, 'train_feature_plot.png'))
    df_test.fillna(-999, inplace=True)
    test_plot_data = feature_plot(df_test, label, path=os.path.join(report_dst, 'test_feature_plot.png'))

    train_plot_data = {k: {'train': v} for k, v in train_plot_data.items()}
    test_plot_data = {k: {'test': v} for k, v in test_plot_data.items()}
    plot_data = defaultdict(dict)
    for k, v in chain(train_plot_data.items(), test_plot_data.items()):
        plot_data[k].update(v)
    simplejson.dump(plot_data, open(os.path.join(report_dst, 'plot_data.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def filter_report(enc, df_train, df_test, label, report_dst, gen_report=True, covariate_shift_eva=True):
    """
    过滤指标得分输出
    :param enc:
    :param score_func:
    :param report_dst:
    :return:
    """
    data = {}
    for encoder_dict in enc.st:
        df = encoder_dict['enc'].features_report
        data[encoder_dict['method']] = table_to_dict(df)

    simplejson.dump(data, open(os.path.join(report_dst, 'feature_indices.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

    df = pd.DataFrame()
    for encoder_dict in enc.st:
        t = encoder_dict['enc'].features_report
        t.columns = pd.MultiIndex.from_product([[encoder_dict['method']], t.columns])
        df = pd.concat([df, t], axis=1)

    t = pd.Series(True, index=df.index)
    for c in df.columns:
        if c[-1].endswith('_support'):
            t = t & df[c]
    df['final_supprt'] = t

    df.round(4).to_pickle(os.path.join(report_dst, 'feature_filter.report'))

    _gen_eda_report(df_train, df_test, label, report_dst, gen_report=gen_report,
                    covariate_shift_eva=covariate_shift_eva)

def optimizer_report(all_df, report_dst):
    """
    每轮迭代参数和效果的输出
    :param all_df:
    :param report_dst:
    :return:
    """
    all_df.to_pickle(os.path.join(report_dst, 'optimizer_result.report'))
    """
    Currently (as of Pandas version 0.18), df.to_dict('records') accesses the NumPy array df.values.
    This property upcasts the dtype of the int column to float so that the array can have a single common dtype.
    After this point there is no hope of returning the desired result -- all the ints have been converted to floats.
    refers: https://stackoverflow.com/questions/37897527/get-python-pandas-to-dict-with-orient-records-but-without-float-cast
    """
    dic = [{col: getattr(row, col) for col in all_df} for row in all_df.itertuples()]
    simplejson.dump(dic, open(os.path.join(report_dst, 'all_params_result.json'), 'w'), ignore_nan=True,
                    cls=simplejsonEncoder)

def trainer_report(enc, df_train, df_val, test_src, test_data_dst, report_dst):
    """
    训练报告
    :param enc:
    :param df_train:
    :param df_val:
    :param df_test:
    :param test_m:
    :param report_dst:
    :return:
    """
    if test_src is not None:
        df_test = enc.transform(test_src)
        df_test.to_pickle(test_data_dst)
        test_m = 'oot'
    else:
        test_m = 'test'

    df = get_data(test_src)

    try:
        df_label = df.pop(enc.label)
        df_label.name = 'y_true'
    except:
        df_label = pd.DataFrame()

    if enc.enc._estimator_type == 'ruler':
        tmp = enc.enc.get_metrics(df, df_label)
        tmp.round(4).to_pickle(os.path.join(report_dst, 'rule_report.report'))
        tmp = table_to_dict(tmp)
        simplejson.dump(tmp, open(os.path.join(report_dst, 'rule_report.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

    try:
        rr = feature_evaluation.ResultReport()
        rr.fit(df_train.y_pred)
        df = rr.transform({'train': df_train, 'val': df_val, test_m: df_test})
        df.to_pickle(os.path.join(report_dst, 'level_report.report'))
        pickle.dump(rr, open(os.path.join(report_dst, 'rr_enc.pkl'), 'wb'))

        eval_plot_path = os.path.join(report_dst, 'trainer_eva_report.png')
        dic_all = eva_plot({'train': [df_train.y_true, df_train.y_pred], 'val': [df_val.y_true, df_val.y_pred],
                            test_m: [df_test.y_true, df_test.y_pred]}, path=eval_plot_path)
        import re
        from collections import defaultdict
        dic = defaultdict(dict)
        token = re.compile('[\d.]+')
        for k in dic_all['roc_curve']:
            for v in dic_all['ks_curve'][k]:
                if v.startswith('KS'):
                    s = float(re.findall(token, v)[0])
                    #             dic[k].append({'ks': s})
                    dic[k]['ks'] = s
        for v in dic_all['roc_curve']['train'].keys():
            k1, k2 = v.split('(')
            k1 = k1.strip()
            s = float(re.findall(token, k2)[0])
            #     dic[k1].append({'auc': s})
            dic[k1]['auc'] = s
        # df = pd.DataFrame.from_dict(dic)
        simplejson.dump(dict(dic), open(os.path.join(report_dst, 'abstract_data.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)
        simplejson.dump(dic_all, open(os.path.join(report_dst, 'eva_data.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

        pd.DataFrame(dic).T.to_pickle(os.path.join(report_dst, 'abstract_data.report'))

        importances = getattr(enc.enc, "feature_importances_", None)
        coef_ = getattr(enc.enc, "coef_", None)
        rules_ = getattr(enc.enc, "rules_", None)
        if not (importances is None and coef_ is None):
            try:
                df_features = \
                    pd.DataFrame(
                        {'feature_names': enc.columns, 'feature_importances': get_feature_importances(enc.enc)})[
                        ['feature_names', 'feature_importances']]
            except Exception as e:
                print(traceback.format_exc())
                df_features = pd.DataFrame()
        elif rules_ is not None:
            try:
                df_features = pd.DataFrame({'rules': enc.enc.rules_})
            except Exception as e:
                print(traceback.format_exc())
                df_features = pd.DataFrame()
        else:
            df_features = pd.DataFrame()

        simplejson.dump(df_features.to_dict(orient='records'),
                        open(os.path.join(report_dst, 'feature_importances.json'), 'w'), ignore_nan=True,
                        cls=simplejsonEncoder)

        pd.DataFrame(df_features).round(4).to_pickle(os.path.join(report_dst, 'feature_importances.report'))
    except Exception as e:
        print(traceback.format_exc())

    report_src = os.path.dirname(os.path.dirname(report_dst))
    reporter_report(report_src)
    return


import datetime
import discord

class MessageUtils:

    @staticmethod
    def convert_time_delta(end: datetime, start: datetime):
        delta = end - start
        hours, remainder = divmod(int(delta.total_seconds()), 3600)
        minutes, seconds = divmod(remainder, 60)
        days, hours = divmod(hours, 24)

        days_text = 'days' if days != 1 else 'day'
        hours_text = 'hours' if hours != 1 else 'hour'
        minutes_text = 'minutes' if minutes != 1 else 'minute'
        seconds_text = 'seconds' if seconds != 1 else 'second'

        if days > 0:
            return f'{days} {days_text}, {hours} {hours_text}, {minutes} {minutes_text}, {seconds} {seconds_text}'
        if hours > 0:
            return f'{hours} {hours_text}, {minutes} {minutes_text}, {seconds} {seconds_text}'
        if minutes > 0:
            return f'{minutes} {minutes_text}, {seconds} {seconds_text}'
        return f'{seconds} {seconds_text}'

    @staticmethod
    def stringify_datetime(time: datetime):
        """
        Returns:
            a readable and formatted version of a datetime object
        """
        return time.strftime('%B %d, %Y at %H:%M:%S UTC')

class ColoredEmbed(discord.Embed):
    def __init__(self, **kwargs):
        super().__init__(colour=0x3dbbe5, **kwargs)