import os
import json
import torch
import logging
import re
import random
import nltk.data

# the telegram package holds everything we need for this tutorial
import telegram
from telegram.ext import Updater
from telegram.ext import CommandHandler
from telegram.ext import MessageHandler, Filters

from src.transformer import Transformer
from dataset import DialogueDataset, Vocab
from src.transformer import Chatbot

def clean_response(response):
    substitutions = [(r' </s>', r''),
                     (r' \.', '.'),
                     (r' !', r'!'),
                     (r' \?', r'?'),
                     (r' ,', r'?'),
                     (r' i ', r' I ')]
                     #('[.|!|?] *[a-z]', )]

    for (pattern, replacement) in substitutions:
        response = re.sub(pattern, replacement, response)

    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    sentences = sent_tokenizer.tokenize(response)
    sentences = [sent.capitalize() for sent in sentences]
    response = ' '.join(sentences)
    return response

class TelegramBot:
    def __init__(self, token):
        """This chatbot takes a Telegram authorization toke and a handler function,
        and deploys a Telegram chatbot to respond to user messages with that token.

        token - a string authorization token provided by @BotFather on Telegram
        handle_func - a function taking update, context parameters which responds to user inputs
        """
        self.token = token
        self.bot = telegram.Bot(token=token)
        self.updater = Updater(token=token)
        self.dispatcher = self.updater.dispatcher
        self.updater.start_polling()

    def stop(self):
        """Stop the Telegram bot"""
        self.updater.stop()

    def add_handler(self, handler):
        """Add a handler function to extend bot functionality"""
        self.dispatcher.add_handler(handler)

class ChatBot:
    def __init__(self, args):
        # get the dir with pre-trained model

        load_dir = os.path.join(args.experiment_dir, args.old_model_dir)

        # initialize, and load vocab
        self.vocab = Vocab()
        vocab_filename = os.path.join(load_dir, "vocab.json")
        self.vocab.load_from_dict(vocab_filename)

        # load configuration
        with open(os.path.join(load_dir, "config.json"), "r") as f:
            config = json.load(f)

        args.response_len = config["response_len"]
        args.history_len = config["history_len"]

        # initialize an empty dataset. used to get input features
        self.dataset = DialogueDataset(None,
                                       history_len=config["history_len"],
                                       response_len=config["response_len"],
                                       vocab=self.vocab,
                                       update_vocab=False)

        # set device
        self.device = torch.device(args.device)

        # initialize model
        model = Transformer(
            config["vocab_size"],
            config["vocab_size"],
            config["history_len"],
            config["response_len"],
            d_word_vec=config["embedding_dim"],
            d_model=config["model_dim"],
            d_inner=config["inner_dim"],
            n_layers=config["num_layers"],
            n_head=config["num_heads"],
            d_k=config["dim_k"],
            d_v=config["dim_v"],
            dropout=config["dropout"],
            pretrained_embeddings=None
        ).to(self.device)

        # load checkpoint
        checkpoint = torch.load(os.path.join(load_dir, args.old_model_name),
                                map_location=self.device)
        model.load_state_dict(checkpoint['model'])

        # create chatbot
        self.chatbot = Chatbot(args, model)

        self.args = args

    def run(self):
        logging.basicConfig(
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
            level=logging.INFO)

        greeting_text = "Hello! I am the Hawkbot! Let me tell you about myself" \
            "... please dont hurt my feelings!" \
                        "  If you would like to reset "\
                        "the conversation, please type '/reset'. "

        # initialize history dictionary for each chat id
        history = dict()

        def greeting(bot, update):
            # reset history, or create new history for chat id
            if update.message.chat_id in history:
                id = "{}_history".format(update.message.chat_id)
                if id in history:
                    history[id].append(history[update.message.chat_id])
                else:
                    history[id] = [history[update.message.chat_id]]
                history[update.message.chat_id].clear()
            else:
                history[update.message.chat_id] = list()

            # send a message
            bot.send_message(update.message.chat_id, greeting_text)

        def respond(bot, update):
            # initialize history for chat if it doesnt exist
            if update.message.chat_id not in history:
                greeting(bot, update)
            else:
                # get message, and add to history
                message = update.message.text
                history[update.message.chat_id].append(message)
                # get response, and add to history
                response = self._print_response(history[update.message.chat_id])
                history[update.message.chat_id].append(response)
                # send response from user
                bot.send_message(update.message.chat_id, clean_response(response))

                with open(self.args.save_filename, 'w') as f:
                    json.dump({"history": history, "args": vars(self.args)},
                              f, indent=4)

        # queries sent to: https://api.telegram.org/bot<token>/METHOD_NAME
        TOKEN = self.args.token

        bot = TelegramBot(TOKEN)
        bot.add_handler(MessageHandler(Filters.text, respond))
        bot.add_handler(CommandHandler('reset', greeting))

    # print the response from the input
    def _print_response(self, history):

        # generate responses
        responses, scores = self._generate_responses(history)
        # chose response
        if self.args.choose_best:
            response = responses[0][0]
        else:
            # pick a random result from the n_best
            idx = random.randint(0, min(self.args.n_best,
                                        self.args.beam_size) - 1)
            response = responses[0][idx]

        # uncomment this line to see all the scores
        # print("scores in log prob: {}\n".format(scores[0]))

        # create output string
        output = ""
        for idx in response[:-1]:
            token = self.vocab.id2token[idx]
            output += "{} ".format(token)
        print(f'{history[-1]} -> {output}')
        return output

    def _generate_responses(self, history):
        # get input features for the dialogue history
        h_seq, h_pos, h_seg = self.dataset.get_input_features(history)

        # get response from model
        response = self.chatbot.translate_batch(h_seq, h_pos, h_seg)
        return response


#!/usr/bin/python

# Requires kernel module bcm2835_gpiomem and udev rule /etc/udev/rules.d/20-gpio.rules
#SUBSYSTEM=="bcm2835-gpiomem", KERNEL=="gpiomem", GROUP="gpio", MODE="0660"
#SUBSYSTEM=="gpio", KERNEL=="gpiochip*", ACTION=="add", PROGRAM="/bin/sh -c 'chown root:gpio /sys/class/gpio/export /sys/class/gpio/unexport ; chmod 220 /sys/class/gpio/export /sys/class/gpio/unexport'"
#SUBSYSTEM=="gpio", KERNEL=="gpio*", ACTION=="add", PROGRAM="/bin/sh -c 'chown root:gpio /sys%p/active_low /sys%p/direction /sys%p/edge /sys%p/value ; chmod 660 /sys%p/active_low /sys%p/direction /sys%p/edge /sys%p/value'"
#further information: https://sourceforge.net/p/raspberry-gpio-python/tickets/115/ and http://wiringpi.com/wiringpi-update-to-2-29/
from __future__ import division
from __future__ import with_statement
import RPi.GPIO as GPIO
import time
from datetime import datetime
import os, sys
import threading

import logging
logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', filename='/tmp/motionInterrupt.log', level=logging.DEBUG)

###Configuration
LCD_POWER_PIN=8
MOTION_PIN=11
LCD_POWER_DETECT_PIN = 24
TIME_UNTIL_TURN_OFF_MIN= 15 # time of no detected motion until the screen is turned off
CHECK_FOR_TURN_OFF_SEC = 30 # sleep time between test for turning off the display

###Global variables

#LCD state
lcd_on = True
#Lock to synchronize switching
lock_switch = threading.Lock()

###Functions

def init():
    global lcd_on
    global last_motion_time

    lcd_on = True

    # SoC als Pinreferenz waehlen
    GPIO.setmode(GPIO.BCM)

    GPIO.setwarnings(False)

    # Pin 24 vom SoC als Input deklarieren und Pull-Down Widerstand aktivieren
    GPIO.setup(MOTION_PIN, GPIO.IN)

    GPIO.setup(LCD_POWER_PIN, GPIO.OUT, initial=GPIO.HIGH)

    GPIO.setup(LCD_POWER_DETECT_PIN, GPIO.IN)

    last_motion_time = datetime.now()

    # Interrupt Event hinzufuegen. Pin 24, auf steigende Flanke reagieren und ISR "Interrupt" deklarieren
    GPIO.add_event_detect(MOTION_PIN, GPIO.FALLING, callback = motion_interrupt, bouncetime = 200)

    logging.info('Configuration: Turning off after %s min, checking time difference each %s s', TIME_UNTIL_TURN_OFF_MIN, CHECK_FOR_TURN_OFF_SEC)

    logging.info('Finished init()')

# ISR
def motion_interrupt(channel):
    # Access global variables
    global last_motion_time
    global lcd_on

    last_motion_time = datetime.now()

    logging.debug("Motion detected")
    #XXX may wait until we received several measures (filtering)

    if not lcd_on:
        logging.info("LCD is turned on again")
        switch_to_lcd_state(1)
        lcd_on = True

def get_lcd_state():
    # returns 0 if display is off (LED=off) and 1 if display is on (LED=on)
    return GPIO.input(LCD_POWER_DETECT_PIN)

def switch_to_lcd_state(target_state):
    """
    Switch the lcd to the given target state
    Keep trying until target state is achieved
    :param target_state: target state of the lcd 0=off 1=on
    """
    global lock_switch
    with lock_switch:
        while get_lcd_state() != target_state:
            press_lcd_power_key()
            time.sleep(1)

# Press Power key of LCD display
def press_lcd_power_key():
    GPIO.output(LCD_POWER_PIN, GPIO.LOW)
    time.sleep(0.1)
    GPIO.output(LCD_POWER_PIN, GPIO.HIGH)
    #print("power button pressed")

def check_idle_time_turn_off():
    global lcd_on
    now = datetime.now()
    diff_min = (now - last_motion_time).total_seconds() / 60

    logging.debug("Current time diff: %s", diff_min)

    if diff_min >= TIME_UNTIL_TURN_OFF_MIN and lcd_on == True:
        logging.info("Idle time exeeded, LCD is turned off")
        switch_to_lcd_state(0)
        lcd_on = False

def main_loop():
    init()
    # endless loop
    while True:
        time.sleep(CHECK_FOR_TURN_OFF_SEC)
        check_idle_time_turn_off()

###Main programm
if __name__ == '__main__':
    #alternative without process
    #main_loop()

    # start check in an own process
    print("Starting subprocess")
    fpid = os.fork()
    if fpid==0:
      # Running as daemon now. PID is fpid
      logging.debug("Daemon process started")
      main_loop()
    else:
        exit() #exit first start process


 #!/usr/bin/python
# -*- coding: utf8 -*-
import pickle
import pandas as pd
import math

dane = pd.read_csv("../data/processed/votes_with_city_wojewodztwo.csv", sep=';', encoding = "utf8")

wroclaw_votes = dane[(dane['Miasto']==u'Wrocław') & ((dane['Prog_1'] > 0)|(dane['Prog_2'] > 0)|(dane['Prog_3'] > 0))]
wroclaw_records = wroclaw_votes.to_records()

# matrix: (vote1.id, vote2.id, projects(vote1) interstesction projects(vote2)
#
votes_to_votes_through_projects = {v[1] : {} for v in wroclaw_records}

project_votes = {}
for vote in wroclaw_records:
    for i in [-3,-4,-5]:
        p = vote[i]
        if p not in project_votes:
            project_votes[p] = []
        project_votes[p].append(vote[1])
del project_votes[0]

# vote id -> postcode
votes_to_codes = { vote[1] : vote[-7].replace('-','') for vote in wroclaw_records }

# (post_code1, post_code2, list of votes that )
codes_from_votes = { vote[-7].replace('-','') : {} for vote in wroclaw_records }

# (post_code1, post_code2, list of votes that )
codes_from_projects = { vote[-7].replace('-','') : {} for vote in wroclaw_records }

all_votes = len(wroclaw_records)

print("Counting votes")

i = 0

step = int(math.ceil(0.1*all_votes))
thresh = step

for vote in wroclaw_records:
    code = vote[-7].replace('-','')
    # projects that vote voted for
    p_v = [vote[i] for i in [-3,-4,-5] if vote[i] > 0]

    # for each project
    for p in p_v:
        # get all votes that voted on the project
        for vid in project_votes[p]:
            # get post code of each vote
            cooccured_code = votes_to_codes[vid]

            # this post code project wise co-occurs with the original votes postcode
            if cooccured_code not in codes_from_votes[code]:
                codes_from_votes[code][cooccured_code] = []

            if cooccured_code not in codes_from_projects[code]:
                codes_from_projects[code][cooccured_code] = []

            # count the votes? that make two postcodes co-occur
            codes_from_votes[code][cooccured_code].append(vid)
            codes_from_projects[code][cooccured_code].append(p)

    if i > thresh:
        print("{} percent done".format(thresh))
        thresh+=step

    i+=1

codes_weighted_by_vote_count = { vote[-7].replace('-','') : {} for vote in wroclaw_records }
codes_weighted_by_project_count = { vote[-7].replace('-','') : {} for vote in wroclaw_records }

print("Counting unique votes")

for k1 in codes_from_votes.keys():
    for k2 in codes_from_votes[k1].keys():
        codes_from_votes[k1][k2] = list(set(codes_from_votes[k1][k2]))
        codes_weighted_by_vote_count[k1][k2] = len(codes_from_votes[k1][k2])

for k1 in codes_from_projects.keys():
    for k2 in codes_from_projects[k1].keys():
        codes_from_projects[k1][k2] = list(set(codes_from_projects[k1][k2]))
        codes_weighted_by_project_count[k1][k2] = len(codes_from_projects[k1][k2])

print("Saving votes")

with open("./postcodes-coocuring-by-votes.pkl","w") as fp:
    pickle.dump(codes_from_votes, fp)

with open("./postcodes-coocuring-by-project.pkl","w") as fp:
    pickle.dump(codes_from_projects, fp)

print("Saving weights")

weights = {
    'codes_weighted_by_vote_count' : codes_weighted_by_vote_count,
    'codes_weighted_by_project_count' : codes_weighted_by_project_count,

}
with open("./postcodes-coocuring-weights.pkl","w") as fp:
    pickle.dump(weights, fp)


#!/usr/bin/env python
from __future__ import print_function, unicode_literals

import re

ip_regex = '''^(25[0-5]|2[0-4][0-9]|[0-1]?[0-9][0-9]?)\.(
            25[0-5]|2[0-4][0-9]|[0-1]?[0-9][0-9]?)\.(
            25[0-5]|2[0-4][0-9]|[0-1]?[0-9][0-9]?)\.(
            25[0-5]|2[0-4][0-9]|[0-1]?[0-9][0-9]?)$'''

mac_regex = '''^([0-9a-f]{4}\.[0-9a-f]{4}\.[0-9a-f]{4})$'''

def show():

    mac1 = "Internet  10.220.88.29           94   5254.abbe.5b7b  ARPA   FastEthernet4"
    mac2 = "Internet  10.220.88.30            3   5254.ab71.e119  ARPA   FastEthernet4"
    mac3 = "Internet  10.220.88.32          231   5254.abc7.26aa  ARPA   FastEthernet4"

    maclist = []
    maclist.append(mac1.split())
    maclist.append(mac2.split())
    maclist.append(mac3.split())

    print(mac1)
    ips = []
    macs = []

    for list in maclist:
        for item in list:
            if re.search(ip_regex, item):
                ips.append(item)
            elif re.search(mac_regex, item):
                macs.append(item)

    # Output
    print("{col1:>20}{col2:>20}".format(col1="IP ADDR",col2="MAC ADDRESS"))
    print("{col1:>20}{col2:>20}".format(col1="-"*20,col2="-"*20))
    index = 0
    for ip in ips:
            print("{col1:>20}{col2:>20}".format(col1=ip, col2=macs[index]))
            index += 1
    return 0


from __future__ import print_function
import collections, bleach, json, itertools
import numpy as np

from SetProcessing import SetProcessing
from utils import unicodeReader, returnDatasets, makeLangPrefixMapping
from utils import unpickleFile as uf
from textblob import TextBlob, Word
from collections import Counter
from time import time
from nltkparsing import *
from wordvectors import *
from tinysegmenter import TinySegmenter
from konlpy.tag import Mecab, Kkma
from rakutenma import RakutenMA

from nltk.tag.stanford import StanfordPOSTagger
from nltk.tag.stanford_segmenter import StanfordSegmenter
from nltk.classify.decisiontree import DecisionTreeClassifier
from nltk.metrics.distance import edit_distance
from nltk.util import ngrams as ng

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.pipeline import FeatureUnion
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer
from sklearn.cluster import KMeans, MiniBatchKMeans
from sklearn.metrics import metrics
from sklearn.metrics import classification_report
from sklearn.preprocessing import Normalizer
from sklearn.decomposition import TruncatedSVD
from sklearn.base import BaseEstimator, TransformerMixin

class SyntacticStructExtraction:
	def __init__(self):
		self.english = ['English']
		self.french = ['French']
		self.spanish = ['Spanish']
		self.japanese = ['Japanese']
		self.korean = ['Korean']
		self.mandarin = ['Mandarin']
		self.lang_index = ['English', 'French', 'Spanish', 'Japanese', 'Korean','Mandarin']

		self.english_freqs_path = 'data/english_freqs.pickle'
		self.french_freqs_path = 'data/french_freqs.pickle'
		self.spanish_freqs_path = 'data/spanish_freqs.pickle'
		self.mandarin_freqs_path = 'data/mandarin_freqs.pickle'
		self.japanese_freqs_path = 'data/japanese_freqs.pickle'
		self.korean_freqs_path = 'data/korean_freqs.pickle'

	def fit(self, x, y=None):
		return self

	def averageSentenceLengthByLanguage(self, entries):
		t0 = time()
		total_entries = len(entries)
		averaged = 0
		for entry in entries:
			averaged += self.averageSentenceLength(entry)
		print("Took %s seconds to return the avg. sent. length by language." % (time()-t0))
		return float(averaged) / total_entries

	def averageSentenceLength(self, entry):
		'''Finds the average sentence length in chars.'''
		sentences = parseSentenceFeatures(entry)
		total_sents = len(sentences)
		all_words_total = 0
		for sent in sentences:
			all_words_total += len(sent)
		return float(all_words_total) / total_sents

	def averageNumberOfTokens(self, entries, eastern=True):
		'''Finds the average number of words in a sentence.'''
		t0 = time()
		entries_count = len(entries)
		wordcount = 0
		for entry in entries:
			if eastern:
				wordcount += len(TinySegmenter().tokenize(entry))
			else:
				wordcount += len(entry.split())
		print("Took %s seconds to return the avg. # of tokens per entry." % (time()-t0))
		print(float(wordcount) / entries_count)
		return float(wordcount) / entries_count

	def spellingErrors(self, sentences):
		spelling_errors_counter = 0
		misspelled_words = []
		for sent in sentences:
			blob = TextBlob(sent)
			correction = blob.correct()
			if correction is not sent:
				for word in sent.split():
					if word not in correction:
						misspelled_words.append(word)
						spelling_errors_counter += 1
		print("There were %s spelling errors." % spelling_errors_counter)
		return misspelled_words

	def confirmSpellcheck(tokens):
		spellcheck = []
		for token in tokens:
			w = Word(token)
			spellcheck.append(w.spellcheck())
		return spellcheck

	def tagWordsInSentences(self, studying, entry):
		'''Tags the part of speech for each word.'''
		jar_path = 'stanford-postagger-full/stanford-postagger.jar'
		if studying in self.english:
			words = parseWordsFromEntry(entry)
			tagged_words = tagWords(words)
			return tagged_words
		elif studying in self.japanese or self.korean or self.mandarin:
			#segmenter = TinySegmenter()
			#words = segmenter.tokenize(entry)
			rm = RakutenMA()
			tagged_words = rm.tokenize(entry)
			#mecab = Mecab()
			#tagged_words = mecab.pos(entry)
			return tagged_words
		else:
			if studying in self.spanish:
				model_path = 'stanford-postagger-full/models/spanish.tagger'
				words = parseWordsFromEntry(entry)
			elif studying in self.french:
				model_path = 'stanford-postagger-full/models/french.tagger'
				words = parseWordsFromEntry(entry)
			postagger = StanfordPOSTagger(model_path, jar_path, encoding='utf8')
			tagged_words = postagger.tag(words)
			return tagged_words

	def findFrequencyOfSequence(self, tagged_words, ngrams=2):
		pos = [tag for word, tag in tagged_words]
		ngramlist = ng(pos, ngrams)
		freqs = Counter(n for n in ngramlist)
		return freqs

	def makeLanguageCounter(self, studying, entries):
		t0 = time()
		freqs_counter = Counter()
		for entry in entries:
			tagged_words = self.tagWordsInSentences(studying, entry)
			freqs = self.findFrequencyOfSequence(tagged_words)
			freqs_counter = freqs_counter + freqs
		print("Took %s seconds to make lang. counter" % (time()-t0))
		#print(freqs_counter)
		return {studying: freqs_counter}

	def makeCounter(self, counters):
		counter_n = 0
		holder = dict()
		for counter_dict in counters:
			lang, counter = counter_dict.popitem()
			counter_n += len(counter.values())
			holder[lang] = counter.most_common(100)
		return holder, counter_n

	def matchLangBySyntaxProb(self, holder, counter_n, studying, entry):
		observations = self.findFrequencyOfSequence(self.tagWordsInSentences(studying, entry))
		length = len(observations)
		observations = observations.most_common(int(length / 4))
		problang = studying
		curr_suma = 0
		for x, y in holder.items():
			for elem, num in y:
				for tag in observations:
					(first, second), count = tag
					if first or second in y:
						suma = float(count) / num
						if suma > curr_suma:
							problang = x
							curr_suma = suma
		return entry, curr_suma, problang

	def transform_sentence_length(self, pairs):
		features = np.recarray(shape=(len(pairs),),
		dtype=[('spoken', object),('sentence_count', object)])
		for i, pair in enumerate(pairs):
			spoken, count = pair
			features['spoken'][i] = spoken
			features['sentence_count'][i] = count
		return features

class GiveawayFeatureExtraction:
	def __init__(self):
		self.clusters = 5

	def scanForOtherLanguages(self, words, target):
		'''Checks to see if other, non-specified languages are in the entry.'''
		langmap = makeLangPrefixMapping()
		langprefs = set()
		for word in words:
			detection = detect(word)
			if detection is not langmap[target]:
				langprefs.add( (word, detection) )
		return langprefs

class SocialFeatureExtraction:
	def __init__(BaseEstimator, TransformerMixin):
		pass

	def topicClustering(self, datalist, language_tag):
		w2v = w2vRetrieve(datalist, language_tag)
		print("Beginning kmeans clustering...")
		t0 = time()
		word_vectors = w2v.syn0
		num_clusters = int(word_vectors.shape[0] / 5)
		kmeans_clustering = KMeans(n_clusters=num_clusters)
		idx = kmeans_clustering.fit_predict(word_vectors)
		print("Finished kmeans in %s seconds" % (time()-t0))
		word_centroid_map = dict(zip( w2v.index2word, idx ))
		for cluster in range(0,10):
			print("\nCluster %s" % cluster)
			words = []
			vals = list(word_centroid_map.values())
			keys = list(word_centroid_map.keys())
			for i in range(len(vals)):
				if vals[i]==cluster:
					words.append(keys[i])
			print(words)

class CorrectionExtraction:
	def __init__(self):
		pass

	def fit(self, x, y=None):
		return self

	def errorEditDistance(self, pairs):
		'''Calculates the edit distance between two strings.'''
		t0 = time()
		total = 0; distances = 0
		for pair in pairs:
			incorrect, correct = pair
			for i, j in zip(incorrect, correct):
				total += 1
				distances += edit_distance(i, j)
		print("Took %s seconds to calc. edit distance" % (time()-t0))
		return float(distances) / total

class TokenFeatures(BaseEstimator, TransformerMixin):
	def fit(self, x, y=None):
		return self

	def transform_token_count(self, pairs):
		features = np.recarray(shape=len(pairs),),
		dtype=[('spoken', object),('token_count', object)]
		for i, pair in enumerate(pairs):
			spoken, count = pair
			features['spoken'][i] = spoken
			features['token_count'][i] = count
		return features

class FeatureGetter(BaseEstimator, TransformerMixin):
	def __init__(self):
		self.item = item

	def fit(self, x, y=None):
		return self

	def transform(self, item):
		return item

class POSFeatures(BaseEstimator, TransformerMixin):
	def fit(self, x, y=None):
		return self

	def transform(self, pairs):
		features = np.recarray(shape=(len(pairs),),
								dtype=[('entry', object),('pos_probability', object),('problang', object)])
		for i, pair in enumerate(pairs):
			entry, prob, problang = pair
			features['entry'][i] = entry
			features['pos_probability'][i] = prob
			features['problang'][i] = problang
		return features

class CorrectionFeatures(BaseEstimator, TransformerMixin):
	def __init__(obj):
		self.obj = obj

	def fit(self, x, y=None):
		return self

	def transform(self, pairs):
		features = np.recarray(shape=(len(pairs),),dtype=[('spoken', object),('edit_dist', object)])
		for i, pair in enumerate(pairs):
			spoken, count = pair
			features['spoken'][i] = spoken
			features['edit_dist'][i] = count
		return features

if __name__ == '__main__':
	train, dev, test = returnDatasets()

	''' Set up the classes. '''
	#gfe = GiveawayFeatureExtraction()
	#sfe = SocialFeatureExtraction()
	sse = SyntacticStructExtraction()
	ce = CorrectionExtraction()
	sp = SetProcessing()

	datalist = sp.convertDataToList(train)
	#dev = sp.convertDataToList(dev)
	#test = sp.convertDataToList(test)
	#merged = sp.mergeLists(train, dev, test)
	#english, french, spanish, japanese, korean, mandarin = sp.returnSplitDatasets(train, 5, False)
	'''Return the individual sets by native language.'''
	'''Takes approx. 1 second.'''
	print("Collecting test sets...")
	western_native, eastern_native = sp.organizeDataByRegion(train)
	english_native, french_native, spanish_native = sp.organizeWesternLanguages(western_native)
	japanese_native, korean_native, mandarin_native = sp.organizeEasternLanguages(eastern_native)

	'''Return the individual sets by language being studied.'''
	'''Takes approx. 1 second.'''
	western_learning, eastern_learning = sp.organizeDataByRegion(train, False)
	english_learning, french_learning, spanish_learning = sp.organizeWesternLanguages(western_learning, False)
	japanese_learning, korean_learning, mandarin_learning = sp.organizeEasternLanguages(eastern_learning, False)

	'''First feature: retrieve word frequencies. USE LEARNING SETS.'''
	'''Can take anywhere from 40 seconds to 7 minutes depending on the language and the set.'''
	#english_entries = sp.returnEntries(english_learning) # English works!
	#french_entries = sp.returnEntries(french_learning) # French works!
	#spanish_entries = sp.returnEntries(spanish_learning) # Spanish works!
	#mandarin_entries = sp.returnEntries(mandarin_learning) # Mandarin works!
	#korean_entries = sp.returnEntries(korean_learning) # Korean works!
	#japanese_entries = sp.returnEntries(japanese_learning)
	print("Gathering feature number one...")
	#freq_list = sse.makeLanguageCounter(japanese_learning[0][sp.STUDYING], japanese_entries)
	#pickle.dump(freq_list, open(sse.spanish_freqs_path, 'wb'))

	#pickle.dump(freq_list, open(sse.english_freqs_path, 'wb'))

	#pickle.dump(freq_list, open(sse.french_freqs_path, 'wb'))

	#pickle.dump(freq_list, open(sse.mandarin_freqs_path, 'wb'))

	#pickle.dump(freq_list, open(sse.japanese_freqs_path, 'wb'))

	#pickle.dump(freq_list, open(sse.korean_freqs_path, 'wb'))

	counter_list = [uf(sse.english_freqs_path), uf(sse.french_freqs_path), uf(sse.spanish_freqs_path),
	uf(sse.japanese_freqs_path), uf(sse.korean_freqs_path), uf(sse.mandarin_freqs_path)]

	holder, counter_n = sse.makeCounter(counter_list)

	lang_prob_path = 'data/langprobs_Train.pickle'
	#problang_pairs = []
	#finding_counter = 0
	#print("Finding language probability pairs...")
	#lang_prob_time = time()
	#for data in datalist:
	#	finding_counter += 1
	#	print("Have checked %s entries at %s" % (finding_counter, (time()-lang_prob_time)))
	#	entry, suma, problang = sse.matchLangBySyntaxProb(holder, counter_n, data[sp.STUDYING], data[sp.ENTRY])
	#	if problang is not None:
	#		problang_pairs.append( (entry, suma, problang) )
	#		pickle.dump(problang_pairs, open(lang_prob_path, 'wb'))

	'''Second feature: error distance between words. USE NATIVE SETS.'''
	'''Can take anywhere from 10 seconds to 9 minutes depending on the language and the set.'''
	print("Gathering feature number two...")
	mandarin_pairs = sp.buildCorrectionPairs(mandarin_native)
	korean_pairs = sp.buildCorrectionPairs(korean_native)
	japanese_pairs = sp.buildCorrectionPairs(japanese_native)
	english_pairs = sp.buildCorrectionPairs(english_native)
	french_pairs = sp.buildCorrectionPairs(french_native)
	spanish_pairs = sp.buildCorrectionPairs(spanish_native)

	english_dist= ce.errorEditDistance(english_pairs)
	mandarin_dist = ce.errorEditDistance(mandarin_pairs)
	korean_dist = ce.errorEditDistance(korean_pairs)
	japanese_dist = ce.errorEditDistance(japanese_pairs)
	french_dist = ce.errorEditDistance(french_pairs)
	spanish_dist = ce.errorEditDistance(spanish_pairs)

	dist_list = [('English', english_dist), ('French', french_dist), ('Spanish', spanish_dist),
	('Japanese', japanese_dist), ('Korean', korean_dist), ('Mandarin', mandarin_dist)]

	#ce.transform_edit_dist(dist_list)

	'''Third feature: average token length per sentence. USE NATIVE SETS.'''
	'''Takes about 5 seconds to 3.5 minutes depending on the language.'''
	print("Gathering feature number three...")
	english_entries = sp.returnEntries(english_native)
	french_entries = sp.returnEntries(french_native)
	spanish_entries = sp.returnEntries(spanish_native)
	mandarin_entries = sp.returnEntries(mandarin_native)
	korean_entries = sp.returnEntries(korean_native)
	japanese_entries = sp.returnEntries(japanese_native)

	mandarin_token = sse.averageNumberOfTokens(mandarin_entries)
	korean_token = sse.averageNumberOfTokens(korean_entries)
	japanese_token = sse.averageNumberOfTokens(japanese_entries)
	english_token = sse.averageNumberOfTokens(english_entries)
	french_token = sse.averageNumberOfTokens(french_entries)
	spanish_token = sse.averageNumberOfTokens(spanish_entries)

	tokens_list = [('English', english_token), ('French', french_token), ('Spanish', spanish_token),
	('Japanese', japanese_token), ('Korean', korean_token), ('Mandarin', mandarin_token)]

	#sse.transform_token_count(tokens_list)
	train_data = sp.convertDataToList(train)
	train_entries, train_langs = sp.returnEntriesWithSpoken(train_data)

	pipeline = Pipeline([
		('union', FeatureUnion(
			transformer_list=[

			# Pipeline for feature #1
			('feature_1', Pipeline([
				('selector', FeatureGetter(uf(lang_prob_path))),
				('word_freqs', POSFeatures()),
				('tfidf', TfidfVectorizer()),
				])),
			# Pipeline for feature 2
			('feature_2', Pipeline([
				('selector', FeatureGetter(dist_list)),
				('edit_dist', CorrectionFeatures()),
				('tfidf', TfidfVectorizer()),
				])),
			# Pipeline for feature 3
			('feature_3', Pipeline([
				('selector', FeatureGetter(tokens_list)),
				('token_count', TokenFeatures()),
				('tfidf', TfidfVectorizer()),
				])),
			# Pipeline for bag of words feature
			('bag_of_words', Pipeline([
				('selector', FeatureGetter(train_entries)),
				('vect', CountVectorizer(ngram_range=(1,1), max_features=500)),
				('tfidf', TfidfTransformer(use_idf=True)),
				])),
			# Weights for the features
			],
			transformer_weights = {
			'feature_1': 1.5,
			'feature_2': 0.5,
			'feature_3': 0.5,
			'feature_4': 1.5,
			},
		)),
		# Decision tree used as classifier
		('clf', DecisionTreeClassifier(max_features=500)),
	])

pipeline_pickle_path = 'data/pipeline.pickle'
pickle.dump(pipeline, open(pipeline_pickle_path, 'wb'))
print("About to run the pipeline...")
train_data = sp.convertDataToList(sp.train)
test_data = sp.convertDataToList(sp.train)
train_entries, train_langs = sp.returnEntriesWithSpoken(train_data)
test_entries, test_langs = sp.returnEntriesWithSpoken(test_data)
pipeline.fit(train_entries, train_langs)
y = pipeline.predict(train_entries)
print(classification_report(y, test_langs))


# -*- coding:utf-8 -*-

from mxnet.gluon import data as gdata
import d2l

class FashionMnistDataLoader:
    def __init__(self, batch_size):
        self.mnist_train = gdata.vision.FashionMNIST(train=True)
        self.mnist_test = gdata.vision.FashionMNIST(train=False)

        print(f'len mnist_train : {len(self.mnist_train)}')
        print(f'len mnist_test : {len(self.mnist_test)}')

        features, label = self.mnist_train[0]
        print(f'check features data shape: {features.shape}, dtype: {features.dtype}')
        print(f'check label data: {label}, type: {type(label)}, dtype: {label.dtype}')

        self.train_iter, self.test_iter = d2l.load_data_fashion_mnist(batch_size)

    # This function has been saved in the d2l package for future use
    def get_fashion_mnist_labels(self, labels):
        text_labels = [
            't-shirt', 'trouser', 'pullover', 'dress', 'coat',
            'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot'
        ]
        return [text_labels[int(i)] for i in labels]

    # This function has been saved in the d2l package for future use
    def show_fashion_mnist(self, images, labels):
        d2l.use_svg_display()
        # Here _ means that we ignore (not use) variables
        _, figs = d2l.plt.subplots(1, len(images), figsize=(12, 12))

        for f, img, lbl in zip(figs, images, labels):
            f.imshow(img.reshape((28, 28)).asnumpy())
            f.set_title(lbl)
            f.axes.get_xaxis().set_visible(False)
            f.axes.get_yaxis().set_visible(False)


import subprocess
from debug import debugout

class Application(object):
    def  __init__(self, command, params):
        self.command = command
        self.proc = None
        self.params = params

    def start(self):
        assert self.proc == None, "application already started"

        cmdline = [self.command]
        for k, v in self.params:
            cmdline.append("-" + k)
            cmdline.append(str(v))

        debugout("starting application", cmdline)
        self.proc = subprocess.Popen(cmdline)

    def wait(self):
        debugout("waiting for application to quit")
        self.proc.wait()

    def kill(self):
        self.proc.kill()


# Code to measure mean fluorescence intensity (instead of spot count) from each ROI.
# The integrated intensity = mean intensity * size of ROI

import os, sys, zarr
import numpy as np
import pandas as pd
from glob import glob
from skimage.measure import regionprops
from skimage.io import imread, imsave

def measure_intensity(lb, img_dir, channel, scale='s2'):
	"""
    Returns spot counts for each ROI.

    lb: segmenntation mask
    img_dir: n5 image file to measure fluorescence intensity
	channel: image channnel, e.g. c0, c1
	scale: image channel, e.g. s0, s1. Default to 's2', 4x4x2 downsampled
    """
	lb_id = np.unique(lb[lb!= 0])
	roi=len(lb_id)
	if type(img_dir)==str:
		# get n5 image data
		im = zarr.open(store=zarr.N5Store(img_dir), mode='r')
		img = im[channel+'/'+scale][:,:,:]
		if channel == 'c3': #default c3 to the channel where bleed-through correction is needed, modify if it does not apply.
			dapi=im['c2/'+scale][...] #default c2 to DAPI channel, change if dapi is in a different channel
			lo=np.percentile(np.ndarray.flatten(dapi),99.5)
			bg_dapi=np.percentile(np.ndarray.flatten(dapi[dapi!=0]),1)
			bg_img=np.percentile(np.ndarray.flatten(img[img!=0]),1)
			dapi_factor=np.median((img[dapi>lo] - bg_img)/(dapi[dapi>lo] - bg_dapi))
			img = np.maximum(0, img - bg_img - dapi_factor * (dapi - bg_dapi)).astype('float32')
			print('bleed_through:',dapi_factor)
			print('DAPI background:',bg_dapi)
			print('c3 background:',bg_img)
		df_mean = pd.DataFrame(data=np.empty([roi,1]), columns=['roi'], dtype=object)
		lb_stat = regionprops(lb,intensity_image=img)
		for i in range(0,roi):
			df_mean.loc[i, 'roi'] = lb_stat[i].label
			df_mean.loc[i, channel] = lb_stat[i].mean_intensity
	else:
		print('Image file is in wrong format')

	return(df_mean)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
from sklearn import tree
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from functools import partial
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix, classification_report,accuracy_score, roc_auc_score, roc_curve, log_loss

# In[2]:

# https://archive.ics.uci.edu/ml/datasets/Bank+Marketing
# dataset using from UCI online
Bank_data = pd.read_csv("bank-full.csv",delimiter = ";")
Bank_data.head()

# In[3]:

Bank_data.dtypes

# In[4]:

cat_attr = Bank_data.columns[Bank_data.dtypes==object].tolist()
cat_attr

# In[5]:

for i in cat_attr:
    print('Unique values for ' + i)
    print(Bank_data[i].unique())
    print('')

# In[6]:

for col in cat_attr:
    plt.figure(figsize=(10,4))
    sns.barplot(Bank_data[col].value_counts().values, Bank_data[col].value_counts().index)
    plt.title(col)
    plt.tight_layout()

# In[7]:

Bank_data.corr()

# In[8]:

missing_values = Bank_data.isnull().mean()*100 # Checking Missing values
missing_values.sum()

# In[9]:

Bank_data[Bank_data.duplicated()].iloc[0:,:] #Checking Duplicate

# In[10]:

# Remove outliers using IQR
def IQR(df,x):
    q1 = df[x].quantile(0.25)
    q3 = df[x].quantile(0.75)
    iqr = q3 - q1
    fence_low = q1 - 1.5 * iqr
    fence_high = q3 + 1.5 * iqr
    cleaned_data = df.loc[(df[x] > fence_low) & (df[x] < fence_high)]
    return cleaned_data

# In[11]:

cleaned1 = IQR(Bank_data,"age")
cleaned1.shape

# In[12]:

cleaned1 = IQR(cleaned1, "campaign")
cleaned1.shape

# In[13]:

cleaned1 = IQR(cleaned1, "balance")
cleaned1.shape

# In[14]:

df = cleaned1.copy()
df
#Generating Dummy Variables for categorical data
df[cat_attr[:-1]] = df[cat_attr[:-1]].astype(str)

num_cattr = []
for i in df.columns:
    if i not in cat_attr:
        num_cattr.append(i)

num_cattr

# In[15]:

df[cat_attr[:-1]] = df[cat_attr[:-1]].astype(str)
df[num_cattr] = df[num_cattr].astype(float)

model_data = pd.get_dummies(df.iloc[:,:-1], drop_first=True).copy()
model_data.columns

# In[16]:

model_data = pd.concat([model_data,df.iloc[:,-1]],axis=1)
model_data.shape

# In[17]:

# Independent Variable
X = model_data.loc[:, ~model_data.columns.isin(['y'])]
#Dependent Variable
y = model_data.loc[:,'y'].copy()
# Dividing data in Training and testing Set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

X_train.shape, X_test.shape

# # Task: 2 Random Forest

# In[18]:

rforest=RandomForestClassifier(n_estimators=100,  random_state=100)
rforest.fit(X_train, y_train)

# In[19]:

preds = rforest.predict(X_test)

# In[20]:

round(accuracy_score(y_test, preds), 2) # Accuracy

# In[21]:

print(classification_report(y_test, preds)) # classification report of random forest

# # Task 3: Get probabilities and make new dataframe

# In[22]:

probabilities = rforest.predict_proba(X_train)
probabilities[:5] # Getting probabilities for training data

# In[23]:

probabilities_test = rforest.predict_proba(X_test)
probabilities_test[:5] # Getting probabilities for testing data

# In[24]:

# Both probabilities are in array so we need to convert them dataframe so we can concat them
probab_df = pd.DataFrame(probabilities)
probab_test_df = pd.DataFrame(probabilities_test)
probab_df.head()

# In[25]:

# Reset index to avoid NAN Value.
X_train.reset_index(drop=True, inplace=True)
probab_df.reset_index(drop=True, inplace=True)
y_train.reset_index(drop=True, inplace=True)

X_test.reset_index(drop=True, inplace=True)
probab_test_df.reset_index(drop=True, inplace=True)
y_test.reset_index(drop=True, inplace=True)

# Concat training data and training probabilities
full = pd.concat([X_train,probab_df,y_train], axis=1)
# Concat Testing data and training probabilities
full_test = pd.concat([X_test,probab_test_df,y_test], axis=1)
full_test.head()

# In[26]:

X_train_new = full.iloc[:,:-1] # New tarining data
X_test_new = full_test.iloc[:,:-1] # New Testing data
X_train_new.head()

# In[27]:

y_train_new = full.y # Training target
y_test_new = full_test.y # Testing target
X_test_new.shape, y_test_new.shape # Shape of testing dta

# # Task 4: Decision Tree Classifier On new Data

# In[28]:

dt = DecisionTreeClassifier()
dt.fit(X_train_new, y_train_new) # Using new data

# In[29]:

dt_pred = dt.predict(X_test_new)
round(accuracy_score(y_test_new, dt_pred), 2) # Accuracy

# In[30]:

print(classification_report(y_test_new, dt_pred))

# In[31]:

fig = plt.figure(figsize=(20,10))
tree.plot_tree(dt, filled=True)
plt.title('Decision Tree Diagram');

# In[32]:

# Make dictionary of all parameter
model_params = {
      'Decision Tree':{
          'model' : DecisionTreeClassifier(),
          'params': {
              'criterion': ['gini','entropy'],
              'splitter':['best', 'random'],
              'max_depth':[1,5,10,20,50,100],
              'max_features':['auto','sqrt','log2'],
          }
      }
}

# In[33]:

# Hyper parameter tunning using GridSearchCV
scores = [] # To make list of best features
for model_name, mp in model_params.items():
    clf_dt = GridSearchCV(mp['model'],mp['params'],cv = 5, return_train_score=False)
    clf_dt.fit(X_train_new,y_train_new)
    scores.append({
        'model': model_name,
        'best_score': clf_dt.best_score_,
        'best_params': clf_dt.best_params_
    })

# In[34]:

scores # List of our best parameter with best score

# In[35]:

preds_clfdt = clf_dt.predict(X_test_new)
round(accuracy_score(y_test_new, preds_clfdt), 2) # accuracy

# In[36]:

print(classification_report(y_test_new, preds_clfdt))

# # Task 5: Decision Tree with original data using best hyper parameter

# In[37]:

clf_dt.best_params_ # Now need to use these all parameter for build Decision tree

# In[38]:

dt_orig = DecisionTreeClassifier(criterion='gini',max_depth=50,max_features='sqrt',splitter='random')
dt_orig.fit(X_train,y_train) # original training and testing data

# In[39]:

dt_orig_pred = dt_orig.predict(X_test)
round(accuracy_score(y_test, dt_orig_pred), 2)

# In[40]:

print(classification_report(y_test,dt_orig_pred))

# # Task 6:

# In[43]:

print(f'Accuracy of Random foreston Original Data : {round(accuracy_score(y_test, preds), 2)}')
print(f'Accuracy of Decision Tree knowledge distillation from random forests : {round(accuracy_score(y_test_new, dt_pred), 2)}')
print(f'Accuracy of Decision Tree using Hyper parameter tunning :{round(accuracy_score(y_test_new, preds_clfdt), 2)}')
print(f'Accuracy of Decision Tree using Hyper parameter tunning  On original data :{round(accuracy_score(y_test_new, preds_clfdt), 2)}')

# In[ ]:

Assignment_files/Knowledge Distillation on Bank Marketing data.py


# -*- coding: utf-8 -*-
"""
Created on 2018/1/3 19:57

@author: JERRY
"""

import hashlib
import zlib
import json
from time import sleep
from threading import Thread
import ssl
import websocket
from datapool.general_config import bitmex_ws

basehost = bitmex_ws

########################################################################
class BitmexApi(object):
    """基于Websocket的API对象"""

    # ----------------------------------------------------------------------
    def __init__(self):
        """Constructor"""
        self.apiKey = ''  # 用户名
        self.secretKey = ''  # 密码
        self.host = ''  # 服务器地址
        self.currency = ''  # 货币类型（usd或者cny）
        self.ws = None  # websocket应用对象
        self.thread = None  # 工作线程
        global x

    #######################
    ## 通用函数
    #######################

    # ----------------------------------------------------------------------
    def readData(self, evt):
        """解压缩推送收到的数据"""
        # 创建解压器
        decompress = zlib.decompressobj(-zlib.MAX_WBITS)

        # 将原始数据解压成字符串
        inflated = decompress.decompress(evt) + decompress.flush()

        # 通过json解析字符串
        data = json.loads(inflated)

        return data

    # ----------------------------------------------------------------------
    def generateSign(self, params):
        """生成签名"""
        l = []
        for key in sorted(params.keys()):
            l.append('%s=%s' % (key, params[key]))
        l.append('secret_key=%s' % self.secretKey)
        sign = '&'.join(l)
        return hashlib.md5(sign.encode('utf-8')).hexdigest().upper()

    # ----------------------------------------------------------------------
    def onMessage(self, ws, evt):
        """信息推送"""
        print('onMessage')
        data = json.loads(evt)

        print(data)

    # ----------------------------------------------------------------------
    def onError(self, ws, evt):
        """错误推送"""
        print('onError')
        print(evt)

    # ----------------------------------------------------------------------
    def onClose(self, ws):
        """接口断开"""
        print('onClose')

    # ----------------------------------------------------------------------
    def onOpen(self, ws):
        """接口打开"""
        print('onOpen')

    # ----------------------------------------------------------------------
    def connect(self, trace=False):
        """连接服务器"""
        self.host = 'wss://www.bitmex.com/realtime'
        websocket.enableTrace(trace)
        self.ws = websocket.WebSocketApp(url = self.host,
                                         on_message=self.onMessage,
                                         on_error=self.onError,
                                         on_close=self.onClose,
                                         on_open=self.onOpen)

        self.thread = Thread(target=self.ws.run_forever)
        self.thread.start()

    # ----------------------------------------------------------------------
    def reconnect(self):
        """重新连接"""
        # 首先关闭之前的连接
        self.close()

        # 再执行重连任务
        self.ws = websocket.WebSocketApp(self.host,
                                         on_message=self.onMessage,
                                         on_error=self.onError,
                                         on_close=self.onClose,
                                         on_open=self.onOpen)

        self.thread = Thread(target=self.ws.run_forever(sslopt={"cert_reqs": False}))
        self.thread.start()

    # ----------------------------------------------------------------------
    def close(self):
        """关闭接口"""
        if self.thread and self.thread.isAlive:
            self.ws.close()
            self.thread.join()

    # ----------------------------------------------------------------------
    def sendMarketDataRequest(self, arglist):
        """发送行情请求"""
        # 生成请求
        d = {}
        d['op'] = 'subscribe'
        d['args'] = arglist

        # 使用json打包并发送
        j = json.dumps(d)

        # 若触发异常则重连
        try:
            self.ws.send(j)
        except websocket.WebSocketConnectionClosedException:
            pass


from tensorflow.keras.datasets import mnist
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import load_model  #←汇入 load_model 函式

#载入 MNIST 资料集
(_, _), (test_images, test_labels) = mnist.load_data()

#资料预处理
x_test = test_images.reshape((10000, 28 * 28)) #←将 10000 笔测试样本做同样的转换
x_test = x_test.astype('float32') / 255       #←将 10000 笔测试标签做同样的转换
y_test  = to_categorical(test_labels)

model = load_model('MnistModel.h5')  #← 由档案载入模型

test_loss, test_acc = model.evaluate(x_test, y_test) #←用测试资料评估成效
print('对测试资料的准确率：', test_acc)


months = (
    {"name": {"ru": ("январь", "января"), "en": ("january",)}, "value": 1},
    {"name": {"ru": ("февраль", "февраля"), "en": ("february",)}, "value": 2},
    {"name": {"ru": ("март", "марта"), "en": ("march",)}, "value": 3},
    {"name": {"ru": ("апрель", "апреля"), "en": ("april",)}, "value": 4},
    {"name": {"ru": ("май", "мая"), "en": ("may",)}, "value": 5},
    {"name": {"ru": ("июнь", "июня"), "en": ("june",)}, "value": 6},
    {"name": {"ru": ("июль", "июля"), "en": ("july",)}, "value": 7},
    {"name": {"ru": ("август", "августа"), "en": ("august",)}, "value": 8},
    {"name": {"ru": ("сентябрь", "сентября"), "en": ("september",)}, "value": 9},
    {"name": {"ru": ("октябрь", "октября"), "en": ("october",)}, "value": 10},
    {"name": {"ru": ("ноябрь", "ноября"), "en": ("november",)}, "value": 11},
    {"name": {"ru": ("декабрь", "декабря"), "en": ("december",)}, "value": 12},
)

genders = {"ru": ("Мужчина", "Женщина"), "en": ("Male", "Female")}
years_months = {"ru": ("год", "лет", "месяц"), "en": ("year", "month")}
born_on = {"ru": ("родился", "родилась"), "en": ("born on",)}
show_more = {"ru": "Показать еще", "en": "Show more"}
citizenship = {
    "citizenship": {"ru": "Гражданство", "en": "Citizenship"},
    "permission": {"ru": "Разрешение", "en": "Permission"},
    "commute": {"ru": "Время в пути ", "en": "Travel"},
}
own_car = {
    "has": {"ru": "Имеется собственный автомобиль", "en": "Own car"},
    "categories": {"ru": "Категории", "en": "Categories"},
}
willing = {"ru": "готов", "en": "willing"}
total = {"ru": "Общий стаж", "en": "Total"}
personal = {
    "title": {"ru": "Личные данные", "en": "Personal"},
    "birthday": {"ru": "Дата рождения", "en": "Date of birth"},
    "location": {"ru": "Место нахождения", "en": "Location"},
    "contacts": {"ru": "Контакты", "en": "Contacts"},
}

SECTION_TITLE_MAX = 40


from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ThreadPoolExecutor
from apscheduler.triggers.interval import IntervalTrigger
from apscheduler.triggers.date import DateTrigger
from apscheduler.triggers.cron import CronTrigger
from apscheduler.events import EVENT_SCHEDULER_SHUTDOWN, EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR, EVENT_JOB_EXECUTED
from django_redis import get_redis_connection
from django.utils.functional import SimpleLazyObject
from django.db import connections
from apps.schedule.models import Task, History
from apps.schedule.utils import send_fail_notify
from apps.notify.models import Notify
from apps.schedule.builtin import auto_run_by_day, auto_run_by_minute
from django.conf import settings
from libs import AttrDict, human_datetime
import logging
import json

SCHEDULE_WORKER_KEY = settings.SCHEDULE_WORKER_KEY

class Scheduler:
    timezone = settings.TIME_ZONE
    week_map = {
        '/': '/',
        '-': '-',
        '*': '*',
        '7': '6',
        '0': '6',
        '1': '0',
        '2': '1',
        '3': '2',
        '4': '3',
        '5': '4',
        '6': '5',
    }

    def __init__(self):
        self.scheduler = BackgroundScheduler(timezone=self.timezone, executors={'default': ThreadPoolExecutor(30)})
        self.scheduler.add_listener(
            self._handle_event,
            EVENT_SCHEDULER_SHUTDOWN | EVENT_JOB_ERROR | EVENT_JOB_MAX_INSTANCES | EVENT_JOB_EXECUTED
        )

    @classmethod
    def covert_week(cls, week_str):
        return ''.join(map(lambda x: cls.week_map[x], week_str))

    @classmethod
    def parse_trigger(cls, trigger, trigger_args):
        if trigger == 'interval':
            return IntervalTrigger(seconds=int(trigger_args), timezone=cls.timezone)
        elif trigger == 'date':
            return DateTrigger(run_date=trigger_args, timezone=cls.timezone)
        elif trigger == 'cron':
            args = json.loads(trigger_args) if not isinstance(trigger_args, dict) else trigger_args
            minute, hour, day, month, week = args['rule'].split()
            week = cls.covert_week(week)
            return CronTrigger(minute=minute, hour=hour, day=day, month=month, day_of_week=week,
                               start_date=args['start'], end_date=args['stop'])
        else:
            raise TypeError(f'unknown schedule policy: {trigger!r}')

    def _handle_event(self, event):
        obj = SimpleLazyObject(lambda: Task.objects.filter(pk=event.job_id).first())
        if event.code == EVENT_SCHEDULER_SHUTDOWN:
            logging.warning(f'EVENT_SCHEDULER_SHUTDOWN: {event}')
            Notify.make_notify('schedule', '1', '调度器已关闭', '调度器意外关闭，你可以在github上提交issue')
        elif event.code == EVENT_JOB_MAX_INSTANCES:
            logging.warning(f'EVENT_JOB_MAX_INSTANCES: {event}')
            send_fail_notify(obj, '达到调度实例上限，一般为上个周期的执行任务还未结束，请增加调度间隔或减少任务执行耗时')
        elif event.code == EVENT_JOB_ERROR:
            logging.warning(f'EVENT_JOB_ERROR: job_id {event.job_id} exception: {event.exception}')
            send_fail_notify(obj, f'执行异常：{event.exception}')
        connections.close_all()

    def _init_builtin_jobs(self):
        self.scheduler.add_job(auto_run_by_day, 'cron', hour=1, minute=20)
        self.scheduler.add_job(auto_run_by_minute, 'interval', minutes=1)

    def _dispatch(self, task_id, command, targets):
        output = {x: None for x in targets}
        history = History.objects.create(
            task_id=task_id,
            status='0',
            run_time=human_datetime(),
            output=json.dumps(output)
        )
        Task.objects.filter(pk=task_id).update(latest_id=history.id)
        rds_cli = get_redis_connection()
        for t in targets:
            rds_cli.rpush(SCHEDULE_WORKER_KEY, json.dumps([history.id, t, command]))

    def _init(self):
        self.scheduler.start()
        self._init_builtin_jobs()
        for task in Task.objects.filter(is_active=True):
            trigger = self.parse_trigger(task.trigger, task.trigger_args)
            self.scheduler.add_job(
                self._dispatch,
                trigger,
                id=str(task.id),
                args=(task.id, task.command, json.loads(task.targets)),
            )

    def run(self):
        rds_cli = get_redis_connection()
        self._init()
        rds_cli.delete(settings.SCHEDULE_KEY)
        logging.warning('Running scheduler')
        while True:
            _, data = rds_cli.brpop(settings.SCHEDULE_KEY)
            task = AttrDict(json.loads(data))
            if task.action in ('add', 'modify'):
                trigger = self.parse_trigger(task.trigger, task.trigger_args)
                self.scheduler.add_job(
                    self._dispatch,
                    trigger,
                    id=str(task.id),
                    args=(task.id, task.command, task.targets),
                    replace_existing=True
                )
            elif task.action == 'remove':
                job = self.scheduler.get_job(str(task.id))
                if job:
                    job.remove()


from datetime import datetime

from simulator import *
from agent import *
from expert import *

sim = Simulator(
        agent_cls = EG,
        expert_cls = MeanReversion,
        start_date = datetime(2000, 1, 1),
        end_date = datetime(2017, 8, 31))

sim.setup(
        data_dir = "../data/djia_test_small/",
        agent_args = {
            "eta": 0.05
            },
        expert_args = {
            "n_obs": 10,
            "threshold": 1.5
            })

sim.run()
sim.generate_summary()


from django.conf.urls import url
from app import views

# 路由分发层 Router - HTTP
#   函数名尽量与url对应，或者遵守一定的命名约定只要便于管理即可
#   注意url从上到下的匹配顺序
#

urlpatterns = [
    url(r'^user$', views.user),
    url(r'^user/login$', views.user_login),
    url(r'^user/logout$', views.user_logout),
    url(r'^user/register$', views.user_register),
    url(r'^user/keepalive$', views.user_keepalive),
    url(r'^tag$', views.tag),
    url(r'^q$', views.q),
    url(r'^a$', views.a),

    url(r'^log$', views.log),
    url(r'^test$', views.test),
    url(r'^index$', views.index),
    url(r'^$', views.index),
]


import argparse
import json

# google sheets imports
import gspread
from oauth2client.service_account import ServiceAccountCredentials

# sendgrid imports
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Bcc, Mail

# markdown
import markdown2 as m

def load_config():
    with open("maillist.json", "r") as fp:
        return json.load(fp)

def client(cred_file_path):
    scope = [
        "https://spreadsheets.google.com/feeds",
        "https://www.googleapis.com/auth/drive",
    ]
    credentials = ServiceAccountCredentials.from_json_keyfile_name(
        cred_file_path, scope
    )
    gc = gspread.authorize(credentials)
    return gc

def spreadsheet(client, name):
    return client.open(name)

def emails(worksheet):
    for row in worksheet.get_all_records():
        if row["Email Address"]:
            yield row["Email Address"]

def render(fp):
    return m.markdown(fp.read())

def send_email(from_addr, subject, html_content, to_emails, api_key):
    sg = SendGridAPIClient(api_key)
    unsubscribe = "Unsubscribe".format(from_addr)
    for counter, email in enumerate(to_emails):
        message = Mail(
            subject=subject,
            from_email=from_addr,
            html_content=f"{html_content}\n{unsubscribe}",
            to_emails=[email],
        )
        try:
            resp = sg.send(message)
            print("{} emails sent".format(counter + 1))
        except Exception as exc:
            print("Could not send email to {} due {}".format(email, str(exc)))

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--markdown",
        "-m",
        required=True,
        help="Markdown content for post.",
        type=argparse.FileType("r"),
    )
    parser.add_argument("--subject", "-s", required=True, help="Subject for the post.")

    # cli arguments
    args = parser.parse_args()

    # config values
    config = load_config()
    api_key = config["SENDGRID_API_KEY"]
    cred_file_path = config["GOOGLE_CRED_FILE"]
    spreadsheet_name = config["SPREADSHEET_NAME"]
    from_address = config["FROM_ADDRESS"]

    gc = client(cred_file_path)
    ss = spreadsheet(gc, spreadsheet_name)
    wks = ss.sheet1

    to_emails = list(emails(wks))
    html_content = render(args.markdown)

    send_email(from_address, args.subject, html_content, to_emails, api_key)


from json import load, dump
from numpy import random, sqrt, array
from os import mkdir
from os.path import exists

def ramdomPesos(x, y):
    pesos = dict()
    pesos = random.uniform(-1.,1.,size=(x,y))/sqrt(x*y)
    aux = dict()
    for i in range(len(pesos)):
        aux[i] = list(pesos[i])
    return aux

def JsontoArray(pesos):
    for i in pesos:
        aux = list()
        for j in range(len(pesos[i])):
            aux.append(pesos[i][str(j)])
        pesos[i] = array(aux)
    return pesos

def DicttoArray1(pesos):
    for i in pesos:
        aux = list()
        for j in range(len(pesos[i])):
            aux.append(pesos[i][j])
        pesos[i] = array(aux)
    return pesos

def ArraytoDict(pesos):
    for i in pesos:
        aux = dict()
        for j in range(len(pesos[i])):
            aux[j] = list(pesos[i][j])
        pesos[i] = aux
    return pesos

class Pesos:
    entrada = 100
    salida = 3

    def __init__(self, oculta, nDataset):
        self.oculta = oculta
        self.setPath(str(nDataset))
        self.cargarPesos()

    def setPesosValidacion(self):
        self.pesosValidacion = self.pesos

    def setPath(self, nDataset):
        path = 'pesos\\'+nDataset
        if not exists(path):
            mkdir(path)
        path = path+"\\"+str(self.entrada)+"-"
        for i in self.oculta:
            path = path + str(i)+"-"
        self.path = path + str(self.salida) + '.json'
        self.pathValidacion = path + str(self.salida) + 'Validacion.json'

    def cargarPesos(self):
        if exists(self.path):
            with open(self.path, 'r') as file_json:
                self.pesos = load(file_json)
            self.pesos = JsontoArray(self.pesos)
        else:
            self.definirPesos()

        if exists(self.pathValidacion):
            with open(self.pathValidacion, 'r') as file_json:
                self.pesosValidacion = load(file_json)
            self.pesosValidacion = JsontoArray(self.pesosValidacion)

    def definirPesos(self):
        self.pesos = dict()
        self.pesos['0'] = ramdomPesos(self.entrada, self.oculta[0])
        x=0
        for i in range((len(self.oculta)-1)):
            self.pesos[str(i+1)] = ramdomPesos(self.oculta[i], self.oculta[i+1])
            x = i+1
        self.pesos[str(x+1)] = ramdomPesos(self.oculta[-1], self.salida)
        self.guardarJson()

    def guardarJson(self, validacion = False):
        if validacion:
            self.pesosValidacion = ArraytoDict(self.pesosValidacion)
            with open(self.pathValidacion, 'w') as file_json:
                dump(self.pesosValidacion, file_json)
            self.pesosValidacion = DicttoArray1(self.pesosValidacion)

        else:
            self.pesos = ArraytoDict(self.pesos)
            with open(self.path, 'w') as file_json:
                dump(self.pesos, file_json)
            self.pesos = DicttoArray1(self.pesos)


import sys

n = int(input())
s = []

for _ in range(n):
    temp = sys.stdin.readline().strip()
    s.append((temp, len(temp)))

s = list(set(s))
s.sort(key=lambda x: (x[1], x[0]))
for i in s:
    print(i[0])


# 用DP解决双十一问题

items = [2, 2, 4, 6, 3]
lowest = 200          # capacity

def double_11_dp(lowest, items):
    # define a matrix to store all the states, shape: num * (weight+1)
    n = len(items)
    weight = 3 * lowest + 1

    states = [[False] * weight for _ in range(n)]

    # initialize the first row for the first item(0-th item)
    states[0][0] = True
    if items[0] <= weight:
        states[0][items[0]] = True

    # iteration to the next item
    for i in range(1, n):
        for j in range(weight+1):
            if states[i-1][j]:
                # NOT add the i-th item
                states[i][j] = True
                # Add the i-th item
                if j + items[i] <= weight:
                    states[i][j+items[i]] = True

    # check the last line state: states[num-1]
    for j in range(lowest, weight, 1):
        if states[len(items)-1][j] == True:
            break
    else:
        return       # no solutions

    for i in range(len(items)-1, 1, -1):
        if j - items[i] >= 0 and states[i-1][j-items[i]] == True:
            print("Bought: {}".format(items[i]))
            j = j - items[i]
        else:
            pass        # did not buy the i-th item
    # check the first row -> the first item, buy or not to buy
    if j != 0:
        print("Bought: {}".format(items[0]))

double_11_dp(lowest, items)


"""
Write a function that takes in a SORTED array of distinct ints as well as a target int. The caveat is that the ints in the array have been shifted by some amount; in other words, they've been moved to the left or to the right by one or more positions. EX--[1, 2, 3, 4] might have turned into [3, 4, 1, 2].

The function should use a variation of the Binary Search algorithm to determine if the target int is contained in the array and should return its index if it is, otherwise -1.

Input:
    array = [45, 61, 71, 72, 73, 0, 1, 21, 33, 37]
    target = 33
Output:
    8

answer - O(N^2)T / O(N)S
"""
import time

def shiftedBinarySearch(arr, target):
    """
    I must not understand this problem cause it seems easy.
    -write the binary search
    -do a little extra math while passing the subarray info
    -slice = [0:end]... and indices = real idx list
    -after calculating middle, calc its idx based on indices
    -or you could have an additional array whose length matches
     the array and stores indices as values, then pass those slices, too

    * you could start by dividing the arr in 2?
    * and deal with each half separately
    """
    # Store the original indices
    indices = [x for x in range(len(arr))]

    return recur_binary_search(arr, indices, target)

def recur_binary_search(arr, indices, target):
    # I'm making:
    #     -left/right/middle pointers
    #     -a subarray from the target eval
    #     -a new indices from that
    # Can I pass the original array? Then pass the L/R indices instead?
    left_idx = 0
    right_idx = len(arr) - 1
    middle_idx = (left_idx + right_idx) // 2

    # Lazy
    if len(arr) == 2:
        if target == arr[0]: return indices[0]
        elif target == arr[1]: return indices[1]
        else: return -1
    elif len(arr) == 1: return indices[0] if arr[0] == target else -1
    elif len(arr) == 0: return -1
    if arr[middle_idx] == target: return indices[middle_idx]

    # One of these is sorted
    if arr[left_idx] < arr[middle_idx]:
        if target >= arr[left_idx] and target <= arr[middle_idx]:
            subarray, indices = get_left(arr, indices, left_idx, middle_idx)
        else:
            subarray, indices = get_right(arr, indices, middle_idx, right_idx)
    else:
        if target >= arr[middle_idx] and target <= arr[right_idx]:
            subarray, indices = get_right(arr, indices, middle_idx, right_idx)
        else:
            subarray, indices = get_left(arr, indices, left_idx, middle_idx)

    return recur_binary_search(subarray, indices, target)

def get_left(arr, indices, left_idx, middle_idx):
    new_arr = arr[left_idx:middle_idx+1]
    new_ind = indices[left_idx:middle_idx+1]
    return new_arr, new_ind

def get_right(arr, indices, middle_idx, right_idx):
    new_arr = arr[middle_idx:right_idx+1]
    new_ind = indices[middle_idx:right_idx+1]
    return new_arr, new_ind

def recur_binary_search_orig(arr, target_value):
    # This works on sorted array
    # But is NOT totally the Binary Search method
    # strictly it's Lidx + Ridx // 2 = middle
    middle_idx = len(arr) // 2
    if len(arr) == 0:
        return -1
    elif arr[middle_idx] == target_value:
        return arr[middle_idx], arr.index(target_value)
    else:
        if arr[middle_idx] < target_value:
            arr_slice = arr[middle_idx + 1:]
        elif arr[middle_idx] > target_value:
            arr_slice = arr[0:middle_idx]
        return recur_binary_search(arr_slice, target_value)

if __name__ == "__main__":

    array = [44, 45, 61, 71, 72, 73, 0, 1, 21, 33, 37]
    target = 33

    print(shiftedBinarySearch(array, target))


# import libraries
import argparse
import logging
import os
import pickle
import random
import numpy as np
import torch
import pandas as pd
from tqdm import tqdm

from transformers import (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer)
from vae_gpt2_model import VAE_config, VAE_GPT2

logger = logging.getLogger(__name__)
MODEL_CLASSES = {
    'gpt2': (GPT2Config, GPT2LMHeadModel, GPT2Tokenizer)}

"""====================== CLASSES AND METHODS DEFINITIONS ======================"""

def set_seed(args):
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

# truncating/padding method
def truncating_padding_sentence(tokens, block_size):
    if (len(tokens) > block_size):
        original_tokens_len = block_size
        tokens = tokens[:block_size]
    else:
        original_tokens_len = len(tokens)
        tokens = tokens + ["[PAD]"]*(block_size - len(tokens))
    return tokens, original_tokens_len
# creating attention mask method
def create_attention_mask(sentence_length, seq_length, gpt2_config, mask_type = "encoder_mask"):
    # args:
    #   sentence_length is length of real text, from [SOS] to <|endoftext|>
    #   seq_length is length with [PAD] (32, 64, 128, ...)

    if mask_type == "encoder_mask":
        mask_one_head = np.zeros([seq_length, seq_length])
        mask_one_head[:, :sentence_length] = 1  # the attention of [PAD] is also the input sentence
        mask_all_heads = [mask_one_head] * gpt2_config.n_head
        mask_all_heads = np.array(mask_all_heads)
    if mask_type == "decoder_mask":
        # attention, the triangular matrix is [seq_length,seq_length+1] becasuse the original has one more "past" token
        mask_one_head = np.tril(np.ones([seq_length,seq_length+1]),1)
        mask_all_heads = [mask_one_head] * gpt2_config.n_head
        mask_all_heads = np.array(mask_all_heads)
    return mask_all_heads

def convert_tensor_inference(model, sentence_embedding, args, device, current_generated_sentences):

    # convert to tensor and put on device
    sentence_embedding_converted = torch.FloatTensor(sentence_embedding).unsqueeze(0).to(device)

    # generate sentence
    generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = sentence_embedding_converted, args = args, device = device)
    generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
    first_endoftext = generated_sample.find("<|endoftext|>")
    generated_sample = str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)])
    count = 1
    while ((generated_sample in current_generated_sentences) and (count<10)):
        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = sentence_embedding_converted, args = args, device = device)
        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
        first_endoftext = generated_sample.find("<|endoftext|>")
        generated_sample = str(generated_sample[:(first_endoftext)])
        count += 1

    current_generated_sentences.append(generated_sample)

    ## DEBUGGING
    # logger.info("len(current_generated_sentences): " + str(len(current_generated_sentences)))
    # logger.info("current_generated_sentences: " + str(current_generated_sentences))
    # logger.info("generated_sample: " + str(generated_sample))
    # print("count+: "+ str(count))

    # print generated sentence sample
    generated_sample = generated_sample[6: ]
    print(str(generated_sample))

    return current_generated_sentences

def inference_test_1(model, args, device):
    print(" === inference_test_1 ===")
    print("Task 1: generate text from input sentence")

    # reading file
    sentences = []
    with open(args.sentences_file, encoding="utf-8") as file_p:
        for sentence in file_p:
            sentences.append(sentence)

    # tokenizing, truncating/padding
    tokenized_sentences = []
    converted_tokenized_sentences = []
    attention_mask_sentences = []
    for sentence in sentences:
        sentence_tokenized = model.tokenizer.tokenize(sentence)
        # encoder_input
        encoder_input = sentence_tokenized
        encoder_input, encoder_input_len = truncating_padding_sentence(encoder_input, args.block_size)
        encoder_input = model.tokenizer.convert_tokens_to_ids(encoder_input)
        encoder_input = np.array(encoder_input)
        # encoder_attention_mask
        encoder_attention_mask = create_attention_mask(encoder_input_len, args.block_size, args.gpt2_config, "encoder_mask")

        tokenized_sentences.append(truncating_padding_sentence(sentence_tokenized, args.block_size))
        converted_tokenized_sentences.append(encoder_input)
        attention_mask_sentences.append(encoder_attention_mask)

    # get average embeddings and inferring
    for i in range(len(sentences)):
        sentence = sentences[i]
        sentence_tokenized = tokenized_sentences[i]
        converted_tokenized_sentence = converted_tokenized_sentences[i]
        attention_mask_sentence = attention_mask_sentences[i]
        sentence_tokenized = model.tokenizer.decode(model.tokenizer.convert_tokens_to_ids(sentence_tokenized), clean_up_tokenization_spaces=True)
        converted_tokenized_sentence = torch.tensor(converted_tokenized_sentence).long().unsqueeze(0).to(device) # adding batch_size dimension
        attention_mask_sentence = torch.tensor(attention_mask_sentence).long().unsqueeze(0).to(device) # adding batch_size dimension

        sentence_embedding, z = model.get_embedding(converted_tokenized_sentence, attention_mask_sentence, device)

        ## DEBUGGING
        # logger.info("converted_tokenized_sentence_1: " + str(converted_tokenized_sentence_1.shape))
        # logger.info("attention_mask_sentence_1: " + str(attention_mask_sentence_1.shape))
        # logger.info("sentence_1_embedding: " + str(len(sentence_1_embedding)))
        # logger.info("sentence_1_embedding[0]: " + str(sentence_1_embedding[0].shape))
        # logger.info("average_embedding: " + str(average_embedding[0][:5]))

        # inferring from average embedding
        generated, decoder_attentions = model.inference(sentence_embedding = sentence_embedding, args = args, device = device)
        generated_sentence =  model.tokenizer.decode(generated[0].tolist(), clean_up_tokenization_spaces=True)

        print("=====")
        print("input_sentence: " + str(sentence_tokenized))
        first_endoftext = generated_sentence.find("<|endoftext|>")
        print("generated_sentence: " + str(generated_sentence[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sentence)]))

    return

def inference_test_2(model, args, device):
    print(" === inference_test_2 ===")
    print("Task 2: generate text from each dimension")

    # ==== Compute the embeddings
    # check cached file
    sentences_directory, sentences_filename = os.path.split(args.sentences_file)
    model_directory = args.output_dir
    cache_embeddings_file = os.path.join(model_directory, 'cached_inference_' + sentences_filename)
    logger.info("cache_embeddings_file: " + cache_embeddings_file)
    if os.path.exists(cache_embeddings_file) and not args.overwrite_cache:
        logger.info("Loading features from cached file %s", cache_embeddings_file)
        with open(cache_embeddings_file, 'rb') as handle:
            # sentences_texts, sentences_embeddings = pickle.load(handle)
            sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)

    # if no cached file
    else:
        # reading file
        print("reading input file.")
        sentences = []
        with open(args.sentences_file, encoding="utf-8") as file_p:
            for count, sentence in enumerate(file_p):
                sentence = sentence.strip("\n")
                sentences.append(sentence)

        # tokenizing, truncating/padding
        print("tokenizing and processing.")
        converted_tokenized_sentences = []
        attention_mask_sentences = []
        for sentence in sentences:
            sentence_tokenized = model.tokenizer.tokenize(sentence)
            # encoder_input
            encoder_input = sentence_tokenized
            encoder_input, encoder_input_len = truncating_padding_sentence(encoder_input, args.block_size)
            encoder_input = model.tokenizer.convert_tokens_to_ids(encoder_input)
            encoder_input = np.array(encoder_input)
            # encoder_attention_mask
            encoder_attention_mask = create_attention_mask(encoder_input_len, args.block_size, args.gpt2_config, "encoder_mask")

            converted_tokenized_sentences.append(encoder_input)
            attention_mask_sentences.append(encoder_attention_mask)

        # get embeddings
        batch_size = args.batch_size
        print("get embeddings with batch size {}.".format(str(batch_size)))
        sentences_ids = []
        sentences_embeddings = []
        sentences_texts = []
        for i in tqdm(list(range(0, len(sentences), batch_size))):
            sentences_text_batch = sentences[i:min(i+batch_size, len(sentences))]
            converted_tokenized_sentence = converted_tokenized_sentences[i:min(i+batch_size, len(sentences))]
            attention_mask_sentence = attention_mask_sentences[i:min(i+batch_size, len(sentences))]
            # convert to tensor, move to device
            converted_tokenized_sentence = torch.tensor(converted_tokenized_sentence).long().to(device)
            attention_mask_sentence = torch.tensor(attention_mask_sentence).long().to(device)
            # get embedding
            sentence_embedding, z = model.get_embedding(converted_tokenized_sentence, attention_mask_sentence, device)
            # append
            if sentences_embeddings == []:
                sentences_ids = list(range(i,min(i+batch_size, len(sentences))))
                sentences_embeddings = sentence_embedding.detach().cpu().numpy()
                sentences_texts = sentences_text_batch
            else:
                sentences_ids.extend(list(range(i,min(i+batch_size, len(sentences)))))
                sentences_embeddings = np.concatenate((sentences_embeddings, sentence_embedding.detach().cpu().numpy()), axis = 0)
                sentences_texts.extend(sentences_text_batch)
        logger.info("sentences_embeddings.shape: " + str(sentences_embeddings.shape))

        # save to file
        with open(cache_embeddings_file, 'wb') as handle:
            pickle.dump([sentences_texts, sentences_embeddings], handle)
            logger.info("dump file to: {}, number of records: {}, file size: {} MB".format(cache_embeddings_file, str(len(sentences_embeddings)), str(os.path.getsize(cache_embeddings_file)/pow(1024,2))))

        # save to file
        with open(cache_embeddings_file, 'wb') as handle:
            pickle.dump([sentences_ids, sentences_texts, sentences_embeddings], handle)
            logger.info("dump file to: {}, number of records: {}, file size: {}".format(cache_embeddings_file, str(len(sentences_embeddings)), str(os.path.getsize(cache_embeddings_file))))
            # save as csv file
            cache_embeddings_file_csv, file_extension = os.path.splitext(cache_embeddings_file)
            cache_embeddings_file_csv += '.csv'
            logger.info("also save to csv file: {}".format(cache_embeddings_file_csv))
            text_df = pd.DataFrame(sentences_texts)
            sentences_ids = [str(sentences_id) for sentences_id in sentences_ids]
            text_df.index = sentences_ids
            text_df.insert(0, 'message_id', sentences_ids)
            text_df.columns = ["message_id", "message"]
            embeddings_df = pd.DataFrame(sentences_embeddings)
            embeddings_df.index = sentences_ids
            embeddings_df.columns = ["COMPONENT_" + str(i) for i in range(sentences_embeddings.shape[1])]
            df = text_df.merge(embeddings_df, left_on = text_df.index, right_on = embeddings_df.index)
            df = df.drop(columns=['key_0'])
            df.message_id = df.message_id.astype(str)
            df.to_csv(cache_embeddings_file_csv, index = False)

    # ==== Analyzing the embeddings
    # method_1 for testing_2
    if args.method == "method_1":
        # method 1 (just use the embedding)
        # analyzing, find std and mean of each hidden dimension
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)

        # set parameters
        std_steps = 5
        std_step_interval = 0.5
        std_random_level = 0
        print("UPDATED!")

        # generate sentence for each hidden dimension
        hidden_size = means.shape[0]
        for i in range(hidden_size):

            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            # explore the positive direction
            for step in range(std_steps, -1, -1):
                std_position = (step+1)*std_step_interval

                print("samples around mean + {}*std:".format(std_position))
                print("generation avoid repeating!")
                generated_samples = []   # avoid repeated generated_sample
                for _ in range(args.generate_num):

                    # sample embedding around embedding + std_position*stds[i]
                    epsilon = np.random.uniform(-std_random_level,std_random_level)
                    embedding_sample = np.copy(means)
                    embedding_sample[i] = embedding_sample[i] + std_position*stds[i] + epsilon*stds[i]
                    embedding_sample = torch.tensor(embedding_sample, device = device).unsqueeze(0)

                    # generate sentence
                    generatied_count = 0
                    while True:
                        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = embedding_sample, args = args, device = device)
                        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
                        generatied_count += 1
                        first_endoftext = generated_sample.find("<|endoftext|>")
                        generated_sample_clean = generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]
                        if (generated_sample_clean not in generated_samples) or generatied_count >= 10:
                            generated_samples.append(generated_sample_clean)
                            break

                    # print generated sentence sample
                    first_endoftext = generated_sample.find("<|endoftext|>")
                    print("generated_sample: " + str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]))

            # explore the negative direction
            for step in range(0, std_steps + 1, 1):
                std_position = (step+1)*std_step_interval

                print("samples around mean - {}*std:".format(std_position))
                for _ in range(args.generate_num):

                    # sample embedding around embedding - std_position*stds[i]
                    epsilon = np.random.uniform(-std_random_level,std_random_level)
                    embedding_sample = np.copy(means)
                    embedding_sample[i] = embedding_sample[i] - std_position*stds[i] + epsilon*stds[i]
                    embedding_sample = torch.tensor(embedding_sample, device = device).unsqueeze(0)

                    # generate sentence
                    generatied_count = 0
                    while True:
                        generated_sample, decoder_attentions_sample = model.inference(sentence_embedding = embedding_sample, args = args, device = device)
                        generated_sample =  model.tokenizer.decode(generated_sample[0].tolist(), clean_up_tokenization_spaces=True)
                        generatied_count += 1
                        first_endoftext = generated_sample.find("<|endoftext|>")
                        generated_sample_clean = generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]
                        if (generated_sample_clean not in generated_samples) or generatied_count >= 10:
                            generated_samples.append(generated_sample_clean)
                            break

                    # print generated sentence sample
                    first_endoftext = generated_sample.find("<|endoftext|>")
                    print("generated_sample: " + str(generated_sample[:(first_endoftext + 13) if first_endoftext>0 else len(generated_sample)]))

    # method_2 for testing_2
    if args.method == "method_2":

        # sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)
        df = pd.DataFrame(np.array(sentences_embeddings))
        df.columns = [str(latent_dim) for latent_dim in range(len(sentences_embeddings[0]))]
        df.index = sentences_ids
        df['text'] = sentences_texts
        print(df.columns)
        print(df.index[:5])

        # compute *std score
        hidden_size = len(sentences_embeddings[0])
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)
        for i in range(hidden_size):
            df['*std_' + str(i)] = round((df[str(i)] - means[i])/stds[i],2)

        ## get unique sentence with highest score for each hidden dimension
        print("(=)(=)(=) unique sentence highest score (=)(=)(=)")
        for i in range(hidden_size):
            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            top_count = 10
            print("(*) top max score: ")
            sorted_df = df.sort_values(by=[str(i)], ascending=False)
            printed_sentences = []
            for index, row in sorted_df.iterrows():
                if row['text'] not in printed_sentences:
                    count = len(sorted_df[sorted_df['text'] == row['text']])
                    print("{} - score: {} - count: {}".format(row['text'], round(row[str(i)],2), count))
                    printed_sentences.append(row['text'])
                if len(printed_sentences)== top_count:
                    break

            print("(*) top min score: ")
            sorted_df = df.sort_values(by=[str(i)], ascending=True)
            printed_sentences = []
            for index, row in sorted_df.iterrows():
                if row['text'] not in printed_sentences:
                    count = len(sorted_df[sorted_df['text'] == row['text']])
                    print("{} - *std score: {} - count: {}".format(row['text'], round(row['*std_' + str(i)],2), count))
                    printed_sentences.append(row['text'])
                if len(printed_sentences)== top_count:
                    break

    # method_3 for testing_2
    if args.method == "method_3":

        # sentences_ids, sentences_texts, sentences_embeddings = pickle.load(handle)
        df = pd.DataFrame(np.array(sentences_embeddings))
        df.columns = [str(latent_dim) for latent_dim in range(len(sentences_embeddings[0]))]
        df.index = sentences_ids
        df['text'] = sentences_texts
        print(df.columns)
        print(df.index[:5])

        # compute *std score
        hidden_size = len(sentences_embeddings[0])
        means = np.mean(sentences_embeddings, axis = 0)
        stds = np.std(sentences_embeddings, axis = 0)
        for i in range(hidden_size):
            df['*std_' + str(i)] = round((df[str(i)] - means[i])/stds[i],2)

        ## get sentence with highest frequency for each hidden dimension
        print("(=)(=)(=) sentence highest frequency (=)(=)(=)")
        for i in range(hidden_size):
            print("=====")
            print("HIDDEN DIMENISON " + str(i) + ":")

            top_count = 8
            std_threshold = 1
            print("(*) top highest frequency above {}*std ".format(str(std_threshold)))
            df_pass_threshold = df.loc[df[str(i)] >= (means[i] + std_threshold*stds[i])]
            df_pass_threshold['text_std'] = df_pass_threshold['text'] + '    ' + df_pass_threshold[('*std_' + str(i))].astype(str)
            print(df_pass_threshold['text_std'].value_counts()[:top_count])
            print("(*) top highest frequency below -{}*std: ".format(str(std_threshold)))
            df_pass_threshold = df.loc[df[str(i)] <= (means[i] - std_threshold*stds[i])]
            df_pass_threshold['text_std'] = df_pass_threshold['text'] + '    ' + df_pass_threshold[('*std_' + str(i))].astype(str)
            print(df_pass_threshold['text_std'].value_counts()[:top_count])

            df_pass_threshold.groupby(['text', '*std_' + str(i)])

    return

"""====================== MAIN FUNCTION ======================"""

# main function
def main():
    parser = argparse.ArgumentParser()

    # dataset/save path parameters
    parser.add_argument("--sentences_file", default=None, type=str,
                        help="An optional input generate data file.")
    parser.add_argument('--overwrite_cache', action='store_true',
                        help="Overwrite the cached training and evaluation sets")
    parser.add_argument("--primals_file", default=None, type=str,
                        help="An optional input generate data file.")
    parser.add_argument("--output_dir", default=None, type=str, required=False,
                        help="The output directory where the model predictions and checkpoints will be written.")

    # model parameters
    parser.add_argument("--gpt2_model_type", default="bert", type=str,
                        help="The model architecture to be fine-tuned.")
    parser.add_argument("--gpt2_model_name_or_path", default="bert-base-cased", type=str,
                        help="The model checkpoint for weights initialization.")
    parser.add_argument("--latent_size", default=-1, type=int, required=True,
                        help="Size of latent VAE layer.")
    parser.add_argument("--do_lower_case", action='store_true',
                        help="Set this flag if you are using an uncased model.")

    # generating parameters
    parser.add_argument("--inference_test", default=0, type=int)
    parser.add_argument("--method", type=str, default=None)
    parser.add_argument("--generate_num", type=int, default=None)
    parser.add_argument("--generate_length", type=int, default=None)
    parser.add_argument("--temperature", type=float, default=1.0)
    parser.add_argument("--top_k", type=int, default=0)
    parser.add_argument("--top_p", type=float, default=0.9)
    parser.add_argument("--block_size", default=-1, type=int,
                        help="Optional input sequence length after tokenization. The training dataset will be truncated in block of this size for training. Default to the model max input length for single sentence inputs (take into account special tokens).")
    parser.add_argument("--batch_size", type=int, default=None)

    # other generating parameters
    parser.add_argument("--no_cuda", action='store_true',
                        help="Avoid using CUDA when available")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")

    # other parameters
    parser.add_argument('--fp16', action='store_true',
                        help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
    parser.add_argument('--fp16_opt_level', type=str, default='O1',
                        help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
                             "See details at https://nvidia.github.io/apex/amp.html")
    parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.")
    parser.add_argument('--server_port', type=str, default='', help="For distant debugging.")

    # parsing parameters
    args = parser.parse_args()

    # =========== checking parameters and setting up  =========== #
    # setting things up
    # Setup CUDA, GPU & distributed training
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")    # CHECK! make sure we use all 3 GPUs
    args.n_gpu = torch.cuda.device_count()
    args.device = device
    # Setup logging
    logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt = '%m/%d/%Y %H:%M:%S',
                        level = logging.INFO)
    # Set seed
    set_seed(args)

    # =========== bulilding model and inferencing  =========== #
    # Building model
    gpt2_config_class, gpt2_class, tokenizer_class = MODEL_CLASSES[args.gpt2_model_type]
    gpt2_config = gpt2_config_class.from_pretrained(args.gpt2_model_name_or_path, cache_dir = None)
    vae_config = VAE_config(latent_size = args.latent_size)
    model = VAE_GPT2(gpt2_config, vae_config, args)

    # Load from checkpoint model
    model.from_pretrained(args)
    if args.block_size <= 0:  # modify args.block_size variable
        args.block_size = model.tokenizer.max_len_single_sentence  # Our input block size will be the max possible for the model
    args.block_size = min(args.block_size, model.tokenizer.max_len_single_sentence)

    # Send model to GPU
    model.to(args.device)

    # Logging info
    logger.info("Inference parameters %s", args)

    # Testing inference
    args.gpt2_config = model.gpt2_config

    if args.inference_test == 1:
        # Inference test 1 - regenerate text
        inference_test_1(model, args, device)
    if args.inference_test == 2:
        # Inference test 2 - sample for each of latent dimension
        inference_test_2(model, args, device)
    if args.inference_test == 3:
        # Inference test 3 - interpolation generating
        inference_test_3(model, args, device)

if __name__ == "__main__":
    main()


# Program to scrape images from a given URL in a very lucid language

from bs4 import BeautifulSoup as BS
import requests
import urllib.request

first_half = 'https://www.google.com/search?q='
left_over = '+images&sxsrf=APq-WBv8a41PjPq3A3_LPfW2oYYjybAW0A:1647082397085&source=lnms&tbm=isch&sa=X&ved=2ahUKEwi5nsaJtMD2AhUSFogKHc-NAp8Q_AUoAXoECAEQAw&biw=811&bih=625&dpr=1'

Obj = input('Enter Object name ')               # enter the name of objects whose image you want to scrape
URL = first_half + Obj + left_over
r = requests.get(URL)
Data = BS(r.text,'html.parser')
ImgList = Data.findAll('img')
#print(ImgList)

for i in range(1,11):           # Put the number of Images you want to scrape
        link=ImgList[i].get('src')
        print(i,link)
        urllib.request.urlretrieve(link,'G:/'+Obj+'_'+str(i)+'.jpg')


from utils import read_file_by_line, read_file_and_split
from typing import List

def parse_instructions(instructions: str) -> (int, int, int):
    out = instructions.split(' ')
    return int(out[1]), int(out[3]) - 1, int(out[5]) - 1

def get_tops(state: List[List[str]]) -> str:
    out = ''
    for pile in state:
        out += pile[0]

def part1(state: List[List[str]], instructions: List[str]) -> str:
    for instruction in instructions:
        count, start, end = parse_instructions(instruction)
        for i in range(0, count):
            temp = state[start].pop(0)
            state[end].insert(0, temp)
    return get_tops(state)

def part2(state: List[List[str]], instructions: List[str]) -> str:
    for instruction in instructions:
        count, start, end = parse_instructions(instruction)
        temp = state[start][:count]
        state[start] = state[start][count:]
        temp.extend(state[end])
        state[end] = temp
    return get_tops(state)

def main():
    instructions = read_file_by_line('test_instructions.txt')
    start_state = read_file_and_split('test_state.txt', ',')
    print(f'part 1: {part1(start_state, instructions)}')
    start_state = read_file_and_split('test_state.txt', ',')
    print(f'part 2: {part2(start_state, instructions)}')

if __name__ == '__main__':
    main()


import urllib3
import config
import logging
import xml.etree.ElementTree as ET
from typing import Union, Optional
from datetime import date
import json

logger = logging.getLogger(__name__)
c_handler = logging.StreamHandler()
f_handler = logging.FileHandler(config.log_file_name)
c_handler.setLevel(logging.INFO)
f_handler.setLevel(logging.INFO)
c_format = logging.Formatter(config.log_c_format)
f_format = logging.Formatter(config.log_f_format)
c_handler.setFormatter(c_format)
f_handler.setFormatter(f_format)

logger.addHandler(c_handler)
logger.addHandler(f_handler)
logger.setLevel(logging.INFO)

class CbrCoursesNotAvailableException(Exception):
    pass

class IncorrectXmlDataException(Exception):
    pass

def get_cbr_courses_xml(main_url: str = config.ext_cbr_site,
                        second_url: str = config.ext_cbr_site_2) -> bytes:
    http_pool = urllib3.PoolManager()
    resp = http_pool.request("GET", main_url)
    if resp.status == 200 and resp.geturl() == main_url:
        xml_data = resp.data
        logger.info("Main url available. Xml data taken.")
    else:
        logger.info("Main url not available. Trying second url...")
        resp = http_pool.request("GET", second_url)
        if resp.status == 200 and resp.geturl() == second_url:
            xml_data = resp.data
            logger.info("Second url available. Xml data taken.")
        else:
            http_pool.clear()
            logger.error("CBR courses not available")
            raise CbrCoursesNotAvailableException("CBR courses not available")
    http_pool.clear()
    return xml_data

def get_course_from_xml(xml_data: Union[bytes, str],
                        valute_charcode: Union[str, bytes] = config.valute_charcode) -> dict:
    try:
        root = ET.fromstring(xml_data)
    except ET.ParseError as e:
        logger.error(e)
        raise

    if root.tag != "ValCurs":
        logger.error("Incorrect xml data")
        raise IncorrectXmlDataException("Incorrect xml data")

    # for ElementTree 1.3+
    valute_el = root.find("./Valute/CharCode[.='" + valute_charcode + "']/..")
    if valute_el is None:
        logger.error("Valute with charcode %s not found!" % valute_charcode)
        raise IncorrectXmlDataException("Valute with charcode %s not found!" % valute_charcode)

    valute_course_el = valute_el.find("./Value")

    try:
        valute_course = float(valute_course_el.text.replace(",", "."))
        date_of_course = root.get("Date")
    except ValueError as e:
        logger.error(e)
        raise

    logger.info("Got course from xml: %f (%s), date: %s" % (valute_course, valute_charcode, date_of_course))
    return {"course": valute_course, "date_of_course": date_of_course, "valute": valute_charcode}

def get_converted_value(value: Union[int, float], course: Union[int, float]) -> float:
    result = value * course
    logger.info("Request value: %f, course: %f, result: %f" % (value, course, result))
    return result

def get_converted_value_for_valute(value: Union[int, float], valute_charcode: str = config.valute_charcode) -> float:
    course = get_course(valute_charcode=valute_charcode)["course"]
    result = value * course
    logger.info("Request value: %f, course: %f, result: %f" % (value, course, result))
    return result

def get_course(valute_charcode: Union[str, bytes] = config.valute_charcode) -> dict:
    xml_data = get_cbr_courses_xml()
    course_data = get_course_from_xml(xml_data, valute_charcode)
    return course_data

def build_result_json(requested_value: Union[int, float, str],
                      result_value: Union[int, float],
                      date_of_course: Union[str, date],
                      course: Union[float, int],
                      valute: str) -> str:
    res = {"requested_value": requested_value,
           "result_value": result_value,
           "date_of_course": date_of_course,
           "course": course,
           "valute": valute
           }
    return json.dumps(res)

def build_error_result_json(error_text: str,
                            requested_value: Optional[Union[int, float, str]] = None,) -> str:
    if requested_value is not None:
        res = {"requested_value": requested_value,
               "error": error_text }
    else:
        res = {"error": error_text}
    return json.dumps(res)