from __future__ import print_function

from datetime import datetime, time, timedelta, date
from dateutil.relativedelta import relativedelta

import logging

import settings
from utils import get_keyboard_from_list, month_choice_keyboard, get_name_cur_and_next_month

from telegram.ext import Updater, CommandHandler, MessageHandler, Filters, ConversationHandler, CallbackQueryHandler
from telegram import ParseMode, ReplyKeyboardMarkup, ReplyKeyboardRemove, InlineKeyboardButton, InlineKeyboardMarkup

import os.path
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials

import secrets

def get_calendar_service():
    SCOPES = ['https://www.googleapis.com/auth/calendar']
    creds = None
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'credentials.json', SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        # with open('token.json', 'w') as token:
        #     token.write(creds.to_json())

    service = build('calendar', 'v3', credentials=creds)
    return service

service = get_calendar_service()

def get_all_avalible_days(start_date):
    next_month_day = start_date + relativedelta(months = 1)
    end_date = next_month_day.replace(day = 1)
    temp_date = start_date
    appointment_days = []
    while temp_date < end_date :
        # print(temp_date, temp_date.isoweekday())
        if temp_date.isoweekday() == 3 or temp_date.isoweekday() == 7:
            if len(available_time_window_list(temp_date)) > 0:
                appointment_days.append(temp_date.day)
        temp_date = temp_date + timedelta(days = +1)
    return appointment_days

def make_calendar_event(start_date):
    # service = get_calendar_service()

    start = start_date.isoformat()
    end = (start_date + timedelta(hours=1)).isoformat()
    id_event = secrets.token_hex(32)

    event_result = service.events().insert(calendarId='kolupaev.vladislav@gmail.com',
        body={
            "summary": 'Automating calendar',
            "description": 'This is a tutorial example of automating google calendar with python',
            "start": {"dateTime": start, "timeZone": 'Europe/Moscow'},
            "end": {"dateTime": end, "timeZone": 'Europe/Moscow'},
            "id" : id_event,
        }
    ).execute()
    return id_event

def get_days_events(start_date):
    # service = get_calendar_service()

    min_time = start_date.isoformat() + 'Z'

    max_time = start_date.replace(hour = 21, minute = 0).isoformat() + 'Z'

    events_result = service.events().list(calendarId='kolupaev.vladislav@gmail.com', timeMin=min_time,
                                        timeMax=max_time, singleEvents=True,
                                        orderBy='startTime').execute()
    events = events_result.get('items', [])

    events_list = []

    for event in events:
        start = event['start']['dateTime']
        end = event['end']['dateTime']
        start = datetime. strptime(start[0:19], "%Y-%m-%dT%H:%M:%S")
        end = datetime. strptime(end[0:19], "%Y-%m-%dT%H:%M:%S")
        event_start_end = {}
        event_start_end['start'] = start
        event_start_end['end'] = end
        events_list.append(event_start_end)
        # print(event_start_end)

    return events_list

def reg_appointment_event(start_date):
    SCOPES = ['https://www.googleapis.com/auth/calendar']
    creds = None
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())
        else:
            flow = InstalledAppFlow.from_client_secrets_file(
                'credentials.json', SCOPES)
            creds = flow.run_local_server(port=0)
        # Save the credentials for the next run
        with open('token.json', 'w') as token:
            token.write(creds.to_json())

    service = build('calendar', 'v3', credentials=creds)
    start = start_date.isoformat()
    end = start_date.replace(hour=start_date.hour + 1).isoformat()
    event = {
    'description': 'Test tg bot event',
    'start': {
        'dateTime': start
    },
    'end': {
        'dateTime': end
    }
    }

    event = service.events().insert(calendarId='kolupaev.vladislav@gmail.com', body=event).execute()

def available_time_window_list(start_date):
    start_date = start_date.replace(second=0, microsecond=0)
    if start_date.date() == date.today() and start_date.time() > time(9, 0):
        cur_time = start_date
        if cur_time.minute > 30:
            cur_time = start_date.replace(hour=start_date.hour + 1, minute=0)
        else:
            cur_time = start_date.replace(minute=30)
    else:
        start_time = time(9, 0)
        cur_time = datetime.combine(start_date.date(), start_time)
    time_window_list = []
    end_time = datetime.combine(start_date.date(), time(20, 0))
    events_list = get_days_events(start_date)
    while cur_time <= end_time:
        time_window = {}
        time_window['start_time'] = cur_time
        time_window['end_time'] = cur_time + timedelta(hours=+1)
        good_window = True
        for events in events_list:
            if (events['start'] >= time_window['start_time']  and events['start'] < time_window['end_time']) or \
                (events['end'] > time_window['start_time']  and events['end'] <= time_window['end_time']) or \
                (events['start'] <= time_window['start_time'] and events['end'] >= time_window['end_time']):
                good_window = False
                break
        if good_window:
            time_window_list.append(str(cur_time.time())[:5])
        cur_time = cur_time + timedelta(minutes=+30)
    return time_window_list

def month_choice(update, context):
    user_data = context.user_data
    user_data.pop('month', None)
    update.message.reply_text(
        "Выберите месяц",
        reply_markup=month_choice_keyboard()
    )
    return "appointment_day_time"

def get_days_from_month(update, context):
    user_data = context.user_data
    if 'day' in user_data:
        user_data.pop('day', None)
    user_month = update.message.text
    cur_month_sub_str = 'Текущий месяц'
    if 'month' not in user_data:
        if cur_month_sub_str in user_month:
            start_date = datetime.today()
            user_data['month'] = 'cur_month'
        else:
            start_date = (datetime.today() + relativedelta(months = 1)).replace(day=1)
            user_data['month'] = 'next_month'
    else:
        if user_data['month'] == 'cur_month':
            start_date = datetime.today()
        else:
            start_date = (datetime.today() + relativedelta(months = 1)).replace(day=1)
    list_of_avaliable_days = get_all_avalible_days(start_date)
    keyboard = get_keyboard_from_list(list_of_avaliable_days, 5)
    keyboard.append(['/back_to_month'])
    keyboard.append(['/cancel'])
    # print(start_date)
    update.message.reply_text(
        "Выберите дату",
        reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True)
    )

def available_time_window_list_handler(update, context):
    user_data = context.user_data
    if 'day' not in user_data:
        user_data['day'] = update.message.text
    if user_data['month'] == 'cur_month':
        start_date = datetime.today()
    else:
        start_date = datetime.today() + relativedelta(months = 1)
    user_day = user_data['day']
    start_date = start_date.replace(day=int(user_day))
    available_time_window = available_time_window_list(start_date)
    keyboard_window_time = get_keyboard_from_list(available_time_window, 5)
    keyboard_window_time.append(['/back_to_day'])
    keyboard_window_time.append(['/cancel'])
    # print(start_date)
    update.message.reply_text(
        "Выберите время",
        reply_markup=ReplyKeyboardMarkup(keyboard_window_time, one_time_keyboard=True)
    )

def comfirm_month_day_time_choice(update, context):
    user_time = update.message.text
    user_data = context.user_data
    user_data['time'] = datetime.strptime(user_time, '%H:%M').time()
    if user_data['month'] == 'cur_month':
        month_name = get_name_cur_and_next_month()[0]
        start_date = date.today()
    else:
        month_name = get_name_cur_and_next_month()[1]
        start_date = date.today() + relativedelta(months = 1)
    start_date = start_date.replace(day=int(user_data['day']))
    if start_date.isoweekday() == 3:
        weekday_name = 'Среда'
    else:
        weekday_name = 'Воскресенье'
    ans_string = f"Ваш прием состоится {int(user_data['day'])} {month_name} ({weekday_name}) в {str(user_data['time'].strftime('%H:%M'))}"
    keyboard = [['/confirm'], ['/back_to_time'], ['/cancel']]
    # print(start_date)
    update.message.reply_text(
        ans_string,
        reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True)
    )

def confirm_month_day_time_app(update, context):
    user_data = context.user_data
    if user_data['month'] == 'cur_month':
        start_date = date.today()
    else:
        start_date = date.today() + relativedelta(months = 1)
    start_date = start_date.replace(day=int(user_data['day']))
    start_date = datetime.combine(start_date, user_data['time'])
    id_event = make_calendar_event(start_date)
    user_data['id_event'] = id_event
    update.message.reply_text(
        'Введите ваши фамилию, имя и отчество'
    )
    # return ConversationHandler.END
    # print(id_event)
    return "contact"

def fullname(update, context):
    user_name = update.message.text
    if len(user_name.split()) < 3:
        update.message.reply_text("Пожалуйста, напишите фамилию, имя и отчество")
        return "contact"
    else:
        context.user_data['contact'] = {"name": user_name}
        update.message.reply_text("Укажите ваш номер телефона в формате 8_________")
        return "contact"

def phone_number(update, context):
    user_phone_number = update.message.text
    print(context.user_data['contact'])
    if len(user_phone_number) != 11:
        update.message.reply_text("Укажите номер в формате 8__________")
        return "contact"
    else:
        context.user_data['contact']['phone_number'] = user_phone_number
        update.message.reply_text("Что вас беспокоит?")
        return "complaint"

def complaint(update, context):
    user_complaint = update.message.text
    context.user_data['contact']['complaint'] = user_complaint
    user_data = context.user_data
    if user_data['month'] == 'cur_month':
        month_name = get_name_cur_and_next_month()[0]
        start_date = date.today()
    else:
        month_name = get_name_cur_and_next_month()[1]
        start_date = date.today() + relativedelta(months = 1)
    start_date = start_date.replace(day=int(user_data['day']))
    if start_date.isoweekday() == 3:
        weekday_name = 'Среда'
    else:
        weekday_name = 'Воскресенье'
    print(user_data['contact'])
    user_text = f"""
Ваш прием состоится {int(user_data['day'])} {month_name} ({weekday_name}) в {str(user_data['time'].strftime('%H:%M'))}
ФИО: {user_data['contact']['name']}
Тел: {user_data['contact']['phone_number']}
Жалобы: {user_data['contact']['complaint']}"""
    keyboard = [['/confirm'], ['/change_cell_number'], ['/cancel']]
    update.message.reply_text(user_text,
        reply_markup=ReplyKeyboardMarkup(keyboard, one_time_keyboard=True), parse_mode=ParseMode.HTML
    )

def cancel(update, context):
    update.message.reply_text(
        'Для записи на прием наберите /start!'
    )
    return ConversationHandler.END

def main():
    mybot = Updater(settings.BOT_KEY, use_context=True)

    dp = mybot.dispatcher

    appointment = ConversationHandler(
        entry_points=[CommandHandler('start', month_choice)],
        states={
            "appointment_day_time" : [
                MessageHandler(Filters.regex('^(Текущий месяц|Следующий месяц)'), get_days_from_month),
                MessageHandler(Filters.regex('^([1-9]|[1,2][0-9]|3[01])$'), available_time_window_list_handler),
                MessageHandler(Filters.regex('^([0-1]?[0-9]|2[0-3]):[0-5][0-9]$'), comfirm_month_day_time_choice),
                CommandHandler('back_to_time', available_time_window_list_handler),
                CommandHandler('back_to_day', get_days_from_month),
                CommandHandler('back_to_month', month_choice),
                CommandHandler('confirm', confirm_month_day_time_app)
                ],
            "contact" : [
                MessageHandler(Filters.regex('^([0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9])$'), phone_number),
                MessageHandler(Filters.text, fullname)
                ],
            "complaint" : [
                MessageHandler(Filters.text, complaint),
                CommandHandler('change_cell_number', fullname)

                ]
        },
        fallbacks=[CommandHandler('cancel', cancel), "contact"]

    )
    dp.add_handler(appointment)

    logging.info("Бот поехал!")

    mybot.start_polling()
    mybot.idle()

if __name__ == "__main__":
    main()


# coding: utf-8

# In[4]:

from sets import Set
import numpy as np

f = open('1as')
in_lines = f.readlines() # 1行毎にファイル終端まで全て読む(改行文字も含まれる)
f.close()
# lines2: リスト。要素は1行の文字列データ

case_count = int(in_lines[0])

numbers = []
outlines = []
for i in range(case_count):
    string = in_lines[i+1]

    last_word = ""

    for j in range(len(string)):
        # last_wordがないならそのまま追加。
        if last_word == "":
            last_word += string[j]
        else:
            #当該文字と今の頭文字を比較
            if string[j] >= last_word[0]:
                #前方に追加
                last_word = string[j] + last_word
            else:
                last_word += string[j]
    outlines.append("Case #%d: %s" % (i+1, str(last_word)))

f = open('out.txt', 'w') # 書き込みモードで開く
f.writelines(outlines) # シーケンスが引数。
f.close()

# In[ ]:


from bdmcore.dsproject import dsProject
from bdmcore.util import setup_logging
import pandas as pd

# setting
setup_logging()
dsp = dsProject(name="kdd2015_dropout")
dsp.define_target_label(['enrollment_id'], ['label'])

# load hash data
hash_f = dsp.load_data_tables('features', sub='hash_', nrows=1)

col_hash = []
for file in hash_f:
    col_hash.append(hash_f[file].columns[1])

# load preprocess data
dsp.feature_tables = {}
all_df = dsp.load_data_tables('features', sub='pre_t')

# hash encoding
for file in all_df:
    for col in all_df[file]:
        if col in col_hash:
            name_h = "hash_%s.csv" % col
            df_h = dsp.load_data_table(name_h, 'features')
            all_df[file] = pd.merge(all_df[file], df_h, how='left', on=col)
            all_df[file][col] = all_df[file][('hash_%s' % col)]
            del all_df[file][('hash_%s' % col)]
    dsp.write_feature((all_df[file]), 'h_%s' % file[:-4])


# -*- coding: utf-8 -*-
# @Time    : 2020/12/10 12:44 下午
# @Author  : Yijia Zheng
# @FileName: train_my_model.py

import sys
sys.path.append('..')
import os
import numpy as np
import pandas as pd
import joblib
import matplotlib.pyplot as plt

#### import models and tools####
from preprocessing.data import load_data, load_reduction_data
from models.tools import pca, plot_confusion_matrix
from models.SVM import MultiSVM
from models.KNN import KNN
from models.BPNN import NeuralNet
from models.RF import RandomForestClassifier

DIM = 60
MODEL_NAME = 'mlp' # choosing from knn, svm, rf, mlp
Reduction_Method = 'lasso' # choosing from pca, lasso, original

result_save_path = '../save/result/'
if not os.path.exists(result_save_path):
    os.makedirs(result_save_path)

#load processed_data
if Reduction_Method=='pca':
    X_train, X_val, X_test, y_train, y_val, y_test = load_data('r50')
    X_train = np.asarray(pca(X_train, DIM)[0])
    X_val = np.asarray(pca(X_val, DIM)[0])
    X_test = np.asarray(pca(X_test, DIM)[0])

elif Reduction_Method=='lasso':
    X_train, X_val, X_test, y_train, y_val, y_test = load_reduction_data('r50')

else:
    assert Reduction_Method=='original'
    X_train, X_val, X_test, y_train, y_val, y_test = load_data('r50')

print('Data process done!')

#######  Train Model ######

if MODEL_NAME == 'svm':
    clf = MultiSVM(C=1, kernel='rbf', gamma=1/X_train.shape[1], decision_function='ovo')
    clf.fit(X_train, y_train)
    clf.save('svm_{}_val.pkl'.format(Reduction_Method))
    y_pred = clf.predict(X_val)
    # np.save(result_save_path+'{}_{}_val'.format(MODEL_NAME, Reduction_Method), y_pred)

elif MODEL_NAME == 'knn':
    model = KNN(X_train, y_train)
    neighbor_list = np.linspace(10, 10, 1)
    best_acc = 0
    # for neighbor_num in map(int, list(neighbor_list)):
    #     y_val_predicted = []
    #     for idx, sample in enumerate(X_val):
    #         print(idx)
    #         y_val_predicted.append(model.fit(sample, neighbor_num))
    #     result = np.vstack((np.asarray(y_val_predicted), y_val)).T
    #     accuracy = len(result[np.where(result[:, 0]==result[:, 1])])/result.shape[0]
    #     print('acc=', accuracy)
    #     if accuracy > best_acc:
    #         best_model = model
    #         best_result = result
    #         best_acc = accuracy
    #         best_num_neighbor = neighbor_num
    #     np.save(result_save_path + '{}_{}_val'.format(MODEL_NAME, Reduction_Method), y_val_predicted)
    for neighbor_num in map(int, list(neighbor_list)):
        y_train_predicted = []
        for idx, sample in enumerate(X_train):
            print(idx)
            y_train_predicted.append(model.fit(sample, neighbor_num))
        result = np.vstack((np.asarray(y_train_predicted), y_train)).T
        accuracy = len(result[np.where(result[:, 0]==result[:, 1])])/result.shape[0]
        print('acc=', accuracy)
        if accuracy > best_acc:
            best_model = model
            best_result = result
            best_acc = accuracy
            best_num_neighbor = neighbor_num

    print('Best {} is from {} neighbors'.format(best_acc, best_num_neighbor))

elif MODEL_NAME == 'rf':
    clf = RandomForestClassifier(n_estimators=100, max_depth=4,
                                 min_samples_split=2,
                                 min_samples_leaf=3,
                                 min_split_gain=0.0,
                                 colsample_bytree="sqrt",
                                 subsample=0.8,
                                 random_state=2018111396)
    clf.fit(pd.DataFrame(X_train), pd.DataFrame(y_train).iloc[:, 0])
    joblib.dump(clf, '../save/my_model/rf_{}_val.pkl'.format(Reduction_Method))
    y_pred = clf.predict(pd.DataFrame(X_val))
    # np.save(result_save_path + '{}_{}_val'.format(MODEL_NAME, Reduction_Method), y_pred)

elif MODEL_NAME == 'mlp':
    out_num = np.max(y_train) + 1
    train_num = len(y_train)
    val_num = len(y_val)
    y_train_sparse = np.zeros((train_num, out_num))
    y_train_sparse[np.arange(0, train_num), y_train] = 1
    y_val_sparse = np.zeros((val_num, out_num))
    y_val_sparse[np.arange(0, val_num), y_val] = 1

    plt.rcParams['figure.figsize'] = (8.0, 10.0)
    plt.rcParams['image.interpolation'] = 'nearest'

    for (act_name, alg_name, color) in zip(['relu', 'relu', 'sigmoid', 'sigmoid'],
                                    ['sgd', 'adam', 'sgd', 'adam'], ['#ADD8E6', '#F08080', '#FAFAD2', '#90EE90']):
        model = NeuralNet(hidden_layers=(100,), batch_size=200, learning_rate=1e-3,
                          max_iter=100, tol=1e-4, alpha=1e-5,
                          activation=act_name, solver=alg_name)
        model.fit(X_train, y_train_sparse)
        y_pred = model.predict(X_val)
        # np.save(result_save_path + '{}_{}_{}_{}_val'.format(MODEL_NAME, Reduction_Method, act_name, alg_name), y_pred)
        print('accuracy:', 1 - np.sum(np.abs(y_pred - y_val_sparse)) / 2 / y_val_sparse.shape[0])
        ax1 = plt.subplot(211)
        plt.plot(model.loss_curve, color=color, label=alg_name+'_'+act_name)
        plt.legend(loc='best')
        plt.title('Loss Curve in Training Set')
        ax2 = plt.subplot(212)
        plt.plot(model.acc_curve, color=color, label=alg_name + '_' + act_name)
    plt.legend(loc='best')
    plt.title('Accuracy Curve in Training Set')
    # plt.savefig('../save/plots/mlp_training.jpg', dpi=200)
    plt.show()

else:
    print('Misuse, please choose from svm, knn, rf, mlp')


import os
import gym
import numpy as np
import copy
import torch
from tensorboardX import SummaryWriter

from ding.config import compile_config
from ding.worker import BaseLearner, Episode1v1Collector, OnevOneEvaluator, NaiveReplayBuffer
from ding.envs import BaseEnvManager, DingEnvWrapper
from ding.policy import PPOPolicy
from dizoo.overcooked.models.overcooked_vac import BaselineVAC
from ding.utils import set_pkg_seed
from dizoo.overcooked.envs import OvercookGameEnv
from dizoo.overcooked.config import overcooked_demo_ppo_config

def wrapped_overcookgame():
    return OvercookGameEnv({})

def main(cfg, seed=0, max_iterations=int(1e10)):
    cfg.exp_name = 'selfplay_demo_ppo'
    cfg = compile_config(
        cfg,
        BaseEnvManager,
        PPOPolicy,
        BaseLearner,
        Episode1v1Collector,
        OnevOneEvaluator,
        NaiveReplayBuffer,
        save_cfg=True
    )
    collector_env_num, evaluator_env_num = cfg.env.collector_env_num, cfg.env.evaluator_env_num
    collector_env = BaseEnvManager(env_fn=[wrapped_overcookgame for _ in range(collector_env_num)], cfg=cfg.env.manager)
    evaluator_env1 = BaseEnvManager(
        env_fn=[wrapped_overcookgame for _ in range(evaluator_env_num)], cfg=cfg.env.manager
    )
    evaluator_env2 = BaseEnvManager(
        env_fn=[wrapped_overcookgame for _ in range(evaluator_env_num)], cfg=cfg.env.manager
    )

    collector_env.seed(seed)
    evaluator_env1.seed(seed, dynamic_seed=False)
    evaluator_env2.seed(seed, dynamic_seed=False)
    set_pkg_seed(seed, use_cuda=cfg.policy.cuda)

    model1 = BaselineVAC(**cfg.policy.model)
    policy1 = PPOPolicy(cfg.policy, model=model1)
    model2 = BaselineVAC(**cfg.policy.model)
    policy2 = PPOPolicy(cfg.policy, model=model2)

    tb_logger = SummaryWriter(os.path.join('./{}/log/'.format(cfg.exp_name), 'serial'))
    learner1 = BaseLearner(
        cfg.policy.learn.learner, policy1.learn_mode, tb_logger, exp_name=cfg.exp_name, instance_name='learner1'
    )
    learner2 = BaseLearner(
        cfg.policy.learn.learner, policy2.learn_mode, tb_logger, exp_name=cfg.exp_name, instance_name='learner2'
    )
    collector = Episode1v1Collector(
        cfg.policy.collect.collector,
        collector_env, [policy1.collect_mode, policy2.collect_mode],
        tb_logger,
        exp_name=cfg.exp_name
    )
    # collect_mode ppo use multimonial sample for selecting action
    evaluator1_cfg = copy.deepcopy(cfg.policy.eval.evaluator)
    evaluator1_cfg.stop_value = cfg.env.stop_value
    evaluator1 = OnevOneEvaluator(
        evaluator1_cfg,
        evaluator_env1, [policy1.collect_mode, policy2.collect_mode],
        tb_logger,
        exp_name=cfg.exp_name,
        instance_name='selfplay_evaluator1'
    )
    evaluator2_cfg = copy.deepcopy(cfg.policy.eval.evaluator)
    evaluator2_cfg.stop_value = cfg.env.stop_value
    evaluator2 = OnevOneEvaluator(
        evaluator2_cfg,
        evaluator_env2, [policy2.collect_mode, policy1.collect_mode],
        tb_logger,
        exp_name=cfg.exp_name,
        instance_name='selfplay_evaluator2'
    )

    for _ in range(max_iterations):
        if evaluator1.should_eval(learner1.train_iter):
            stop_flag1, reward = evaluator1.eval(learner1.save_checkpoint, learner1.train_iter, collector.envstep)
            tb_logger.add_scalar('selfplay1_evaluator_step/reward_mean', reward, collector.envstep)
        if evaluator2.should_eval(learner1.train_iter):
            stop_flag2, reward = evaluator2.eval(learner1.save_checkpoint, learner1.train_iter, collector.envstep)
            tb_logger.add_scalar('selfplay2_evaluator_step/reward_mean', reward, collector.envstep)
        if stop_flag1 and stop_flag2:
            break
        train_data, _ = collector.collect(train_iter=learner1.train_iter)
        for i in range(cfg.policy.learn.update_per_collect):
            learner1.train(train_data[0], collector.envstep)
            learner2.train(train_data[1], collector.envstep)

if __name__ == "__main__":
    main(overcooked_demo_ppo_config)


# Copyright (c) James Percent, Byron Galbraith and Unlock contributors.
# All rights reserved.
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    1. Redistributions of source code must retain the above copyright notice,
#       this list of conditions and the following disclaimer.
#
#    2. Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#
#    3. Neither the name of Unlock nor the names of its contributors may be used
#       to endorse or promote products derived from this software without
#       specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from unlock.state.state import UnlockState, TimerState
from unlock.util import Trigger
from random import Random

class CueStateMachine(UnlockState):
    def __init__(self, rand, trials, cue_states, rest_state, indicate_state):
        super(CueStateMachine, self).__init__()
        self.cue_states = cue_states
        self.rest_state = rest_state
        self.indicate_state = indicate_state
        self.rand = rand
        self.trials = trials
        self.count = 0
        self.state = rest_state
        self.next_cue_state = 0

    def __transition__(self, next_state):
        self.state.stop()
        next_state.last = self.state
        next_state.start()
        self.state = next_state
        return self.state.trigger

    def indicate_rest(self):
        return self.__transition__(self.rest_state)

    def rest_cue(self):
        self.count += 1
        if self.count == self.trials:
            return Trigger.Complete
        if self.rand != None:
            next_ = self.__transition__(self.cue_states[self.rand.randint(0, len(self.cue_states)-1)])
        else:
            next_ = self.__transition__(self.cue_states[self.next_cue_state])
            self.next_cue_state += 1
            if self.next_cue_state == len(self.cue_states):
                self.next_cue_state = 0
        return next_

    def cue_indicate(self):
        return self.__transition__(self.indicate_state)

    def cue_rest(self):
        return self.__transition__(self.rest_state)

    def process_command(self, command):
        return self.state.process_command(command)

    def start(self):
        self.state.start()

class CueState(UnlockState):
    def __init__(self, trigger, trial_time_state, transition_fn=None):
        super(CueState, self).__init__()
        self.state = False
        self.trigger = trigger
        self.trial_time_state = trial_time_state
        self.transition_fn = transition_fn

    def get_state(self):
        return self.state

    def start(self):
        assert self.transition_fn != None
        self.trial_time_state.begin_timer()
        self.state = True

    def stop(self):
        self.state = False

    def process_command(self, command):
        assert self.transition_fn != None
        ret = Trigger.Null
        self.trial_time_state.update_timer(command.delta)
        if self.trial_time_state.is_complete():
            ret = self.transition_fn()
        return ret

# XXX - this could use a refactoring.  It was put together quickly on a deadline.  In particular, the views should not maintain
#       the position information.  It should be managed by the model, and when it changes, it should be sent to the view for
#       rendering.  The classes in util.Observer should be used for this.
class DynamicPositionCueState(CueState):
    def __init__(self, trigger, trial_time_state, screen_height, height, screen_width, width, radius = 1, transition_fn=None):
        super(DynamicPositionCueState, self).__init__(trigger, trial_time_state, transition_fn)
        self.screen_height = screen_height
        self.screen_width = screen_width
        self.height = height
        self.width = width
        self.radius = radius
        self.x = self.screen_width / 2
        self.y = self.screen_height / 2

    def get_state(self):
        return self.x, self.y, self.state
 #
 #self.reset_state = CueState(self.state_id['reset'], TextDraw('+', self.text, viewport.controller.draw), self.trigger.send, reset_duration, None)
 #
 #       position_x = img.width // 2
 #       self.left_indicator = VisualizationState(self.state_id['indicator'], ImageDraw(reset_image_filename, anchor_x, anchor_y, position_x, position_y), self.trigger.send, indicator_duration)
 #       self.left_indicator.drawer.screen = screen
 #
 #       position_x = viewport.window.width - img.width // 2
 #       self.right_indicator = VisualizationState(self.state_id['indicator'], ImageDraw(reset_image_filename, anchor_x, anchor_y, position_x, position_y), self.trigger.send, indicator_duration)
 #
 #       position_x = viewport.window.width // 2
 #       position_y = viewport.window.height - img.height // 2
 #       self.up_indicator = VisualizationState(self.state_id['indicator'], ImageDraw(reset_image_filename, anchor_x, anchor_y, position_x, position_y), self.trigger.send, indicator_duration)
 #
 #       position_y = img.height // 2
 #       self.down_indicator = VisualizationState(self.state_id['indicator'], ImageDraw(reset_image_filename, anchor_x, anchor_y, position_x, position_y), self.trigger.send, indicator_duration)
 #

    def start(self):
        #change position
        if self.last.trigger == Trigger.Up:
            self.x = self.screen_width / 2
            self.y =  (self.screen_height - (self.screen_height/2 - (self.screen_height/2*self.radius))) - self.height / 2
        elif self.last.trigger == Trigger.Right:
            self.x =  (self.screen_width - (self.screen_width/2 - (self.screen_width/2*self.radius))) - self.width / 2
            self.y = self.screen_height / 2
        elif self.last.trigger == Trigger.Down:
            self.x = self.screen_width / 2
            self.y = self.height / 2 + ((self.screen_height/2) - (self.screen_height/2) * self.radius)
        elif self.last.trigger == Trigger.Left:
            self.x = self.width / 2 + ((self.screen_width/2) - (self.screen_width/2) * self.radius)
            self.y = self.screen_height / 2

        super(DynamicPositionCueState, self).start()

class TimedStimulusCueState(UnlockState):
    def __init__(self, timed_stimulus):
        super(TimedStimulusCueState, self).__init__()
        self.timed_stimulus = timed_stimulus
        self.trigger = Trigger.Stop

    def start(self):
        assert self.transition_fn != None
        self.timed_stimulus.start()
        self.state = self.timed_stimulus.get_state()

    def stop(self):
        self.timed_stimulus.stop()
        self.state = None

    def process_command(self, command):
        assert self.transition_fn != None
        ret = Trigger.Null
        trigger_value = self.timed_stimulus.process_command(command)
        if trigger_value == Trigger.Stop:
            ret = self.transition_fn()
        else:
            self.state = self.timed_stimulus.get_state()

        return ret

# XXX - this should be refactored.  this should decorate the TimedStimulusCueState
#       with multiple sequenctial timed stimuli.
class MultipleSequentialTimedStimuliCueState(UnlockState):

    def __init__(self):
        super(MultipleSequentialTimedStimuliCueState, self).__init__()
        self.pos = -1
        self.stimuli = []
        self.trigger = None
        self.stimulus = None

    def add_stimulus(self, trigger, stimulus):
        self.stimuli.append((trigger, stimulus))

    def setup_next_stimulus(self):
        self.pos = (self.pos + 1) % len(self.stimuli)
        self.trigger, self.stimulus = self.stimuli[self.pos]

    def start(self):
        assert self.transition_fn != None
        assert len(self.stimuli) > 0
        self.setup_next_stimulus()
        self.stimulus.start()
        self.state = self.stimulus.get_state()

    def stop(self):
        assert len(self.stimuli) > 0
        self.stimulus.stop()
        self.state = None

    def process_command(self, command):
        assert self.transition_fn != None
        assert len(self.stimuli) > 0
        ret = Trigger.Null
        trigger_value = self.stimulus.process_command(command)
        if trigger_value == Trigger.Stop:
            ret = self.transition_fn()
        else:
            ret = trigger_value
            self.state = self.stimulus.get_state()
        return ret


# -*- coding: utf-8 -*-

from django.http import HttpResponse
from django.shortcuts import redirect, render, render_to_response
from django.template.context import RequestContext
from django.views.generic import View
from django.contrib.auth.models import User,Group
from brandline.modelforms import FormCreator
from django import forms
import simplejson
import datetime
from django.contrib.auth import authenticate, login, logout
from django.contrib.auth.hashers import make_password
from django.core.mail import send_mail, BadHeaderError
from brandline.settings import *
from django.core.mail import send_mail, BadHeaderError
from django.core.exceptions import ValidationError
from admins.models import *
from django.views.decorators.csrf import csrf_exempt
from django.core.mail import send_mail
from django.template.defaultfilters import slugify
import time
import os
from django.conf import settings
import mimetypes
from django.contrib.sites.shortcuts import get_current_site
from django.core.mail import EmailMultiAlternatives
from django.db.models import Q
from PIL import Image,ImageFont,ImageDraw
from django.contrib.sites.models import *
from django.http import JsonResponse

from django.core.files.base import ContentFile
from io import BytesIO
from base64 import b64decode
from PIL import Image,ImageFont,ImageDraw

class ImageProcess:

    def djangoImg(self):

        if self.imagencode and 'static/' not in self.imagencode:
            imagen = self.imagencode.decode('base64')
            originals = Image.open(BytesIO(imagen))
            name_imagen = 'logo.%s'%originals.format.lower()
            imagenobj = ContentFile(imagen,name_imagen)
        else:
            imagenobj = None

        return imagenobj

class Index(View):

    template = 'registroapp/inicio.html'

    def get(self, request,secty=None,lugy=None,columna=None):
        context = {}
        f = FormCreator()
        current_site = get_current_site(request)
        setts = Pagesettings.objects.filter(sitio=current_site).first()
        context['pagesetts'] = setts
        data = request.GET.copy()
        response = render(request, self.template, context)
        return response

Index = Index.as_view()

class addFont(View):

    def post(self,request):
        current_site = get_current_site(request)
        data = request.POST.copy()
        archivex = data.get('fuente_file')
        fname = data.get('fuente_name')
        fusage = data.get('funete_usage')
        ffname = data.get('fuente_fname')
        formats, filestr = archivex.split(';base64,')
        ffile = ContentFile(b64decode(filestr),name=ffname)
        f = Fuente.objects.filter(font_type=fusage)
        f.delete()
        f = Fuente(**{'font_name':fname,'font_type':fusage,'sitio':current_site})
        f.save()
        ff = Fontfile(**{'fuente_file':ffile,'fuentepk':f})
        ff.save()
        context = {'furi':ff.fuente_file.name}
        return JsonResponse(context)

addFont = addFont.as_view()

class getSettings(View):

    def get(self,request):
        current_site = get_current_site(request)
        pgst = Pagesettings.objects.filter(sitio=current_site).first()
        #assert False,pgst.confiuracion
        context = {}
        context['setts'] = simplejson.loads(pgst.configuracion)
        return JsonResponse(context)

getSettings = getSettings.as_view()

class saveSettings(View):

    def get(self,request):

        data = request.GET.copy()
        configuracion = {}
        colores = [] #request.GET.getlist('colores')
        fuentes = [] #
        colorsite = data.get('colorsite','{}')
        combinaciones = [] #request.GET.getlist('combinaciones')

        for f in request.GET.getlist('fuentes'):
            fuentes.append(simplejson.loads(f))

        for cr in request.GET.getlist('colores'):
            colores.append(simplejson.loads(cr))

        for cb in request.GET.getlist('combinaciones'):
            combinaciones.append(simplejson.loads(cb))

        domainname = data.get('domain',get_current_site(request))
        reloadornot = data.get('domain')

        configuracion['fuentes'] = fuentes
        configuracion['colores'] = colores
        configuracion['colorsite'] = simplejson.loads(colorsite)
        configuracion['combinaciones'] = combinaciones
        configuracion['template'] = data.get('template')
        configuracion = simplejson.dumps(configuracion)

        sitio = Site.objects.get(domain=domainname)
        if sitio:

            pg,fails = Pagesettings.objects.get_or_create(sitio=sitio)
            pg.configuracion = configuracion
            pg.save()

        context = {'saved':pg.pk,'datos':pg.configuracion}
        return JsonResponse(data)

saveSettings = saveSettings.as_view()

class Register(View):
    """Delete company."""
    template = 'mainapp/register.html'
    f = FormCreator()
    forma = f.form_to_model(modelo=User,fields=['first_name','last_name','username','password'])

    def post(self,request):
        context = {}
        data = request.POST.copy()
        dominio = data.get('dominio',None)
        fname = data.get('first_name',None)
        lname = data.get('last_name',None)
        imagen = data.get('imagen',None)
        pk = data.get('pk',None)
        #profile Sitio
        img = ImageProcess()
        img.imagencode = imagen
        logo = img.djangoImg()

        if pk:
            instanced = User.objects.get(pk=pk)
        else:
            instanced = None

        valides = ['dominio','last_name','first_name']

        forma = self.forma(data,instance=instanced)

        for x in valides:
            valida = data.get(x,None)
            if not valida:
                forma.errors[x]='Este campo es obligatorio'

        if forma.is_valid():
            # sAVING SITE
            originalD = request.META['HTTP_HOST']
            sdomain= '%s.%s'%(dominio,originalD)
            sitio,fails = Site.objects.get_or_create(domain=sdomain)
            pgsttngs,pgfail = Pagesettings.objects.get_or_create(sitio=sitio)
            if logo:
                pgsttngs.logo = logo
                pgsttngs.save()
            # SAVING USER
            savex = forma.save()
            savex.is_active=True
            savex.is_staff = True
            savex.set_password(data.get('password'))
            savex.save()

            if instanced:
                return JsonResponse({'saved':savex.pk,'domain':sdomain})

            # RELATING SITE AND USER
            up = UserProfile(myuser=savex,mysite=sitio)
            up.save()
            #CREATING SITE
            patron = NGINX_PATRON%(dominio,request.META['HTTP_HOST'],BASE_DIR)
            filename = '%s/%s.conf'%(BASE_DIR,dominio)
            F = open(filename,'w')
            F.write(patron)
            F.close()
            paz = 'saruman'
            escribe =  'mv %s /etc/nginx/conf.d/'%(filename)
            renueva = '/etc/init.d/nginx reload '
            os.system('echo %s|sudo -S %s' % (paz, escribe))

            cs = CreateSite(data.get('dominio'),sitio)
            cs.create_section()
            cs.create_modulo()
            redirector = 'http://%s.%s/'%(dominio,originalD)
            os.system('echo %s|sudo -S %s' % (paz, renueva))
            context['saved']=savex.pk
            context['callback']='golink'
            context['domain'] = sdomain
            context['datos'] = {'ruta':redirector}

        else:
            context['errors']=forma.errors

        context = simplejson.dumps(context)
        return HttpResponse(context)

    def get(self,request):
        context = {}

        for x,y in self.forma.base_fields.iteritems():
            #y.widget_attrs={'class':'monolito'}
            (y.widget.attrs.update({'class':'form-control'}))

        context['forma']=self.forma
        response = render(request, self.template, context)
        return response

Register = Register.as_view()

class CreateSite:

    def __init__(self,domain=None,sitio=None):
        self.domain=domain
        self.sitio = sitio
        self.section = None

    def create_section(self):
        title = self.domain
        orden = 1
        slug = slugify(title)
        sitio = self.sitio
        params = {'sec_name':title,'orden':orden,'sec_slug':slug,'sitio':sitio,'ishome':True}
        s = Section(**params)
        s.save()
        self.section = s
        return s

    def create_modulo(self):
        modulos = [{'secpk':self.section,'slugmd':'color',
                    'icono':'icon-iconos-brandline_portada',
                    'config':"{}",
                    'orden':0,
                    'mdtitle':self.section.sec_slug,
                    'fijo':True
                  }]

        for m in modulos:
            md = Modulo(**m)
            md.save()

            subitm = ITMS['color']
            subitm['modulopk']=md
            subitm['title'] = self.section.sec_name
            subitm['publicado']=True
            subitm['config'] = simplejson.dumps({'backcolor':'#4c4c4c',
                           'type_a':'color',
                           'altura':'25',
                           'fonttype':'Arial',
                           'fontcolor':'#fff',
                           'fontsize':'20',
                           'textalg':'center'
                         })
            subm = MD_grid(**subitm)
            subm.save()

        return m


'''
To run this code and the other code like it that were submitted, the final_proj_funcs program will need to be in the same folder
and then this code just needs to be exceuted.
The data file location can be edited below when the funciton is called
'''

import pandas as pd
import numpy as np
from final_proj_funcs import *
import tensorflow as tf
from tensorflow.keras import (
    models, losses, optimizers,
    layers, preprocessing,
)
from sklearn.model_selection import (
    train_test_split, KFold,
    StratifiedKFold, cross_val_score
)
from sklearn.linear_model import LogisticRegression
from sklearn import tree, ensemble, svm
from sklearn.metrics import (
    accuracy_score, precision_score,
    confusion_matrix, recall_score
)
import spacy, nltk
import unicodedata
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
def whitespace_tokenizer(line):
    return line.split()
def get_stem(text):
    stemmer = nltk.porter.PorterStemmer()
    text = ' '.join([stemmer.stem(word) for word in text.split()])
    return text

def remove_accents(text):
    new_text = unicodedata.normalize('NFKD', text).encode('ascii', \
        'ignore').decode('utf-8', 'ignore')
    return new_text

def remove_encoding(text):
    new_text = text.replace('\x92', "'")
    new_text = new_text.replace('\x94', '"')
    new_text = new_text.replace('\x93', '"')
    return new_text

stop_file = '../mallet_en_stoplist.txt'
proc_file = '../new_legis_proc_jargon_stopwords.txt'

stop_words = load_stopwords(stop_file)
proc_words = load_stopwords(proc_file)

def model_run(file_path, stopwords_dec, test_size, pred_thresh):

    alt_df = pd.read_csv('../data/wcpr_essays.csv', encoding="ISO-8859-1")
    alt_df['Target'] = pd.Series(np.where(alt_df['cNEU'] == 'y', 1, 0))
    alt_df['TEXT'] = alt_df['TEXT'].str.replace('\x92', "'")
    alt_df['TEXT'] = alt_df['TEXT'].str.replace('\x94', '"')
    alt_df['TEXT'] = alt_df['TEXT'].str.replace('\x93', '"')
    alt_df['Status_Length'] = pd.Series([len(alt_df['TEXT'][i].split()) for i in range(len(alt_df))])
    alt_df = alt_df[alt_df['Status_Length'] > 4]
    alt_target = alt_df['Target']
    alt_status = alt_df[['TEXT' ,'#AUTHID']]

    valid_feat_strings = convert_lines_to_feature_strings(np.array(alt_status['TEXT']), stop_words, \
        proc_words, remove_stopword_bigrams=stopwords_dec, include_trigrams=True)

    df = pd.read_csv(file_path, \
        encoding="ISO-8859-1")

    df['Target'] = pd.Series(np.where(df['cNEU'] == 'y', 1, 0))
    df['STATUS'] = df['STATUS'].str.replace('\x92', "'")
    df['STATUS'] = df['STATUS'].str.replace('\x94', '"')
    df['STATUS'] = df['STATUS'].str.replace('\x93', '"')
    # df['STATUS'] = df['STATUS'].map(remove_accents)
    # df['STATUS'] = df['STATUS'].map(check_punc)
    # df['STATUS'] = df['STATUS'].map(remove_numbers)
    df['STATUS'] = df['STATUS'].map(get_stem)
    df['Status_Length'] = pd.Series([len(df['STATUS'][i].split()) for i in range(len(df))])
    df = df[df['Status_Length'] > 4]

    target = df['Target']
    status = df[['STATUS' ,'#AUTHID']]
    dist_users = df[['#AUTHID', 'Target']].drop_duplicates()

    x_train, x_test, y_train, y_test = train_test_split(dist_users['#AUTHID'], dist_users['Target'],\
        test_size=test_size, random_state=4, stratify=dist_users['Target'])

    train_data = pd.concat([x_train, y_train], axis=1)
    test_data = pd.concat([x_test, y_test], axis=1).sort_values(by=['#AUTHID'])

    x_train_feat = pd.merge(status, train_data, on='#AUTHID',how='inner')
    x_test_feat = pd.merge(status, test_data, on='#AUTHID',how='inner')

    x_train_pp = np.array(x_train_feat['STATUS'])
    x_test_pp = np.array(x_test_feat['STATUS'])

    x_train_feat_strings = convert_lines_to_feature_strings(x_train_pp, stop_words, \
        proc_words, remove_stopword_bigrams=stopwords_dec, include_trigrams=True)
    x_test_feat_string = convert_lines_to_feature_strings(x_test_pp, stop_words, \
        proc_words, remove_stopword_bigrams=stopwords_dec, include_trigrams=True)

    x_features_train, training_vectorizer = convert_text_into_features(x_train_feat_strings, \
        stop_words, whitespace_tokenizer)
    x_test_transformed = training_vectorizer.transform(x_test_feat_string)
    valid_transformed = training_vectorizer.transform(valid_feat_strings).toarray()
    valid_y = np.array(alt_target)

    x_train_mod = x_features_train.toarray()
    x_test_mod = x_test_transformed.toarray()
    y_train_mod = np.array(x_train_feat['Target'])

    if stopwords_dec == True:
        input_size = 26718
    else:
        input_size = 62690

    test_mod = models.Sequential([
        layers.Input(shape=(x_train_mod.shape[1],)),
        layers.Dense(32, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dropout(0.2),
        layers.Dense(1, activation='sigmoid')
    ])

    metric = tf.keras.metrics.Precision()
    optim = optimizers.RMSprop(lr=0.0001)
    test_mod.compile(optimizer='adam',
                    loss='binary_crossentropy',
                    metrics=[metric])
    callback = tf.keras.callbacks.EarlyStopping(monitor='precision', patience=3)
    hist = test_mod.fit(x_train_mod, y_train_mod, epochs=10, steps_per_epoch=100, callbacks=[callback])
    out_shape = test_mod.predict(x_test_mod).shape
    y_hat = pd.Series(np.round(test_mod.predict(x_test_mod).reshape(out_shape[0],)))
    # y_hat = test_mod.predict(x_test_mod).reshape(out_shape[0],)
    # y_hat = pd.Series(np.round(test_mod.predict(x_test_mod).reshape(2825,)))

    x_test_feat['Preds'] = y_hat

    grouped_preds = x_test_feat.groupby('#AUTHID')['Preds'].sum().reset_index().sort_values(by=['#AUTHID'])
    grouped_counts = x_test_feat.groupby('#AUTHID')['Preds'].count().reset_index().sort_values(by=['#AUTHID'])

    joined_preds = pd.merge(grouped_preds, grouped_counts, \
        suffixes=('_preds', '_counts'),on='#AUTHID', how='inner')
    # grouped_preds['Final_pred'] = pd.Series(np.round(y_hat))
    joined_preds['Final_pred'] = pd.Series(\
        np.where((joined_preds['Preds_preds'] / joined_preds['Preds_counts']) >= pred_thresh, 1, 0))

    valid_preds = np.round(test_mod.predict(valid_transformed).reshape(2467,))
    print(accuracy_score(test_data['Target'], joined_preds['Final_pred']))
    print(precision_score(test_data['Target'], joined_preds['Final_pred']))
    print(recall_score(test_data['Target'], joined_preds['Final_pred']))
    # print(valid_preds)
    print(accuracy_score(valid_y, valid_preds))
    print(precision_score(valid_y, valid_preds))
    print(recall_score(valid_y, valid_preds))
    # return x_train_mod

model_run('../data/wcpr_mypersonality.csv', False, 0.3, 0.6)


#!/usr/bin/env python3
import argparse
from datetime import datetime, timedelta
from enum import Enum
import gevent
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
import glob
import json
import os
from progress.bar import IncrementalBar
import re
import requests
from requests_toolbelt import sessions
from subprocess import Popen, PIPE, STDOUT
import sys
from time import sleep
import urllib.parse

VERSION_STRING="2021.03.18.3"

OPENREC=r'^(?:https?:\/\/)?(?:www\.)?openrec\.tv\/(?P<type>[^\/]+?)\/(?P<id>[^\/]+)$'
VALID_LIVE_ID=r'^(?P<id>[a-z0-9]+?)$'
FULL_SIZE_IMG=r'\.w[0-9]{1,}\.ttl[0-9]{1,}\.(?P<ext>[a-z]{1,4})\?'
FULL_SIZE_REP=r'.\g<ext>?q=100&quality=100&'
CLEAN_FILENAME_KINDA=r'[^\w\-_\. \[\]\(\)]'
API="https://public.openrec.tv/external/api/v5/"

class LogLevel(Enum):
    BASIC=1
    VERBOSE=2

class DownloadBar(IncrementalBar):
    suffix="%(percent).1f%% ETA %(time_remaining)s"
    @property
    def time_remaining(self):
        ts = self.eta_td.total_seconds()
        m = ts % 60
        s = ts / 60
        return f"{int(m)}:{int(s):02}"

class StreamDownloader():
    def __init__(self, playlist_base):
        self.m3u8_session = sessions.BaseUrlSession(base_url=playlist_base)
        adapter = requests.adapters.HTTPAdapter(pool_connections=5, pool_maxsize=5, max_retries=10)
        self.m3u8_session.mount('http://', adapter)
        self.m3u8_session.mount('https://', adapter)
        self.success = True
        self.failed_list = []
        self.completed = {}

    def run(self, stream_filename, ts_list, download_bar):
        self.stream_filename = stream_filename
        self.stream_file = open(os.path.join(args.directory, f"{stream_filename}.ts.tmp"), "ab")
        self.ts_count = len(ts_list)
        self.download_bar = download_bar
        join_thread = gevent.spawn(self._append_file)
        self._download_segments(ts_list)
        join_thread.join()

    def _download_segments(self, ts_list):
        Pool(10).map(self._download_worker, ts_list)
        if not self.success:
            ts_list = self.failed_list
            self.failed_list = []
            self._download_segments(ts_list)

    def _download_worker(self, ts_tuple):
        ts_segment = ts_tuple[0]
        ts_index = ts_tuple[1]
        for retry in range(0, 5):
            try:
                ts_r = self.m3u8_session.get(ts_segment)
                if ts_r.ok:
                    segment_filename = f"{self.stream_filename}.seg{ts_index}"
                    with open(os.path.join(args.directory, segment_filename), 'wb') as segment_file:
                        segment_file.write(ts_r.content)
                    self.completed[ts_index] = segment_filename
                    return
            except:
                print_log("download worker", f"failed to download {ts_segment}, retrying ({retry}/5)...")
        self.success = False

    def _append_file(self):
        index = 0
        while index < self.ts_count:
            segment_filename = self.completed.get(index, '')
            if segment_filename:
                with open(os.path.join(args.directory, segment_filename), 'rb') as segment_file:
                    self.stream_file.write(segment_file.read())
                os.remove(os.path.join(args.directory, segment_filename))
                self.download_bar.next()
                index += 1
            else:
                sleep(0.05)

def dl_channel(s, channel_id):
    # check for argument that cannot be used
    if args.list_formats:
        print_log(f"channel:{channel_id}", "cannot use argument --list-formats with channel link")
        return

    # get base channel data for checking validity
    init_response = s.get(f'channels/{channel_id}')
    if not init_response.ok:
        print_log(f"channel:{channel_id}", "failed to get channel information")
        print_log(f"channel:{channel_id}", f"API response returned status code {init_response.status_code}", LogLevel.VERBOSE)
        return
    c_init_json = init_response.json()
    if "status" in c_init_json:
        print_log(f"channel:{channel_id}", "failed to get channel information")
        print_log(f"channel:{channel_id}", f"API body returned status code {c_init_json['status']}: {c_init_json['message']}", LogLevel.VERBOSE)
        return

    # string to use in output channel filenames
    channel_string = re.sub(CLEAN_FILENAME_KINDA, "_", f"{c_init_json['name']} [{c_init_json['id']}]").strip()

    # use search api to retrieve more complete json data
    search_query_param = urllib.parse.urlencode({"search_query": c_init_json["name"]})
    search_response = s.get(f'search-users?{search_query_param}')

    # use /channels json if /search-users json doesn't exist for some reason
    c_json = c_init_json
    if search_response.ok:
        found_full_json = False
        for user_result in search_response.json():
            if user_result["name"] == c_init_json["name"]:
                c_json = user_result
                found_full_json = True
                break
        if not found_full_json:
            print_log(f"channel:{channel_id}", f"failed to get complete channel information from API")
    else:
        print_log(f"channel:{channel_id}", f"failed to get complete channel information from API")
        print_log(f"channel:{channel_id}", f"API response returned status code {search_response.status_code}", LogLevel.VERBOSE)

    if args.write_info_json:
        info_filename = f"{channel_string}.info.json"
        print_log(f"info:{channel_id}", f"writing channel information to '{info_filename}'")
        with open(os.path.join(args.directory, f"{info_filename}.tmp"), "w") as channel_info:
            channel_info.write(json.dumps(c_json))
        os.rename(os.path.join(args.directory, f"{info_filename}.tmp"), os.path.join(args.directory, info_filename))

    if args.write_thumbnail:
        # icon (avatar)
        full_size_icon_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, c_json["icon_image_url"])
        icon_format = urllib.parse.parse_qs(full_size_icon_url)["format"][0]
        icon_filename = f"{channel_string}-icon.{icon_format}"
        print_log(f"icon:{channel_id}", f"writing channel icon to '{icon_filename}'")
        icon_response = requests.get(full_size_icon_url)
        if icon_response.ok:
            with open(os.path.join(args.directory, f"{icon_filename}.tmp"), "wb") as channel_icon:
                channel_icon.write(icon_response.content)
            os.rename(os.path.join(args.directory, f"{icon_filename}.tmp"), os.path.join(args.directory, icon_filename))
        else:
            print_log(f"icon:{channel_id}", "failed to retrieve channel icon")
            print_log(f"icon:{channel_id}", f"API response returned status code {icon_response.status_code}", LogLevel.VERBOSE)

        # cover (banner)
        full_size_cover_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, c_json["cover_image_url"])
        cover_format = urllib.parse.parse_qs(full_size_cover_url)["format"][0]
        cover_filename = f"{channel_string}-cover.{cover_format}"
        print_log(f"cover:{channel_id}", f"writing channel cover to '{cover_filename}'")
        cover_response = requests.get(full_size_icon_url)
        if cover_response.ok:
            with open(os.path.join(args.directory, f"{cover_filename}.tmp"), "wb") as channel_cover:
                channel_cover.write(cover_response.content)
            os.rename(os.path.join(args.directory, f"{cover_filename}.tmp"), os.path.join(args.directory, cover_filename))
        else:
            print_log(f"icon:{channel_id}", "failed to retrieve channel cover")
            print_log(f"icon:{channel_id}", f"API response returned status code {cover_response.status_code}", LogLevel.VERBOSE)

    movie_list_response = s.get(f"movies?channel_ids={channel_id}").json()
    for movie_index in range(len(movie_list_response)):
        print_log(f"channel:{channel_id}", f"downloading video {movie_index + 1} of {len(movie_list_response)}")
        dl_movie(s, movie_list_response[movie_index]["id"])

def dl_movie(s, movie_id):
    # check if we can skip
    if args.download_archive:
        if os.path.isfile(args.download_archive):
            with open(args.download_archive, "r") as archive_file:
                skip_ids = archive_file.read().splitlines()
            if movie_id in skip_ids:
                print_log(f"movie:{movie_id}", "already recorded in archive")
                return

    # get video data and check validity
    movie_response = s.get(f"movies/{movie_id}")
    if not movie_response.ok:
        print_log(f"info:{movie_id}", "failed to get movie information")
        print_log(f"info:{movie_id}", f"API response returned status code {movie_response.status_code}", LogLevel.VERBOSE)
        return
    m_json = movie_response.json()
    if "status" in m_json:
        print_log(f"info:{movie_id}", "failed to get movie information")
        print_log(f"info:{movie_id}", f"API body returned status code {m_json['status']}: {m_json['message']}")
        return

    # string to use in output video names
    movie_string = re.sub(CLEAN_FILENAME_KINDA, "_", f"{m_json['title']} [{m_json['id']}]").strip()

    # remove ad info because who cares about ads
    m_json.pop("ad")

    # api only gives public m3u8 by default. we dont care :^)
    # derive m3u8 link from json[media][url_public]
    # these links might not actually exist, so downloading relies on values from playlist.m3u8
    m_json["media"]["url"] = urllib.parse.urljoin(m_json["media"]["url_public"], "playlist.m3u8")
    m_json["media"]["url_audio"] = urllib.parse.urljoin(m_json["media"]["url_public"], "aac.m3u8")
    m_json["media"]["url_source"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_source/chunklist.m3u8")
    m_json["media"]["url_low_latency"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_low/chunklist.m3u8")
    m_json["media"]["url_ull"] = urllib.parse.urljoin(m_json["media"]["url_public"], "chunklist_144p/chunklist.m3u8")

    formats_list = get_m3u8_info(m_json["media"]["url"])

    if args.list_formats:
        print_log(f"movie:{movie_id}", f"available formats:")
        print_formats(formats_list)
        return

    downloading_format = None
    for format_settings in formats_list:
        if args.format in {format_settings["media"]["NAME"], format_settings["media"]["GROUP-ID"]}:
            downloading_format = urllib.parse.urljoin(m_json["media"]["url_public"], format_settings["location"])

    if args.write_info_json:
        info_filename = f"{movie_string}.info.json"
        print_log(f"info:{movie_id}", f"writing video information to '{info_filename}'")
        with open(os.path.join(args.directory, f"{info_filename}.tmp"), "w") as movie_info:
            movie_info.write(json.dumps(m_json))
        os.rename(os.path.join(args.directory, f"{info_filename}.tmp"), os.path.join(args.directory, info_filename))

    if args.write_thumbnail:
        full_size_thumb_url = re.sub(FULL_SIZE_IMG, FULL_SIZE_REP, m_json["thumbnail_url"])
        thumbnail_format = urllib.parse.parse_qs(full_size_thumb_url)["format"][0]
        thumbnail_filename = f"{movie_string}.{thumbnail_format}"
        print_log(f"thumbnail:{movie_id}", f"writing thumbnail to '{thumbnail_filename}'")
        thumb_response = requests.get(full_size_thumb_url)
        if thumb_response.ok:
            with open(os.path.join(args.directory, f"{thumbnail_filename}.tmp"), "wb") as movie_thumbnail:
                movie_thumbnail.write(thumb_response.content)
            os.rename(os.path.join(args.directory, f"{thumbnail_filename}.tmp"), os.path.join(args.directory, thumbnail_filename))
        else:
            print_log(f"thumbnail:{movie_id}", "failed to retrieve video thumbnail")
            print_log(f"thumbnail:{movie_id}", f"API response returned status code {thumb_response.status_code}", LogLevel.VERBOSE)

    if args.write_live_chat:
        dl_live_chat(s, movie_id, movie_string, m_json["started_at"])

    if not args.skip_download:
        if downloading_format:
            dl_m3u8_video(movie_id, movie_string, downloading_format)
        else:
            print_log(f"movie:{movie_id}", f"could not find video format '{args.format}'. to view all available formats, use --list-formats")
    else:
        print_log(f"movie:{movie_id}", f"skipping download")

def dl_m3u8_video(movie_id, movie_filename, m3u8_link):
    movie_path = os.path.join(args.directory, f"{movie_filename}")
    # don't re-download
    if os.path.isfile(f"{movie_path}.ts") or os.path.isfile(f"{movie_path}.mp4"):
        print_log(f"movie:{movie_id}", f"already downloaded")
    # can't resume downloads (atm), so remove progress
    else:
        if os.path.isfile(f"{movie_path}.ts.tmp"):
            os.remove(f"{movie_path}.ts.tmp")
            for seg_file in os.listdir(args.directory):
                if re.fullmatch(r"^" + f"{re.escape(movie_filename)}" + r"\.seg[0-9]{1,}$", seg_file):
                    os.remove(seg_file)

        # get necessary variables ready
        playlist_base = urllib.parse.urljoin(m3u8_link, ".")
        m3u8_text = requests.get(m3u8_link).text
        ts_list = [n for n in m3u8_text.splitlines() if n and not n.startswith("#")]
        ordered_ts_list = list(zip(ts_list, [n for n in range(len(ts_list))]))

        # run the downloader
        print_log(f"movie:{movie_id}", f"writing video to '{movie_filename}.ts'")
        stream_downloader = StreamDownloader(playlist_base)
        download_bar = DownloadBar(f"[movie:{movie_id}]", max=len(ts_list))
        stream_downloader.run(movie_filename, ordered_ts_list, download_bar)

        # if success, check if converting
        if stream_downloader.success:
            download_bar.finish()
            os.rename(f"{movie_path}.ts.tmp", f"{movie_path}.ts")
            if not args.skip_convert:
                mpeg_convert(os.path.join(args.directory, f"{movie_filename}"))
        else:
            print_log(f"movie:{movie_id}", f"failed to download")
            return
    if args.download_archive:
        with open(args.download_archive, "a") as archive_file:
            archive_file.write(f"{movie_id}\n")

def mpeg_convert(file_path):
    print_log("mpeg-convert", f"converting video to '{movie_filename}.mp4'")
    ffmpeg_list = ["ffmpeg", "-i", f"{file_path}.ts", "-acodec", "copy", "-vcodec", "copy", f"{file_path}.mp4"]
    try:
        ffmpeg_process = Popen(ffmpeg_list, stdout=PIPE, stderr=PIPE)
        stdout, stderr = ffmpeg_process.communicate()
    except Exception:
        print_log("mpeg-convert", "failure in executing ffmpeg")
        return
    os.remove(f"{file_path}.ts")

# based on parts of https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py
def get_m3u8_info(playlist_link):
    m3u8_info = []
    m3u8_text = requests.get(playlist_link).text
    media_details = None
    format_details = None
    for line in m3u8_text.splitlines():
        if line.startswith("#EXT-X-MEDIA:"):
            media_details = parse_m3u8_attributes(line)
            #media_type, group_id, name = media.get("TYPE"), media.get("GROUP-ID"), media.get("NAME")
        elif line.startswith("#EXT-X-STREAM-INF:"):
            format_details = parse_m3u8_attributes(line)
        elif not line.startswith("#"):
            if line.endswith(".m3u8"):
                if not media_details:
                    print_log("get-m3u8-info", f"could not find media details for stream with format '{str(format_details)}'", LogLevel.VERBOSE)
                elif not format_details:
                    print_log("get-m3u8-info", f"could not find format details for stream with media '{str(media_details)}'", LogLevel.VERBOSE)
                else:
                    m3u8_info += [{"location": line, "media": media_details, "format": format_details}]
                    media_details = None
                    format_details = None
            else:
                print_log("get-m3u8-info", f"unexpected line in m3u8 file: '{line}'", LogLevel.VERBOSE)
    return m3u8_info

# https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py#L5495
def parse_m3u8_attributes(attrib):
    info = {}
    for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
        if val.startswith("\""):
            val = val[1:-1]
        info[key] = val
    return info

def print_formats(formats_list):
    print(f"{'NAME':<7} {'GROUP-ID':<8} {'RESOLUTION':<10} {'FPS':<4} {'TBR':<6} {'CODECS':<24}")
    print(f"{'-' * 7} {'-' * 8} {'-' * 10} {'-' * 4} {'-' * 6} {'-' * 24}")
    for format_settings in formats_list:
        print(f"{format_settings['media']['NAME']:<7} " +
        f"{format_settings['media']['GROUP-ID']:<8} " +
        f"{format_settings['format']['RESOLUTION']:<10} " +
        f"{format_settings['format']['FRAME-RATE']:<4} " +
        f"{str(int(float(format_settings['format']['BANDWIDTH']) / 1000))+'k':<6} " +
        f"{format_settings['format']['CODECS']:<24}")

def dl_live_chat(s, movie_id, movie_filename, started_at):
    live_chat_filename = os.path.join(args.directory, f"{movie_filename}.live_chat.json")
    if os.path.isfile(live_chat_filename):
        print_log(f"live-chat:{movie_id}", "already downloaded")
        return
    # no resuming downloads, so just remove the temp
    elif os.path.isfile(f"{live_chat_filename}.tmp"):
        os.remove(f"{live_chat_filename}.tmp")

    # read datetime object and remove utc offset for using in the chat API
    chat_dt = datetime.strptime(started_at, "%Y-%m-%dT%H:%M:%S%z")
    chat_dt = (chat_dt - chat_dt.utcoffset()).replace(tzinfo=None)
    chat_response = s.get(f"movies/{movie_id}/chats?from_created_at={chat_dt.isoformat()}.000Z&is_including_system_message=false")

    print_log(f"live-chat:{movie_id}", f"writing live chat to {movie_filename}.live_chat.json")
    with open(f"{live_chat_filename}.tmp", "w") as live_chat_file:
        # loop until blank response
        while chat_response.ok and len(chat_response.json()) > 0:
            last_post_time = datetime.strptime(chat_response.json()[-1]["posted_at"], "%Y-%m-%dT%H:%M:%S%z")
            last_post_time = (last_post_time - last_post_time.utcoffset()).replace(tzinfo=None)
            # stop if this is the same date as previously requested
            if last_post_time.isoformat() == chat_dt.isoformat():
                break
            for chat_line in chat_response.json():
                live_chat_file.write(f"{json.dumps(chat_line)}\n")
            # use timestamp of last chat post retrieved to fill next url
            chat_dt = last_post_time
            chat_response = s.get(f"movies/{movie_id}/chats?from_created_at={chat_dt.isoformat()}.000Z&is_including_system_message=false")
    if not chat_response.ok:
        print_log(f"live-chat:{movie_id}", f"unexpected ending with API response status code {chat_response.status_code}", LogLevel.VERBOSE)
    os.rename(f"{live_chat_filename}.tmp", live_chat_filename)

def print_log(component, message, level=LogLevel.BASIC):
    if level == LogLevel.VERBOSE and not args.verbose:
        return
    print(f"[{component}] {message}")

def get_arguments():
    parser.add_argument("--version", action="store_true", help="print version string and exit")
    parser.add_argument("-V", "--verbose", action="store_true", help="print debugging information")
    parser.add_argument("-d", "--directory", type=str, help="save directory (defaults to current)", default=os.getcwd())
    parser.add_argument("--download-archive", metavar="FILE", type=str, help="download only videos not listed in the archive file and record the IDs of downloaded videos")
    parser.add_argument("--write-info-json", action="store_true", help="write metadata to .info.json file")
    parser.add_argument("--write-thumbnail", action="store_true", help="write thumbnail to image file")
    parser.add_argument("--write-live-chat", action="store_true", help="write live chat comments to .live_chat.json file")
    parser.add_argument("-f", "--format", type=str, help="video format, specified by either NAME or GROUP-ID (defaults to Source)", default="Source")
    parser.add_argument("-F", "--list-formats", action="store_true", help="print available format details for a video and exit")
    parser.add_argument("--skip-download", action="store_true", help="do not download the video")
    parser.add_argument("--skip-convert", action="store_true", help="do not use ffmpeg to convert the MPEG-TS stream to MPEG-4")
    parser.add_argument("links", metavar="LINK", nargs="*", help="openrec channel or video link(s)/ids")
    return parser.parse_args()

def main():
    api_session = sessions.BaseUrlSession(base_url=API)
    if args.version:
        print(VERSION_STRING)
        return
    if len(args.links) == 0:
        parser.print_usage()
    elif not os.path.isdir(args.directory):
        os.makedirs(args.directory)
    for link in args.links:
        # is openrec link
        openrec_m = re.search(OPENREC, link)
        id_m = re.search(VALID_LIVE_ID, link)
        if openrec_m:
            t = openrec_m.group("type")
            if t == "user":
                dl_channel(api_session, openrec_m.group("id"))
            elif t == "live":
                dl_movie(api_session, openrec_m.group("id"))
            else:
                print_log("openrec", f"unknown link type \'{t}\'")
        elif id_m:
            dl_movie(api_session, id_m.group("id"))
        else:
            print_log("openrec", f"invalid link or id \'{link}\'")

parser = argparse.ArgumentParser()
args = get_arguments()

if __name__ == "__main__":
    main()


# Copyright 2008-2019 pydicom authors. See LICENSE file for details.
"""Use the `numpy <https://numpy.org/>`_ package to convert supported *Overlay
Data* to a :class:`numpy.ndarray`.

**Supported transfer syntaxes**

* 1.2.840.10008.1.2 : Implicit VR Little Endian
* 1.2.840.10008.1.2.1 : Explicit VR Little Endian
* 1.2.840.10008.1.2.1.99 : Deflated Explicit VR Little Endian
* 1.2.840.10008.1.2.2 : Explicit VR Big Endian

**Supported data**

The numpy handler supports the conversion of data in the (60xx,3000)
*Overlay Data* element to a :class:`~numpy.ndarray` provided the
related :dcm:`Overlay Plane<part03/sect_C.9.2.html>` and :dcm:`Multi-frame
Overlay<part03/sect_C.9.3.html>` module elements have values given in the
table below.

+------------------------------------------------+--------------+
| Element                                        | Supported    |
+-------------+---------------------------+------+ values       |
| Tag         | Keyword                   | Type |              |
+=============+===========================+======+==============+
| (60xx,0010) | OverlayRows               | 1    | N > 0        |
+-------------+---------------------------+------+--------------+
| (60xx,0011) | OverlayColumns            | 1    | N > 0        |
+-------------+---------------------------+------+--------------+
| (60xx,0015) | NumberOfFramesInOverlay   | 1    | N > 0        |
+-------------+---------------------------+------+--------------+
| (60xx,0100) | OverlayBitsAllocated      | 1    | 1            |
+-------------+---------------------------+------+--------------+
| (60xx,0102) | OverlayBitPosition        | 1    | 0            |
+-------------+---------------------------+------+--------------+

"""

import warnings

try:
    import numpy as np
    HAVE_NP = True
except ImportError:
    HAVE_NP = False

import pydicom.uid
from pydicom.pixel_data_handlers.numpy_handler import unpack_bits

HANDLER_NAME = 'Numpy Overlay'

DEPENDENCIES = {
    'numpy': ('http://www.numpy.org/', 'NumPy'),
}

SUPPORTED_TRANSFER_SYNTAXES = [
    pydicom.uid.ExplicitVRLittleEndian,
    pydicom.uid.ImplicitVRLittleEndian,
    pydicom.uid.DeflatedExplicitVRLittleEndian,
    pydicom.uid.ExplicitVRBigEndian,
]

def is_available():
    """Return ``True`` if the handler has its dependencies met.

    .. versionadded:: 1.4
    """
    return HAVE_NP

def supports_transfer_syntax(transfer_syntax):
    """Return ``True`` if the handler supports the `transfer_syntax`.

    .. versionadded:: 1.4

    Parameters
    ----------
    transfer_syntax : uid.UID
        The Transfer Syntax UID of the *Pixel Data* that is to be used with
        the handler.
    """
    return transfer_syntax in SUPPORTED_TRANSFER_SYNTAXES

def get_expected_length(elem, unit='bytes'):
    """Return the expected length (in terms of bytes or pixels) of the *Overlay
    Data*.

    .. versionadded:: 1.4

    +------------------------------------------------+-------------+
    | Element                                        | Required or |
    +-------------+---------------------------+------+ optional    |
    | Tag         | Keyword                   | Type |             |
    +=============+===========================+======+=============+
    | (60xx,0010) | OverlayRows               | 1    | Required    |
    +-------------+---------------------------+------+-------------+
    | (60xx,0011) | OverlayColumns            | 1    | Required    |
    +-------------+---------------------------+------+-------------+
    | (60xx,0015) | NumberOfFramesInOverlay   | 1    | Required    |
    +-------------+---------------------------+------+-------------+

    Parameters
    ----------
    elem : dict
        A :class:`dict` with the keys as the element keywords and values the
        corresponding element values (such as ``{'OverlayRows': 512, ...}``)
        for the elements listed in the table above.
    unit : str, optional
        If ``'bytes'`` then returns the expected length of the *Overlay Data*
        in whole bytes and NOT including an odd length trailing NULL padding
        byte. If ``'pixels'`` then returns the expected length of the *Overlay
        Data* in terms of the total number of pixels (default ``'bytes'``).

    Returns
    -------
    int
        The expected length of the *Overlay Data* in either whole bytes or
        pixels, excluding the NULL trailing padding byte for odd length data.
    """
    length = elem['OverlayRows'] * elem['OverlayColumns']
    length *= elem['NumberOfFramesInOverlay']

    if unit == 'pixels':
        return length

    # Determine the nearest whole number of bytes needed to contain
    #   1-bit pixel data. e.g. 10 x 10 1-bit pixels is 100 bits, which
    #   are packed into 12.5 -> 13 bytes
    return length // 8 + (length % 8 > 0)

def reshape_overlay_array(elem, arr):
    """Return a reshaped :class:`numpy.ndarray` `arr`.

    .. versionadded:: 1.4

    +------------------------------------------------+--------------+
    | Element                                        | Supported    |
    +-------------+---------------------------+------+ values       |
    | Tag         | Keyword                   | Type |              |
    +=============+===========================+======+==============+
    | (60xx,0010) | OverlayRows               | 1    | N > 0        |
    +-------------+---------------------------+------+--------------+
    | (60xx,0011) | OverlayColumns            | 1    | N > 0        |
    +-------------+---------------------------+------+--------------+
    | (60xx,0015) | NumberOfFramesInOverlay   | 1    | N > 0        |
    +-------------+---------------------------+------+--------------+

    Parameters
    ----------
    elem : dict
        A :class:`dict` with the keys as the element keywords and values the
        corresponding element values (such as ``{'OverlayRows': 512, ...}``)
        for the elements listed in the table above.
    arr : numpy.ndarray
        A 1D array containing the overlay data.

    Returns
    -------
    numpy.ndarray
        A reshaped array containing the overlay data. The shape of the array
        depends on the contents of the dataset:

        * For single frame data (rows, columns)
        * For multi-frame data (frames, rows, columns)

    References
    ----------

    * DICOM Standard, Part 3, Sections :dcm:`C.9.2<part03/sect_C.9.2.html>`
      and :dcm:`C.9.3<part03/sect_C.9.3.html>`
    * DICOM Standard, Part 5, :dcm:`Section 8.2<part05/sect_8.2.html>`
    """
    if not HAVE_NP:
        raise ImportError("Numpy is required to reshape the overlay array.")

    nr_frames = elem['NumberOfFramesInOverlay']
    nr_rows = elem['OverlayRows']
    nr_columns = elem['OverlayColumns']

    if nr_frames < 1:
        raise ValueError(
            "Unable to reshape the overlay array as a value of {} for "
            "(60xx,0015) 'Number of Frames in Overlay' is invalid."
            .format(nr_frames)
        )

    if nr_frames > 1:
        return arr.reshape(nr_frames, nr_rows, nr_columns)

    return arr.reshape(nr_rows, nr_columns)

def get_overlay_array(ds, group):
    """Return a :class:`numpy.ndarray` of the *Overlay Data*.

    .. versionadded:: 1.4

    Parameters
    ----------
    ds : Dataset
        The :class:`Dataset` containing an Overlay Plane module and the
        *Overlay Data* to be converted.
    group : int
        The group part of the *Overlay Data* element tag, e.g. ``0x6000``,
        ``0x6010``, etc. Must be between 0x6000 and 0x60FF.

    Returns
    -------
    np.ndarray
        The contents of (`group`,3000) *Overlay Data* as an array.

    Raises
    ------
    AttributeError
        If `ds` is missing a required element.
    ValueError
        If the actual length of the overlay data doesn't match the expected
        length.
    """
    transfer_syntax = ds.file_meta.TransferSyntaxUID
    # The check of transfer syntax must be first
    if transfer_syntax not in SUPPORTED_TRANSFER_SYNTAXES:
        raise NotImplementedError(
            "Unable to convert the overlay data as the transfer syntax "
            "is not supported by the numpy overlay data handler."
        )

    # Check required elements
    elem = {
        'OverlayData': ds.get((group, 0x3000), None),
        'OverlayBitsAllocated': ds.get((group, 0x0100), None),
        'OverlayRows': ds.get((group, 0x0010), None),
        'OverlayColumns': ds.get((group, 0x0011), None),
    }

    missing = [kk for kk, vv in elem.items() if vv is None]
    if missing:
        raise AttributeError(
            "Unable to convert the overlay data as the following required "
            "elements are missing from the dataset: " + ", ".join(missing)
        )

    # Grab the element values
    elem = {kk: vv.value for kk, vv in elem.items()}

    # Add in if not present
    nr_frames = ds.get((group, 0x0015), None)
    if nr_frames is None:
        elem['NumberOfFramesInOverlay'] = 1
    else:
        elem['NumberOfFramesInOverlay'] = nr_frames.value

    # Calculate the expected length of the pixel data (in bytes)
    #   Note: this does NOT include the trailing null byte for odd length data
    expected_len = get_expected_length(elem)

    # Check that the actual length of the pixel data is as expected
    actual_length = len(elem['OverlayData'])

    # Correct for the trailing NULL byte padding for odd length data
    padded_expected_len = expected_len + expected_len % 2
    if actual_length < padded_expected_len:
        if actual_length == expected_len:
            warnings.warn(
                "The overlay data length is odd and misses a padding byte.")
        else:
            raise ValueError(
                "The length of the overlay data in the dataset ({} bytes) "
                "doesn't match the expected length ({} bytes). "
                "The dataset may be corrupted or there may be an issue "
                "with the overlay data handler."
                .format(actual_length, padded_expected_len)
            )
    elif actual_length > padded_expected_len:
        # PS 3.5, Section 8.1.1
        msg = (
            "The length of the overlay data in the dataset ({} bytes) "
            "indicates it contains excess padding. {} bytes will be removed "
            "from the end of the data"
            .format(actual_length, actual_length - expected_len)
        )
        warnings.warn(msg)

    # Unpack the pixel data into a 1D ndarray, skipping any trailing padding
    nr_pixels = get_expected_length(elem, unit='pixels')
    arr = unpack_bits(elem['OverlayData'])[:nr_pixels]

    return reshape_overlay_array(elem, arr)


from flask import Flask, render_template, request, send_file, redirect, url_for

app = Flask(__name__)

if __name__ == '__main__':
    app.run()

@app.route('/', methods=['GET'])
def index():
    if request.method == 'GET':
        return render_template('index.html')

@app.route('/static/resume.pdf', methods=['GET'])
def resume():
    if request.method == 'GET':
        return redirect(url_for('index'))


from typing import List
class Solution:
    def permute(self, nums: List[int]) -> List[List[int]]:
        n = len(nums)
        if n <= 0:
            return []
        used = [False] * n
        res = []
        path = []
        self.dfs(nums, n,0,used,res,path)
        return res

    def dfs(self, nums:List[int], n:int, depth:int, used:List[bool],res:List[List[int]],path:List[int]):
        if n == depth:
            res.append(path.copy())
            return res
        for i in range(n):
            if used[i] == False:
                path.append(nums[i])
                used[i] = True
                self.dfs(nums,n,depth + 1,used,res,path)
                used[i] = False
                path.pop()
a = Solution()
print(a.permute([1,2,3]))


#coding=utf-8

# python (渣打马拉松大赛)
# 分类： python 2013-08-05 13:08 564人阅读 评论(5) 收藏 举报
# 题目：http://blog.csdn.net/mrpre/article/details/9765979
# 题目就加减，挺简单的，所以没通过后缀表达式解析
# 现在已经写好了 中缀转后缀 表达式的解析算法。比赛结束后再贴出来。
# 此程序:在解释器版本Python 2.7.3 中通过
# 假设1:不知道规则，不知道规则个数，不知道规则单位类型,
# 假设2:不知道计算个数，不知道计算单位类型，不知道加减运算符个数
# 仅加减，无优先级，故无需采用堆栈计算出后缀表达式的方法，来解析算术表达式，采用递归解析即可

# WARNING:
#官方给的题目(input.txt)有问题
#按英美通行的一些格式规则，大于1或小于-1，单位用复数；-1和1之间，用单数。
#但是实际运用中，英语国家习惯是只有等于1或-1时用单数，其余都用复数，
#但在github给出的input文件中，却综合出现。例如出现了 "0.15 yard" 以及"0.9 miles"

#file start

global line #Limit the number of lines

#functiuon used to add/open "input.txt" to list(initialization )
def init():
    list_tmp=[]
    split_tmp=split=0
    for line in open("input.txt") :
        if (line!='\n'):
            list_tmp.append(line)
            split_tmp=split_tmp+1
        else:
            split=split_tmp
    return list_tmp,split

#add new rule to dict ,promote the ability to mach
def renew(dic_rule):
    list_tmp=[]
    for i in dic_rule:
        list_tmp.append(i)
    for i in list_tmp:
        dic_rule[i[0:len(i)-1]]=dic_rule[i]
    return dic_rule

#extract rule
def extractrule(list_rule):
    dict_tmp={}
    for i in list_rule:
        number_eql=i.index("=")
        number_unitm=i.index("m\n")
        dict_tmp[i[2:(number_eql-1)]]=i[(number_eql+2):(number_unitm-1)]
    return dict_tmp

#calculator
def calculator(list_quen):
   global line
   line=0
   for i in list_quen:
       if i.find("+")>0 or i.find("-")>0:
           compute(i)
           line=line+1
       else:
           answer=convert(i)
           answer=round(answer,2)
           line=line+1
           fp.write(str(answer)+" m\n")

#convert
def convert(i):
    sp=i.find(" ")
    length=len(i)
    if(i[length-1]==" "or i[length-1]=="\n"): #forma unit
        unit=i[sp+1:length-1]
    else:
        unit=i[sp+1:length]

    value=i[0:sp]
    value=value.replace(" ","")#forma value
    value=eval(value)

    if unit=="feet" or unit=="inches":
        if unit=="feet":unit="foot"
        if unit=="inches":unit="inch"
    else:
        unit=unit[0:len(unit)-1]

    answer=eval(dic_rule[unit])*value
    return answer

#change (recursive function)
def change(st):
    global answer_change
    if(st.count("+ ")>1 and st.count("- ")==0):
        list_tmp=st.split("+ ")
        change(str(list_tmp[0]))
        change(str(list_tmp[1]))

    if(st.count("+ ")==0 and st.count("- ")>1):
        list_tmp=st.split("+ ")
        change(str(list_tmp[0]))
        change(str(list_tmp[1]))

    if(st.count("+ ")==1 and st.count("- ")==0):
        list_tmp=st.split("+ ")
        answer=convert(list_tmp[0])+convert(list_tmp[1])
        answer_change=answer_change+answer
    if(st.count("- ")==1 and st.count("+ ")==0):
        list_tmp=st.split("- ")
        answer=convert(list_tmp[0])-convert(list_tmp[1])
        answer_change=answer_change+answer
    if(st.count("+ ")==0 and st.count("- ")==0):
        st_tmp=convert(st)
        answer_change=answer_change+st_tmp
    if(st.count("+ ")>0 and st.count("- ")>0):
        list_tmp=st.split("+ ")
        change(str(list_tmp[0]))
        change(str(list_tmp[1]))

#add and substract
def compute(i):
    global answer_change
    global line
    afchange_str=change(i)
    answer_change=round(answer_change,2)#with accuracy of 2 decimal points
    fp.write(str(answer_change)+" m")#output to "output.txt"
    if line<len(list_quen)-1:
        fp.write("\n")                 #Don't add "\n" to last line
    answer_change=0#reset answer_change ,preparing for next question

#output personal information
def outputinfo():
    fp.write("xxxxxxxxxx@126.com \n\n")

#main
fp=open("output.txt","w")
answer_change=0 #global variability used to conserve the answer

list_main=init()#init
split=list_main[1]
length=len(list_main[0])

list_rule=list_main[0][0:split]#extract information about rule to list
list_quen=list_main[0][split:length]#generate question to list
dic_rule=extractrule(list_rule) #generate rule to dict
dic_rule=renew(dic_rule) #采用模糊匹配(允许结尾字母的出错)

outputinfo()    #output personal info
calculator(list_quen)   #compute
fp.close()  #close output.txt
#file end
#print
#print "ALL:",list_main
#print "RULE origin:",list_rule
#print "RULE answer:",dic_rule
#print "quen:",list_quen


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jan 22 22:49:32 2017

@author: maida

Plot bivariate Gaussian
"""

import numpy as np
import matplotlib.pyplot as plt

# covar matrix must be positive definite, or
# symmetric with an inverse for Gaussian distribution
covarMat = np.array([[1.0, 0.0], [0.0, 6.0]])  # 2x2 array
covarInv = np.linalg.inv(covarMat)

def gaussValue(x):  # x is 2-element column vec
    return np.exp(-0.5 * np.dot(np.dot(x.T,
                                       covarInv),
                                x))

# create a range from -6 to 6 in 0.2 increments. Length is 61.
pltRange = np.linspace(-6, 6, num=61, endpoint=True)

# array to hold the values for a gaussian function
# index z arrays by [i,j)
# indices are from 0 to 60
z1 = np.zeros((61, 61))

# for a 2D array, (0,0) is top left
# also 1st component of array index is y (vertical)

# Generate values
i = 0
for x in pltRange:
    j = 0
    for y in pltRange:
        pt1 = np.array([[x], [y]])  # column vector. 2x1 array.
        z1[j, i] = gaussValue(pt1)
        j = j + 1
    i = i + 1

z1 = np.flipud(z1)

plt.figure()
CS = plt.contour(pltRange, pltRange, z1)
plt.grid()
plt.show()


import tensorflow as tf
import GeoSimilarity.src.sampling as sampling

class TestSampling(tf.test.TestCase):

    def test_extract_neighbor(self):
        print('Test neighbor extraction')
        n_channels = 4
        n_frames = 12
        img_size = 1024
        patch_size = 64
        channels_frames = n_channels * n_frames
        neighborhood_factor = 2

        data = tf.random_normal(shape=(img_size, img_size, channels_frames))

        neighborhood, index = sampling.extract_neighborhood(data, img_size, patch_size, channels_frames, neighborhood_factor)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(10):
                np_data, np_neighborhood, np_index = sess.run([data, neighborhood, index])
                print(np_index)

                assert (np_neighborhood - np_data[np_index[0]:np_index[0] + patch_size * neighborhood_factor,
                                          np_index[1]:np_index[1] + patch_size * neighborhood_factor, :]).sum() == 0

    def test_extract_time_series(self):
        print('Test time series extraction')
        n_channels = 4
        n_frames = 12
        img_size = 1024
        patch_size = 64
        channels_frames = n_channels * n_frames
        neighborhood_factor = 2

        data = tf.random_normal(shape=(img_size, img_size, channels_frames))

        neighborhood, index = sampling.extract_neighborhood(data,
                                                            img_size,
                                                            patch_size,
                                                            channels_frames,
                                                            neighborhood_factor)

        time_series, neighbor_time_series, new_index0, new_index1 = sampling.extract_time_series(neighborhood,
                                                                                patch_size,
                                                                                channels_frames,
                                                                                neighborhood_factor)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(10):
                np_neighborhood, np_time_series, np_neighbor_time_series,\
                np_new_index0, np_new_index1 = sess.run([neighborhood,
                                                         time_series,
                                                         neighbor_time_series,
                                                         new_index0,
                                                         new_index1])
                print(np_new_index0)
                print(np_new_index1)

                assert (np_time_series - np_neighborhood[np_new_index0[0] * patch_size:(np_new_index0[0] + 1) * patch_size,
                                          np_new_index0[1] * patch_size:(np_new_index0[1] + 1) * patch_size, :]).sum() == 0

                assert (np_neighbor_time_series - np_neighborhood[np_new_index1[0] * patch_size:(np_new_index1[0] + 1) * patch_size,
                                                  np_new_index1[1] * patch_size:(np_new_index1[1] + 1) * patch_size, :]).sum() == 0

    def test_extract_imgs(self):
        print('Test image extraction')
        n_channels = 4
        n_frames = 12
        img_size = 1024
        patch_size = 64
        channels_frames = n_channels * n_frames
        neighborhood_factor = 2

        data = tf.random_normal(shape=(img_size, img_size, channels_frames))

        neighborhood, index = sampling.extract_neighborhood(data,
                                                            img_size,
                                                            patch_size,
                                                            channels_frames,
                                                            neighborhood_factor)

        time_series, neighbor_time_series, new_index0, new_index1 = sampling.extract_time_series(neighborhood,
                                                                                patch_size,
                                                                                channels_frames,
                                                                                neighborhood_factor)
        image_o, image_f, index_o, index_f = sampling.extract_same_location_images(time_series, n_channels, n_frames, patch_size)
        image_t = sampling.extract_same_time_image(neighbor_time_series, n_channels, patch_size, index_o)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(10):
                np_time_series, np_neighbor_time_series, np_image_o,\
                np_image_f, np_image_t, np_index_o, np_index_f = sess.run([time_series,
                                                                           neighbor_time_series,
                                                                           image_o,
                                                                           image_f,
                                                                           image_t,
                                                                           index_o,
                                                                           index_f])
                print(np_index_o)
                print(np_index_f)

                assert (np_image_o - np_time_series[:, :, n_channels * np_index_o:n_channels * (np_index_o + 1)]).sum() == 0
                assert (np_image_f - np_time_series[:, :, n_channels * np_index_f:n_channels * (np_index_f + 1)]).sum() == 0
                assert (np_image_t - np_neighbor_time_series[:, :, n_channels * np_index_o:n_channels * (np_index_o + 1)]).sum() == 0

    def test_extract_tuples(self):
        print('Test tuple extraction')
        batch_size = 2
        n_channels = 4
        n_frames = 12
        img_size = 1024
        patch_size = 64
        channels_frames = n_channels * n_frames
        neighborhood_factor = 2
        n_patches = 64

        inputs = tf.random_normal(shape=(batch_size, img_size, img_size, channels_frames))

        reference_batch, same_location_batch, same_time_batch = sampling.extract_tuples(inputs, n_patches, img_size,
                                                                                patch_size, n_channels, neighborhood_factor)

        with self.test_session() as sess:
            sess.run(tf.global_variables_initializer())
            for i in range(10):
                np_reference_batch, np_same_location_batch, np_same_time_batch = sess.run([reference_batch,
                                                                                same_location_batch,
                                                                                same_time_batch])
                print(np_reference_batch.shape)
                print(np_same_location_batch.shape)
                print(np_same_time_batch.shape)


import scrapy

class Safcec(scrapy.Spider):
    name = "safcec"
    #allowed_domains = ["domains.com"]
    start_urls = ['http://www.safcec.org.za/search/newsearch.asp?bst=&cdlGroupID=&txt_country=&txt_statelist=&txt_state=&ERR_LS_20180416_103724_19330=txt_state%7CLocation%7C20%7C0%7C%7C0']

    def parse(self, response):
        #First portion scrapes LISTS
        #looks for href urls to scrape the details on their pages and not the root page itself
        urls = response.css('a[onmouseout="HideMiniProfile();"]::attr(href)').extract()
        #Join then Visits the scraped URLS
        for url in urls:
            #this finds the last half of a url and joins it to the base url
            url = response.urljoin(url)
            #now it yields all the completed urls contained in the root page and parses their details
            yield scrapy.Request(url=url, callback=self.parse_details)

        #Scrapes data:
        for item in response.css('div.pathway'):
            item = {
                'detail 1': item.css('div.pathway').extract(),
                'detail 2': item.css('div.pathway').extract(),
            }
            yield item

        #follow pagination links
        next_page_url = response.css('li.next > a::attr(href)').extract_first()
        if next_page_url:
            next_page_url = response.urljoin(next_page_url)
            yield scrapy.Request(url=next_page_url, callback=self.parse)

    def parse_details(self, response):
        yield {
            'detail 1': response.css('div.detail::text').extract(),
            'detail 2': response.xpath('xpath').extract(),
        }

    def CharShiftDecrypt(self, strInVal, shiftval, shiftCharSet):
        #encrypted string, shit in value (6), shift character string (abceABCD...)
        strInString = strInVal
        intInString = len(strInString)
        strCharSet = shiftCharSet
        intCharSetLen = len(strCharSet)
        strOutVal = ''

        for c in strInString:
            nextchar = c
            try:
                nextindex = strCharSet.index(nextchar)
            except:
                nextindex = None
            if nextindex: #found it
                nextindex = (nextindex - int(shiftval)) % intCharSetLen

                if nextindex < 0:
                    nextindex = nextindex + intCharSetLen
                elif nextindex >= intCharSetLen:
                    nextindex = nextindex - intCharSetLen

                strOutVal += strCharSet[nextindex]

            else:
                strOutVal += nextchar

        return strOutVal;