import os
import platform
import re
import requests

from kodi_six import xbmc, xbmcgui, xbmcaddon
from kodi_six.utils import py2_encode

from six.moves import urllib_parse

from lt2http.logger import log
from lt2http.osarch import PLATFORM
from lt2http.addon import ADDON, ADDON_NAME, ADDON_ICON

try:
    import simplejson as json
except ImportError:
    import json

def translatePath(path):
    try:
        from xbmcvfs import translatePath
    except ImportError:
        from xbmc import translatePath

    return translatePath(path)

def notify(message, header=ADDON_NAME, time=5000, image=ADDON_ICON):
    sound = ADDON.getSetting('do_not_disturb') == 'false'
    dialog = xbmcgui.Dialog()
    return dialog.notification(getLocalizedLabel(header), getLocalizedLabel(message), toUtf8(image), time, sound)

def dialog_ok(message, header=ADDON_NAME):
    dialog = xbmcgui.Dialog()
    return dialog.ok(getLocalizedLabel(header), getLocalizedLabel(message))

def getLocalizedText(text):
    try:
        return re.sub(r'LOCALIZE\[(\d+)\]', getLocalizedStringMatch, text)
    except:
        return text

def getLocalizedLabel(label):
    try:
        if "LOCALIZE" not in label:
            return py2_encode(label)
        if ";;" not in label and label.endswith(']'):
            return py2_encode(getLocalizedString(int(label[9:-1])))
        else:
            parts = label.split(";;")
            translation = getLocalizedString(int(parts[0][9:14]))
            for i, part in enumerate(parts[1:]):
                if part[0:8] == "LOCALIZE":
                    parts[i + 1] = getLocalizedString(int(part[9:14]))
                else:
                    parts[i + 1] = py2_encode(parts[i + 1])

            return py2_encode(translation % tuple(parts[1:]), 'utf-8', 'ignore')
    except Exception as e:
        log.error("Cannot decode the label: %s, Error: %s" % (label, e))
        return label

def getLocalizedStringMatch(match):
    try:
        return ADDON.getLocalizedString(int(match.group(1)))
    except:
        return match.group(1)

def getLocalizedString(stringId):
    try:
        return py2_encode(ADDON.getLocalizedString(stringId), 'utf-8', 'ignore')
    except:
        return stringId

def getLt2HttpLocalizedString(stringId):
    LT2HTTP_ADDON = xbmcaddon.Addon('service.lt2http')
    try:
        return py2_encode(LT2HTTP_ADDON.getLocalizedString(stringId), 'utf-8', 'ignore')
    except:
        return stringId

def toUtf8(string):
    try:
        if isinstance(string, bytes):
            string = string.decode("utf-8")
        return py2_encode(string)
    except:
        return string

def system_information():
    build = xbmc.getInfoLabel("System.BuildVersion")
    log.info("System information: %(os)s_%(arch)s %(version)s" % PLATFORM)
    log.info("Kodi build version: %s" % build)
    log.info("OS type: %s" % platform.system())
    log.info("uname: %s" % repr(platform.uname()))
    return PLATFORM

def getShortPath(path):
    if PLATFORM["os"] == "windows":
        return getWindowsShortPath(path)
    return path

def getWindowsShortPath(path):
    try:
        import ctypes
        import ctypes.wintypes

        ctypes.windll.kernel32.GetShortPathNameW.argtypes = [
            ctypes.wintypes.LPCWSTR,  # lpszLongPath
            ctypes.wintypes.LPWSTR,  # lpszShortPath
            ctypes.wintypes.DWORD  # cchBuffer
        ]
        ctypes.windll.kernel32.GetShortPathNameW.restype = ctypes.wintypes.DWORD

        buf = ctypes.create_unicode_buffer(1024)  # adjust buffer size, if necessary
        ctypes.windll.kernel32.GetShortPathNameW(path, buf, len(buf))

        return buf.value
    except:
        return path

def download_github_folder(repo, folder, destination):
    contents_url = "https://api.github.com/repos/%s/contents/%s" % (repo, folder)
    log.info("Downloading repo content for folder: %s, to folder: %s" % (contents_url, destination))
    try:
        with requests.get(contents_url) as r:
            lists = json.loads(r.content, parse_int=str)

            downloaded = 0
            for i in lists:
                if 'download_url' not in i:
                    continue

                dest = os.path.join(destination, urllib_parse.unquote(i['download_url'].rsplit('/', 1)[1]))
                log.info("Downloading file '%s' to '%s'" % (i['download_url'], dest))

                with requests.get(i['download_url'], stream=True) as rd:
                    rd.raise_for_status()
                    with open(dest, 'wb') as f:
                        for chunk in rd.iter_content(chunk_size=8192):
                            f.write(chunk)
                downloaded += 1
            return downloaded > 0
    except Exception as e:
        log.error("Could not get list of files from github: %s" % e)
        raise

def download_current_version(repo):
    contents_url = "https://api.github.com/repos/%s/git/refs/tags" % (repo)
    log.debug("Downloading repo releases: %s" % (contents_url))
    try:
        with requests.get(contents_url) as r:
            lists = json.loads(r.content, parse_int=str)

            ref = lists[-1]["ref"]
            if ref.find('/'):
                return ref.rsplit('/', 1)[1]

            return ""
    except Exception as e:
        log.error("Could not get list of tags from github: %s" % e)
        raise

def read_current_version(dest_dir):
    p = os.path.join(dest_dir, "version")
    if os.path.exists(p):
        try:
            with open(p, 'r') as file:
                return file.read().replace('\n', '')
        except:
            pass
    return ""

def write_current_version(dest_dir, version):
    p = os.path.join(dest_dir, "version")
    try:
        with open(p, 'w') as file:
            file.write(version)
            file.close()
    except:
        pass


# Combining all the ML elements into a pipeline

from sklearn.tree import DecisionTreeRegressor # Select a model (right now the model doesn't matter)
from sklearn.model_selection import train_test_split # Train-test split
from sklearn.model_selection import KFold # K-fold Cross validation
import sklearn.preprocessing as pp
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
import sklearn.metrics as m
from sklearn.model_selection import cross_validate

# Data management
import pandas as pd

# Data
from gapminder import gapminder as gap

# Let's predict life expectancy given GDP and population
y = gap['lifeExp'] # Clarify outcome
X = gap[['gdpPercap','pop']] # Clarify predictors

# %% -----------------------------------------

# split data
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.25, random_state=123)

# %% -----------------------------------------

# Combine preprocessing and model together so all steps are run at once
pipe = Pipeline([('scaler', pp.StandardScaler()), ('dtree', DecisionTreeRegressor())])

# Now we can run everything in one pass
pipe.fit(X_train,y_train)

# Make our prediction
y_hat = pipe.predict(X_train)

# See how we did.
m.mean_squared_error(y_train,y_hat)
m.r2_score(y_train,y_hat)

# How well do we do out of sample?
y_hat = pipe.predict(X_test)
m.mean_squared_error(y_test,y_hat)
m.r2_score(y_test,y_hat)

# Not so great. Over fit.

# %% -----------------------------------------

# Fold Cross validation into this process (to estimate the test error using the training data)

kf = KFold(n_splits=5) # Set the number of folds
k_splits = kf.split(X_train) # Create the split generator
scores = [] # Empty list to retain scores

# Loop through the splits, refit the model, and store performance
for train_ind, val_ind in k_splits:

    # Fit Model on the training slits
    pipe.fit(X_train.iloc[train_ind,:],y_train.iloc[train_ind])

    # Predict using the validation splits
    yhat = pipe.predict(X_train.iloc[val_ind,:])

    # Calculate performance scores
    mse = m.mean_squared_error(y_train.iloc[val_ind],yhat)
    r2 = m.r2_score(y_train.iloc[val_ind],yhat)

    # save
    scores.append((mse,r2))

# Convert into DF
dat_scores = pd.DataFrame(scores,columns=["mse","r2"])

# Estimate of test performance
dat_scores.mean(axis=0)

# %% -----------------------------------------

# Is there a quicker way to do this??? Yes!

scores_we_care_about = {"r2":"r2",'mse':'neg_mean_squared_error'}
# List of all the relevant keys: m.SCORERS.keys()

# Use the cross_validation method to validate.
scores = cross_validate(pipe,X=X_train,y=y_train,
                        scoring=scores_we_care_about,
                        cv=5)

# Predicted fit on the test data test error
scores['test_r2'].mean()
-1*scores['test_mse'].mean()

# ISSUE: Note there is one issue with the above. If we want to compare the performance
# of different models (which we often want to do), then we need to ensure that
# the splits for each fold are the same across models (else the difference
# in performance between two models might just be statistical fluke given
# how the data was scrambled).

# How do we get around this? Use the KFold generator to generate consistent folds
# that we can use again and again

# By seeding the random state, we can ensure the same folds are generated each time.
gen_folds = KFold(n_splits=5,random_state=111,shuffle=True)

scores = cross_validate(pipe,X=X_train,y=y_train,
                        scoring=scores_we_care_about,
                        cv=gen_folds) # Just drop the generator in for the cv arg

# %% -----------------------------------------

# Can we do better by adding in country information?

y = gap['lifeExp'] # Clarify outcome
X = gap[['country','gdpPercap','pop']] # Clarify predictors

# split data
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.25, random_state=123)

# Different preprocessing transformations given different variable types.
numeric_transformer = Pipeline(steps=[('scaler', pp.StandardScaler())])
categorical_transformer = Pipeline(steps=[('onehot', pp.OneHotEncoder())])

# Combine the different preprocessing instruction, mapping them to specific variables
preprocessor = ColumnTransformer(
    transformers=[('num', numeric_transformer, ['pop','gdpPercap']),
                  ('cat', categorical_transformer, ['country'])]
    )

# Now put it all together into a single pipeline
pipe2 = Pipeline([("preproces",preprocessor), ('dtree', DecisionTreeRegressor())])

# Use the cross_validation method to validate.
scores2 = cross_validate(pipe2,X=X_train,y=y_train,
                        scoring=scores_we_care_about,
                        cv=gen_folds)

# Predicted fit on the test data test error. Country info adds a lot!
scores2['test_r2'].mean()
-1*scores2['test_mse'].mean()


from django.contrib.auth.mixins import LoginRequiredMixin
from django.shortcuts import render

# Create your views here.
from django.views.generic.base import View  # 视图类

from apps.operations.form import UserFavForm, CommentForm  # 表单验证类
from django.http import JsonResponse
from apps.operations.models import UserFavorite, CourseComments, Banner  # 用户收藏和用户评论模型类
from apps.courses.models import Course    #课程类
from apps.organizations.models import CourseOrg  #机构类
from apps.organizations.models import Teacher   #教师类

#收藏
class AddFavView(View):
    '''
    用户收藏实现：
    前台使用ajax请求post方法，把该请求的数据存到数据库中,这相当于提交了一个表单
    1.当用户点击收藏按钮时，ajax请求提交，也就是进入该视图类了。
    2.首先判断用户是否登录，如果没有登陆，那么就跳转到login页面进行登录。
    3.如果用户登录了，那么就实例化表单验证，
      如果该表单对象是不合法的，那么就返回错误信息
      如果该表单对象是合法的，那么就会获取该表单中的fav_id和fav_type进一步操作。
    4,获取到以后到数据库中查询用户是否已经收藏了该数据，也就是在数据库中查询是否有这条收藏记录，
      如果有，那么就删除该条记录，(从已收藏到未收藏)
      如果没有，就插入该条记录。(未收藏到已收藏)

    '''
    # POST方法
    def post(self, request, *args, **kwargs):
        # 先判断用户是否登录
        if not request.user.is_authenticated:
            return JsonResponse({
                'status': 'fail',
                'msg': '用户未登录'
            })
        # 实例化表单验证类
        user_fav_form = UserFavForm(request.POST)
        if user_fav_form.is_valid():  # 如果是合法的
            fav_id = user_fav_form.cleaned_data['fav_id']
            fav_type = user_fav_form.cleaned_data['fav_type']

            #判断用户是否已经收藏
            exist = UserFavorite.objects.filter(user=request.user,fav_id = fav_id,fav_type=fav_type)

            if exist:     #如果已经收藏，就删除该信息
                exist.delete()

                #当数据库中已经有该记录时，就删除该记录，改为未收藏状态。
                if fav_type ==1:
                    course = Course.objects.get(id=fav_id)
                    course.fav_nums-=1
                    course.save()
                elif fav_type == 2:
                    courseorg = CourseOrg.objects.get(id=fav_id)
                    courseorg.fav_nums -= 1
                    courseorg.save()
                elif fav_type == 3:
                    teacher = Teacher.objects.get(id=fav_id)
                    teacher.fav_nums-=1
                    teacher.save()
                return JsonResponse({
                    'status':'success',
                    'msg':'收藏'
                })
            else:
                #添加数据，创建对象
                userfav = UserFavorite()
                userfav.user = request.user
                userfav.fav_id = fav_id
                userfav.fav_type = fav_type
                userfav.save()
                return JsonResponse({
                    'status': 'success',
                    'msg': '已收藏'
                })
        else:
            return JsonResponse({
                'status': 'fail',
                'msg': '参数错误'
            })

#发表评论
class CommentView(View):

    def post(self, request, *args, **kwargs):
        # 先判断用户是否登录
        if not request.user.is_authenticated:
            return JsonResponse({
                'status': 'fail',
                'msg': '用户未登录'
            })
        #实例化表单
        comment_form = CommentForm(request.POST)
        if comment_form.is_valid():
            comments = comment_form.cleaned_data['comments']
            course = comment_form.cleaned_data['course']

            ccmodel = CourseComments()
            ccmodel.comments = comments
            ccmodel.user = request.user
            ccmodel.course = course
            ccmodel.save()
            return JsonResponse({
                'status': 'success',
                'msg': '评论成功'
            })
        else:
            return JsonResponse({
                'status': 'fail',
                'msg': '评论参数错误'
            })

#首页
class IndexView(View):
    def get(self, request, *args, **kwargs):
        '''
        首页展示
        :param request:
        :param args:
        :param kwargs:
        :return:
        '''
        #轮播图，查询轮播图并按照index顺序排序
        banners = Banner.objects.all().order_by('index')

        #公开课，除去banner
        courses = Course.objects.filter(is_banner=False)[:7]

        # 小banner
        banner_courses = Course.objects.filter(is_banner=True)[:4]

        # 课程机构加载
        course_orgs = CourseOrg.objects.all()[:15]

        return render(request, 'index.html', {
            "banners": banners,         #轮播图
            "courses": courses,         #公开课
            'banner_courses':banner_courses,       #公开课中的轮播图
            "course_orgs": course_orgs,        #课程机构
        })


from git import Repo, Actor
from datetime import datetime, timedelta

repo = Repo.init('.')
git = repo.git
git.update_ref("HEAD", d=True)
index = repo.index
actor = Actor("Ciarán Ó hAoláin", "ciaran@cohaolain.ie")

with open('art.csv') as art_file:
    art = list(
        map(lambda x: list(map(lambda y: 1 if y else 0, x.strip().split(","))),
            art_file.readlines()))

for i, weekday in enumerate(art):
    if len(weekday) < 52:
        weekday.extend([0 for _ in range(52 - len(weekday))])
if len(art) < 7:
    art.extend([[0 for _ in range(52)] for _ in range(7 - len(art))])

art = list(map(list, zip(*art)))

flattened_art = [item for sublist in art for item in sublist]

start_date = (datetime.today() - timedelta(days=365)
              ).replace(microsecond=0)

while start_date.weekday() != 6:
    start_date -= timedelta(days=1)

time = start_date
while flattened_art:
    if flattened_art.pop(0):
        for _ in range(100):
            index.commit(str(time), author=actor, committer=actor,
                         commit_date=time.isoformat(),
                         author_date=time.isoformat())
    time += timedelta(days=1)


#!/usr/bin/env python

from testbase import *

class modifyUserTest(TestWithUser):
    def runTest(self):
        ur = self.I.UserRecord(-1, self.gid, self.name,
                               self.name, '/bin/bash')
        ur.uid = self.um.create(ur)

        ur2 = self.um.getByName(self.name)
        self.assertEqual(ur, ur2)

        ur.fullname = self.name + ', modified'
        self.um.modify(ur)

        ur2 = self.um.getByName(self.name)
        self.assertEqual(ur, ur2)

        self.um.delete(ur.uid)

tests = load(modifyUserTest)

if __name__ == '__main__':
    main(tests)


import random

def quick_sort(arr):
    if len(arr) <= 1:
        return arr

    pivot = arr[0]
    less = [i for i in arr[1:] if i <= pivot]
    greater = [i for i in arr[1:] if i > pivot]

    return quick_sort(less) + [pivot] + quick_sort(greater)

print(quick_sort([int(random.random() * 10000) for i in range(10000, 0, -1)]))

def merge_sort(arr, operations=0):
    length = len(arr)
    if length <= 1:
        return arr, 1
    pivot_index = length // 2
    greater = [i for i in arr if i >= arr[pivot_index]]
    less = [i for i in arr if i < arr[pivot_index]]

    return quick_sort(less) + quick_sort(greater),

print(merge_sort(list(range(10))))


# -*- coding: utf-8 -*-
"""
Created on Wed Aug 29 11:32:29 2018

@author: semyanov
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

file_name = 'C:/Users/semyanov/Desktop/gaussian_fwhm.txt'

N=1000
x = np.linspace(-10, 10, N)

"""
reference:
https://www.originlab.com/doc/Origin-Help/Gaussian-Function-FitFunc
Meanings:
y0 = base
xc = center
A = area
w = FWHM
"""
def gaussian_fwhm(y0, xc, A, w, x):
    C=4*np.log(2)
    pi=np.pi
    y = y0 + (A*np.exp((-C*(x-xc)**2)/(w*w))/(w*np.sqrt(pi/C)))
    return y

y=gaussian_fwhm(0.0, 0, 1.0, 2.0, x)

plt.figure(1)
plt.plot(x[0:N-1], y[0:N-1], 'g-', linewidth= 1.0)
plt.show()

data = np.stack((x, y), axis=-1)
df = pd.DataFrame({'Column1':data[:,0],'Column2':data[:,1]})
df.to_csv(file_name, sep='\t')


from pythonwhat.State import State
from pythonwhat.Reporter import Reporter

from IPython.core.interactiveshell import InteractiveShell

class MockProcess:
    """Mock the processes used in pythonbackend to evaluate code."""
    def __init__(self, **kwargs):
        self.shell = InteractiveShell(user_ns = kwargs)

    def executeTask(self, task):
        return task(self.shell)

def create_state(solution_code="", student_code="", pre_exercise_code="",
                 student_process = None, solution_process = None,
                 raw_student_output="",
                 reporter = None):
    """Create a state for testing SCTs.

    Custom processes are required for SCTs that execute code (e.g. has_equal_value).
    See MockProcess above.
    """
    if reporter is None: reporter = Reporter()
    Reporter.active_reporter = reporter
    state =  State(full_student_code = student_code,
                   full_solution_code = solution_code,
                   **locals())

    # only used to get converters :/
    State.root_state = state

    return state

def create_x_state(stu_val, sol_val, **kwargs):
    s = create_state(student_process = MockProcess(x = stu_val),
                     solution_process = MockProcess(x = sol_val),
                     **kwargs)

    return s


import os

from collections import defaultdict

import seaborn as sns
import pandas as pd
import numpy as np

import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rcParams['font.family'] = 'sans-serif'
mpl.rcParams['font.sans-serif'] = 'DejaVu Sans'
mpl.rcParams['pdf.fonttype'] = 42

from .. utils import other as uo

def main_clust_all_umap(args, argparser):
    path_file_out = os.path.join(args.PATH, "res", "deg")
    if not os.path.exists(path_file_out):
        os.makedirs(path_file_out)

    file_dict = {
        'epcy' : "prediction_capability.xls",
        'deseq2' : "deseq2_genes.xls",
        'edger' : "edger_genes.xls",
        'limma' : "limma_voom_genes.xls",
    }

    # create dict with search params
    search_params_dict = {
        'methods' : args.METHODS,
        'designs' : args.DESIGN
    }

    dir_design = os.path.join(args.PATH, "all")
    df_design = uo.get_design(args, path=dir_design, replace=False)

    df_exp = uo.get_exp(args, args.MATRIX)
    df_exp = df_exp[df_design["sample"]]
    list_genes = df_exp.index.tolist()

    path_out = os.path.join(args.OUTDIR, "clust_all", "umap")
    if not os.path.exists(path_out):
        os.makedirs(path_out)

    dict_umap = uo.cluster_umap(args, df_exp, df_design, path_out, "no_method", "all_design", "all_exp", metric='euclidean')

    dict_mcc = defaultdict(list)
    for method in search_params_dict['methods']:
        gene_ids_keep = []
        for design in search_params_dict['designs']:
            print("##{}, {}".format(method.upper(), design))
            dir_design = os.path.join(args.PATH, design)
            df_diff = uo.read_diff_table(args, file_dict[method], method, dir_design, args.PVALUE, list_genes)
            gene_ids_keep += df_diff["ID"][:10].tolist()

            if method == "epcy":
                for mcc in df_diff['KERNEL_MCC'][:10].tolist():
                    dict_mcc['design'].append(design)
                    dict_mcc['MCC'].append(mcc)

        gene_ids_keep = list(set(gene_ids_keep))
        print(len(gene_ids_keep))
        df_exp_method = df_exp.loc[gene_ids_keep]
        df_exp_method.sort_index(inplace=True)

        print(str(df_exp_method.shape[0]) + " " + str(df_exp_method.shape[1]))
        dict_umap_tmp = uo.cluster_umap(args, df_exp_method, df_design, path_out, method, "all_design", "selected_exp", metric='euclidean')
        for key in dict_umap_tmp.keys():
            dict_umap[key] = dict_umap[key] + dict_umap_tmp[key]

    # create subfolder for log and res
    fig_dir = os.path.join(args.OUTDIR, "clust_all", "umap")
    if not os.path.exists(fig_dir):
        os.makedirs(fig_dir)

    # convert dict to df for analysis
    df_res = pd.DataFrame(dict_umap)
    csv_out = os.path.join(fig_dir, "umap_table.csv")
    df_res.to_csv(csv_out, index=False, sep="\t")

    # plot results
    plt_fig = sns.catplot(data=df_res, x="top", y="full_ARS", hue="method", col="design", kind="point", facet_kws=dict(subplot_kws=dict(ylim=[-0.05,1.05])))
    fig_out =  os.path.join(fig_dir, "full_ARS." + args.EXT)
    plt_fig.savefig(fig_out)
    plt.close()

    plt_fig = sns.catplot(data=df_res, x="top", y="full_MIS", hue="method", col="design", kind="point", facet_kws=dict(subplot_kws=dict(ylim=[-0.05,1.05])))
    fig_out =  os.path.join(fig_dir, "full_MIS." + args.EXT)
    plt_fig.savefig(fig_out)
    plt.close()

    plt_fig = sns.catplot(data=df_res, x="top", y="ARS", hue="method", col="design", kind="point", facet_kws=dict(subplot_kws=dict(ylim=[-0.05,1.05])))
    fig_out =  os.path.join(fig_dir, "ARS." + args.EXT)
    plt_fig.savefig(fig_out)
    plt.close()

    plt_fig = sns.catplot(data=df_res, x="top", y="MIS", hue="method", col="design", kind="point", facet_kws=dict(subplot_kws=dict(ylim=[-0.05,1.05])))
    fig_out =  os.path.join(fig_dir, "MIS." + args.EXT)
    plt_fig.savefig(fig_out)
    plt.close()

    plt_fig = sns.catplot(data=df_res, x="top", y="p_clustered", hue="method", col="design", kind="point", facet_kws=dict(subplot_kws=dict(ylim=[-0.05,1.05])))
    fig_out =  os.path.join(fig_dir, "p_clustered." + args.EXT)
    plt_fig.savefig(fig_out)
    plt.close()

    df_res = pd.DataFrame(dict_mcc)
    plt_fig = sns.boxplot(data=df_res, x="design", y="MCC")
    plt_fig.set_xticklabels(plt_fig.get_xticklabels(),rotation=30)
    plt_fig.set(ylim=(-0.05, 1.05))
    fig_out =  os.path.join(fig_dir, "mcc_boxplot." + args.EXT)
    plt_fig = plt_fig.get_figure()
    plt_fig.savefig(fig_out)
    plt.close()


import os
import re

mp4_playlist = "/Volumes/Movies/music/playlists/certi1"
mp3_playlist = "/Volumes/Movies/music/playlists/certi2"

mp3path = "/Volumes/Movies/music/dnbMP3"

failed = []

def split_vid_name(name):
    try:
        v = name.split(".")[-2].split("-")
        if (len(v[-1]) == 11):
            return v[-1]
        elif (len(v[-1]) == 9): # soundcloud
            t, s = add_split_vid_name(v[-2], v[-1])
            if (t):
                return s
            else:
                return s
        else:
            t, s = add_split_vid_name(v[-2], v[-1])
            if (t):
                return s
            else:
                v_s2 = f"{v[-3]}-{s}"
                if (len(v_s2) == 11):
                    return v_s2
                else:
                    v_s3 = f"{v[-4]}-{v_s2}"
                    if (len(v_s3) == 11):
                        return v_s3
                    else:
                        failed.append({"error":"not size 11", "name":name, "split":v})
                        return False
    except IndexError as e:
        if (name == ".DS_Store"):
            return False
        else:
            failed.append({"error":e, "name":name, "split":v})
            return False

def add_split_vid_name(a, b):
    v_s = f"{a}-{b}"
    if (len(v_s) == 11):
        return (True, v_s)
    return (False, v_s)

def get_downloaded_vids():
    downloaded_vid_names = os.listdir(mp4_playlist)
    vid_ids_all = list(map(split_vid_name, downloaded_vid_names))
    return list(zip(vid_ids_all, downloaded_vid_names))

def copy_vid_id(z):
    id, name = z
    name_split = name.split(".")
    name = "".join(name_split[:-1])
    name = safe_filename(name)
    cmd = f'cp "{mp3path}/{name}.mp3" "{mp3_playlist}/{name}.mp3"'
    os.system(cmd)

def safe_filename(s: str, max_length: int = 255) -> str:
    # Characters in range 0-31 (0x00-0x1F) are not allowed in ntfs filenames.
    ntfs_characters = [chr(i) for i in range(0, 31)]
    characters = [
        r'"',
        r"\#",
        r"\$",
        r"\%",
        r"'",
        r"\*",
        r"\,",
        r"\.",
        r"\/",
        r"\:",
        r'"',
        r"\;",
        r"\<",
        r"\>",
        r"\?",
        r"\\",
        r"\^",
        r"\|",
        r"\~",
        r"\\\\",
    ]
    pattern = "|".join(ntfs_characters + characters)
    regex = re.compile(pattern, re.UNICODE)
    filename = regex.sub("", s)
    return filename[:max_length].rsplit(" ", 0)[0]

def main():
    vids = get_downloaded_vids()
    for i in range(len(vids)):
        copy_vid_id(vids[i])

    print("failes")
    print(failed)

if __name__ == '__main__':
    main()
    # print(safe_filename("s.p.y dljknfdv .mp3"))


import torch

import torch.nn as nn
import torch.nn.functional as F

from dcn.dcn_v2 import DCN

class Center_Angle_Output_Module(nn.Module):
    """ Channel attention module"""
    def __init__(self):
        super(Center_Angle_Output_Module, self).__init__()

        self.x_linear = nn.Sequential(nn.Linear(in_features=40, out_features=1))
        self.y_linear = nn.Sequential(nn.Linear(in_features=40, out_features=1))
        self.z_linear = nn.Sequential(nn.Linear(in_features=64, out_features=1))

        self.angle_output = nn.Sequential(DCN(64, 1, kernel_size=1, stride=1, padding=0), nn.MaxPool2d(40, stride=1))

    def forward(self, semantic_map, roi_center):
        m_batchsize, C, h, w = semantic_map.size()

        x_zplane = F.avg_pool2d(semantic_map.permute(0, 2, 3, 1), kernel_size=[w, C]).reshape(m_batchsize, -1)
        y_zplane = F.avg_pool2d(semantic_map.permute(0, 3, 2, 1), kernel_size=[h, C]).reshape(m_batchsize, -1)
        z_zplane = F.avg_pool2d(semantic_map, kernel_size=[w, h]).reshape(m_batchsize, -1)

        x_bias = self.x_linear(x_zplane)
        y_bias = self.y_linear(y_zplane)
        z_bias = self.z_linear(z_zplane)

        final_center = torch.cat([x_bias, z_bias, y_bias], dim=1) + roi_center
        final_angle = self.angle_output(semantic_map).reshape(m_batchsize, -1)

        return final_center, final_angle


from flask import Flask,redirect
from flask import render_template
from flask import request
import with_excel as we
import conf as c
import json
import pdb
import sys

dataList_values={}
app = Flask(__name__)
_DEBUG=True
# _DEBUG=False

@app.route('/getAllRecords')
def getAllRecords():
    l=[]
    new_record=we.project_data()
    # new_record.readRecords(l)
    the_captions=[]
    the_datas=[]
    the_captions,the_datas=new_record.getRecords()
    return render_template('getAllRecords.html',the_captions=the_captions,the_datas=the_datas,file_name=new_record.filename,succeed="0")

@app.route('/setDataTableLanguage')
def setDataTableLanguage():
    return render_template('chinese.json')

@app.route('/')
def index():
    # new_record=we.project_data()
    # return render_template('index.html',file_name=new_record.filename)

    l=[]
    new_record=we.project_data()
    # new_record.readRecords(l)
    the_captions=[]
    the_datas=[]
    the_captions,the_datas=new_record.getRecords()
    return render_template('getAllRecords.html',the_captions=the_captions,the_datas=the_datas,file_name=new_record.filename,succeed="0")

@app.route('/writeRecord',methods=['GET','POST'])
def writeRecord():
    '''
    处理数据增加的部分
    '''
    if request.method=='POST':
        new_record=we.project_data()
        if (new_record.init_success ==False):
            return
        # 得到行数信息
        editRow=request.form.get('editRow')
        if (editRow==''):
            editRow=-2
        else:
            editRow=int(editRow)
        # 得到三个日期数据
        error_date=request.form.get('error_date')
        propose_date=request.form.get('propose_date')
        actual_date=request.form.get('actual_date')

        if len(error_date)>1:
            c.dic_date['error_year'],c.dic_date['error_month'],c.dic_date['error_day'] =error_date.split('-')
        else:
            c.dic_date['error_year'], c.dic_date['error_month'],c.dic_date['error_day']='','',''

        if len(propose_date)>1:
            c.dic_date['propose_year'],c.dic_date['propose_month'],c.dic_date['propose_day'] =propose_date.split('-')
        else:
            c.dic_date['propose_year']=''
            c.dic_date['propose_month']=''
            c.dic_date['propose_day']=''

        if len(actual_date)>1:
            c.dic_date['actual_year'],c.dic_date['actual_month'],c.dic_date['actual_day'] =actual_date.split('-')
        else:
            c.dic_date['actual_year']=''
            c.dic_date['actual_month']=''
            c.dic_date['actual_day']=''

        # 得到其它数据
        for i in c.dic_error_information.keys():
            try:
                c.dic_error_information[i]=request.form.get(i)
            except:
                print('在i='+i+'时候出错了')

        # 得到人员信息
        list_persons=new_record.getPeopleList()
        dic_persons={}
        for x in list_persons:
            dic_persons[x]=request.form.get(x)
            if dic_persons[x]==None:
                dic_persons[x]=''

        # 写入数据
        new_record.writePeopleStatus(dic_persons,editRow)
        new_record.testWrite2(c.dic_error_information,c.dic_date,editRow)
        # return render_template('success.html')
        # new_record.readRecords(l)
        the_captions=[]
        the_datas=[]
        the_captions,the_datas=new_record.getRecords()
        return render_template('getAllRecords.html',the_captions=the_captions,the_datas=the_datas,file_name=new_record.filename,succeed="1")
        # return render_template('success.html')

@app.route('/getOneRecord')
def getOneRecord():
    '''
    读取已有的记录
    '''
    # print(dataList_values)
    r=request.args.get('editRow')
    if (r==None):
        r=-2
    new_record=we.project_data()
    mydata={}
    mydata=new_record.readOneRecord(r)
    myconf={}
    myconf=new_record.readDataList()
    list_persons=new_record.getPeopleList()
    dic_persons=new_record.getPeopleStatus(r)
    return render_template('getOneRecord.html',dic_value=mydata,editRow=r,dataList=myconf,dic_persons=dic_persons,list_persons=list_persons,file_name=new_record.filename)

# 程序启动
if __name__ == '__main__':
    if (_DEBUG==True):
        app.debug=True
    else:
        app.debug=False
    app.run(host='0.0.0.0',port=5000)
# vim : se fdc=2 :


"""AppliGate URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from django.conf import settings
from django.conf.urls.static import static

# from django.conf.urls import url
from rest_framework import routers

from company_app import views
from user_app.views import ProfileStatusView

urlpatterns = [

    # ProfileCompany for company
    path("company/profile/<str:username>/", views.ProfileCompanyView.as_view(), name="personalCompany"),
    # path("company/profile/<str:username>/", ProfileStatusView.as_view(), name="profileStatusView"),
    path("jobofferlistings/<int:page>", views.JobOfferListingView.as_view(), name="jobOfferListing"),
    # path("company/joboffer", views.JobOfferListingView.as_view(), name="jobOffer"),
    path("company/joboffer/createjoboffer", views.JobOfferCreateOfferView.as_view(), name="jobOfferCreateJobOffer"),
    path("company/joboffer/apply/<int:id>", views.JobApplicationView.as_view(), name="jobApply"),
    path("applications/<int:page>", views.JobOfferUserAppliedListingView.as_view(), name="jobUserApplications"),

    path("company/joboffer/info", views.JobOfferCompanyView.as_view(), name="jobOfferCompanyData"),
    path("company/joboffer/top", views.JobOfferTopView.as_view(), name="JobOfferTopData"),
    path("company/joboffer/info/<int:id>", views.JobOfferCompanyView.as_view(), name="jobOfferCompanyDataId"),
    path("company/joboffer/top/<int:id>", views.JobOfferTopView.as_view(), name="JobOfferTopDataId"),
    path("company/joboffer/topmore", views.JobOfferTopMoreView.as_view(), name="JobOfferTopMoreData"),
    path("company/joboffer/topmore/<int:id>", views.JobOfferTopMoreView.as_view(), name="JobOfferTopMoreDataId"),

    path("company/joboffer/topcolors", views.JobOfferTopColorsView.as_view(), name="JobOfferTopColorsData"),
    path("company/joboffer/topcolors/<int:id>", views.JobOfferTopColorsView.as_view(), name="JobOfferTopColorsDataId"),

    path("company/joboffer/skill", views.JobOfferSkillView.as_view(), name="JobOfferSkillData"),
    path("company/joboffer/skill/<int:id>", views.JobOfferSkillView.as_view(), name="JobOfferSkillDataId"),
    path("company/joboffer/skill/delete/<int:item_id>", views.JobOfferSkillView.as_view(), name="JobOfferSkillDataId"),

    path("company/joboffer/responsibility", views.JobOfferResponsibilityView.as_view(), name="JobOfferResponsibilityData"),
    path("company/joboffer/responsibility/<int:id>", views.JobOfferResponsibilityView.as_view(), name="JobOfferResponsibilityDataId"),
    path("company/joboffer/responsibility/edit/<int:item_id>", views.JobOfferResponsibilityView.as_view(), name="JobOfferResponsibilityDataId"),
    path("company/joboffer/responsibility/delete/<int:item_id>", views.JobOfferResponsibilityView.as_view(), name="JobOfferResponsibilityDataId"),

    path("company/joboffer/requirement", views.JobOfferRequirementView.as_view(), name="JobOfferRequirementData"),
    path("company/joboffer/requirement/<int:id>", views.JobOfferRequirementView.as_view(), name="JobOfferRequirementDataId"),
    path("company/joboffer/requirement/edit/<int:item_id>", views.JobOfferRequirementView.as_view(), name="JobOfferRequirementDataId"),
    path("company/joboffer/requirement/delete/<int:item_id>", views.JobOfferRequirementView.as_view(), name="JobOfferRequirementDataId"),

    path("company/joboffer/weoffer", views.JobOfferWhatWeOfferView.as_view(), name="JobOfferWhatWeOfferData"),
    path("company/joboffer/weoffer/<int:id>", views.JobOfferWhatWeOfferView.as_view(), name="JobOfferWhatWeOfferDataId"),
    path("company/joboffer/weoffer/edit/<int:item_id>", views.JobOfferWhatWeOfferView.as_view(), name="JobOfferWhatWeOfferDataId"),
    path("company/joboffer/weoffer/delete/<int:item_id>", views.JobOfferWhatWeOfferView.as_view(), name="JobOfferWhatWeOfferDataId"),

    path("company/joboffer/application", views.JobOfferApplicationView.as_view(), name="JobOfferApplicationData"),
    path("company/joboffer/application/<int:id>", views.JobOfferApplicationView.as_view(), name="JobOfferApplicationDataId"),
    path("company/joboffer/application/edit/<int:item_id>", views.JobOfferApplicationView.as_view(), name="JobOfferApplicationDataId"),
    path("company/joboffer/application/delete/<int:item_id>", views.JobOfferApplicationView.as_view(), name="JobOfferApplicationDataId"),

    path("company/joboffer/jobofferstatus", views.JobOfferStatusView.as_view(), name="JobOfferStatuData"),
    path("company/joboffer/jobofferstatus/<int:id>", views.JobOfferStatusView.as_view(), name="JobOfferStatuDataId"),

    path("company/joboffer/deleteoffer/<int:pk>", views.JobOfferDeleteOfferView.as_view(), name="JobOfferDeleteOffer"),

    path("company/joboffer/about", views.JobOfferAboutView.as_view(), name="JobOfferAbout"),
    path("company/joboffer/about/<int:id>", views.JobOfferAboutView.as_view(), name="JobOfferAboutId"),

    # path("company/jobofferlistings/<str:username>/", views.JobOfferListing.as_view(), name="jobOfferListingIsername"),
    # list job offers created by company user
    path("company/myJobOffers/<int:page>", views.MyJobOffersListingView.as_view(), name="myJobOffers"),
    # list job offers created by company user for profile display
    path("company/myJobOffers/<str:username>/", views.MyJobOffersListingView.as_view(), name="myJobOffersProfile"),
    path("company/joboffer/applicants/<int:offer_id>/<int:page>", views.JobOfferAppliedForOfferListingView.as_view(), name="myJobOffersProfile"),

    #Assess job offer button response
    path("joboffer/profile/assess/<str:username>/<int:offer_id>", views.JobOfferAssessView.as_view(), name="JobOfferAssess"),

    # path("company/joboffercreator/", views.ProfileCompanyView.as_view(), name="personalCompany"),

]


import telebot
import os
from telebot.async_telebot import AsyncTeleBot

# import test1
# import test2

# API_TOKEN = '1986393023:AAGmgNDQVTn_MmBBP1Y8DMJqLVSxwvQOvV8'
API_TOKEN = '5648589910:AAFArjtVL_BcnCKzRbJcKRpnlfRJV97YLWo'
# bot = telebot.TeleBot(API_TOKEN)
bot = AsyncTeleBot(API_TOKEN)

async def String_to_Command(string):
    while string.find(' ') != -1:
        string = '/' + string[0:string.find(' ')] + '_' + string[string.find(' ') + 1:len(string)]
    print(string)
    return string

async def make_markup_wo_upload_file():
    markup = telebot.types.InlineKeyboardMarkup()
    buttonA = telebot.types.InlineKeyboardButton('Остановить добавление', callback_data='stop')
    # buttonD = telebot.types.InlineKeyboardButton('Настройки', callback_data='settings')

    markup.row(buttonA)
    # markup.row(buttonD)

    return markup

async def make_markup():
    markup = telebot.types.InlineKeyboardMarkup()
    # buttonA = telebot.types.InlineKeyboardButton('Остановить добавление', callback_data='stop')
    buttonB = telebot.types.InlineKeyboardButton('Загрузить документ с номерами', callback_data='upload')

    markup.row(buttonB)
    # markup.row(buttonA)

    return markup

# @bot.message_handler(content_types=['text'])
# async def message(message):
#     await bot.send_message(message.chat.id, message.text)

# Handle '/start'
@bot.message_handler(commands=['start'])
async def start_message(message):
    """
    :param message: class 'telebot.types.Message'
    Обработчик команды start, выдает пользователю меню для выбора дальнейших действий
    """
    await bot.send_message(message.chat.id, '\U00002694    Привет! Я позволю тебе управлять добавлением пользователей в канал'
                                      'телеграм из файла. \n'
                                      '\U00002694    Для удобства пользования не нужно отправлять мне сообщения '
                                      '(за исключениемм файлов), достаточно делать выбор из предложенных'
                                      'вариантов.')

    markup = await make_markup()

    await bot.send_message(message.chat.id, 'Выберите необходимый пункт',
                     reply_markup=markup)

@bot.message_handler(content_types=['document'])
async def get_file(message):
    files_dir = f"users_files/{message.chat.id}"
    file_name = message.document.file_name
    file_extension = file_name.split('.')[-1]
    if file_extension != 'xlsx':
        markup = make_markup()
        await bot.send_message(message.chat.id, 'Бот умеет обрабатывать только xlsx файлы(', reply_markup=markup)
    else:
        file_info = await bot.get_file(message.document.file_id)
        downloaded_file = await bot.download_file(file_info.file_path)
        if os.path.exists(files_dir):
            counter = len(os.listdir(files_dir))
            with open(f"{files_dir}/file_{counter}.xlsx", 'wb') as new_file:
                new_file.write(downloaded_file)
        else:
            os.mkdir(files_dir)
            with open(f"{files_dir}/file_0.xlsx", 'wb') as new_file:
                new_file.write(downloaded_file)
        markup = await make_markup_wo_upload_file()
        await bot.send_message(message.chat.id, 'Файл успешно загружен. Сейчас начнется добавление в канал', reply_markup=markup)

        # await test1.job('users_files/484704240/file_0.xlsx')
        # await test1.('users_files/484704240/file_0.xlsx')

@bot.callback_query_handler(func=lambda call: True)
async def handle(call):
    # global app
    if call.data == 'stop':
        try:
            files_dir = f"users_files/{call.message.chat.id}"
            if os.path.exists(files_dir):
                counter = len(os.listdir(files_dir)) - 1
                os.remove(f"{files_dir}/file_{counter}.xlsx")
                await bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id, text='Файл удален. Если вы захотите продолжить, то файл придется загрузить заново')
        except:
            await bot.send_message(chat_id=call.message.chat.id, text='К сожалению, файлы не найдены(')
    elif call.data == 'upload':
        await bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id, text='Загрузите файл в формате .xlsx')
    elif call.data == 'amount':
        markup = telebot.types.InlineKeyboardMarkup()
        buttonA = telebot.types.InlineKeyboardButton('Остановить добавление', callback_data='stop')
        buttonB = telebot.types.InlineKeyboardButton('Загрузить документ с номерами', callback_data='upload')

        markup.row(buttonB)
        markup.row(buttonA)

        await bot.edit_message_text(chat_id=call.message.chat.id, message_id=call.message.message_id, text='Выберите'
                                                                                'необходимый'
                                                                                'пункт',
                              reply_markup=markup)

import asyncio
asyncio.run(bot.polling())


import serial
import torch
from DNN import dnn
import re
import cv2
import numpy as np

def init_process(nums):

    num1 = float(nums[0])
    if(num1>3.3):
        num1 = 0
    num2 = float(nums[1])
    if (num2 > 3.3):
        num2 = 0
    num3 = float(nums[2])
    if (num3 > 3.3):
        num3 = 0
    num4 = float(nums[3])
    if (num4 > 3.3):
        num4 = 0

        # nums = [float(col1[i]), float(col2[i]), float(col3[i]), float(col4[i])]
    nums = [num1, num2, num3, num4]
    nums = np.array(nums)
    nums = torch.tensor(nums, dtype=torch.float)

    return nums

if __name__ == '__main__':

    s = serial.Serial('com6', 115200)

    flag = 0
    cnt = 1

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    model_x = dnn().to(device)
    model_y = dnn().to(device)
    model_x.load_state_dict(torch.load("model/20230413/dnn3(x).pkl"), False)
    model_x.eval()
    model_y.load_state_dict(torch.load("model/20230413/dnn3(y).pkl"), False)
    model_y.eval()
    while True:
        st = ''
        while True:
            try:
                char = str(s.read(), 'utf-8')

                st = st + char
            except:
                continue
            if (char == '\n'):
                break
        if (st[0] == 'd'):
            flag = 1
        else:
            continue
        if (flag == 1):
                # try:
                #     st1 = re.findall(r'\d+\.?\d*', st)
                #     nums = [float(st1[0]), float(st1[1]), float(st1[2]), float(st1[3])]
                #
                #     print(st1)
                #     print(nums)
                #     img = getImg(nums)
                #
                #     image = Image.fromarray(img).convert('RGB')
                #
                #
                #     img_tensor = transform(image)
                #     # print(img_tensor.shape)
                #     outputs = model(img_tensor)
                #     print(outputs)
                #     predicted = torch.max(outputs.data, 1)[1].data
                #     print(predicted)
                # except:
                #     print("no data")
            st1 = re.findall(r'\d+\.?\d*', st)
            nums = [float(st1[0]), float(st1[1]), float(st1[2]), float(st1[3])]

                # print(st1)
                # print(nums)
            nums = init_process(nums).to(device)
                # print(img_tensor.shape)
            nums = torch.unsqueeze(nums, 0)
            outputs_x = model_x(nums)
            outputs_y = model_y(nums)

            predicted_x = torch.max(outputs_x.data, 1)[1].data
            pos_x = predicted_x.cpu().item()
            predicted_y = torch.max(outputs_y.data, 1)[1].data
            pos_y = predicted_y.cpu().item()
                # print("x: ", pos_x)
                # print("y: ", pos_y)
            send_str = "x:" + str(pos_x) + " y:" + str(pos_y)
            print(send_str)
                # s.write(send_str.encode('utf-8'))
                # s_send_str = ''
                # while True:
                #     try:
                #         char = str(s_send.read(), 'utf-8')
                #         s_send_str = s_send_str + char
                #     except:
                #         continue
                #     if (char == '\n'):


#!/usr/bin/env python
# coding: utf-8

# # LIBRARIES

# In[141]:

# Data manipulation
import numpy as np
import pandas as pd
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 99)

# Plotting
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

# File system manangement
import os

# sklearn preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split

# Suppress warnings
import warnings
warnings.filterwarnings("ignore")

# # IMPORTING DATA

# In[142]:

df_train = pd.read_csv('application_train.csv')
df_test = pd.read_csv('application_test.csv')
print("Train Data Shape b4 adding target col : ",df_train.shape)
print("Test Data Shape b4 adding target col : ",df_test.shape)

# In[143]:

df_train["source"] = "train"
df_test["source"] = "test"
print("Train Data Shape aftr adding target col : ",df_train.shape)
print("Test Data Shape aftr adding target col : ",df_test.shape)

# In[144]:

df = pd.concat([df_train,df_test])

# # EXPLORING DATA

# We can easily convert the dataset into a pandas dataframe to perform exploratory data analysis. Simply pass in the dataset.data as an argument to pd.DataFrame(). We can view the first 5 rows in the dataset using head() function.

# In[145]:

df.shape

# find the dimension of given data

# In[146]:

df.info()

# In[147]:

df.columns

# We can check the datatype of each column using dtypes to make sure every column has numeric datatype. If a column has different datatype such as string or character, we need to map that column to a numeric datatype such as integer or float. For this dataset, luckily there is no such column.

# In[148]:

df.dtypes

# In[149]:

df.sample()

# In[150]:

df.SK_ID_CURR.nunique()

# In[151]:

df.head()

# # REMOVING COLUMNS

# In[152]:

lst=[]
lst=df.columns
row=df.shape[0]
cols=[]
len(cols)

# In[153]:

# Removing columns which has more than 60% of NA Values .
[cols.append(i) for i in lst if df[i].isnull().sum()/row*100 > 60]

# In[154]:

len(cols)

# In[155]:

cols

# In[156]:

cols_to_drop = [
               #has more than 60% of NA Values .
                'OWN_CAR_AGE',
                'YEARS_BUILD_AVG',
                'COMMONAREA_AVG',
                'FLOORSMIN_AVG',
                'LIVINGAPARTMENTS_AVG',
                'NONLIVINGAPARTMENTS_AVG',
                'YEARS_BUILD_MODE',
                'COMMONAREA_MODE',
                'FLOORSMIN_MODE',
                'LIVINGAPARTMENTS_MODE',
                'NONLIVINGAPARTMENTS_MODE',
                'YEARS_BUILD_MEDI',
                'COMMONAREA_MEDI',
                'FLOORSMIN_MEDI',
                'LIVINGAPARTMENTS_MEDI',
                'NONLIVINGAPARTMENTS_MEDI',
                'FONDKAPREMONT_MODE',

                #all null / constant / others
                'FLAG_DOCUMENT_2',
                'FLAG_DOCUMENT_3',
                'FLAG_DOCUMENT_4',
                'FLAG_DOCUMENT_5',
                'FLAG_DOCUMENT_6',
                'FLAG_DOCUMENT_7',
                'FLAG_DOCUMENT_8',
                'FLAG_DOCUMENT_9',
                'FLAG_DOCUMENT_10',
                'FLAG_DOCUMENT_11',
                'FLAG_DOCUMENT_12',
                'FLAG_DOCUMENT_13',
                'FLAG_DOCUMENT_14',
                'FLAG_DOCUMENT_15',
                'FLAG_DOCUMENT_16',
                'FLAG_DOCUMENT_17',
                'FLAG_DOCUMENT_18',
                'FLAG_DOCUMENT_19',
                'FLAG_DOCUMENT_20',
                'FLAG_DOCUMENT_21',
                'AMT_REQ_CREDIT_BUREAU_HOUR',
                'AMT_REQ_CREDIT_BUREAU_DAY',
                'AMT_REQ_CREDIT_BUREAU_WEEK',
                'AMT_REQ_CREDIT_BUREAU_MON',
                'AMT_REQ_CREDIT_BUREAU_QRT',
                'AMT_REQ_CREDIT_BUREAU_YEAR'

]

# In[157]:

data = df.drop(cols_to_drop, axis=1)

# In[158]:

data.shape

# In[159]:

data.head()

# In[160]:

data.isnull().any()

# In[161]:

data.isna().sum()

# In[162]:

data.duplicated().sum()

# In[163]:

data.describe(include='all').T

# # EXPLORATORY DATA ANALYSIS

# ## Correlation Check

# Finding correlation between attributes is a highly useful way to check for patterns in the dataset.

# In[164]:

# Distribution graphs (histogram/bar graph) of column data
def plotPerColumnDistribution(df, nGraphShown, nGraphPerRow):
    nunique = df.nunique()
    df = df[[col for col in df if nunique[col] > 1 and nunique[col] < 50]] # For displaying purposes, pick columns that have between 1 and 50 unique values
    nRow, nCol = df.shape
    columnNames = list(df)
    nGraphRow = (nCol + nGraphPerRow - 1) / nGraphPerRow
    plt.figure(num = None, figsize = (6 * nGraphPerRow, 8 * nGraphRow), dpi = 80, facecolor = 'w', edgecolor = 'k')
    for i in range(min(nCol, nGraphShown)):
        plt.subplot(nGraphRow, nGraphPerRow, i + 1)
        columnDf = df.iloc[:, i]
        if (not np.issubdtype(type(columnDf.iloc[0]), np.number)):
            valueCounts = columnDf.value_counts()
            valueCounts.plot.bar()
        else:
            columnDf.hist()
        plt.ylabel('counts')
        plt.xticks(rotation = 90)
        plt.title(f'{columnNames[i]} (column {i})')
    plt.tight_layout(pad = 1.0, w_pad = 1.0, h_pad = 1.0)
    plt.show()

# data.corr
corrmat=data.corr()
plt.figure(figsize=(22,19))
sns.heatmap(corrmat,vmax=0.8,square=True)
plt.show()

# In[165]:

#Using Pearson Correlation
plt.figure(figsize=(12,10))
cor = data.corr()
sns.heatmap(cor, annot=True, cmap=plt.cm.Reds)
plt.show()

# if there are pairs of features that have a high correlation, only one will be taken. The correlation value that is used as a benchmark as a high correlation is uncertain, generally the number 0.7 is used.

# In[166]:

corr_matrix = data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
to_drop_hicorr = [column for column in upper.columns if any(upper[column] > 0.7)]
to_drop_hicorr

# ## Target Variable Distribution

# In[167]:

data.head()

# In[168]:

pd.options.display.float_format = '{:,.2f}'.format
x = data['TARGET'].value_counts()
print('The number of defaults in the target variable is {} whereas the number of repayments is {}'.format(x.loc[0],x.loc[1]))
x.plot.pie()

# ## Missing Value Cheking

# Sometimes, in a dataset we will have missing values such as NaN or empty string in a cell. We need to take care of these missing values so that our machine learning model doesn’t break. To handle missing values, there are three approaches followed.
# When it comes time to build our machine learning models, we will have to fill in these missing values (known as imputation). In later work, we will use models such as XGBoost that can handle missing values with no need for imputation. Another option would be to drop columns with a high percentage of missing values, although it is impossible to know ahead of time if these columns will be helpful to our model. Therefore, we will keep all of the columns for now.

# In[169]:

check_missing = data.isnull().sum() * 100 / data.shape[0]
check_missing[check_missing > 0].sort_values(ascending=False)

# In[170]:

data.drop(to_drop_hicorr, axis=1, inplace=True)

# In[171]:

pd.options.display.float_format = '{:,.2f}'.format
df_missing_val = pd.DataFrame(columns = ['column','number of missing vals'])

for column in data.columns:
    missing_val = data[column].isnull().sum()
    x = {'column':column,'number of missing vals':missing_val}
    df_missing_val= df_missing_val.append(x, ignore_index=True)
df_missing_val = df_missing_val[df_missing_val['number of missing vals'] !=0]
df_missing_val = df_missing_val.sort_values(by='number of missing vals',ascending=False)
print('Below is a table showing the number of missing values per column. The numbers are quite significant for some features. However, we shall deal withbthis at a later stage')

df_missing_val.head(10)

# ## Corelation without Variable

# In[172]:

cor_target = abs(data.corr())
#Selecting highly correlated features
relevant_features = cor_target[cor_target<0.3]
relevant_features

# In[173]:

type(relevant_features)
relevant_features.items

# ## Checking Categorical Features

# In[174]:

data.select_dtypes(include='object').nunique()

# Most of the categorical variables have a relatively small number of unique entries. We will need to find a way to deal with these categorical variables!
#

# In[175]:

data.select_dtypes(exclude='object').nunique()

# In[176]:

for col in data.select_dtypes(include='object').columns.tolist():
    print(data[col].value_counts(normalize=True)*100)
    print('\n')

# # DEALING WITH CATEGORICAL VARIABLES

# ## One Hot Encoding

# One-hot encoding: create a new column for each unique category in a categorical variable. Each observation recieves a 1 in the column for its corresponding category and a 0 in all other new columns.

# In[177]:

train_data = pd.get_dummies(data)
test_data = pd.get_dummies(data)
print('The number of training features and variables is: {}'.format(train_data.shape))
print('The number of test features and variables is: {}'.format(test_data.shape))

# ## Label Encoding

# In[178]:

le = LabelEncoder()
le_count = 0

# Iterate through the columns
for col in train_data:
    if train_data[col].dtype == 'object':
        # If 2 or fewer unique categories
        if len(list(train_data[col].unique())) <= 2:
            # Train on the training data
            le.fit(train_data[col])
            # Transform both training and testing data
            train_data[col] = le.transform(train_data[col])
            test_data[col] = le.transform(test_data[col])

            # Keep track of how many columns were label encoded
            le_count += 1

print('%d columns were label encoded.' % le_count)

# ## Aligning the variables in the test and train datasets

# There need to be the same features (columns) in both the training and testing data. One-hot encoding has created more columns in the training data because there were some categorical variables with categories not represented in the testing data. To remove the columns in the training data that are not in the testing data, we need to align the dataframes. First we extract the target column from the training data (because this is not in the testing data but we need to keep this information). When we do the align, we must make sure to set axis = 1 to align the dataframes based on the columns and not on the rows!

# In[179]:

target_col = train_data['TARGET']
train_data,test_data = train_data.align(test_data,join='inner',axis=1)
train_data['TARGET'] = target_col
print('The number of training features and variables after alignment is: {}'.format(train_data.shape))
print('The number of test features and variables after alignment is: {}'.format(test_data.shape))

# The training and testing datasets now have the same features which is required for machine learning. The number of features has grown significantly due to one-hot encoding. At some point we probably will want to try dimensionality reduction (removing features that are not relevant) to reduce the size of the datasets.

# # Modeling

# In[180]:

target_col.shape

# In[181]:

target_col.head()

# In[182]:

from sklearn.model_selection import train_test_split
train_test_split(train_data,target_col)

# In[183]:

x_train, x_test, y_train, y_test = train_test_split(train_data,target_col)
print (x_train.shape, y_train.shape)
print (x_test.shape, y_test.shape)

# In[184]:

x_train, x_test, y_train, y_test = train_test_split(train_data,target_col, train_size=0.8, random_state=20)
print (x_train.shape, y_train.shape)
print (x_test.shape, y_test.shape)

# In[185]:

scaler = MinMaxScaler().fit(X)
scaled_X = scaler.transform(X)

# In[186]:

seed      = 42
test_size = 0.20

X_train, X_test, Y_train, Y_test = train_test_split(scaled_X, Y, test_size = test_size, random_state = seed)

print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)

# In[187]:

ftr_app = train_data.drop(['SK_ID_CURR', 'TARGET'], axis=1)
target_app = train_data['TARGET']

train_x, valid_x, train_y, valid_y = train_test_split(ftr_app, target_app, test_size=0.3, random_state=2020)
train_x.shape, valid_x.shape

Alia Tungga Dewi_VIX_Home Credit Indonesia.py


import numpy as np
import torch
import os
from collections import defaultdict
from datetime import datetime as dt

from torch.utils.data import DataLoader
from anomaly_detection.model.base import BaseModel
from anomaly_detection.model.one_class import DeepSVDD
from anomaly_detection.model.reconstruction import AutoEncoder as AE, DAGMM
from anomaly_detection.model.shallow import OCSVM
from anomaly_detection.trainer.one_class import DeepSVDDTrainer
from anomaly_detection.trainer.reconstruction import AutoEncoderTrainer as AETrainer, DAGMMTrainer
from anomaly_detection.trainer.shallow import OCSVMTrainer
from anomaly_detection.utils.utils import average_results
from anomaly_detection.datamanager.base import AbstractDataset
from anomaly_detection.datamanager.dataset import IEEEFraudDetection

available_models = [
    "DAGMM",
    "DeepSVDD",
    "OC-SVM",
]
available_datasets = [
    "IEEEFraudDetection",
]

def store_results(results: dict, params: dict, model_name: str, dataset: str, dataset_path: str, results_path: str = None):
    output_dir = results_path or f"./results/{dataset}"
    if not os.path.exists(output_dir):
        os.mkdir(output_dir)
    fname = output_dir + f'/{model_name}_results.txt'
    with open(fname, 'a') as f:
        hdr = "Experiments on {}\n".format(dt.now().strftime("%d/%m/%Y %H:%M:%S"))
        f.write(hdr)
        f.write("-".join("" for _ in range(len(hdr))) + "\n")
        f.write(f'{dataset} ({dataset_path.split("/")[-1].split(".")[0]})\n')
        f.write(", ".join([f"{param_name}={param_val}" for param_name, param_val in params.items()]) + "\n")
        f.write("\n".join([f"{met_name}: {res}" for met_name, res in results.items()]) + "\n")
        f.write("-".join("" for _ in range(len(hdr))) + "\n")
    return fname

def store_model(model, model_name: str, dataset: str, models_path: str = None):
    output_dir = models_path or f'../models/{dataset}/{model_name}/{dt.now().strftime("%d_%m_%Y_%H_%M_%S")}"'
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    model.save(f"{output_dir}/model")

model_trainer_map = {
    # Deep Models
    "AE": (AE, AETrainer),
    "DAGMM": (DAGMM, DAGMMTrainer),
    "DeepSVDD": (DeepSVDD, DeepSVDDTrainer),
    # Shallow Models
    "OC-SVM": (OCSVM, OCSVMTrainer),
}

def resolve_model_trainer(
        model_name: str,
        dataset: AbstractDataset,
        n_epochs: int,
        batch_size: int,
        learning_rate: float,
        device: str
):
    model_trainer_tuple = model_trainer_map.get(model_name, None)
    assert model_trainer_tuple, "Model %s not found" % model_name
    model, trainer = model_trainer_tuple
    model = model(
        dataset_name=dataset.name,
        in_features=dataset.in_features,
        n_instances=dataset.n_instances,
        device=device
    )
    trainer = trainer(
        model=model,
        lr=learning_rate,
        n_epochs=n_epochs,
        batch_size=batch_size,
        device=device
    )

    return model, trainer

def train_model(
        model: BaseModel,
        model_trainer,
        train_ldr: DataLoader,
        test_ldr: DataLoader,
        n_runs: int,
        thresh: float,
        device: str,
        model_path: str,
        test_mode: bool
):
    # Training and evaluation on different runs
    all_results = defaultdict(list)

    if test_mode:
        for model_file_name in os.listdir(model_path):
            model = BaseModel.load(f"{model_path}/{model_file_name}")
            model = model.to(device)
            model_trainer.model = model
            print("Evaluating the model on test set")
            # We test with the minority samples as the positive class
            y_test_true, test_scores = model_trainer.test(test_ldr)
            print("Evaluating model")
            results = model_trainer.evaluate(test_scores, y_test_true)
            for k, v in results.items():
                all_results[k].append(v)
    else:
        for i in range(n_runs):
            _ = model_trainer.train(train_ldr)
            # We test with the minority samples as the positive class
            y_test_true, test_scores = model_trainer.test(test_ldr)
            results = model_trainer.evaluate(test_scores, y_test_true)
            print(results)
            for k, v in results.items():
                all_results[k].append(v)
            store_model(model, model.name, "IEEEFraudDetection", model_path)
            model.reset()

    # Compute mean and standard deviation of the performance metrics
    print("Averaging results ...")
    return average_results(all_results)

def train(
        model_name: str,
        dataset_path: str,
        batch_size: int,
        n_runs: int,
        n_epochs: int,
        learning_rate: float,
        results_path: str,
        models_path: str,
        test_mode: bool
):
    print("Training model %s for %d runs of %d epochs each" % (model_name, n_runs, n_epochs))
    # Dynamically load the Dataset instance
    dataset = IEEEFraudDetection(dataset_path)
    anomaly_thresh = int(np.ceil((1 - dataset.anomaly_ratio) * 100))
    device = "cuda" if torch.cuda.is_available() else "cpu"

    # split data in train and test sets
    # we train only on the majority class
    train_ldr, test_ldr = dataset.loaders(batch_size=batch_size, seed=42)

    # check path
    for p in [results_path, models_path]:
        if p:
            assert os.path.exists(p), "Path %s does not exist" % p

    model, model_trainer = resolve_model_trainer(
        model_name=model_name,
        dataset=dataset,
        batch_size=batch_size,
        n_epochs=n_epochs,
        learning_rate=learning_rate,
        device=device,
    )
    res = train_model(
        model=model,
        model_trainer=model_trainer,
        train_ldr=train_ldr,
        test_ldr=test_ldr,
        n_runs=n_runs,
        device=device,
        thresh=anomaly_thresh,
        model_path=models_path,
        test_mode=test_mode
    )
    print(res)
    params = dict(
        {"BatchSize": batch_size, "Epochs": n_epochs, "Threshold": anomaly_thresh},
        **model.get_params()
    )
    # Store the average of results
    fname = store_results(res, params, model_name, dataset.name, dataset_path, results_path)
    print(f"Results stored in {fname}")


#i am stupid so i will brute force it
satisfy = []
for i in range(387638,919123):
    num = str(i)

    one = False
    two = True

    for n in range(0,len(num)):
        try:
            if num[n + 1] < num[n]:
                two = False
                break
            if num[n] == num[n + 1]:
                one = True
            # if num[n] == num[n + 1] and num[n+1] == num[n+2]:
            #     one = False
        except:
            pass

    if one and two:
        satisfy.append(num)

ans = 0

for i in satisfy:
    count = {}

    for index,s in enumerate(i):
        if s in count and s == i[index - 1]:
            count[s] += 1
        else:
            count[s] = 1

    for n in count:
        if count[n] == 2:
            ans += 1
            break
    print(count)

print(ans)


import tkinter as tk
from tkinter.font import Font
from utils import utils
from PIL import ImageTk, Image
from lolwidgets.runepage import RuneSheet
from lolwidgets.championcard import ChampionFrame
from lolwidgets.abilities import AbilitiesTable
from lolwidgets.items import ItemBuildFrame
from lolwidgets.skinchooser import SkinChooser
from lolwidgets.utils import *

class Build(tk.Frame):
	def __init__(self, parent, vertical_space: int = 20, horizontal_space: float = 20, ability_cell_size: int = 40, champion_frame_size: int = 110, item_size: float = 64, control_bar_font_size: int = 16, *args, **kwargs) -> None:
		tk.Frame.__init__(self, parent, *args, **kwargs)
		self.champion = None
		self.backend = self.winfo_toplevel().backend
		self.spacing = Spacing(vertical=vertical_space, horizontal=horizontal_space)

		self.rune_sheet = RuneSheet(self, self.backend.get_runes(), vertical_space=15)
		self.abilities = AbilitiesTable(self, cell_size=int((self.rune_sheet.width - ability_cell_size) / 19), border_size=2)
		self.width = int(self.rune_sheet.width + self.abilities.width + self.spacing.horizontal * 3)
		self.champion_frame = ChampionFrame(self, width=self.width, size=champion_frame_size, backend=self.backend, background=self['bg'])
		self.data = ControlFrame(self, lane_navigation=self.backend.navigation_icons['lane_navigation'], width=self.rune_sheet.width, font_size=control_bar_font_size, func=self.select_lane, background=self['bg'])

		self.starting_items = ItemBuildFrame(self, title='Starting Items', width=self.rune_sheet.width, height=item_size, title_width=150, background=self['bg'])
		self.full_build = ItemBuildFrame(self, title='Full Build', width=self.rune_sheet.width, height=item_size, title_width=150, background=self['bg'])

		self.height = int(self.champion_frame.height + self.data.height + self.rune_sheet.height + vertical_space)
		self.skins = SkinChooser(self, width=self.abilities.width, height=self.height - self.champion_frame.height - self.abilities.height - self.starting_items.height - self.full_build.height - 5 * self.spacing.vertical)
		self.previous_selected = ''

	def set_champion(self, champion) -> None:
		self.champion = champion
		self.data.set_lanes(champion['build'].keys())
		self.champion_frame.set_champion(champion['name'], champion['image'])
		self.abilities.set_abilities(champion['abilities'])
		self.select_lane(champion['default_lane'])
		self.skins.set_skins(self.backend.get_skins())

	def select_lane(self, lane: str) -> None:
		if self.champion is None:
			return
		self.rune_sheet.select_runes(*self.champion['build'][lane][0]['runes'])
		self.champion_frame.set_spells(self.champion['build'][lane][0]['spells'])
		self.abilities.set_ability_order(self.champion['build'][lane][0]['ability_order'])
		self.starting_items.set_items(self.champion['build'][lane][0]['items'][0])
		self.full_build.set_items(self.champion['build'][lane][0]['items'][1])
		self.data.set_values(self.champion['build'][lane][0]['wr'], self.champion['build'][lane][0]['br'], self.champion['build'][lane][0]['pr'])
		self.data.select_lane(lane)
		self.previous_selected = lane

		self.backend.set_everything(self.champion['build'][lane][1])

	def place(self, x=0, y=0) -> None:
		super().place(x=x, y=y, height=self.height, width=self.width)
		y = 0
		self.champion_frame.place(x=0, y=0)
		y += self.champion_frame.height
		self.data.place(x=self.spacing.horizontal, y=y)
		y += self.data.height
		self.rune_sheet.place(x=self.spacing.horizontal, y=y)
		x = self.spacing.horizontal * 2 + self.rune_sheet.width
		y = self.champion_frame.height + self.spacing.vertical
		self.abilities.place(x=x, y=y)
		y += self.abilities.height + self.spacing.vertical
		self.starting_items.place(x=x, y=y)
		y += self.starting_items.height + self.spacing.vertical
		self.full_build.place(x=x, y=y)
		y += self.full_build.height + self.spacing.vertical
		self.skins.place(x=x, y=y)

	def place_forget(self) -> None:
		super().place_forget()
		self.rune_sheet.place_forget()
		self.champion_frame.place_forget()
		self.abilities.place_forget()
		self.starting_items.place_forget()
		self.full_build.place_forget()
		self.data.place_forget()

	def select_skin(self, skinid):
		self.skins.select_skin(skinid)

class ControlFrame(tk.Frame):
	def __init__(self, parent, lane_navigation, width: float, font_size: int, horizontal_spacing: float = 20, func: callable = lambda *args: None, *args, **kwargs) -> None:
		tk.Frame.__init__(self, parent, *args, **kwargs)
		self.horizontal_spacing = horizontal_spacing
		self.font = Font(size=font_size)
		self.height = self.font.metrics('linespace') * 2
		self.width = width

		self.pick_rate = tk.Label(self, text='PR:', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.pick_rate_value = tk.Label(self, text='100%', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.win_rate = tk.Label(self, text='WR:', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.win_rate_value = tk.Label(self, text='100%', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.ban_rate = tk.Label(self, text='BR:', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.ban_rate_value = tk.Label(self, text='100%', font=self.font, background=self['bg'], foreground='white', anchor='w')
		self.lane_images = [ImageTk.PhotoImage(Image.open('images/Top.png').resize((self.height, self.height)))]
		self.laneFrame = LaneFrame(self, lane_navigation=lane_navigation, height=self.height, background=self['bg'], func=func)

	def select_lane(self, lane):
		self.laneFrame.select_lane(lane)

	def set_values(self, win_rate, ban_rate, play_rate):
		self.win_rate_value['text'] = f'{win_rate:.2f}%'
		self.ban_rate_value['text'] = f'{ban_rate:.2f}%'
		self.pick_rate_value['text'] = f'{play_rate:.2f}%'

	def set_lanes(self, lane_list):
		self.laneFrame.set_active_buttons(lane_list)

	def place(self, x=0, y=0) -> None:
		super().place(x=x, y=y, height=self.height, width=self.width)

		x = 0
		self.pick_rate.place(x=x, y=0, height=self.height, width=self.font.measure(self.pick_rate['text']) + self.horizontal_spacing // 4)
		x += self.font.measure(self.pick_rate['text']) + self.horizontal_spacing // 2
		self.pick_rate_value.place(x=x, y=0, height=self.height, width=self.font.measure(self.pick_rate_value['text']) + self.horizontal_spacing)

		x += self.font.measure(self.pick_rate_value['text']) + self.horizontal_spacing
		self.win_rate.place(x=x, y=0, height=self.height, width=self.font.measure(self.win_rate['text']) + self.horizontal_spacing // 4)
		x += self.font.measure(self.win_rate['text']) + self.horizontal_spacing // 4
		self.win_rate_value.place(x=x, y=0, height=self.height, width=self.font.measure(self.win_rate_value['text']) + self.horizontal_spacing)

		x += self.font.measure(self.win_rate_value['text']) + self.horizontal_spacing
		self.ban_rate.place(x=x, y=0, height=self.height, width=self.font.measure(self.ban_rate['text']) + self.horizontal_spacing // 4)
		x += self.font.measure(self.ban_rate['text']) + self.horizontal_spacing // 4
		self.ban_rate_value.place(x=x, y=0, height=self.height, width=self.font.measure(self.ban_rate_value['text']) + self.horizontal_spacing)

		x = self.width - self.laneFrame.width
		self.laneFrame.place(x=x, y=0)

	def place_forget(self) -> None:
		super().place_forget()
		self.pick_rate.place_forget()
		self.pick_rate_value.place_forget()
		self.win_rate.place_forget()
		self.win_rate_value.place_forget()
		self.ban_rate.place_forget()
		self.ban_rate_value.place_forget()
		self.laneFrame.place_forget()

class LaneFrame(tk.Frame):
	def __init__(self, parent, lane_navigation, height: float, horizontal_spacing=5, func: callable = lambda *args: None, *args, **kwargs) -> None:
		tk.Frame.__init__(self, parent, *args, **kwargs)
		self.lane_func = func
		self.height = height
		self.width = 0
		self.horizontal_spacing = horizontal_spacing
		self.button_list = [LaneButton(self, lane=lane, active_icon=lane_navigation[lane]['enabled'], inactive_icon=lane_navigation[lane]['disabled'], size=int(self.height), borderwidth=0, background=self['bg'], activebackground=self['bg'], func=self.toggle_lane) for lane in utils.lanes if lane in lane_navigation]
		self.active_buttons = []
		self.previous_selected = -1

	def select_lane(self, lane):
		if self.previous_selected != -1:
			self.button_list[self.previous_selected].set_active(False)
			self.button_list[self.previous_selected]['bg'] = self['bg']
		self.button_list[utils.lane_indexes[lane]].set_active(True)
		self.button_list[utils.lane_indexes[lane]]['bg'] = utils.widget_color
		self.previous_selected = utils.lane_indexes[lane]

	def toggle_lane(self, lane):
		if self.previous_selected == utils.lane_indexes[lane]:
			return
		self.lane_func(lane)

	def set_active_buttons(self, lanes):
		self.active_buttons = []
		for lane in lanes:
			self.active_buttons.append(utils.lane_indexes[lane])
		self.active_buttons.sort()
		self.width = (self.height + self.horizontal_spacing) * len(self.active_buttons) - self.horizontal_spacing

	def place(self, x: float = 0, y: float = 0) -> None:
		super().place(x=x, y=y, width=(self.height + self.horizontal_spacing) * len(self.active_buttons), height=self.height)
		x = 0
		for i in self.active_buttons:
			self.button_list[i].place(x=x, y=0, height=self.height, width=self.height)
			x += self.height + self.horizontal_spacing

class LaneButton(tk.Button):
	def __init__(self, parent, active_icon, inactive_icon, size: int, lane: str, func: callable = lambda *args: None, *args, **kwargs):
		tk.Button.__init__(self, parent, *args, **kwargs)
		self.size = size
		self.lane = lane
		self.is_active = False
		self.active_image = ImageTk.PhotoImage(Image.open(active_icon).resize((size, size)))
		self.inactive_image = ImageTk.PhotoImage(Image.open(inactive_icon).resize((size, size)))
		self.size = size
		self.func = func
		self['image'] = self.inactive_image
		self['command'] = self.callback

	def set_active(self, status: bool) -> None:
		self.is_active = status
		if status:
			self['image'] = self.active_image
		else:
			self['image'] = self.inactive_image

	def callback(self):
		self.func(self.lane)


import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.nn.parallel
from torch.nn.utils import clip_grad_norm_
import torch.optim
from torch.utils.data import DataLoader
import numpy as np
import datetime
import time
import os
import math

from ops.models import TSN
from ops.dataset import TSNDataSet
from ops.transforms import *
from ops.utils import *
import cl_methods.cl_utils as cl_utils
import cl_methods.distillation as cl_dist
import cl_methods.classifer as classifier
from cl_methods import tcd
import copy

def _train(args, train_loader, model, criterion, optimizer, epoch, age, lambda_0=[0.5,0.5], model_old=None, importance_list=None):
    batch_time = AverageMeter()
    data_time = AverageMeter()
    losses = AverageMeter()

    losses_ce = AverageMeter()
    losses_kd_logit = AverageMeter()
    losses_att = AverageMeter()
    losses_div = AverageMeter()

    top1 = AverageMeter()

    loss = torch.tensor(0.).cuda()
    loss_kd_logit = torch.tensor(0.).cuda()
    loss_att = torch.tensor(0.).cuda()
    loss_div = torch.tensor(0.).cuda()

    if args.no_partialbn:
        model.module.partialBN(False)
    else:
        model.module.partialBN(True)

    # switch to train mode
    model.train()

    if model_old:
        model_old.eval()

    end = time.time()
    for i, (input, target, _) in enumerate(train_loader):
        # measure data loading time
        data_time.update(time.time() - end)

        input = input.cuda()
        target = target.cuda()

        # compute output
        outputs = model(input=input,t_div=args.t_div)
        preds = outputs['preds']
        feat = outputs['feat']
        int_features = outputs['int_features']

        if args.loss_type == 'bce':
            target_ = cl_utils.convert_to_one_hot(target, preds.size(1))
        else:
            target_ = target

        if age > 0:
            if args.cl_type == 'DIST':
                with torch.no_grad():
                    outputs_old = model_old(input=input)
                    preds_old = outputs_old['preds']
                    feat_old = outputs_old['feat']
                    int_features_old = outputs_old['int_features']

                old_size = preds_old.size(1)
                preds_base = torch.sigmoid(preds_old)

                if args.fc == 'lsc':
                    loss_ce = cl_dist.nca_loss(preds, target_)
                else:
                    loss_ce = criterion(preds, target_)

                loss_kd_logit = cl_dist.lf_dist_tcd(feat,feat_old,factor=importance_list[-1] if importance_list else None)
                loss_att = cl_dist.feat_dist(int_features,int_features_old,args,factor=importance_list[:-1] if importance_list else None)
                loss = lambda_0[0] * loss_ce + lambda_0[1] * loss_kd_logit + args.lambda_1 * loss_att

                #del preds_old, feat_old, int_features_old
                #del feat, int_features
            else:
                loss_ce = criterion(preds, target_)
                loss = loss_ce
        else:
            if args.fc == 'lsc':
                loss_ce = cl_dist.nca_loss(preds, target_)
            else:
                loss_ce = criterion(preds, target_)
            loss = loss_ce

        if args.t_div:
            loss_div = outputs['t_div'].sum()/input.size(0)
            loss = loss + args.lambda_2 * loss_div

        # compute gradient and do SGD step
        optimizer.zero_grad()
        loss.backward()

        if args.clip_gradient is not None:
            total_norm = clip_grad_norm_(model.parameters(), args.clip_gradient)

        optimizer.step()

        # measure accuracy and record loss
        prec1 = accuracy(preds.data, target, topk=(1,))
        losses.update(loss.item(), input.size(0))
        losses_ce.update(loss_ce.item(), input.size(0))
        losses_kd_logit.update(loss_kd_logit.item(),input.size(0))
        losses_att.update(loss_att.item(), input.size(0))
        losses_div.update(loss_div.item(),input.size(0))
        top1.update(prec1[0].item(), input.size(0))

        #del preds

        # measure elapsed time
        batch_time.update(time.time() - end)
        end = time.time()

        #printGPUInfo()

        if i % args.print_freq == 0:
            print(datetime.datetime.now())
            output = ('Epoch: [{0}][{1}/{2}], lr: {lr:.5f}\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Loss CE {loss_ce.val:.4f} ({loss_ce.avg:.4f})\t'
                      'Loss KD (Logit) {loss_kd_logit.val:.4f} ({loss_kd_logit.avg:.4f})\t'
                      'Loss KD (Feature) {loss_att.val:.4f} ({loss_att.avg:.4f})\t'
                      'Loss DIV {loss_div.val:.4f} ({loss_div.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses, loss_ce=losses_ce, loss_kd_logit=losses_kd_logit,
                loss_att=losses_att, loss_div=losses_div,
                top1=top1, lr=optimizer.param_groups[-1]['lr'] * 0.1))
            print(output)
        #torch.cuda.empty_cache()

def _validate(args, val_loader, model, criterion, age):
    batch_time = AverageMeter()
    top1 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    end = time.time()
    with torch.no_grad():
        for i, (input, target, _) in enumerate(val_loader):
            input = input.cuda()
            target = target.cuda()

            # compute output
            outputs = model(input=input)
            output = outputs['preds']

            target_one_hot = cl_utils.convert_to_one_hot(target, output.size(1))

            # measure accuracy and record loss
            prec1 = accuracy(output.data, target, topk=(1,))

            top1.update(prec1[0].item(), input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % args.print_freq == 0:
                output = ('Test: [{0}/{1}]\t'
                          'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                          'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                    i, len(val_loader), batch_time=batch_time,
                    top1=top1))

    output = ('Testing Results: Prec@1 {top1.avg:.3f} '
              .format(top1=top1))
    print(output)

    return top1.avg

def train_task(args, age, current_task, current_head, class_indexer, model_old=None, prefix=None):
    if age == 1 and args.dataset=='hmdb51' and args.fc=='cc':
        args.lr = args.lr * 0.1
    hook = False
    if age > 0 and args.cl_type=='DIST':
        hook = True
    if age > 0 and args.exemplar:
        exemplar_dict = load_exemplars(args)
        exemplar_list = exemplar_dict[age-1]
        exemplar_per_class = len(exemplar_list[0])

    else:
        exemplar_dict = {}
        exemplar_list = None
        exemplar_per_class = 0

    if age > 0 and args.use_importance:
        importance_dict = load_importance(args)
        importance_temp = importance_dict[age-1]
        importance_list = []
        for i in importance_temp:
            importance_list.append(i)
    else:
        importance_dict = {}
        importance_list = None

    # Construct TSM Models
    model = TSN(args, num_class=current_head if age==0 else current_head-len(current_task), #modality='RGB',
            fc_lr5=not (args.tune_from and args.dataset in args.tune_from),
            apply_hooks=hook, age=age, training=True,
            cur_task_size=len(current_task))#,exemplar_segments=args.exemplar_segments)

    scale_size = model.scale_size
    input_size = model.input_size

    normalize = GroupNormalize(model.input_mean, model.input_std)
    train_augmentation = model.get_augmentation(flip=False if 'something' in args.dataset or 'jester' in args.dataset else True)
    transform_rgb = torchvision.transforms.Compose([
                       train_augmentation,
                       Stack(roll=(args.arch in ['BNInception', 'InceptionV3'])),
                       ToTorchFormatTensor(div=(args.arch not in ['BNInception', 'InceptionV3'])),
                       normalize,
                   ])
    lambda_0 = 0.0

    if age > 0:
        print("Load the Previous Model")
        ckpt_path = os.path.join(args.root_model, args.dataset, str(args.init_task), str(args.nb_class), '{:03d}'.format(args.exp), 'task_{:03d}.pth.tar'.format(age-1))

        print(ckpt_path)
        sd = torch.load(ckpt_path)
        sd = sd['state_dict']
        state_dict = dict()
        for k, v in sd.items():
            state_dict[k[7:]] = v

        model.load_state_dict(state_dict)

        # Prepare Old Model to Distill
        if args.cl_type == 'DIST' or args.cl_type == 'REG':
            print("Copy the old Model")
            model_old = copy.deepcopy(model)
            model_old.eval()
            lambda_0 = cl_utils.set_lambda_0(len(current_task),current_head-len(current_task),args)
            print('lambda_0  : {}'.format(lambda_0))

        # Increment the classifier
        print("Increment the Model")
        model.increment_head(current_head,age)
    print(model.new_fc)
    #print(args.train_list)
    #print(prefix)
    # Construct DataLoader
    train_dataset = TSNDataSet(args.root_path, args.train_list, current_task, class_indexer, num_segments=args.num_segments,
                new_length=1, modality='RGB',image_tmpl=prefix, transform=transform_rgb, dense_sample=args.dense_sample,
                exemplar_list=exemplar_list, is_entire=(args.store_frames=='entire'), #nb_val=args.nb_val,
                exemplar_per_class=exemplar_per_class, current_head=current_head,
                diverse_rate=args.diverse_rate, cl_method=args.cl_method, age=age)

    train_loader = DataLoader(train_dataset, batch_size=args.train_batch_size,
                            shuffle=True, num_workers=args.workers,
                            pin_memory=True, drop_last=True)

    print("DataLoader Constructed : Train {}".format(len(train_loader)))

    policies = model.get_optim_policies()

    # Wrap the model with DataParallel module
    model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
    if model_old:
        model_old = torch.nn.DataParallel(model_old, device_ids=args.gpus).cuda()

    optimizer = torch.optim.SGD(policies,
                                args.lr,
                                momentum=args.momentum,
                                weight_decay=args.weight_decay)
    if args.loss_type == 'nll':
        criterion = nn.CrossEntropyLoss().cuda()
    elif args.loss_type == 'bce':
        criterion = nn.BCEWithLogitsLoss().cuda()

    print("Optimizer Constructed")

    if age > 0 and args.fc in ['cc','lsc']:
        transform_ex = torchvision.transforms.Compose([
                                                GroupScale(scale_size),
                                                GroupCenterCrop(input_size),
                                                Stack(roll=(args.arch in ['BNInception', 'InceptionV3'])),
                                                ToTorchFormatTensor(div=(args.arch not in ['BNInception', 'InceptionV3'])),
                                                normalize,
                                                ])

        dataset_for_embedding = TSNDataSet(args.root_path, args.train_list, current_task, class_indexer,
                                num_segments=args.num_segments, random_shift=False, new_length=1,
                                modality='RGB', image_tmpl=prefix, transform=transform_ex,
                                dense_sample=args.dense_sample)

        loader_for_embedding = DataLoader(dataset_for_embedding, batch_size=args.train_batch_size,
                            shuffle=False, num_workers=args.workers,
                            pin_memory=True, drop_last=False)

        cl_utils.init_cosine_classifier(model,current_task,class_indexer,loader_for_embedding,args)

    best_prec1 = 0
    best_epoch = 0
    # Train the model for the current task
    for epoch in range(args.start_epoch, args.epochs):
        _adjust_learning_rate(args, optimizer, epoch, args.lr_type, args.lr_steps)
        print("Learning rate adjusted")

        _train(args, train_loader, model, criterion, optimizer, epoch, age,
                lambda_0=lambda_0, model_old=model_old, importance_list=importance_list)
        if (epoch + 1) % args.eval_freq == 0 or epoch == args.epochs - 1:
            if args.fc in ['cc','lsc']:
                print('Sigma : {}, Eta : {}'.format(model.module.new_fc.sigma, model.module.new_fc.eta))

            state = {'epoch': epoch + 1,
                'arch': args.arch,
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(),
                'best_prec1': best_prec1,}
            save_checkpoint(args, age, state)

    if args.use_importance:
        tcd.update_importance(args,model,train_loader,criterion)
        save_importance(args,age,model,importance_dict)

    torch.cuda.empty_cache()
    del model

def _adjust_learning_rate(args, optimizer, epoch, lr_type, lr_steps):
    """Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
    if lr_type == 'step':
        decay = 0.1 ** (sum(epoch >= np.array(lr_steps)))
        lr = args.lr * decay
        decay = args.weight_decay
    elif lr_type == 'cos':
        import math
        lr = 0.5 * args.lr * (1 + math.cos(math.pi * epoch / args.epochs))
        decay = args.weight_decay
    else:
        raise NotImplementedError
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr * param_group['lr_mult']
        param_group['weight_decay'] = decay * param_group['decay_mult']


import argparse
from librec_auto.core import read_config_file
from librec_auto.core.util import Status, StudyStatus
import webbrowser

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

matplotlib.use('Agg')  # For non-windowed plotting

viz_file_pattern_bar = "viz-bar-{}.jpg"
viz_file_pattern_box = "viz-box-{}.jpg"
viz_html_filename = "viz.html"
exp_dir_pattern = "exp[0-9][0-9][0-9]"

# Things we need from each experiment run
# name (dir), params that changed, values of changing parameters, metrics measured,

def get_metric_info(files):
    metric_info = {}

    for sub_paths in files.get_exp_paths_iterator():
        status = Status(sub_paths)

        if status.is_completed():
            params = status._params
            vals = status._vals
            log = status._log

            metric_info[status._name] = (params, vals, log)

    return metric_info

def metric_values_float(log, metric):
    str_values = log.get_metric_values(metric)['cv_results']
    return [float(val) for val in str_values]

def create_bar(path, metric_name, params, settings, metric_values, is_bbo):
    x_range = range(0, len(settings))

    fig, ax = plt.subplots()
    ax.bar(x_range, metric_values, width=0.3)
    ax.set_ylabel(metric_name)
    ax.set_title('{} by\n{}'.format(metric_name, params))
    ax.set_xticks(x_range)
    ax.set_xticklabels(settings)
    if is_bbo: plt.xticks(rotation=90, fontsize=8)

    filename = path / viz_file_pattern_bar.format(metric_name)
    fig.savefig(str(filename), bbox_inches="tight")
    plt.close()

    return filename

def create_bars(path, study, is_bbo):
    # Nasim: add list to it because in python 3 it returns a view, so it doesn't have indexing, you can't access it.
    # metric_names = study.get_metric_names()
    metric_names = study.get_metric_names()
    bar_paths = []
    experiments = sorted(study._experiments.keys())

    # iterate over all metrics used in study
    for metric in metric_names:
        # param_string: for graph title
        param_string = ', '.join(study.get_exp_params())
        settings = []
        metric_vals = study.get_metric_averages(metric)

        # iterate over all experiments to get average value of metric
        for exp in experiments:
            # print(f'{exp} params: {study.get_exp_param_values(exp)}')
            param_vals_list = study.get_exp_param_values(exp)
            vals = [str(val) for (_, val) in param_vals_list]
            settings.append('\n'.join(vals))

        bar_paths.append(
            create_bar(path, metric, param_string, settings, metric_vals, is_bbo))

    return bar_paths

def create_box(path, metric, params, settings, fold_values, is_bbo):
    fig, ax = plt.subplots()
    ax.boxplot(fold_values)
    ax.set_ylabel(metric)
    ax.set_xticklabels(settings)
    if is_bbo: plt.xticks(rotation=90, fontsize=8)
    ax.set_title('{} distribution by\n{}'.format(metric, params))

    filename = path / viz_file_pattern_box.format(metric)

    fig.savefig(str(filename), bbox_inches="tight")
    plt.close()
    return filename

def create_boxes(path, study, is_bbo):
    metric_names = study.get_metric_names()
    experiments = sorted(study._experiments.keys())
    box_paths = []

    for metric in metric_names:
        param_string = ', '.join(study.get_exp_params())
        settings = []
        fold_vals = []

        for exp in experiments:
            param_vals_list = study.get_exp_param_values(exp)
            vals = [str(val) for (_, val) in param_vals_list]
            settings.append('\n'.join(vals))
            fold_vals.append(study.get_metric_folds(exp, metric))

        box_paths.append(
            create_box(path, metric, param_string, settings, fold_vals, is_bbo))

    return box_paths

PAGE_TEMPLATE = '
Study results
{}'
METRIC_TEMPLATE = '
Metric: {}
{}'
IMAGE_TEMPLATE = ''

def create_html(path, study, bars, boxes):
    html = PAGE_TEMPLATE
    metric_chunks = []
    metric_names = study.get_metric_names()

    if boxes is None:
        for name, bar in zip(metric_names, bars):
            images = IMAGE_TEMPLATE.format(bar.name)
            metric_chunks.append(METRIC_TEMPLATE.format(name, images))
    else:
        for name, bar, box in zip(metric_names, bars, boxes):
            images = IMAGE_TEMPLATE.format(bar.name) + IMAGE_TEMPLATE.format(
                box.name)
            metric_chunks.append(METRIC_TEMPLATE.format(name, images))

    output = html.format('\n'.join(metric_chunks))

    filename = path / viz_html_filename

    with open(filename, 'w') as out_file:
        out_file.write(output)

    return filename

def create_graphics(config, display):
    files = config.get_files()
    study_status = StudyStatus(config)

    bars = create_bars(files.get_post_path(), study_status,
                       (config.get_bbo_steps() is not None))

    if config.cross_validation(
    ) > 1:  # Box plot only makes sense for cross-validation
        boxes = create_boxes(files.get_post_path(), study_status,
                             config.get_bbo_steps() is not None)
    else:
        boxes = None

    if display:
        html_file = create_html(files.get_post_path(), study_status, bars,
                                boxes)
        webbrowser.open('file://' + str(html_file.absolute()),
                        new=1,
                        autoraise=True)

def read_args():
    """
    Parse command line arguments.
    :return:
    """
    parser = argparse.ArgumentParser(
        description='Generic post-processing script')
    parser.add_argument('conf', help='Path to configuration file')
    parser.add_argument('--browser',
                        help='Show graphics in browser',
                        choices=['true', 'false'])

    input_args = parser.parse_args()
    return vars(input_args)

if __name__ == '__main__':
    args = read_args()
    config = read_config_file(args['conf'], ".")

    print(f"librec-auto: Creating summary visualizations")

    display = args['browser'] == 'true'

    create_graphics(config, display)