import datetime
import os
from typing import List

from analyze_result import AnalyzeResult
from analyzer import Analyzer
# from cache import Cache
from api.fmg_api import FmgAPI
from sender import Sender
from stock_history_reader import StockHistoryReader
from telegram_bot import init_telegram_bot

'''
tasks:
-) post when `choosen` companies lost more than 3% of price for day
-) print by query /get
-) parse macrotrends (or any other) & compare P/E (take trailing p/e only)
-) simulation of algorithm from random data in the past & show profit from strategy
'''

init_telegram_bot()

def read_good_companies():
    with open("data/good_companies.txt") as fp:
        symbols: [str] = fp.readlines()
        symbols = [x.rstrip() for x in symbols]
        return symbols

def start():
    years = 15
    history = StockHistoryReader.read_history()
    tickers = history.get_tickers()

    result: List[AnalyzeResult] = []
    for ticker in tickers:
        now = datetime.datetime.now()  # be able to change this
        analyzer = Analyzer(ticker, history.get_history(ticker), now, years)
        is_good_company = analyzer.analyze()
        if is_good_company is True:
            pe: float = FmgAPI.get_quote_metrics(ticker)['pe']
            if pe is None or pe > 40:
                continue
            percent_pe_less_than_history = analyzer.get_percent_pe_less_than_history(pe)

            current_ev_over_ebitda = FmgAPI.get_key_metrics(ticker)['enterpriseValueOverEBITDATTM']
            if current_ev_over_ebitda is None:
                current_ev_over_ebitda = 100
            percent_ev_to_ebitda_less_than_history = analyzer.get_percent_ev_to_ebitda_less_than_history(
                current_ev_over_ebitda)

            analyze_result = AnalyzeResult(ticker, pe, percent_pe_less_than_history, current_ev_over_ebitda,
                                           percent_ev_to_ebitda_less_than_history)  # be able to get the old one
            result.append(analyze_result)

    print("Analysis is processed")

    result = sorted(result,
                    key=lambda cur: (cur.percent_pe_less_than_history is None, cur.percent_pe_less_than_history))
    # result = sorted(result, key=lambda cur: (
    #     cur.percent_ev_to_ebitda_less_than_history is None, cur.percent_ev_to_ebitda_less_than_history))

    for res in result:
        print(res.ticker, res.pe, res.percent_pe_less_than_history, res.current_ev_over_ebitda,
              res.percent_ev_to_ebitda_less_than_history)

    additional_info = []
    for ticker in result:
        additional_info.append("some additional info will be here later")

    Sender.send_result(result, additional_info)

if __name__ == '__main__':
    print("start main")
    print("FINANCIAL_MODELING_GREP_API:", os.environ['FINANCIAL_MODELING_GREP_API'])
    try:
        start()
    except:
        var = traceback.format_exc()
        print("Whew!", var, "occurred.")
    finally:
        print("Finished")
        os._exit(0)


import os
import csv
print(os.getcwd())  # 상대 경로를 설정할 때 기준 경로

# 파일 읽기
try:
    file = open('./data/test.txt', encoding='utf-8')  # 파일 깨지는 지 확인하고 인코딩 방식 추가
    content = file.read()  # 전체 읽기
    print(content)
except Exception as e:
    print("파일 처리 중 예외 발생", e)
finally:
    file.close()

# 줄 단위로 파일 읽기
try:
    file1 = open('./data/test1.txt', encoding='utf-8')  # 파일 깨지는 지 확인하고 인코딩 방식 추가
    for line in file1:
        print(line)
except Exception as e:
    print("파일 처리 중 예외 발생", e)
finally:
    file.close()

# with 사용
with open('./data/test1.txt', encoding='utf-8') as file:
    for line in file:
        print(line)
        print()

# 텍스트 파일을 읽어서 list로 변환
# 마지막 데이터에는 \n 추가됨
# 마지막 데이터는 \n 제거해야 함
data = []
with open('./data/경상북도 울릉군_대회정보_20211013.csv', encoding='cp949') as file2:
    for line in file2:
        ar = line.split(",")
        data.append(ar)
print(data)

data1 = []
with open('./data/경상북도 울릉군_대회정보_20211013.csv', encoding='cp949') as file2:
    # 줄 단위로 읽어서 list를 만들어주는 reader 객체를 생성
    rdr = csv.reader(file2)
    for line in rdr:
        data1.append(line)
print(data1)

data2 = []
# w 모드로 열면 기존 내용을 지우고 기록하고 a 모드로 열면 기존 내용 뒤에 추가함
with open('./data/경상북도 울릉군_대회정보_20211013.csv', 'w', encoding='cp949') as file3:
    wr = csv.writer(file)
    wr.writerow(['경상북도','울릉군','대회정보'])
    wr.writerow(['20211013'])


row =int(input("ENter the number of row :"))

for i in range(1,row+1):
   pat =""
   space =""
   for j in range(row,i,-1):
       space+=" "
   for k in range(0,2*i-1):
       pat+="*"
   print(space+pat)


import os, re, sys, json

import ssl
import requests.packages.urllib3.util.ssl_
requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = 'ALL'

#count file output
outCountFile = 'outCount.xls'
lastopenfile = ''

#if count fix num
iscountvalid = 1
outPutNum = 10

#specific index
global fromIndex

#index
count = 0

#match the star text
def matchStarText(text):
    find = 0
    searchObj = re.search( r'.*stargazers">.*', text, re.M|re.I)
    if searchObj:
        find = 1
    return find

#match the star text
def matchForkText(text):
    find = 0
    searchObj = re.search( r'.*class="social-count">.*', text, re.M|re.I)
    if searchObj:
        find = 1
    return find

#drop the special char in line nums
def formatNumber(text):
    text = text.strip()
    text = text.replace(',','')
    return text

#open spec file read git url
def openAndReadGitUrl(path,count):
    json_file = open(path, "r")
    json_data = json.load(json_file)
    gitname = json_data["name"]
    giturl = 'none'
    if 'git' in json_data["source"]:
        giturl = json_data["source"]["git"]

    print ("%s\t%s" %(gitname,giturl))
    if giturl != 'none' and giturl.startswith('http'):
        gitsummary = 'none'
        if 'summary' in json_data:
            gitsummary = json_data['summary'].replace('\n','')
        gitDescription = 'none'
        if 'description' in json_data:
            gitDescription = json_data['description'].replace('\n','')
        gitstar = '-'
        gitfork = '-'

        try:
            fp = requests.get(giturl,timeout=15)
            if fp.status_code == requests.codes.ok:
                match = 0
                for line in fp.iter_lines():
                    line = line.decode('utf-8')
                    if match == 1:
                        gitstar =  formatNumber(line)
                        match = 0
                    if match == 2:
                        gitfork = formatNumber(line)
                        break

                    if matchStarText(line) == 1:
                        match = 1
                    if matchForkText(line) == 1:
                        match = 2
                out = '%d\t%s\t\t%s\t\t%s\t\t%s\t\t%s\t\t%s\n' %(count,gitname,gitstar,gitfork,giturl,gitsummary,gitDescription)
                out = out.encode('ascii', 'ignore')
                print (out.decode('utf-8'))
                file = open(outCountFile,'a')
                out = out.decode('utf-8')
                file.write(out)
        except requests.exceptions.HTTPError as e:
            print ("ERROR:%s" %(e))
        except requests.exceptions.Timeout as e:
            print ("ERROR:%s" %(e))
        except requests.exceptions.RequestException as e:
            print ("ERROR:%s" %(e))

# print all spec file
def listFileList(rootDir):
    global lastopenfile, count
    for file in os.listdir(rootDir):
        if iscountvalid and count >= outPutNum:
            break
        path = os.path.join(rootDir, file)
        if os.path.isdir(path):
            dirs = os.listdir(path)
            newpath = os.path.join(path, dirs[len(dirs)-1])
            if os.path.isdir(newpath):
                listFileList(newpath)
        else:
            if (count+1) >= fromIndex:
                openAndReadGitUrl(path, count+1)
            else:
                print ("index %d was jumped...fromIndex is %d" %(count+1,fromIndex))
            count += 1

if __name__ == '__main__':
    argc = len(sys.argv)
    if argc == 2:
        if os.path.exists(outCountFile):
            os.remove(outCountFile)
        fromIndex = 1
        listFileList(sys.argv[1])
    else:
        if argc == 3:
            fromIndex = int(sys.argv[2])
            listFileList(sys.argv[1])
        else:
            print ('input error, please input path like (python specGitStars.py ~/.cocoapods/repos/master/Specs)')


import requests
import json
import time

#check flask api
api_url = "api/"
port = "5000"

#check fastapi api
# api_url = "api/"
# port = "8000"

#check torchServe api
# api_url = "predictions/bert_trimsweep"
# port = "8080"

url = 'http://127.0.0.1:' + port + '/' + api_url

test_texts = ["Wir hatten hier ein sehr gutes Essen.",
              "Zusammenfassung: guter Service, leckeres (höheres) Essen und schöne Lage.",
              "All dies zu einem sehr vernünftigen Preis!",
              "Das Essen war einfach schlecht.",
              "Wir hatten hier ein paar super Cocktails."]

start_time = time.time()
for text in test_texts:
    j_data = json.dumps({"text": text})
    headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
    r = requests.post(url, data=j_data, headers=headers)
    print(r, r.text)

print("Requests took %0.3f s" % (time.time() - start_time))


# coding:utf-8

import sys
ver=sys.version

if "2." in ver:
    from Tkinter import *
    print("tk = ", ver)
    import tkFileDialog as filedialog

elif "3." in ver:
    from tkinter import *
    print("tk = ", ver)
    from tkinter import filedialog

root=Tk("Text Editor")

text01=Text(root,
            font="Arial 18",
            bg='lightgray',
            fg='green')
text01.grid()
#======================

def save_as():
    global text01
    text = text01.get("1.0", "end-1c")

    save_file=filedialog.asksaveasfilename()

    #f01=open(savelocation, "w+")
    with open(save_file, "w+") as f01:

        f01.write(text)
        #f01.close()

button=Button(root, text="Save",
                command=save_as,
                font="Arial 18",
                bg='yellow',
                fg='red')

button.grid()
#======================================================

def FontHelvetica():

    global text01
    text01.config(font="Helvetica")

def FontCourier():
    global text01
    text01.config(font="Courier")

#------------------------
font=Menubutton(root, text="Font",
                font="Arial 18",
                # bg='yellow',
                fg='blue')

font.grid()
#------------------------

font.menu=Menu(font, tearoff=0)
font["menu"]=font.menu

Helvetica=IntVar()
arial=IntVar()
times=IntVar()
Courier=IntVar()

font.menu.add_checkbutton(label="Courier", variable=Courier,
command=FontCourier)
font.menu.add_checkbutton(label="Helvetica", variable=Helvetica,
command=FontHelvetica)

root.mainloop()


import numpy as np
import matplotlib.pyplot as plt
import matplotlib

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
plt.rcParams["font.size"] = 16

counts = []

with open("../build/braid_counts.txt", "r") as input_file:
    for _ in range(100):
        counts.append(int(input_file.readline().split()[1]))

fig, ax = plt.subplots(figsize = (7, 4))
plt.hist(counts, bins = range(1,101))
plt.xlim(1,101)
fig.subplots_adjust(left=0.08, right=0.95, bottom=0.16, top=0.95)
plt.xlabel("Number of homotopy classes")
plt.savefig("./histogram.pdf")
plt.show()


import json
import os
import shutil
import logging
import csv
import mimetypes
from uuid import uuid4

from django.http import HttpResponse, JsonResponse,FileResponse, \
    HttpResponseRedirect
from django.template import loader
from django.conf import settings
from django.shortcuts import render
from django.core.exceptions import ValidationError
from django.contrib.auth.decorators import login_required
from django.contrib import messages

from . import utils
from .tasks import run_model_task
from .models import CellMetadata

logger = logging.getLogger('django')
guest_sess_prefix_str = 'guest_session:'

def get_guest_session(request):
    storage = messages.get_messages(request)

    for message in storage:
        if message.tags == 'info':
            if message.message.startswith(guest_sess_prefix_str):
                s_len = len(guest_sess_prefix_str)
                sess_str = message.message[s_len:]
                storage.used = False
                return sess_str

    return ''

# Create your views here.
def index(request, session=''):
    #import sys
    #sys.path.append("/home/docker/pycharm-debug")
    #import pydevd
    #pydevd.settrace('172.17.0.1', port=21000, suspend=False)

    storage = messages.get_messages(request)
    storage.used = True
    # make sure session messages and species messages are removed
    for message in storage:
        if message.tags == 'info':
            # consume it so as to clear it
            temp_str = message.message

    if session:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, session)
        if os.path.isdir(ws_path):
            messages.info(request, guest_sess_prefix_str + session)
            status_msg = 'You are in your temporary guest playground where you can explore your own data in conjunction ' \
                         'with system-provided data as specified in the guest workspace you created. The guest data will ' \
                         'exist on the server for at least 12 hours, then will be cleaned up nightly from the system.'
        else:
            status_msg = ''
    else:
        status_msg = ''

    template = loader.get_template('cc_core/index.html')
    context = {
        'SITE_TITLE': settings.SITE_TITLE,
        'status_msg': status_msg
    }
    return HttpResponse(template.render(context, request))

def about(request):
    template = loader.get_template('cc_core/about.html')
    context = {}
    return HttpResponse(template.render(context, request))

def help(request):
    template = loader.get_template('cc_core/help.html')
    context = {}
    return HttpResponse(template.render(context, request))

def cell_data_meta(request, filename):
    meta_rec = CellMetadata.objects.all().filter(cell_filename=filename).first()

    if meta_rec:
        mdict = meta_rec.metadata_dict
    else:
        mdict = {}

    context = {'metadata_dict': mdict,
               "download_url": "/download/" + filename + "/"
               }
    template = loader.get_template('cc_core/cell_meta.html')
    return HttpResponse(template.render(context, request))

def download_all_data(request):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    prof_conf_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH, settings.WORKSPACE_CONFIG_FILENAME)
    profs = utils.get_profile_list(prof_conf_name)

    context = {'dataset_list': utils.get_dataset_list(profs),
               'model_list': utils.get_model_list(profs),
               "download_url": "/download/"
               }
    template = loader.get_template('cc_core/download_all_data.html')
    return HttpResponse(template.render(context, request))

def download_all(request):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    filename = settings.DATA_ZIP_FILENAME
    full_path_filename = os.path.join(ws_path, filename)
    if not os.path.isfile(full_path_filename):
        # create the zipped file that contain all data
        utils.zip_all_data(ws_path, filename)

    # obtain mime_type to set content_type
    mtype = 'application/zip'
    # obtain file size
    stat_info = os.stat(full_path_filename)
    flen = stat_info.st_size
    f = open(full_path_filename, 'r')
    response = FileResponse(f, content_type=mtype)
    response['Content-Disposition'] = 'attachment; filename="{name}"'.format(name=filename)
    response['Content-Lengtsh'] = flen
    return response

def download(request, filename):
    _, ext = os.path.splitext(filename)
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    if ext == '.csv':
        file_full_path = os.path.join(ws_path, settings.CELL_DATA_PATH, filename)
    elif ext == '.xml':
        file_full_path = os.path.join(ws_path, settings.MODEL_INPUT_PATH, filename)
    else:
        messages.error(request, "Only dataset file in csv format and model file in xml format can be downloaded")
        return HttpResponseRedirect(request.META['HTTP_REFERER'])

    # obtain mime_type to set content_type
    mtype = 'application-x/octet-stream'
    mime_type = mimetypes.guess_type(filename)
    if mime_type[0] is not None:
        mtype = mime_type[0]

    # obtain file size
    stat_info = os.stat(file_full_path)
    flen = stat_info.st_size
    f = open(file_full_path, 'r')
    response = FileResponse(f, content_type=mtype)
    response['Content-Disposition'] = 'attachment; filename="{name}"'.format(name=filename)
    response['Content-Lengtsh'] = flen
    return response

def get_profile_list(request):
    """
    It is invoked by an AJAX call, so it returns json object that holds data set list
    """
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    profile_config_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH,
                                       settings.WORKSPACE_CONFIG_FILENAME)
    profile_list = utils.get_profile_list(profile_config_name)

    return HttpResponse(json.dumps(profile_list), content_type='application/json')

def get_profile(request):
    index = int(request.POST['index'])
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    profile_config_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH,
                                       settings.WORKSPACE_CONFIG_FILENAME)
    profile = utils.get_profile_list(profile_config_name)[index]
    data = {}

    data['name'] = profile['name']
    data['description'] = profile['description']

    if 'models' in profile:
        data['modelList'] = [m['fileName'] for m in profile['models']]

    if 'cellData' in profile:
        data['datasetList'] = [d['fileName'] for d in profile['cellData']]

    return HttpResponse(json.dumps(data), content_type='application/json')

def add_user_workspace(request):
    profiles = utils.get_profile_list()
    cell_data_names, model_data_names = utils.get_all_cell_and_model_file_names(
        profiles=profiles)
    context = {
        'guest_session_id': uuid4().hex,
        'cell_data_names': cell_data_names,
        'model_data_names': model_data_names
    }
    return render(request, 'cc_core/add-profile.html', context)

@login_required()
def add_or_delete_profile_request(request):
    profiles = utils.get_profile_list()
    referer = request.META['HTTP_REFERER']
    is_add_profile = False
    is_delete_profile = False
    if 'add_profile' in referer:
        is_add_profile = True
    if 'delete_profile' in referer:
        is_delete_profile = True

    if not is_add_profile and not is_delete_profile:
        storage = messages.get_messages(request)
        for message in storage:
            if message.tags == 'info':
                if message.message == 'AddProfile':
                    is_add_profile = True
                if message.message == 'DeleteProfile':
                    is_delete_profile = True

    if is_add_profile:
        cell_data_names, model_data_names = utils.get_all_cell_and_model_file_names(
            profiles=profiles)
        context = {
            'guest_session_id': '',
            'cell_data_names': cell_data_names,
            'model_data_names': model_data_names
        }
        return render(request, 'cc_core/add-profile.html', context)

    if is_delete_profile:
        profile_names = set()
        for p in profiles:
            profile_names.add(p['name'])
        context = {
            'profile_data_names': profile_names
        }
        return render(request, 'cc_core/delete-profile.html', context)

def delete_profile_from_server(request):
    # delete profile data from server
    sel_profile_name = request.POST.get('profile_sel_name', '')

    if sel_profile_name:
        try:
            utils.delete_profile(sel_profile_name)
            template = loader.get_template('cc_core/index.html')
            context = {
                'SITE_TITLE': settings.SITE_TITLE,
                'status_msg': 'Congratulations! The selected profile has been deleted successfully.'
            }
            return HttpResponse(template.render(context, request))
        except Exception as ex:
            messages.error(request, ex.message)
            messages.info(request, 'DeleteProfile')
            return HttpResponseRedirect(request.META['HTTP_REFERER'])

def add_profile_to_server(request):
    # create profile data to write to profile json file
    data = {}
    sess_id = request.POST.get('guest_session_id', '')
    if sess_id:
        messages.info(request, guest_sess_prefix_str + sess_id)
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_id)
        if not os.path.exists(ws_path):
            os.makedirs(os.path.join(ws_path, settings.CELL_DATA_PATH))
            os.makedirs(os.path.join(ws_path, settings.MODEL_INPUT_PATH))
            os.makedirs(os.path.join(ws_path, settings.MODEL_OUTPUT_PATH))
            os.makedirs(os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH))
    else:
        ws_path = settings.WORKSPACE_PATH

    try:
        data['name'] = request.POST.get('pname')
        if not data['name']:
            messages.error(request, "Please input a profile name.")
            messages.info(request, 'AddProfile')
            return HttpResponseRedirect(request.META['HTTP_REFERER'])

        data['description'] = request.POST.get('pdesc')
        if not data['description']:
            messages.error(request, "Please input a profile description.")
            messages.info(request, 'AddProfile')
            return HttpResponseRedirect(request.META['HTTP_REFERER'])

        cell_data_list = []
        cdfiles = request.FILES.getlist('cell_files')

        req_md_elems = []
        if cdfiles:
            req_md_elems = utils.get_required_metadata_elements()

        cdselnames = request.POST.getlist('cell_sel_names')
        filename_to_idx = {}
        idx = 0
        for cdfile in cdfiles:
            source = cdfile.file.name
            target = os.path.join(ws_path, settings.CELL_DATA_PATH, cdfile.name)
            # check to make sure the uploaded new file names don't have conflict with existing files
            if os.path.isfile(target):
                # file with the same file name already exists - raise validation error and ask user
                # to change file name to avoid file conflict
                messages.error(request, "The file " + cdfile.name +
                               " already exists on the server. Please rename this file "
                               "before uploading to the server to avoid file conflict.")
                messages.info(request, 'AddProfile')
                return HttpResponseRedirect(request.META['HTTP_REFERER'])

            with open(source, 'r') as fp:
                csv_data = csv.reader(fp)
                # validate the dataset metadata to have all required metadata elements before
                # adding the dataset into the server
                try:
                    _, _ = utils.read_metadata_from_csv_data(cdfile.name, csv_data,
                                                                 req_md_elems)
                except ValidationError as ex:
                    messages.error(request, ex.message)
                    messages.info(request, 'AddProfile')
                    return HttpResponseRedirect(request.META['HTTP_REFERER'])

            shutil.copy(source, target)
            cell_data_list.append({'fileName':cdfile.name})
            filename_to_idx[cdfile.name] = idx
            idx += 1

        for cdselname in cdselnames:
            cell_data_list.append({'fileName': cdselname})
            if sess_id:
                # copy the selected data set from system workspace to guest workspace
                source = os.path.join(settings.WORKSPACE_PATH, settings.CELL_DATA_PATH, cdselname)
                target = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_id, settings.CELL_DATA_PATH, cdselname)
                shutil.copy(source, target)
            filename_to_idx[cdselname] = idx
            idx += 1

        cdnames = request.POST.get('cdname')
        if cdnames:
            if ';' in cdnames:
                cdname_list = cdnames.split(';')
            else:
                cdname_list = [cdnames.strip()]
        else:
            cdname_list = []

        for cdname in cdname_list:
            name_pair = cdname.split(':')
            cell_data_list[filename_to_idx[name_pair[0]]]['name'] = name_pair[1]

        cddescs = request.POST.get('cddesc')
        if cddescs:
            if ';' in cddescs:
                cddesc_list = cddescs.split(';')
            else:
                cddesc_list = [cddescs.strip()]
        else:
            cddesc_list = []

        for cddesc in cddesc_list:
            desc_pair = cddesc.split(':')
            cell_data_list[filename_to_idx[desc_pair[0]]]['description'] = desc_pair[1]

        data['cellData'] = cell_data_list

        filename_to_idx = {}
        idx = 0
        model_data_list = []
        mdfiles = request.FILES.getlist('model_files')
        for mdfile in mdfiles:
            source = mdfile.file.name
            target = os.path.join(ws_path, settings.MODEL_INPUT_PATH, mdfile.name)

            if os.path.isfile(target):
                # file with the same file name already exists - raise validation error and ask user
                # to change file name to avoid file conflict
                messages.error(request, "The file " + mdfile.name +
                               " already exists on the server. Please rename this file "
                               "before uploading to the server to avoid file conflict.")
                messages.info(request, 'AddProfile')
                return HttpResponseRedirect(request.META['HTTP_REFERER'])

            shutil.copy(source, target)
            model_data_list.append({'fileName': mdfile.name})
            filename_to_idx[mdfile.name] = idx
            idx += 1

        mdselnames = request.POST.getlist('model_sel_names')
        for mdselname in mdselnames:
            model_data_list.append({'fileName': mdselname})
            if sess_id:
                # copy the selected model from system workspace to guest workspace
                source = os.path.join(settings.WORKSPACE_PATH, settings.MODEL_INPUT_PATH, mdselname)
                target = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_id, settings.MODEL_INPUT_PATH, mdselname)
                shutil.copy(source, target)
            filename_to_idx[mdselname] = idx
            idx += 1

        new_created_model_name = request.POST.get('new_sbml_model_file_name', '')
        if new_created_model_name:
            model_data_list.append({'fileName': new_created_model_name})
            filename_to_idx[new_created_model_name] = idx
            idx += 1

        mdnames = request.POST.get('mdname')
        if mdnames:
            if ';' in mdnames:
                mdname_list = mdnames.split(';')
            else:
                mdname_list = [mdnames.strip()]
        else:
            mdname_list = []

        for mdname in mdname_list:
            name_pair = mdname.split(':')
            model_data_list[filename_to_idx[name_pair[0]]]['name'] = name_pair[1]

        mddescs = request.POST.get('mddesc')
        if mddescs:
            if ';' in mddescs:
                mddesc_list = mddescs.split(';')
            else:
                mddesc_list = [mddescs.strip()]
        else:
            mddesc_list = []

        for mddesc in mddesc_list:
            desc_pair = mddesc.split(':')
            model_data_list[filename_to_idx[desc_pair[0]]]['description'] = desc_pair[1]

        data['models'] = model_data_list

        if not cell_data_list and not model_data_list:
            messages.error(request, "Please upload or select some data - you cannot leave both "
                                    "dataset and model data empty in the new profile")
            messages.info(request, 'AddProfile')
            return HttpResponseRedirect(request.META['HTTP_REFERER'])

        pname_list = data['name'].split()
        profile_file_name = '_'.join(pname_list)
        profile_file_name = profile_file_name + '.json'
        profile_file_full_path = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH, profile_file_name)
        with open(profile_file_full_path, 'w') as out:
            out.write(json.dumps(data, indent=4))

        # update configuration json file
        profile_list_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH, settings.WORKSPACE_CONFIG_FILENAME)
        if os.path.isfile(profile_list_name):
            with open(profile_list_name, 'r') as f:
                json_profile_data = json.load(f)
        else:
            json_profile_data = []

        json_profile_data.append({'fileName': profile_file_full_path,
                                  'group': 1})

        with open(profile_list_name, 'w') as f:
            f.write(json.dumps(json_profile_data, indent=4))

        if sess_id:
            return HttpResponseRedirect('/guest/' + sess_id)
        else:
            template = loader.get_template('cc_core/index.html')
            context = {
                'SITE_TITLE': settings.SITE_TITLE,
                'status_msg': 'Congratulations! The new workspace has been added successfully.'
            }
            return HttpResponse(template.render(context, request))
    except Exception as ex:
        messages.error(request, ex.message)
        messages.info(request, 'AddProfile')
        return HttpResponseRedirect(request.META['HTTP_REFERER'])

def get_dataset_list(request):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    prof_conf_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH, settings.WORKSPACE_CONFIG_FILENAME)
    profs = utils.get_profile_list(prof_conf_name)
    ds_list = utils.get_dataset_list(profs)
    return HttpResponse(json.dumps(ds_list), content_type='application/json')

def get_dataset(request, id):
    filename = id
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    cell_data_filename = os.path.join(ws_path, settings.CELL_DATA_PATH, filename)
    _, csv_str = utils.load_cell_data_csv_content(cell_data_filename)
    return JsonResponse({'csv': csv_str})

def get_model_list(request):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    prof_conf_name = os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH, settings.WORKSPACE_CONFIG_FILENAME)
    profs = utils.get_profile_list(prof_conf_name)
    md_list = utils.get_model_list(profs)
    return HttpResponse(json.dumps(md_list), content_type='application/json')

def get_model(request, id):
    filename = id
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    model_file = os.path.join(ws_path, settings.MODEL_INPUT_PATH, filename)
    return JsonResponse(utils.load_model_content(model_file))

def create_sbml_model(request):
    # request must be a JSON request
    num_g1 = request.POST.get('num_G1', None)
    rate_g1 = request.POST.get('rate_G1', None)
    num_s = request.POST.get('num_S', None)
    rate_s = request.POST.get('rate_S', None)
    num_g2m = request.POST.get('num_G2M', None)
    rate_g2m = request.POST.get('rate_G2M', None)
    sbml_fname = request.POST.get('sbml_file_name', '')

    sess_str = request.POST.get('sess_id', '')
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
        if not os.path.exists(ws_path):
            os.makedirs(os.path.join(ws_path, settings.CELL_DATA_PATH))
            os.makedirs(os.path.join(ws_path, settings.MODEL_INPUT_PATH))
            os.makedirs(os.path.join(ws_path, settings.MODEL_OUTPUT_PATH))
            os.makedirs(os.path.join(ws_path, settings.WORKSPACE_CONFIG_PATH))
    else:
        ws_path = settings.WORKSPACE_PATH

    response_data = {}

    if num_g1 and rate_g1 and num_s and rate_s and num_g2m and rate_g2m and sbml_fname:
        try:
            utils.createSBMLModel_CC_serial(int(num_g1), float(rate_g1),
                                            int(num_s), float(rate_s),
                                            int(num_g2m), float(rate_g2m),
                                            os.path.join(ws_path, settings.MODEL_INPUT_PATH, sbml_fname))
        except Exception as ex:
            response_data["error"] = ex.message + ' for creating SBML model'
            return HttpResponse(json.dumps(response_data), content_type='application/json')

    else:
        response_data["error"] = 'Invalid input parameters for creating SBML model'
        return HttpResponse(json.dumps(response_data), content_type='application/json')

    response_data["new_model_filename"] = sbml_fname

    return HttpResponse(json.dumps(response_data), content_type='application/json')

def delete_sbml_model(request, filename, sess_id = ''):
    response_data = {}

    sess_str = sess_id
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    full_fname = os.path.join(ws_path, settings.MODEL_INPUT_PATH, filename.encode("utf-8"))
    try:
        if os.path.isfile(full_fname):
            # delete this model file
            os.remove(full_fname)
            response_data["new_model_filename"] = filename
            return HttpResponse(json.dumps(response_data), content_type='application/json')
        else:
            response_data["error"] = 'The SBML model data file does not exist'
            return HttpResponse(json.dumps(response_data), content_type='application/json')

    except Exception as ex:
        response_data["error"] = 'Invalid SBML model data file name'
        return HttpResponse(json.dumps(response_data), content_type='application/json')

def send_parameter(request):
    data = {}
    data['species'] = request.POST['species']
    data['value'] = request.POST['value']

    return HttpResponse(json.dumps(data), content_type='application/json')

def run_model(request, filename=''):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH

    if filename:
        full_fname = os.path.join(ws_path, settings.MODEL_INPUT_PATH, filename)
        id_to_names, name_to_ids, species, phases = \
            utils.extract_species_and_phases_from_model(full_fname)

        # put species into session messages so that species can be filtered out in simulation progress return
        messages.info(request, 'species:' + ','.join(species))

        traj = request.POST.get('trajectories', 1)
        # use a big end time value which is used by stochpy to indicate end time is not used
        # but rather simulation will end when the last phase is done in the simulation
        if 'species' not in request.POST:
            sp_name_to_val_list = []
        else:
            sp_name_to_val_list = json.loads(request.POST['species'])

        if 'parameters' not in request.POST:
            sp_name_infl_para_list = []
        else:
            sp_name_infl_para_list = json.loads(request.POST['parameters'])

        pid_list = utils.extract_parameter_ids(full_fname)

        sp_id_infl_para_dict = {}
        for p_dict_item in sp_name_infl_para_list:
            phase = p_dict_item.get('phase', None)
            name1 = p_dict_item['upstream'] # species name which is an influencer
            name2 = p_dict_item['downstream'] # species or phase name which is an influencee
            val = p_dict_item['value']
            pid2 = 'p_' + name1 + '_' + name2
            if phase:
                pid1 = 'z_' + name1 + '_' + name2 + '_' + phase
                if pid1 in pid_list:
                    para_id = pid1
                elif pid2 in pid_list:
                    para_id = pid2
                else:
                    para_id = ''
            elif pid2 in pid_list:
                para_id = 'p_' + name1 + '_' + name2
            else:
                para_id = ''
            if para_id:
                sp_id_infl_para_dict[para_id] = val

        sp_id_to_val_dict = {}
        for item in sp_name_to_val_list:
            sp_name = item['species']
            sp_id = name_to_ids[sp_name]
            sp_id_to_val_dict[sp_id] = item['value']
            if 'degradation' in item:
                sp_deg = item['degradation']
                for ph in phases:
                    para_id = 'a_' + sp_name + '_' + ph
                    if para_id in pid_list:
                        sp_id_infl_para_dict[para_id] = sp_deg

        task = run_model_task.apply_async((ws_path, filename, id_to_names, species, phases, traj,
                                           sp_id_to_val_dict, sp_id_infl_para_dict),
                                          countdown=1)
        context = {
            'task_id': task.task_id,
        }
    else:
        context = {
            'task_id': '',
        }

    return HttpResponse(json.dumps(context), content_type='application/json')

def get_model_result(request, filename):
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    file_full_path = os.path.join(ws_path, settings.MODEL_OUTPUT_PATH, filename)
    with open(file_full_path, 'rb') as model_output_file:
        model_result_json = json.load(model_output_file)
        response = JsonResponse(model_result_json)
        return response

    # return bad request if the json file cannot be served above
    return HttpResponse(status=400)

def check_task_status(request):
    '''
    A view function to tell the client if the asynchronous run_model() task is done.
    Args:
        request: an ajax request to check for model run status
    Returns:
        JSON response to return result from asynchronous task run_model()
    '''
    task_id = request.POST.get('task_id')
    result = run_model_task.AsyncResult(task_id)
    ret_result = {}
    if result.ready():
        try:
            get_result = result.get()
        except Exception as ex:
            ret_result['result'] = None
            ret_result['error'] = ex.message
            return HttpResponse(json.dumps(ret_result),
                                content_type="application/json")

        ret_result['result'] = get_result
        ret_result['error'] = None
        ret_result['progress'] = None
        return HttpResponse(json.dumps(ret_result),
                            content_type="application/json")
    else:
        # read progress generated by simulation
        pstr = ''
        sp_list = []
        storage = messages.get_messages(request)
        for message in storage:
            if message.tags == 'info':
                prefix_str = 'species:'
                if message.message.startswith(prefix_str):
                    p_len = len(prefix_str)
                    sp_list = message.message[p_len:].split(',')
                    storage.used = False
                    break
        pfilename = os.path.join(settings.WORKSPACE_PATH, settings.MODEL_OUTPUT_PATH, 'progress' + task_id + '.txt')
        if os.path.isfile(pfilename):
            with open(pfilename, 'r') as pf:
                pline = pf.readline()
                pinfos = pline.split(',')
                # filtering out species from returned simulation progress
                if len(pinfos) == 2:
                    if not sp_list:
                        pstr = pline
                    elif pinfos[0] not in sp_list:
                        pstr = pline
        if pstr:
            return HttpResponse(json.dumps({"result": None, "error": None, "progress": pstr}),
                                content_type="application/json")
        else:
            return HttpResponse(json.dumps({"result": None}),
                                content_type="application/json")

def terminate_model_run(request):
    '''
    A view function to terminate model run task at user's request.
    Args:
        request: an ajax request to terminate model run task
    Returns:
        JSON response to indicate success or failure
    '''
    try:
        task_id = request.POST.get('task_id')
        run_model_task.AsyncResult(task_id).revoke(terminate=True)
    except Exception as ex:
        return JsonResponse({'status':'failure', 'message': ex.message}, status=500)

    return JsonResponse({'status': 'success'})

def extract_parameters(request, filename):
    """
    extract parameters from the model file and return parameters as downloadable JSON file
    :param request: a request in the form of /phases/<model_file_name>
    :param filename: the model file name
    :return: Downloadable JSON file that contain parameters
    """

    return_object = {}
    sess_str = get_guest_session(request)
    if sess_str:
        ws_path = os.path.join(settings.GUEST_WORKSPACE_PATH, sess_str)
    else:
        ws_path = settings.WORKSPACE_PATH
    try:
        model_file = os.path.join(ws_path, settings.MODEL_INPUT_PATH, filename.encode("utf-8"))
        paras_list = utils.extract_parameters(model_file)
        return_object['parameters'] = paras_list
        jsondump = json.dumps(return_object)
        response = HttpResponse(jsondump, content_type='text/json')
        out_file_name = os.path.splitext(filename)[0] + "_parameters.json"
        response['Content-Disposition'] = 'attachment; filename="' + out_file_name + '"'
        return response
    except Exception as ex:
        return_object['error'] = ex.message
        jsondump = json.dumps(return_object)
        response = HttpResponse(jsondump, content_type='text/json')
        response['Content-Disposition'] = 'attachment; filename=error.json'
        return response


from tensorflow import keras as k
from collections import deque

import tensorflow as tf
import numpy as np

class ReplayBuffer(object):

    def __init__(self, buffer_size=1024):
        self.buffer_size = buffer_size
        self.buffer = deque()

    def add(self, s0, a, r, t, s1):
        to_add = (s0, a, r, t, s1)

        if len(self.buffer) < self.buffer_size:
            self.buffer.append(to_add)
        else:
            self.buffer.popleft()
            self.buffer.append(to_add)

    def __len__(self):
        return len(self.buffer)

    def sample_batch(self, batch_size):
        if batch_size > len(self.buffer):
            batch_size = len(self.buffer)
        indices = np.random.permutation(np.arange(len(self.buffer)))[:batch_size]

        s0, a, r, t, s1 = [], [], [], [], []
        for i in indices:
            s0.append(self.buffer[i][0])
            a.append(self.buffer[i][1])
            r.append(self.buffer[i][2])
            t.append(self.buffer[i][3])
            s1.append(self.buffer[i][4])

        r = tf.reshape(r, (-1, 1))
        return tf.convert_to_tensor(s0), tf.convert_to_tensor(a), tf.convert_to_tensor(r), tf.convert_to_tensor(t), tf.convert_to_tensor(s1)

    def clear(self):
        self.buffer = deque()

class ActorNetwork(object):

    def __init__(self, state_space, action_space, tau, noise_mean=0, noise_sigma=0.2, optimizer=k.optimizers.Adam(), action_range=(0, 1), layer_size=(400, 300)):
        input0, output0 = self.create_model(state_space, action_space, layer_size)
        self.model = k.models.Model(input0, output0)

        input0, output0 = self.create_model(state_space, action_space, layer_size)
        self.target_model = k.models.Model(input0, output0)
        self.target_model.set_weights(self.model.get_weights())

        self.tau = tau
        self.optimizer = optimizer
        self.action_range = action_range
        self.default_range = (-1, 1)

        self.noise_mean = noise_mean
        self.noise_sigma = noise_sigma

    def create_model(self, state_space, action_space, layer_size):
        input0 = k.layers.Input(state_space)

        x = k.layers.Dense(layer_size[0])(input0)
        x = k.layers.BatchNormalization()(x)
        x = k.layers.Activation("relu")(x)

        x = k.layers.Dense(layer_size[1])(x)
        x = k.layers.BatchNormalization()(x)
        x = k.layers.Activation("relu")(x)

        kernel_initializer = k.initializers.RandomUniform(minval=-0.003, maxval=0.003)
        output0 = k.layers.Dense(action_space, activation="tanh", kernel_initializer=kernel_initializer)(x)

        return input0, output0

    def fit(self, s, critic):
        with tf.GradientTape() as g:
            a = self.predict(s)
            v = critic.predict([s, a])
            loss = -tf.reduce_mean(v)

        gradients = g.gradient(loss, self.model.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))

        return loss

    def predict(self, s, training=False):
        return self.scale_action(self.model(s, training=training))

    def predict_target(self, s, training=False):
        return self.scale_action(self.target_model(s, training=training))

    def update_target(self):
        weights = self.model.weights
        target_weights = self.target_model.weights

        for layer_w, target_layer_w in zip(weights, target_weights):
            target_layer_w.assign(self.tau * layer_w + (1 - self.tau) * target_layer_w)

    def scale_action(self, a):
        original_scope = self.default_range[1] - self.default_range[0]
        target_scope = self.action_range[1] - self.action_range[0]
        return (((a - self.default_range[0]) * target_scope) / original_scope) + self.action_range[0]

    @tf.function
    def get_action_with_noise(self, s, mean=None, sigma=None):
        if mean is None:
            mean = self.noise_mean
        if sigma is None:
            sigma = self.noise_sigma

        action = self.predict(s)
        action += tf.random.normal(shape=action.shape, mean=mean, stddev=sigma, dtype=tf.float32)

        return tf.clip_by_value(action, *self.action_range)

class CriticNetwork(object):

    def __init__(self, state_space, action_space, tau, optimizer=k.optimizers.Adam(), layer_size=(400, 300)):
        inputs, output0 = self.create_model(state_space, action_space, layer_size)
        self.model = k.models.Model(inputs, output0)

        inputs, output0 = self.create_model(state_space, action_space, layer_size)
        self.target_model = k.models.Model(inputs, output0)
        self.target_model.set_weights(self.model.get_weights())

        self.tau = tau
        self.optimizer = optimizer

    def create_model(self, state_space, action_space, layer_size):
        input0 = k.layers.Input(state_space)
        input1 = k.layers.Input(action_space)

        x = k.layers.Concatenate(axis=1)([input0, input1])

        x = k.layers.Dense(layer_size[0])(x)
        x = k.layers.BatchNormalization()(x)
        x = k.layers.Activation("relu")(x)

        x = k.layers.Dense(layer_size[1])(x)
        x = k.layers.BatchNormalization()(x)
        x = k.layers.Activation("relu")(x)

        kernel_initializer = k.initializers.RandomUniform(minval=-0.003, maxval=0.003)
        output0 = k.layers.Dense(1, kernel_initializer=kernel_initializer)(x)

        return [input0, input1], output0

    def fit(self, state_input, y):
        model_loss = k.losses.MeanSquaredError()
        with tf.GradientTape() as g:
            y_pred = self.predict(state_input)
            loss = model_loss(y, y_pred)

        gradients = g.gradient(loss, self.model.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))

        return loss

    def predict(self, state_input, training=False):
        return self.model(state_input, training=training)

    def predict_target(self, state_input, training=False):
        return self.target_model(state_input, training=training)

    def update_target(self):
        weights = self.model.weights
        target_weights = self.target_model.weights

        for layer_w, target_layer_w in zip(weights, target_weights):
            # weight update is implicit
            target_layer_w.assign(self.tau * layer_w + (1 - self.tau) * target_layer_w)

@tf.function
def ddpg_update(samples, actor, critic, GAMMA):
    s0, a, r, t, s1 = samples
    if len(a.shape) != 2:
        a = tf.reshape(a, (-1, 1))

    # get q and target ys
    target_q = critic.predict_target([s1, actor.predict_target(s1)])
    t = tf.reshape(1.0 - tf.cast(t, tf.float32), (-1, 1))
    r = tf.cast(r, tf.float32)
    yi = r + t * GAMMA * target_q

    # fit critic & actor
    critic_loss = critic.fit([s0, a], yi)
    actor_loss = actor.fit(s0, critic)

    # update target networks
    critic.update_target()
    actor.update_target()

@tf.function
def get_rewards(y_pred_unlabel, y_pred_valid, losses_delta):
    # reshape tensors
    y_pred_unlabel = tf.reshape(y_pred_unlabel, (y_pred_unlabel.shape[0], 1, -1))
    y_pred_valid = tf.reshape(y_pred_valid, (1, y_pred_valid.shape[0], -1))

    # tile tensors to the same size
    y_pred_unlabel = tf.tile(y_pred_unlabel, (1, y_pred_valid.shape[1], 1))
    y_pred_valid = tf.tile(y_pred_valid, (y_pred_unlabel.shape[0], 1, 1))

    # compute inverse ED and normalize
    inverse_ED = 1 - tf.sqrt(tf.reduce_sum(tf.pow(y_pred_unlabel - y_pred_valid, 2), axis=-1))
    inverse_ED_normalized = inverse_ED / tf.reshape(tf.reduce_sum(inverse_ED, axis=1), (-1, 1))
    inverse_ED_normalized = tf.clip_by_value(inverse_ED_normalized, 0.0, 1.0)

    # prepare losses and get rewards
    losses_delta = tf.tile(tf.reshape(losses_delta, (1, -1)), (y_pred_unlabel.shape[0], 1))
    r = tf.reduce_sum(inverse_ED_normalized * losses_delta, axis=1)
    return r


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 16 08:58:59 2021

@author: -
"""

import requests
import lxml.etree as etree

if __name__ == '__main__':
    headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'}
    url='https://www.taobao.com'
    url1='https://bj.58.com/ershoufang/'
    res=requests.get(url=url1, headers=headers)
    res.encoding=res.apparent_encoding
    # with open('../video-word/url-tb.txt','w',encoding='utf-8') as fp:
    #     fp.write(res.text)

    tree=etree.HTML(res.text)

    print(res.text)


from PyQt5.QtWidgets import QApplication, QPushButton, QStackedWidget, QDialog, QVBoxLayout, QLabel
import sys
from PyQt5 import QtGui
from PyQt5.QtGui import QIcon, QFont

class StackWidget(QDialog):
    def __init__(self):
        super().__init__()
        self.title = "Pyqt5 StackdWidget"
        self.top = 500
        self.left = 500
        self.width = 600
        self.height = 200
        self.iconName = "transistor.jpg"

        self.setWindowIcon(QtGui.QIcon(self.iconName))
        self.setWindowTitle(self.title)
        self.setGeometry(self.left, self.top, self.width, self.height)

        self.StackedWidget()

    def StackedWidget(self):
        vbox = QVBoxLayout()
        self.stackWidget = QStackedWidget()
        vbox.addWidget(self.stackWidget)

        for x in range (0,8):
            label = QLabel(f'Stacked child {str(x)}')
            label.setFont(QFont("Sanserif", 15))
            label.setStyleSheet("color:red")

            self.stackWidget.addWidget(label)

            self.button = QPushButton(f'Stack {str(x)}')
            self.button.setStyleSheet("background-color:green")
            self.button.page = x

            self.button.clicked.connect(self.btn_clicked)

            vbox.addWidget(self.button)

        self.setLayout(vbox)

    def btn_clicked(self):
        self.button = self.sender()
        self.stackWidget.setCurrentIndex(self.button.page - 1)

if __name__ == "__main__":
    App = QApplication(sys.argv)
    stackwidget = StackWidget()
    stackwidget.show()
    sys.exit(App.exec())


# -*- coding: utf-8 -*-
# Author: Lei
import random
import pysam
from collections import Counter
import click

@click.command(context_settings=dict(help_option_names=['-h', '--help'], show_default=True))
@click.argument('genome_bam_path', type=click.STRING)
@click.argument('trans_bam_1_path', type=click.STRING)
@click.argument('trans_bam_2_path', type=click.STRING)

@click.option('-p', '--prefix', default='split_tmp', type=click.STRING,
              help='output prefix')
def bam_corresponding(genome_bam_path, trans_bam_1_path, trans_bam_2_path, prefix = 'split_tmp'):
    """
    Random split a bam file into two bam files based on read names.

    The two output files have the same number of reads.
    All the alignment of a single read will be put into the same file.
    """
    genome_file = pysam.AlignmentFile(genome_bam_path, 'rb')
    bam_1 = pysam.AlignmentFile(trans_bam_1_path, 'rb')
    bam_2 = pysam.AlignmentFile(trans_bam_2_path, 'rb')

    out1 = f'{prefix}.1.bam'
    out2 = f'{prefix}.2.bam'
    out0 = f'{prefix}.0.bam'

    outfile1 = pysam.AlignmentFile(out1, 'wb', template = genome_file)
    outfile2 = pysam.AlignmentFile(out2, 'wb', template = genome_file)
    outfile0 = pysam.AlignmentFile(out0, 'wb', template = genome_file)

    name = Counter()

    for line in genome_file:
        name[line.query_name] = 0
    for line in bam_1:
        name[line.query_name] = 1
    for line in bam_2:
        name[line.query_name] = 2

    genome_file = pysam.AlignmentFile(genome_bam_path, 'rb')
    for line in genome_file:
        if name[line.query_name] == 1:
            outfile1.write(line)
        if name[line.query_name] == 2:
            outfile2.write(line)
        if name[line.query_name] == 0:
            outfile0.write(line)

    genome_file.close()
    outfile1.close()
    outfile2.close()
    outfile0.close()

    return

if __name__ == '__main__':
    bam_corresponding()


import numpy as np
import pandas as pd
from time import time
from sklearn import metrics

class KMeans:
    def predict(self, data, centroids):
        data = np.array(data)
        distances = np.sqrt(((data - centroids[:, np.newaxis]) ** 2).sum(axis=2))
        clusters = np.argmin(distances, axis=0)
        inertia = np.sum(distances)
        return clusters, inertia

    def cluster(self, data, k, iters=10):
        data = np.array(data)
        copied_data = data.copy()
        np.random.shuffle(copied_data)
        centroids = copied_data[:k]
        for i in range(iters):
            # closest centroids to data points
            clusters, _ = self.predict(data, centroids)
            # new centroids
            new_centroids = np.array([data[clusters == k].mean(axis=0) for k in range(centroids.shape[0])])
            centroids = new_centroids
        return centroids, clusters

def read_data(path):
    df = pd.read_csv(path, header=None, sep='\t')
    data = df.iloc[:, :-1]
    labels = df.iloc[:, -1]
    return data, labels

def summary(data, labels, preds, time, inertia):
    n_samples, n_features = data.shape

    print("n_samples %d, \t n_features %d"
          % (n_samples, n_features))

    print(82 * '_')
    print('init\t\ttime\tinertia\thomo\tcompl\tv-meas\tARI\tAMI\tsilhouette')

    def bench_k_means(time, inertia, preds, data):

        print('%-9s\t%.2fs\t%i\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f'
              % ("k-means", time, inertia,
                 metrics.homogeneity_score(labels, preds),
                 metrics.completeness_score(labels, preds),
                 metrics.v_measure_score(labels, preds),
                 metrics.adjusted_rand_score(labels, preds),
                 metrics.adjusted_mutual_info_score(labels, preds),
                 metrics.silhouette_score(data, preds,
                                          metric='euclidean',
                                          sample_size=100)))

    bench_k_means(time, inertia, preds, data)

def call_kmeans(train_data, test_data, test_labels):
    km = KMeans()
    t0 = time()
    centroids, clusters = km.cluster(train_data, 24)
    time_elapsed = time() - t0
    clusters, inertia = km.predict(test_data, centroids)
    summary(test_data, test_labels, clusters, time_elapsed, inertia)

if __name__ == "__main__":
    path_test = "pamap_easy.test.txt"
    path_train = "pamap_easy.train.txt"

    test_data, test_labels = read_data(path_test)
    train_data, train_labels = read_data(path_train)

    call_kmeans(train_data, test_data, test_labels)


import sublime, sublime_plugin

def fixed_set_layout(window, layout):
	#A bug was introduced in Sublime Text 3, sometime before 3053, in that it
	#changes the active group to 0 when the layout is changed. Annoying.
	active_group = window.active_group()
	window.set_layout(layout)
	num_groups = len(layout['cells'])
	window.focus_group(min(active_group, num_groups-1))

class NewWindowWithAlloyFilesCommand(sublime_plugin.WindowCommand):
	def run(self):
		layout = {}
		layout['cells'] = [[0.0,0.0,1.0,1.0],[1.0,0.0,2.0,2.0],[0.0,1.0,1.0,2.0]]
		layout['cols'] = [0.0,0.774979105688,1.0]
		layout['rows'] = [0.0,0.69627419099,1.0]
		fname = sublime.active_window().active_view().file_name()
		sublime.active_window().run_command("new_window")
		new_window = sublime.active_window()
		fixed_set_layout(new_window, layout)
		new_window.focus_group(2)
		new_window.open_file(fname.replace("/controllers/", "/views/").replace(".js", ".xml"))
		new_window.focus_group(1)
		new_window.open_file(fname.replace("/controllers/", "/styles/").replace(".js", ".tss"))
		new_window.focus_group(0)
		new_window.open_file(fname)


"""
由于不知道迭代器和生成器的长度，所以无法执行普通的切片操作
使用itertools.islice()函数
"""
import itertools

def count(n):
    x = 1
    while x <= n:
        yield x
        x += 1

cd = count(10)

for x in itertools.islice(cd, 5, 7):
    print(x, end=' ') # 6 7

"""
需要注意的是islice()产生的是一个迭代器，但是它丢弃了索引之前的元素
索引之后的元素还会继续输出
"""

for x in cd:
    print(x, end=' ') # 8 9 10

迭代器和生成器/迭代器_切片操作_P123.py


"""
Simple formatting on strings. Further string formatting code is in trans.py.
"""

import re
import sys
from functools import lru_cache
from typing import Final, List, Match, Pattern

from black._width_table import WIDTH_TABLE
from blib2to3.pytree import Leaf

STRING_PREFIX_CHARS: Final = "furbFURB"  # All possible string prefix characters.
STRING_PREFIX_RE: Final = re.compile(
    r"^([" + STRING_PREFIX_CHARS + r"]*)(.*)$", re.DOTALL
)
FIRST_NON_WHITESPACE_RE: Final = re.compile(r"\s*\t+\s*(\S)")
UNICODE_ESCAPE_RE: Final = re.compile(
    r"(?P<backslashes>\\+)(?P<body>"
    r"(u(?P<u>[a-fA-F0-9]{4}))"  # Character with 16-bit hex value xxxx
    r"|(U(?P<U>[a-fA-F0-9]{8}))"  # Character with 32-bit hex value xxxxxxxx
    r"|(x(?P<x>[a-fA-F0-9]{2}))"  # Character with hex value hh
    r"|(N\{(?P<N>[a-zA-Z0-9 \-]{2,})\})"  # Character named name in the Unicode database
    r")",
    re.VERBOSE,
)

def sub_twice(regex: Pattern[str], replacement: str, original: str) -> str:
    """Replace `regex` with `replacement` twice on `original`.

    This is used by string normalization to perform replaces on
    overlapping matches.
    """
    return regex.sub(replacement, regex.sub(replacement, original))

def has_triple_quotes(string: str) -> bool:
    """
    Returns:
        True iff @string starts with three quotation characters.
    """
    raw_string = string.lstrip(STRING_PREFIX_CHARS)
    return raw_string[:3] in {'"""', "'''"}

def lines_with_leading_tabs_expanded(s: str) -> List[str]:
    """
    Splits string into lines and expands only leading tabs (following the normal
    Python rules)
    """
    lines = []
    for line in s.splitlines():
        # Find the index of the first non-whitespace character after a string of
        # whitespace that includes at least one tab
        match = FIRST_NON_WHITESPACE_RE.match(line)
        if match:
            first_non_whitespace_idx = match.start(1)

            lines.append(
                line[:first_non_whitespace_idx].expandtabs()
                + line[first_non_whitespace_idx:]
            )
        else:
            lines.append(line)
    return lines

def fix_docstring(docstring: str, prefix: str) -> str:
    # https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation
    if not docstring:
        return ""
    lines = lines_with_leading_tabs_expanded(docstring)
    # Determine minimum indentation (first line doesn't count):
    indent = sys.maxsize
    for line in lines[1:]:
        stripped = line.lstrip()
        if stripped:
            indent = min(indent, len(line) - len(stripped))
    # Remove indentation (first line is special):
    trimmed = [lines[0].strip()]
    if indent < sys.maxsize:
        last_line_idx = len(lines) - 2
        for i, line in enumerate(lines[1:]):
            stripped_line = line[indent:].rstrip()
            if stripped_line or i == last_line_idx:
                trimmed.append(prefix + stripped_line)
            else:
                trimmed.append("")
    return "\n".join(trimmed)

def get_string_prefix(string: str) -> str:
    """
    Pre-conditions:
        * assert_is_leaf_string(@string)

    Returns:
        @string's prefix (e.g. '', 'r', 'f', or 'rf').
    """
    assert_is_leaf_string(string)

    prefix = ""
    prefix_idx = 0
    while string[prefix_idx] in STRING_PREFIX_CHARS:
        prefix += string[prefix_idx]
        prefix_idx += 1

    return prefix

def assert_is_leaf_string(string: str) -> None:
    """
    Checks the pre-condition that @string has the format that you would expect
    of `leaf.value` where `leaf` is some Leaf such that `leaf.type ==
    token.STRING`. A more precise description of the pre-conditions that are
    checked are listed below.

    Pre-conditions:
        * @string starts with either ', ", <prefix>', or <prefix>" where
        `set(<prefix>)` is some subset of `set(STRING_PREFIX_CHARS)`.
        * @string ends with a quote character (' or ").

    Raises:
        AssertionError(...) if the pre-conditions listed above are not
        satisfied.
    """
    dquote_idx = string.find('"')
    squote_idx = string.find("'")
    if -1 in [dquote_idx, squote_idx]:
        quote_idx = max(dquote_idx, squote_idx)
    else:
        quote_idx = min(squote_idx, dquote_idx)

    assert (
        0 <= quote_idx < len(string) - 1
    ), f"{string!r} is missing a starting quote character (' or \")."
    assert string[-1] in (
        "'",
        '"',
    ), f"{string!r} is missing an ending quote character (' or \")."
    assert set(string[:quote_idx]).issubset(
        set(STRING_PREFIX_CHARS)
    ), f"{set(string[:quote_idx])} is NOT a subset of {set(STRING_PREFIX_CHARS)}."

def normalize_string_prefix(s: str) -> str:
    """Make all string prefixes lowercase."""
    match = STRING_PREFIX_RE.match(s)
    assert match is not None, f"failed to match string {s!r}"
    orig_prefix = match.group(1)
    new_prefix = (
        orig_prefix.replace("F", "f")
        .replace("B", "b")
        .replace("U", "")
        .replace("u", "")
    )

    # Python syntax guarantees max 2 prefixes and that one of them is "r"
    if len(new_prefix) == 2 and "r" != new_prefix[0].lower():
        new_prefix = new_prefix[::-1]
    return f"{new_prefix}{match.group(2)}"

# Re(gex) does actually cache patterns internally but this still improves
# performance on a long list literal of strings by 5-9% since lru_cache's
# caching overhead is much lower.
@lru_cache(maxsize=64)
def _cached_compile(pattern: str) -> Pattern[str]:
    return re.compile(pattern)

def normalize_string_quotes(s: str) -> str:
    """Prefer double quotes but only if it doesn't cause more escaping.

    Adds or removes backslashes as appropriate. Doesn't parse and fix
    strings nested in f-strings.
    """
    value = s.lstrip(STRING_PREFIX_CHARS)
    if value[:3] == '"""':
        return s

    elif value[:3] == "'''":
        orig_quote = "'''"
        new_quote = '"""'
    elif value[0] == '"':
        orig_quote = '"'
        new_quote = "'"
    else:
        orig_quote = "'"
        new_quote = '"'
    first_quote_pos = s.find(orig_quote)
    if first_quote_pos == -1:
        return s  # There's an internal error

    prefix = s[:first_quote_pos]
    unescaped_new_quote = _cached_compile(rf"(([^\\]|^)(\\\\)*){new_quote}")
    escaped_new_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){new_quote}")
    escaped_orig_quote = _cached_compile(rf"([^\\]|^)\\((?:\\\\)*){orig_quote}")
    body = s[first_quote_pos + len(orig_quote) : -len(orig_quote)]
    if "r" in prefix.casefold():
        if unescaped_new_quote.search(body):
            # There's at least one unescaped new_quote in this raw string
            # so converting is impossible
            return s

        # Do not introduce or remove backslashes in raw strings
        new_body = body
    else:
        # remove unnecessary escapes
        new_body = sub_twice(escaped_new_quote, rf"\1\2{new_quote}", body)
        if body != new_body:
            # Consider the string without unnecessary escapes as the original
            body = new_body
            s = f"{prefix}{orig_quote}{body}{orig_quote}"
        new_body = sub_twice(escaped_orig_quote, rf"\1\2{orig_quote}", new_body)
        new_body = sub_twice(unescaped_new_quote, rf"\1\\{new_quote}", new_body)
    if "f" in prefix.casefold():
        matches = re.findall(
            r"""
            (?:(?<!\{)|^)\{  # start of the string or a non-{ followed by a single {
                ([^{].*?)  # contents of the brackets except if begins with {{
            \}(?:(?!\})|$)  # A } followed by end of the string or a non-}
            """,
            new_body,
            re.VERBOSE,
        )
        for m in matches:
            if "\\" in str(m):
                # Do not introduce backslashes in interpolated expressions
                return s

    if new_quote == '"""' and new_body[-1:] == '"':
        # edge case:
        new_body = new_body[:-1] + '\\"'
    orig_escape_count = body.count("\\")
    new_escape_count = new_body.count("\\")
    if new_escape_count > orig_escape_count:
        return s  # Do not introduce more escaping

    if new_escape_count == orig_escape_count and orig_quote == '"':
        return s  # Prefer double quotes

    return f"{prefix}{new_quote}{new_body}{new_quote}"

def normalize_unicode_escape_sequences(leaf: Leaf) -> None:
    """Replace hex codes in Unicode escape sequences with lowercase representation."""
    text = leaf.value
    prefix = get_string_prefix(text)
    if "r" in prefix.lower():
        return

    def replace(m: Match[str]) -> str:
        groups = m.groupdict()
        back_slashes = groups["backslashes"]

        if len(back_slashes) % 2 == 0:
            return back_slashes + groups["body"]

        if groups["u"]:
            # \u
            return back_slashes + "u" + groups["u"].lower()
        elif groups["U"]:
            # \U
            return back_slashes + "U" + groups["U"].lower()
        elif groups["x"]:
            # \x
            return back_slashes + "x" + groups["x"].lower()
        else:
            assert groups["N"], f"Unexpected match: {m}"
            # \N{}
            return back_slashes + "N{" + groups["N"].upper() + "}"

    leaf.value = re.sub(UNICODE_ESCAPE_RE, replace, text)

@lru_cache(maxsize=4096)
def char_width(char: str) -> int:
    """Return the width of a single character as it would be displayed in a
    terminal or editor (which respects Unicode East Asian Width).

    Full width characters are counted as 2, while half width characters are
    counted as 1.  Also control characters are counted as 0.
    """
    table = WIDTH_TABLE
    codepoint = ord(char)
    highest = len(table) - 1
    lowest = 0
    idx = highest // 2
    while True:
        start_codepoint, end_codepoint, width = table[idx]
        if codepoint < start_codepoint:
            highest = idx - 1
        elif codepoint > end_codepoint:
            lowest = idx + 1
        else:
            return 0 if width < 0 else width
        if highest < lowest:
            break
        idx = (highest + lowest) // 2
    return 1

def str_width(line_str: str) -> int:
    """Return the width of `line_str` as it would be displayed in a terminal
    or editor (which respects Unicode East Asian Width).

    You could utilize this function to determine, for example, if a string
    is too wide to display in a terminal or editor.
    """
    if line_str.isascii():
        # Fast path for a line consisting of only ASCII characters
        return len(line_str)
    return sum(map(char_width, line_str))

def count_chars_in_width(line_str: str, max_width: int) -> int:
    """Count the number of characters in `line_str` that would fit in a
    terminal or editor of `max_width` (which respects Unicode East Asian
    Width).
    """
    total_width = 0
    for i, char in enumerate(line_str):
        width = char_width(char)
        if width + total_width > max_width:
            return i
        total_width += width
    return len(line_str)


import os
import matplotlib.pyplot as plt
import mpld3
from document_clustering.visualization.toptoolbar import TopToolbar

class Plotter:
    colors = [(0, '#1b9e77'), (1, '#d95f02'), (2, '#7570b3'), (3, '#e7298a'), (4, '#ffb400'), (5, '#ffff00'), (6, '#000000')]
    names = [(0, 'A'),(1, 'B'),(2, 'C'),(3, 'D'),(4, 'E'),(5, 'F'),(6, 'G')]
    cluster_colors = {}
    cluster_names = {}
    output_dir = ''

    def __init__(self, output_dir):
        self.output_dir = output_dir

    def set_palette(self, k):
        self.cluster_colors = dict(self.colors[:k])
        self.cluster_names = dict(self.names[:k])

    def plot_png(self, data_frame):
        # #group by cluster
        groups = data_frame.groupby('label')

        # set up plot
        fig, ax = plt.subplots(figsize=(17, 9)) # set size
        ax.margins(0.05) # Optional, just adds 5% padding to the autoscaling

        #iterate through groups to layer the plot
        #note that I use the cluster_name and cluster_color dicts with the 'name' lookup to return the appropriate color/label
        for name, group in groups:
            ax.plot(group.x, group.y, marker='o', linestyle='', ms=12,
                    label=self.cluster_names[name], color=self.cluster_colors[name],
                    mec='none')
            ax.set_aspect('auto')
            ax.tick_params(\
                axis= 'x',          # changes apply to the x-axis
                which='both',      # both major and minor ticks are affected
                bottom='off',      # ticks along the bottom edge are off
                top='off',         # ticks along the top edge are off
                labelbottom='off')
            ax.tick_params(\
                axis= 'y',         # changes apply to the y-axis
                which='both',      # both major and minor ticks are affected
                left='off',      # ticks along the bottom edge are off
                top='off',         # ticks along the top edge are off
                labelleft='off')

        ax.legend(numpoints=1)  #show legend with only 1 point

        #add label in x,y position with the label as the film title
        for i in range(len(data_frame)):
            ax.text(data_frame.ix[i]['x'], data_frame.ix[i]['y'], data_frame.ix[i]['name'], size=8)

        # plt.savefig(self.dir_name + 'clusters_small_noaxes.png', dpi=200)
        plt.show() #show the plot
        plt.close()

    def plot_html(self, data_frame):
        #group by cluster
        groups = data_frame.groupby('label')

        #define custom css to format the font and to remove the axis labeling
        css = """
        text.mpld3-text, div.mpld3-tooltip {
          font-family:Arial, Helvetica, sans-serif;
        }

        g.mpld3-xaxis, g.mpld3-yaxis {
        display: none; }

        svg.mpld3-figure {
        margin-left: -10px;}
        """

        # Plot 14 6
        fig, ax = plt.subplots(figsize=(10,6)) #set plot size
        ax.margins(0.03) # Optional, just adds 5% padding to the autoscaling

        #iterate through groups to layer the plot
        #note that I use the cluster_name and cluster_color dicts with the 'name' lookup to return the appropriate color/label
        for name, group in groups:
            points = ax.plot(group.x, group.y, marker='o', linestyle='', ms=12,
                             label=self.cluster_names[name], mec='none',
                             color=self.cluster_colors[name])
            ax.set_aspect('auto')
            labels = [i for i in group.name]

            #set tooltip using points, labels and the already defined 'css'
            tooltip = mpld3.plugins.PointHTMLTooltip(points[0], labels, voffset=10, hoffset=10, css=css)
            #connect tooltip to fig
            mpld3.plugins.connect(fig, tooltip, TopToolbar())

            #set tick marks as blank
            ax.axes.get_xaxis().set_ticks([])
            ax.axes.get_yaxis().set_ticks([])

            #set axis as blank
            ax.axes.get_xaxis().set_visible(False)
            ax.axes.get_yaxis().set_visible(False)

        ax.legend(numpoints=1) #show legend with only one dot

        html = mpld3.fig_to_html(fig)

        out_file = os.path.join(self.output_dir, 'index.html')

        with open(out_file, 'w') as file_:
            file_.write(html)

        plt.close()

    def plot_dendrogram(self, dendrogram):

        fig, ax = plt.subplots(figsize=(18, 24)) # set size 15, 20
        ax = dendrogram;

        plt.tick_params(\
            axis= 'x',          # changes apply to the x-axis
            which='both',      # both major and minor ticks are affected
            bottom='off',      # ticks along the bottom edge are off
            top='off',         # ticks along the top edge are off
            labelbottom='off')

        #uncomment below to save figure
        # plt.savefig(self.dir_name + 'ward_clusters.png', dpi=200) #save figure as ward_clusters

        plt.show()
        plt.close()


"""Contains the preferences class and load_preferences funciton."""

from typing import Dict, Tuple, Optional, Union, TYPE_CHECKING

from dill import load, dump
from discord import Member

if TYPE_CHECKING:
    from lib.logic.Player import Player

class Preferences:
    """Stores a member's global preferences.

    Parameters
    ----------
    member : Union[Player, Member]
        The person whose preferences are being stored.

    Attributes
    ----------
    aliases : Dict[str, str]
        Command aliases: [alias, command].
    nick : str
        The bot nickname.
    pronouns : Tuple[str, str, str, str, str, bool]
        A list of pronouns.
    emergency_vote : Tuple[int, Optional[int]]
        A global emergency vote.
    specific_emergencys : Dict[int, Tuple[bool, int]]
        A list of bot-specific emergency votes.
    id : int
        The member's discord id.
    """

    aliases: Dict[str, str]
    emergency_vote: Tuple[int, Optional[int]]
    specific_emergencys: Dict[int, Tuple[bool, int]]
    nick: str

    def __init__(self, member: Union["Player", Member]):
        self.aliases = {}
        self.nick = member.display_name
        self.pronouns = (
            "they",
            "them",
            "their",
            "theirs",
            "themselves",
            True,
        )
        self.emergency_vote = (0, None)
        self.specific_emergencys = {}
        self.id = member.id

    def save_preferences(self):
        """Save preferences."""
        with open("resources/preferences/" + str(self.id) + ".pckl", "wb") as file:
            dump(self, file)

    def get_emergency_vote(self, bot_id: int) -> Tuple[int, Optional[int]]:
        """Generate the (potentially bot-specific) emergency vote.

        Parameters
        ----------
        bot_id : int
            The bot's ID number.

        Returns
        -------
        Tuple[int, Optional[int]]
            The default vote.
        """
        try:
            return self.specific_emergencys[bot_id]
        except KeyError:
            return self.emergency_vote

def load_preferences(member: Union["Player", Member]) -> Preferences:
    """Load a member's preferences.

    Parameters
    ----------
    member : Union[Player, Member]
        The preferences to load.

    Returns
    -------
    Preferences
        The member's preferences.
    """
    try:
        with open("resources/preferences/" + str(member.id) + ".pckl", "rb") as file:
            return load(file)
    except FileNotFoundError:
        return Preferences(member)


import time
import argparse
import json
import os
from typing import List, Dict
from azure.ai.ml.identity import AzureMLOnBehalfOfCredential
from azure.identity import DefaultAzureCredential
from archai.common.store import ArchaiStore
from azure.ai.ml import MLClient

class JobCompletionMonitor:
    """ This helper class uses the ArchaiStore to monitor the status of some long running
    training operations and the status of the Azure ML pipeline those jobs are running in
    and waits for them to finish (either successfully or with a failure)"""
    def __init__(self, store : ArchaiStore, ml_client : MLClient, metric_keys: List[str], pipeline_id=None, timeout=3600, throw_on_failure_rate=0.1):
        """
        Initialize a JobCompletionMonitor instance.
        :param store: an instance of ArchaiStore to monitor the status of some long running training operations
        :param ml_client: an instance of MLClient to check the status of the Azure ML pipeline those jobs are running in
        :param metric_keys: a list of column names to monitor and return from the Azure table.
        :param pipeline_id: (optional) the ID of the Azure ML pipeline to monitor, if not provided we can get this from the ArchaiStore.
        :param timeout: (optional) the timeout in seconds
        :param throw_on_failure_rate: (optional) what ratio of jobs failed (between 0 and 1) should result in raising an an exception.
        Zero means throw exception on any failure.
        This is handy if you want to allow the search to continue even when a small percentage of jobs fails.
        Default is 0.1, or 10% or more of jobs failed will raise an exception.
        """
        self.store = store
        self.ml_client = ml_client
        self.timeout = timeout
        self.pipeline_id = pipeline_id
        self.metric_keys = metric_keys
        self.throw_on_failure_rate = throw_on_failure_rate

    def _check_entity_status(self, waiting, completed):
        failed = 0
        for i in range(len(waiting) - 1, -1, -1):
            id = waiting[i]
            e = self.store.get_status(id)
            if self.pipeline_id is None and 'pipeline_id' in e:
                self.pipeline_id = e['pipeline_id']
            if e is not None and 'status' in e and (e['status'] == 'complete' or e['status'] == 'failed'):
                del waiting[i]
                completed[id] = e
                if e['status'] == 'failed':
                    error = e['error']
                    print(f'Training job {id} failed with error: {error}')
                    failed += 1
                else:
                    if len(self.metric_keys) > 0 and self.metric_keys[0] in e:
                        key = self.metric_keys[0]
                        metric = e[key]
                        print(f'Training job {id} completed with {key} = {metric}')
                    else:
                        print(f'Training job {id} completed')
        return failed

    def _get_model_results(self, model_ids, completed):
        # stitch together the models.json file from our status table.
        print('Top model results: ')
        models = []
        interesting_columns = self.metric_keys + ['status', 'error', 'epochs']
        for id in model_ids:
            row = {'id': id}
            e = completed[id] if id in completed else {}
            for key in interesting_columns:
                if key in e:
                    row[key] = e[key]
            models += [row]

        return {
            'models': models
        }

    def _cancel_waiting_list(self, waiting, pipeline_status):
        # cancel any remaining jobs in the waiting list by marking an error status on the entity
        for i in range(len(waiting) - 1, -1, -1):
            id = waiting[i]
            del waiting[i]
            e = self.store.get_status(id)
            if 'error' not in e:
                e['error'] = f'Pipeline {pipeline_status}'
            if 'status' not in e or e['status'] != 'complete':
                e['status'] = pipeline_status.lower()
            self.store.merge_status_entity(e)

    def _get_pipeline_status(self):
        # try and get the status of the Azure ML pipeline, it returns strings like
        # 'Completed', 'Failed', 'Running', 'Preparing', 'Canceled' and so on.
        try:
            if self.pipeline_id is not None:
                train_job = self.ml_client.jobs.get(self.pipeline_id)
                if train_job is not None:
                    return train_job.status
        except Exception as e:
            print(f'Error getting pipeline status for pipeline {self.pipeline_id}: {e}')

    def wait(self, model_ids: List[str]) -> List[Dict[str, str]]:
        """
        Wait for all the training jobs to finish and return a list of dictionaries
        containing details about each model, including their training validation accuracies.
        :param model_ids: a list of training job IDs
        :return: a list of dictionaries containing details about each model
        """
        completed = {}
        waiting = list(model_ids)
        start = time.time()
        failed = 0

        while len(waiting) > 0:
            failed += self._check_entity_status(waiting, completed)
            if len(waiting) == 0:
                break

            # check the overall pipeline status just in case training jobs failed to even start.
            pipeline_status = self._get_pipeline_status()
            if pipeline_status is not None:
                if pipeline_status == 'Completed':
                    # ok, all jobs are done, which means if we still have waiting tasks then they failed to
                    # even start.
                    self._cancel_waiting_list(waiting, 'failed to start')
                elif pipeline_status == 'Failed' or pipeline_status == 'Canceled':
                    self._cancel_waiting_list(waiting, pipeline_status)

            if len(waiting) > 0:
                if time.time() > self.timeout + start:
                    break
                print("AmlTrainingValAccuracy: Waiting 20 seconds for partial training to complete...")
                time.sleep(20)

        # awesome - they all completed!
        if len(completed) == 0:
            if time.time() > self.timeout + start:
                raise Exception(f'Partial Training Pipeline timed out after {self.timeout} seconds')
            else:
                raise Exception('Partial Training Pipeline failed to start')

        failure_rate = float(failed) / float(len(model_ids))
        if failure_rate > self.throw_on_failure_rate:
            raise Exception(f'Partial Training Pipeline failure rate {failure_rate} exceeds allowed threshold of {self.throw_on_failure_rate}')

        results = self._get_model_results(model_ids, completed)

        timespan = time.strftime('%H:%M:%S', time.gmtime(time.time() - start))
        print(f'Training: Distributed training completed in {timespan} ')
        print(f'Training: returning {len(results)} results:')
        print(json.dumps(results, indent=2))
        return results

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', help='bin hexed config json info for MLClient')
    parser.add_argument('--timeout', type=int, help='pipeline timeout in seconds (default 1 hour)', default=3600)
    parser.add_argument('--model_path', required=True, help='mounted path containing the pending.json file')
    parser.add_argument('--output', required=True, help='folder to write the results to')
    parser.add_argument('--metrics', type=str, help='metrics to return from the azure table')

    args = parser.parse_args()
    output = args.output
    timeout = args.timeout
    model_path = args.model_path
    metrics = []
    if args.metrics:
        metrics = [x.strip() for x in args.metrics.split(',')]

    print(f"Monitor running with model_path={model_path}")
    if not os.path.isdir(model_path):
        raise Exception("### directory not found")

    models_file = os.path.join(model_path, 'pending.json')
    if not os.path.isfile(models_file):
        raise Exception("### 'pending.json' not found in --model_path")

    models = json.load(open(models_file))
    model_ids = [m['id'] for m in models['models']]

    identity = AzureMLOnBehalfOfCredential()
    if args.config:
        print("Using AzureMLOnBehalfOfCredential...")
        workspace_config = str(bytes.fromhex(args.config), encoding='utf-8')
        print(f"Config: {workspace_config}")
        config = json.loads(workspace_config)
    else:
        print("Using DefaultAzureCredential...")
        config_file = "../.azureml/config.json"
        print(f"Config: {config_file}")
        config = json.load(open(config_file, 'r'))
        identity = DefaultAzureCredential()

    subscription = config['subscription_id']
    resource_group = config['resource_group']
    workspace_name = config['workspace_name']
    storage_account_key = config['storage_account_key']
    storage_account_name = config['storage_account_name']

    ml_client = MLClient(
        identity,
        subscription,
        resource_group,
        workspace_name
    )

    store = ArchaiStore(storage_account_name, storage_account_key)

    monitor = JobCompletionMonitor(store, ml_client, metrics, timeout=timeout)
    results = monitor.wait(model_ids)
    if output is not None:
        # save the results with updated validation accuracies to models.json
        with open(os.path.join(output, 'models.json'), 'w') as f:
            f.write(json.dumps(results, indent=2))

if __name__ == "__main__":
    main()


from django import forms
from .models import Audit
from django.contrib.auth.models import User
from django.contrib.auth.forms import UserCreationForm

class AuditForm(forms.ModelForm):
    class Meta:
        model = Audit
        fields = ['date_added', 'audit_zone', 'type_of_risk', 'level_of_risk']
        labels = {
            'date_added': 'Дата аудита',
            'audit_zone': 'Зона аудита',
            'type_of_risk': 'Оцениваемый риск',
            'level_of_risk': 'Уровень риска',
        }

class SignUpForm(UserCreationForm):
    required_css_class = "field"
    error_css_class = "error"

    class Meta(UserCreationForm.Meta):
        model = User
        fields = UserCreationForm.Meta.fields + ('first_name', 'last_name')
        labels = {
            'username': 'Логин',
        }


#Problem Statement

"""Predicting the salary for a job position is crucial for a business' HR & talent function for optimizing compensation strategy and talent retention in a highly competetive labour market. The aim of this project is to build a salary prediction model for existing and future job seekers by examining an existing dataset of job postings.

The analysis is aimed at explaining every step of the process from defining the problem, discovering dataset, developing model and deploying into production. The model applies data transformation and machine learning on features such as work experience, Job Type, Majors, Industry Type, Degree and Miles from metropolis. The final aim is to predict salary for a job posting based on these available features.

The dataset includes available features or labelled columns for analysis as follows:

Job ID/jobId : Given Job ID for the role
Company ID : Company ID for the respective Job ID advertised
Degree : Applicant's qualification/degree
Major : Degree Specialization
Industry : Job ID's categorized industry such as Oil, Auto, Health, Finance etc.
Experience (Years) : Requried Experience for the role
Miles from Metropolis : Distance of the job location in miles from the nearest metropolitan city
Salary : In x1000 dollars of the respective Job ID
"""

# Commented out IPython magic to ensure Python compatibility.
import pandas as pd
import sklearn as sk
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
# %matplotlib inline

from scipy.stats import norm
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LinearRegression

#loading data into a pandas dataframe

from google.colab import drive
drive.mount('/content/drive')

url1 = '/content/drive/My Drive/Demo/train_dataset.csv'
train_dataset_df = pd.read_csv(url1)

url2 = '/content/drive/My Drive/Demo/test_dataset.csv'
test_dataset_df = pd.read_csv(url2)

url3 = '/content/drive/My Drive/Demo/train_salaries.csv'
train_salaries_df = pd.read_csv(url3)

#Renaming columns for ease of exploration

train_dataset_df.rename(columns={'jobId':'Job ID', 'companyId':'Company ID',
                                  'jobType':'Job Type','degree':'Degree','major':'Major',
                                  'industry':'Industry','yearsExperience':'Experience (Years)',
                                 'milesFromMetropolis':'Miles from Metropolis'}, inplace = True)

test_dataset_df.rename(columns={'jobId':'Job ID', 'companyId':'Company ID',
                                  'jobType':'Job Type','degree':'Degree','major':'Major',
                                  'industry':'Industry','yearsExperience':'Experience (Years)',
                                 'milesFromMetropolis':'Miles from Metropolis'}, inplace = True)

train_salaries_df.rename(columns={'jobId':'Job ID','salary':'Salary'}, inplace = True)

train_dataset_df.info()

print(train_dataset_df.shape)
print(test_dataset_df.shape)
print(train_salaries_df.shape)

"""Both train and test datasets have the same lengths and data types"""

train_dataset_df.head()

test_dataset_df.info()

train_salaries_df.info()

train_salaries_df.head()

"""As discussed earlier, we've seen the feature training and target sets contain the same Job IDs except that the latter includes out target variable 'Salary'. We would like to see the data sorted by its Job ID all in one place instead of having to open two datasets."""

#Merging the training features and salaries (target) dataset along Job ID column
train_merged = pd.merge(train_dataset_df, train_salaries_df, on ='Job ID', how = 'inner')
train_merged.head()

#Checking if any Salary attributes have '0' value
len(train_merged[train_merged['Salary']==0])

#Displaying the rows that have '0' as their salary values
train_merged[train_merged['Salary']==0]

#Removing rows with Salary value 0

train_merged = train_merged[train_merged.Salary!= 0]

print(train_merged.shape)
train_merged.reset_index(drop = True).head()

#Checking for duplicate values

train_merged.duplicated().any()

#Visualizing Salary
f, ax = plt.subplots(1,2,figsize=(18,6))
sns.distplot(train_merged['Salary'], ax=ax[0], bins=40, kde=True, norm_hist=True)
ax[0].axvline(np.mean(train_merged['Salary']), color='black')
ax[0].axvline(np.median(train_merged['Salary']), color='darkblue', linestyle='--')
ax[0].set_title('Salary Histogram',fontsize = 14)
sns.boxplot(train_merged['Salary'], ax=ax[1], color='gold')
ax[1].set_title('Salary Boxplot', fontsize = 14)
f.suptitle('Distribution of Salary', fontsize = 18)

print("The Black line in Dist. Plot shows the mean at: ",round(train_merged['Salary'].mean()))
print("The Dotted line in the Dist. Plot shows the median at: ", round(train_merged['Salary'].median()))

#Getting rid of redundant variables from the dataset

train_merged = train_merged.drop('Job ID', axis = 1)

test_dataset_df = test_dataset_df.drop('Job ID', axis = 1)
test_dataset_df = test_dataset_df.drop('Company ID', axis = 1)

train_merged.head()

#One hot encode categorical data in the dataset

train_merged = pd.get_dummies(train_merged)
train_merged.head()

#Dividing data into attributes and labels

x = train_merged.drop('Salary', axis = 1)
#To avoid model overfitting, which happens if the target parameter 'Salary' is
#not removed from training data

y = train_merged['Salary']

#Performing the split (Training - 80%, Testing - 20%)

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.20, random_state=40)

print("Nmber of training samples:",x_train.shape[0])
print("Nmber of test samples :", x_test.shape[0])

#Creating Linear Regressor Object
lr = LinearRegression()

#Fitting model using x and y attributes
lr.fit(x_train, y_train)

#Making Prediction such that y_hat gives an array of Target Value (Salary)
y_hat = lr.predict(x_test)
print("The first 5 predictied salaries: ", y_hat[0:5])

#Having established a baseline model, we can predict Salaries in the test set

y_hat=lr.predict(x_test) #Predicting the training data
y_hat

#Evaluation with MSE
mse = mean_squared_error(y_test, y_hat)
print("Mean Squared Error is: ", mse)

print('The Accuracy of the model is: ', lr.score(x,y))