import logging
import json

from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.http import Http404, HttpResponseNotFound
from django.shortcuts import render
from django.views import View
from haystack.views import SearchView as _SearchView
# from django_redis.serializers import json
from youko_djT import settings
from . import constants
from utils.json_fun import to_json_data
from utils.res_code import Code,error_map
from . import models

# Create your views here.

logger = logging.getLogger('django')

class IndexView(View):
    """

    """
    def get(self,request):
        """

        :param request:
        :return:
        """
        tags = models.Tag.objects.only('id', 'name').filter(is_delete=False)
        hot_news = models.HotNews.objects.select_related('news').only('news__title', 'news__image_url',).filter(is_delete=False).order_by('priority', '-news__clicks')[0:constants.SHOW_HOTNEWS_COUNT]

        return render(request, 'news/index.html', locals())

'''
1、创建类视图
2、校验参数
3、从数据库中查询新闻列表数据
4、序列化数据
5、返回给前端
'''

# 1、创建类视图
class NewsListView(View):
    """
    creat news list view
    route:/news/
    # 1、创建类视图
    """

    def get(self,request):
        # 2、校验参数
        try:
            tag_id =int(request.GET.get('tag_id', 0))
        except Exception as e:
            logger.error('标签错误,\n{}'.format(e))
            tag_id = 0
        try:
            page =int(request.GET.get('page', 1))
        except Exception as e:
            # 若用户访问的页数大于实际页数，则返回最后一页数据
            logger.error("当前页数错误：\n{}".format(e))
            page = 1

        # 3、从数据库中查询新闻列表数据
        news_queryset = models.News.objects.select_related('tag','author'). \
            only('title', 'digest', 'image_url', 'update_time', 'tag__name', 'author__username')
        # title , digest, image_url,update__time,author__name,tag__name

        # if models.Tag.objects.only('id').filter(is_delete=False,id=tag_id).exists():
        #     news = news_queryset.filter(is_delete=False,tag_id=tag_id)
        # else:
        #     news = news_queryset.filter(is_delete=False)

        news = news_queryset.filter(is_delete=False, tag_id=tag_id)or \
            news_queryset.filter(is_delete=False)

        paginator = Paginator(news, constants.PER_PAGE_NEWS_COUNT)
        try:
            news_info = paginator.page(page)
        except EmptyPage:
            # 若用户访问的页数大于实际页数，则返回最后一页数据
            logging.info("用户访问的页数大于总页数。")
            news_info = paginator.page(paginator.num_pages)

        # 序列化输出
        news_info_list = []
        for n in news_info:
            news_info_list.append({
                'id': n.id,
                'title': n.title,
                'digest': n.digest,
                'image_url': n.image_url,
                'tag_name': n.tag.name,
                'author': n.author.username,
                'update_time': n.update_time.strftime('%Y年%m月%d日 %H:%M'),
            })

        # 创建返回前端的数据
        data = {
            'total_pages':paginator.num_pages,
            'news': news_info_list
        }

        return to_json_data(data=data)

'''
1、创建类视图
2、
'''
class NewsBanner(View):
    """
    creat news banners view
    route: /news/banners/
    """
    def get(self, request):
        banners = models.Banner.objects.select_related('news').only('image_url', 'news__id', 'news__title'). \
            filter(is_delete=False).order_by('priority','-update_time', '-id')[0:constants.SHOW_BANNER_COUNT]

        # 序列化输出
        banners_info_list = []
        for b in banners:
            banners_info_list.append({
                'image_url': b.image_url,
                'news_id': b.news.id,
                'news_title': b.news.title,
            })
            # 创建返回给前端的数据
        data = {
            'banners': banners_info_list
        }

        return to_json_data(data=data)

class SerchView(_SearchView):
    # 模板文件
    template = 'news/search.html'

    # 重写响应方式，如果请求参数q为空，返回模型News的热门新闻数据，否则根据参数q搜索相关数据
    def create_response(self):
        kw = self.request.GET.get('q', '')
        if not kw:
            show_all = True
            hot_news = models.HotNews.objects.select_related('news').only('news__title', 'news__image_url', 'news__id').filter(is_delete=False).order_by('priority', '-news__clicks')

            paginator = Paginator(hot_news, settings.HAYSTACK_SEARCH_RESULTS_PER_PAGE)
            try:
                page = paginator.page(int(self.request.GET.get('page', 1)))

            except PageNotAnInteger:
                # 如果参数page的数据类型不是整型，则返回第一页数据
                page = paginator.page(1)
            except EmptyPage:
                # 用户访问的页数大于实际页数，则返回最后一页的数据
                page = paginator.page(paginator.num_pages)
            return render(self.request, self.template, locals())
        else:
            show_all = False
            qs = super(SerchView, self).create_response()
            return qs

    # def get(self, request):
    #     """
    #
    #     :param request:
    #     :return:
    #     """
    #     return render(request, 'news/search.html')

class NewsDetailView(View):
    """

    """
    def get(self, request, news_id):

        news = models.News.objects.select_related('tag', 'author').only('title', 'content', 'update_time', 'tag__name', 'author__username').filter(is_delete=False, id=news_id).first()

        if news:
            comments = models.Comments.objects.select_related('author', 'parent').only('content', 'author__username', 'update_time', 'parent__author__username', 'parent__content').filter(is_delete=False, news_id=news_id)
            # 序列化输出
            comments_list = []
            for comm in comments:
                comments_list.append(comm.to_dict_data())
            return render(request,'news/news_detail.html', locals())
        else:
            raise Http404("<新闻{}>不存在".format(news_id))
            # return HttpResponseNotFound('
Page not found
')
            # return render(request, '404.html')

# 加一个装饰器或者继承一个拓展类,以判断用户是否登录情况
# method_decologin_required
# 或者class NewsCommentView(LoginRequiredMinxin, View):
class NewsCommentView(View):
    """
    create new comment view
    route: '/news/<int:news_id>/comments/'

    """
    def post(self, request, news_id):
        if not request.user.is_authenticated:
            return to_json_data(errno=Code.SESSIONERR, errmsg=error_map[Code.SESSIONERR])
        if not models.News.objects.only('id').filter(is_delete=False,id=news_id).exists():
            return to_json_data(errno=Code.PARAMERR, errmsg="新闻不存在！")
        try:
            json_data = request.body
            if not json_data:
                return to_json_data(errno=Code.PARAMERR, errmsg="参数为空，请重新输入")
            dict_data = json.loads(json_data.decode("utf8"))
        except Exception as e:
            logger.info("错误信息: \n{}".format(e))
            return to_json_data(errno=Code.UNKOWNERR, errmsg=error_map[Code.UNKOWNERR])

        content = dict_data.get('content')
        if not content:
            return to_json_data(errno=Code.PARAMERR, errmsg="评论内容为空！")

        parent_id = dict_data.get('parent_id')
        try:
            if parent_id:
                parent_id = int(parent_id)
                if not models.Comments.objects.only('id').filter(is_delete=False,id=parent_id,news_id=news_id).exists():
                    return to_json_data(errno=Code.PARAMERR, errmsg=error_map[Code.PARAMERR])

        except Exception as e:
            logger.info("前端传过来的parent_id异常：\n{}".format(e))
            return to_json_data(errno=Code.PARAMERR, errmsg="未知异常")

        # 保存到数据库
        news_comment = models.Comments()
        news_comment.content = content
        news_comment.news_id = news_id
        news_comment.author = request.user
        news_comment.parent_id = parent_id if parent_id else None
        news_comment.save()

        return to_json_data(data=news_comment.to_dict_data())

    # def post(self, request, news_id):
    #
    #
    #     if not request.user.is_authenticated:
    #         return to_json_data(errno=Code.SESSIONERR, errmsg=error_map[Code.SESSIONERR])
    #     if not models.News.objects.only('id').filter(is_delete=False, id=news_id).exists():
    #         return to_json_data(errno=Code.PARAMERR, errmsg="新闻不存在！")
    #
    #
    #
    #     # 从前端获取参数
    #     json_data = request.body
    #     if not json_data:
    #         return to_json_data(errno=Code.PARAMERR, errmsg=error_map[Code.PARAMERR])
    #
    #
    #     # 将json转化为dict
    #     dict_data = json.loads(json_data.decode('utf-8'))
    #
    #     content = dict_data.get('content')
    #     if not content:
    #         return to_json_data(errno=Code.PARAMERR, errmsg="评论内容不能为空！")
    #     parent_id = dict_data.get('parent_id')
    #
    #     try:
    #         if parent_id:
    #             parent_id = int(parent_id)
    #             if not models.Comments.objects.only('id').filter(is_delete=False, id=parent_id, news_id=news_id).exists():
    #                 return to_json_data(errno=Code.PARAMERR, errmsg=error_map[Code.PARAMERR])
    #
    #
    #     except Exception as e:
    #         logging.info("前端传过来的parent_id异常：\n{}".format(e))
    #         return to_json_data(errno=Code.PARAMERR, errmsg="未知异常")
    #
    #
    #     # 保存到数据库
    #     news_comment = models.Comments()
    #     news_comment.content = content
    #     news_comment.news_id=news_id
    #     news_comment.author = request.user
    #     news_comment.parent_id = parent_id if parent_id else None
    #     news_comment.save()
    #
    #     return to_json_data(data=news_comment.to_dict_data())


from nextcord import application_command, Interaction
from nextcord.ext import commands

from bics_bot.embeds.useful_links_embed import UsefulLinksEmbed

class UsefulLinksCmd(commands.Cog):
    """This class represents the command </useful_links>

    The </useful_links> command will send a message back with a list of some of
    the most used links for BICS.

    Attributes:
        client: Required by the API, not directly utilized.
    """

    def __init__(self, client):
        self.client = client

    @application_command.slash_command(
        description="Links that might be useful",
    )
    async def useful_links(self, interaction: Interaction):
        """
        This method represents the </useful_links> command which sends a
        message with a list of some ofthe most used links for BICS.

        Args:
            interaction: Required by the API. Gives meta information about
              the interaction.
        """
        await interaction.response.send_message(
            embed=UsefulLinksEmbed(), ephemeral=True
        )

def setup(client):
    """Function used to setup nextcord cogs"""
    client.add_cog(UsefulLinksCmd(client))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

from hamcrest import not_none
from hamcrest import has_entry
from hamcrest import assert_that
from hamcrest import has_entries

import fudge

from zope.component.hooks import getSite

from zope.securitypolicy.interfaces import IPrincipalRoleManager

from nti.app.products.acclaim.client_models import AcclaimOrganization
from nti.app.products.acclaim.client_models import AcclaimOrganizationCollection

from nti.app.testing.application_webtest import ApplicationLayerTest

from nti.app.testing.decorators import WithSharedApplicationMockDS

from nti.dataserver.authorization import ROLE_ADMIN
from nti.dataserver.authorization import ROLE_SITE_ADMIN

from nti.dataserver.tests import mock_dataserver

class TestIntegration(ApplicationLayerTest):

    default_origin = 'http://mathcounts.nextthought.com'

    def _assign_role_for_site(self, role, username, site=None):
        role_manager = IPrincipalRoleManager(site or getSite())
        role_name = getattr(role, "id", role)
        role_manager.assignRoleToPrincipal(role_name, username)

    @WithSharedApplicationMockDS(users=True, testapp=True)
    @fudge.patch('nti.app.products.acclaim.client._AcclaimInitializationUtility._get_organizations')
    def test_integration(self, mock_get_orgs):
        """
        Test enabling acclaim integration and editing.
        """
        org_collection = AcclaimOrganizationCollection()
        org = AcclaimOrganization(organization_id='test_org_id')
        org_collection.organizations = [org]

        mock_get_orgs.is_callable().returns(org_collection)
        admin_username = 'acclaim_int@nextthought.com'
        site_admin_username = 'acclaim_site_admin'
        other_username = 'acclaim_int_other'
        with mock_dataserver.mock_db_trans(self.ds):
            self._create_user(other_username)
            self._create_user(admin_username)
            self._assign_role_for_site(ROLE_ADMIN, admin_username)
            self._create_user(site_admin_username)
        with mock_dataserver.mock_db_trans(self.ds, site_name="mathcounts.nextthought.com"):
            self._assign_role_for_site(ROLE_SITE_ADMIN, site_admin_username)

        admin_env = self._make_extra_environ(admin_username)
        other_env = self._make_extra_environ(other_username)
        site_admin_env = self._make_extra_environ(site_admin_username)

        def _get_acclaim_int(username, env):
            url = "/dataserver2/users/%s/Integration/Integrations" % username
            res = self.testapp.get(url, extra_environ=env)
            res = res.json_body
            acclaim_int = next((x for x in res['Items'] if x.get('Class') == 'AcclaimIntegration'), None)
            return acclaim_int

        acclaim_int = _get_acclaim_int(site_admin_username, site_admin_env)
        assert_that(acclaim_int, not_none())
        enable_href = self.require_link_href_with_rel(acclaim_int, 'enable')

        acclaim_int = _get_acclaim_int(admin_username, admin_env)
        assert_that(acclaim_int, not_none())
        self.require_link_href_with_rel(acclaim_int, 'enable')

        # Enable integration
        self.testapp.post_json(enable_href,
                               {'authorization_token': 'acclaim_authorization_token'},
                               extra_environ=other_env,
                               status=403)

        res = self.testapp.post_json(enable_href,
                                     {'authorization_token': 'acclaim_authorization_token'},
                                     extra_environ=site_admin_env)
        res = res.json_body
        acclaim_href = res.get('href')
        assert_that(acclaim_href, not_none())
        assert_that(res, has_entries('CreatedTime', not_none(),
                                     'NTIID', not_none(),
                                     'authorization_token', '******************ion_token',
                                     'Creator', 'acclaim_site_admin',
                                     'Last Modified', not_none(),
                                     'organization', has_entry('organization_id', 'test_org_id')))

        disconnect_href = self.require_link_href_with_rel(res, 'disconnect')

        # Can not re-enable
        self.testapp.post_json(enable_href,
                               {'authorization_token': 'acclaim_authorization_token'},
                               extra_environ=site_admin_env,
                               status=422)

        self.testapp.put_json(acclaim_href,
                              {'authorization_token': 'new_token'},
                              extra_environ=other_env,
                              status=403)

        # Change orgs
        org = AcclaimOrganization(organization_id='test_org_id2')
        org_collection.organizations = [org]
        # Edit and update toke also fetches new orgs
        res = self.testapp.put_json(acclaim_href,
                                    {'authorization_token': 'new_token'},
                                    extra_environ=site_admin_env)
        res = res.json_body
        assert_that(res, has_entries('authorization_token', '******ken',
                                     'organization', has_entry('organization_id', 'test_org_id2')))

        self.testapp.get(acclaim_href, extra_environ=other_env, status=403)
        self.testapp.get(acclaim_href, extra_environ=admin_env)
        self.testapp.get(acclaim_href, extra_environ=site_admin_env)

        self.testapp.delete(disconnect_href, extra_environ=other_env, status=403)
        self.testapp.delete(disconnect_href, extra_environ=admin_env)
        self.testapp.delete(disconnect_href, extra_environ=site_admin_env, status=404)


import re,datetime
def list_con(ss):
     word='Ai'
     pattern = r'(^|[^\w]){}([^\w]|$)'.format(word)
     pattern = re.compile(pattern, re.IGNORECASE)
     matches = re.search(pattern, ss)
     if matches==None:
         return False
     else:
         return True

ab=[]
ba=[]
def Tell_time():
    import datetime
    hour =int(datetime.datetime.now().hour)
    minutes =int(datetime.datetime.now().minute)
    seconds=int(datetime.datetime.now().second)
    return f"{hour}:{minutes}:{seconds}"

def show_ai_news():
    with open("database.csv","r+") as f:
        aaa =f.readlines()
        for i in range(len(aaa)):
            if(list_con(aaa[i])):
                ab.append(aaa[i])
        if len(ab)>=1:
            with open("ai_database.csv","r+") as f:
                a=f.readlines()
                cc=[]
                for i in range(len(a)):
                    if a[i].startswith("Time") or a[i].startswith('\n'):
                        continue
                    else:
                        cc.append(a[i])

                if len(cc)!=len(ab):

                    with open('ai_database.csv',"a") as f:
                        f.write(f"Time and Date -:{datetime.date.today()} {Tell_time()}\n")
                        for i in range(len(ab)):
                            if ab[i] not in a:
                                f.write(f"{ab[i]}\n")
                    return True
        else:
            return False

def ai_news_data_base():
    pass

if __name__=="__main__":
    print(show_ai_news())


import time, random, decimal
from utils.imagesearch import imagelocate
from pyautogui import press, moveTo, click
from PIL import Image

def keyboard(key):
    press(key)

    # moveTo(x,y,duration=num_seconds)
    # click(button=button)
    # click(x=moveToX, y=moveToY, clicks=num_of_clicks, interval=secs_between_clicks, button='left')

def image_find(image,wait=2,clicky=False,precision=.9):
    im = Image.open(image)
    width, height = im.size
    pause = decimal.Decimal(random.randrange(35,90))/100
    # print(pause)

    timeout = time.time() + wait # 17 seconds from now
    while True:
        pos = imagelocate(image,.1)
        x = ((pos[0]/2)+(width/4))
        y = ((pos[1]/2)+(height/4))

        test = 0

        # timer countdown
        if test == 5 or time.time() > timeout:
            break
        test = test - 1

        if pos[0] != -1:
            # print("x and y : ", x, y)

            time.sleep(pause)
            if clicky == True:
                moveTo(x, y, duration=.2)
                click(x, y)

            return True

        else:
            print("image not found")
            return False


from turtle import *

k_base_box = [(0, 0), (0, 1), (1, 1), (1, 0)]
k_offset_in_pixels = (-150, -300)

def goto_offset(p):
    goto(p[0] + k_offset_in_pixels[0], p[1] + k_offset_in_pixels[1])

def draw_lines(lines, desired_width, desired_color):
    prev_width = width()
    color(desired_color)
    width(desired_width)

    for line in lines:
        goto_offset(line[0])
        down()
        begin_fill()
        goto_offset(line[1])
        end_fill()
        up()

    width(prev_width)

def draw_line(p0, p1, desired_width, desired_color):
    prev_width = width()
    color(desired_color)
    width(desired_width)
    goto_offset(p0)
    down()
    begin_fill()
    goto_offset(p1)
    end_fill()
    up()
    width(prev_width)

def draw_square(start, size, fill_color, pen_color):
    fillcolor(fill_color)
    pencolor(pen_color)
    goto_offset(start)
    down()
    begin_fill()
    for p in k_base_box:
        goto_offset((start[0] + p[0] * size, start[1] + p[1] * size))
    end_fill()
    up()

def init():
    ht()
    speed(0)
    shape("classic")
    resizemode("user")
    shapesize(1, 1)
    width(1)
    up()
    color(0, 0, 0)
    colormode(255)
    return "EVENTLOOP"

def main_init(onleftclick, onmiddleclick, onrightclick):
    init()
    onscreenclick(onleftclick, 1)
    onscreenclick(onmiddleclick, 2)
    onscreenclick(onrightclick, 3)

if __name__ == "__main__":
    msg = init()
    print(msg)
    mainloop()


from pytket.circuit import Circuit
from pytket.extensions.qiskit import tk_to_qiskit

def s0_circ(n_orbitals: int, n_electrons: int):
    """
    Args:
        n_orbitals (int) = number of qubits/spin orbitals from MolecularData
        n_electrons (int) = number of electrons from MolecularData
    Returns:
        circ (pytket.circuit.Circuit object) = in Hartree-Fock ground state
    """
    circ = Circuit(n_orbitals)

    for i in range(n_electrons):
        circ.X(i) #rotates from |0> to |1>

    return circ

def t1_circ(n_orbitals: int, n_electrons: int):
    """
    Args:
        n_orbitals (int) = number of qubits/spin orbitals from MolecularData
        n_electrons (int) = number of electrons from MolecularData
    Returns:
        circ (pytket.circuit.Circuit object) = in T1 configuration from HOMO->LUMO
    """
    circ = Circuit(n_orbitals)

    for i in range(n_electrons-1): # for all but one electrons
        circ.X(i) # rotates from |0> to |1>
    circ.X(n_electrons) # rotate from |0> to |1> for the triplet spin orbital

    return circ

ref_circ_library = {
    's0': s0_circ,
    't1': t1_circ
}

if __name__ == "__main__":
    n_orbitals = 12
    n_electrons = 4
    ref = 's0'

    gen_ref_circ = ref_circ_library[ref]

    print(tk_to_qiskit(gen_ref_circ(n_orbitals, n_electrons)))


# -*- coding: utf-8 -*-
"""
Created on Tue Jan 25 10:52:40 2022

This script is used to segment the text to sentence which is suitable for input of NER models

@author: Meiqi
"""

import spacy
import json
import pandas as pd
import time
import re
import os
import operator

def corpus_output(file_id, sec_id, subsec_id, sent_id, text):
    output_dict = {'corpus': [], 'section': [], 'text': []}

    subsec_str = str(subsec_id).zfill(2)+str(sent_id).zfill(3)
    output_dict['corpus'].append(file_id)
    output_dict['section'].append(sec_id + subsec_str)
    output_dict['text'].append(text)

    return pd.DataFrame(output_dict)

def annotation_output(file_id, sec_id, subsec_id, sent_id, start, end, entity):
    output_dict = {'corpus': [], 'section': [], 'start': [], 'end': [], 'enzyme':[]}

    subsec_str = str(subsec_id).zfill(2)+str(sent_id).zfill(3)
    output_dict['corpus'].append(file_id)
    output_dict['section'].append(sec_id + subsec_str)
    output_dict['start'].append(start)
    output_dict['end'].append(end)
    output_dict['enzyme'].append(entity)

    return pd.DataFrame(output_dict)
def files(fileName):
    fdlist = ['GWAS', 'MWAS', 'Microbiome', 'proteomics']
    for fdname in fdlist:
        folderPath = "D:/BMR-DS/Project 1/Dataset/{}".format(fdname)
        flist = os.listdir(folderPath)
        if fileName in flist:
            abbrevPath = 'D:/BMR-DS/Project 1/Dataset/{}-Abbreviations'.format(fdname)
            print(fdname)
            return folderPath, abbrevPath

if __name__ == '__main__':

    # segment the sentences
    nlp = spacy.load("en_core_web_sm")
    doc = nlp("This is a sentence. This is another sentence.")
    assert doc.has_annotation("SENT_START")
    # output files
    outputText_path = "D:/BMR-DS/Project 1/Dataset/Annalysis.txt"
    outputAnnot_path = "D:/BMR-DS/Project 1/Dataset/AnnotAnnalysis.txt"
    df_text = pd.DataFrame()
    df_annot = pd.DataFrame()

    sectionType = {'methods section': 'M',
                   'results section': 'R',
                   'discussion section': 'D',
                   'textual abstract section': 'A',
                   'introduction section': 'I',
                   'references section': 'F'
                   }
    file_num = 0
    startTime = time.time()
    annonum = 0
    '''
    fmpath = 'D:/BMR-DS/Project 1/Dataset/ManualAnnotationFiles.txt'
    fm = open(fmpath, 'r')
    fmnames = []
    for i in fm.readlines():
        fmnames.append(i.replace('\n', '.json'))
    fm.close()
    print(fmnames)
    '''
    enzdict = {}
    fdlist = ['GWAS', 'MWAS', 'Microbiome', 'proteomics']
    for fdname in fdlist:
        folderPath = "D:/BMR-DS/Project 1/Dataset/{}".format(fdname)
        fmnames = os.listdir(folderPath)

        abbrevPath = 'D:/BMR-DS/Project 1/Dataset/{}-Abbreviations'.format(fdname)

        for fileName in fmnames:
            if re.match('(.*).json$', fileName):
                fileName = fileName.replace('_bioc_annotated.json', '_bioc_annotated.json')
            else:
                continue
            if re.match('(.*).json$', fileName):
                filePath = '/'.join([folderPath, fileName])
                with open(filePath, 'r', encoding = 'utf-8') as file:
                    mainText = json.load(file)
                file.close()
                abbrePath = '/'.join([abbrevPath, fileName.replace('bioc_annotated', 'abbreviations')])
                with open(abbrePath, 'r', encoding = 'utf-8') as abbrefile:
                    abbreText = json.load(abbrefile)
                abbrefile.close()

            else:
                continue

            file_num += 1
            fileID = fileName.split('_')[0]
            print('\r'+'Processing {} ({}/{})'.format(fileID, str(file_num), str(len(fmnames))), flush = True)

            sectionDict = {'M': -1, 'R': -1, 'D': -1, 'A': -1, 'I': -1, 'F': -1}
            textList = mainText['documents'][0]['passages']
            abbreDirt = abbreText['documents'][0]['passages']
            abbreList = {}
            for item in abbreDirt:
                abbreList[item['text_short']] = item['text_long_1']

            for text in textList:
                section = text['infons']['iao_name_1']
                if section in sectionType:
                    sectionDict[sectionType[section]] += 1
                elif 'iao_name_2' in text['infons']:
                    elsesection = text['infons']['iao_name_2']
                    if elsesection in sectionType:
                        section = elsesection
                        sectionDict[sectionType[section]] += 1
                else:
                    continue
                #print(section)
                offset = int(text['offset'])
                # Find the annotations
                annotation = text['annotations']
                annotList = []
                if len(annotation):
                    for i in annotation:
                        #annotList[i['text']] = i['locations'][0]['offset']
                        annotList.append((i['text'], i['locations'][0]['offset'], i['infons']['identifier']))
                        entity = i['text']
                        identifier = i['text']
                        identifier = identifier.lower().strip(',').strip(';').strip('"').strip("'").strip(':').strip('-').strip(']').strip(')').strip('[').strip('(').strip(',').strip(';').strip('"').strip("'").strip(':').strip('-').rstrip('s')
                        identifier = identifier.strip('')
                        if identifier in enzdict:
                            enzdict[identifier] += 1
                        else:
                            enzdict[identifier] = 1
                        '''
                        if identifier in enzdict:
                            if entity in enzdict[identifier]:
                                enzdict[identifier][entity] += 1
                            else:
                                enzdict[identifier][entity] = 1
                        else:
                            enzdict[identifier] = {entity:1}
                        annonum += 1
                        '''

                '''annotList = sorted(annotList, key = lambda x: x[1])
                annonum += len(annotList)
                if len(annotList) == 0:
                    continue

                # print(annotList)
                # Get the main text
                mainText = text['text']
                doc = nlp(mainText)
                sents = [str(s) for s in doc.sents]
                sentStart = offset

                sent_id = -1

                for sent in sents:
                    bias = 0     # Replace the abbreviation with defination
                    flag = False # To ensure which sentence has enzymes
                    #print(sent)
                    sent_id += 1
                    sentEnd = sentStart + len(sent)
                    for entity, loc, identifier in annotList:
                        if loc < sentStart:
                            continue
                        # Check if current sentence includes entities
                        elif loc >= sentStart and loc < sentEnd:
                            annotStart = loc - sentStart + bias
                            annotEnd = annotStart + len(entity)
                            # Check if the location is right
                            if entity == sent[annotStart : annotEnd]:
                                flag = True
                                # Replace the abbreviation using its defination
                                if entity in abbreList:
                                    bias = len(abbreList[entity]) - len(entity)
                                    sent = sent[:annotStart] + sent[annotStart:].replace(entity, abbreList[entity], 1)
                                    entity = abbreList[entity]
                                    annotEnd = annotStart + len(entity)
                                #df_annot = df_annot.append(annotation_output(fileID, sectionType[section], sectionDict[sectionType[section]], sent_id, annotStart, annotEnd, entity), ignore_index = True)

                        else:
                            break
                    #if flag:
                        #df_text = df_text.append(corpus_output(fileID, sectionType[section], sectionDict[sectionType[section]], sent_id, sent.replace('\n', '')), ignore_index = True)
                    #sentStart = sentEnd + 1

        df_text.sort_values(['corpus', 'section'], ignore_index = True, inplace = True)
        df_text = df_text.astype({'corpus': str, 'section': str, 'text': str})
        df_text.to_csv(outputText_path, encoding = 'utf_8_sig', sep = '\t', header = False, index = False)

        df_annot.sort_values(['corpus', 'section', 'start'], ignore_index=True, inplace=True)
        df_annot = df_annot.astype({'corpus': str, 'section': str, 'start': int, 'end': int, 'enzyme': str})
        df_annot.to_csv(outputAnnot_path, encoding = 'utf_8_sig', sep = '\t', header = False, index = False)

    '''
    print("Successfully segment the sentences in {} seconds.".format(time.time() - startTime))
    f = open('D:/BMR-DS/Project 1/Dataset/enzyme_annalysis.json', 'w')
    json.dump(enzdict, f, indent = 4)
    f.close()


class DirectedGraph:
    def __init__(self):
        self.nodes = set()
        self.edges = set()

    def add_edge(self, nodeA, nodeB):
        self.nodes.add(nodeA)
        self.nodes.add(nodeB)
        self.edges.add((nodeA, nodeB))

    def get_neighbours(self, node):
        return {edge[1] for edge in self.edges if edge[0] == node}

    def path_between(self, nodeA, nodeB):
        searched_nodes = {}
        unsearched_nodes = [nodeA]
        path_exists = False
        while unsearched_nodes and not path_exists:
            for node in unsearched_nodes:
                unsearched_nodes.remove(node)
                new_batch = self.get_neighbours(node)
                if nodeB in new_batch:
                    path_exists = True
                    break
                x = [node for node in new_batch if node not in searched_nodes]
                unsearched_nodes.extend(x)

        return path_exists

    def __str__(self):
        return str(self.nodes) + '\n' + str(self.edges)


class Error(Exception):
    def __init__(self, message=""):
        self.message = message

class SecurityError(Exception):
    def __init__(self, message=""):
        self.message = message

class ExternalApiError(Exception):

    def __init__(self, message, url, method, path, status_code=None, reason=None):
        self.message = message
        self.url = url
        self.method = method
        self.path = path
        self.status_code = status_code
        self.reason = reason

    def __str__(self):
        message = "%s %s%s: %s" % (self.method, self.url, self.path, self.message)
        if self.status_code != None:
            message += ", code = %s" % self.status_code
        if self.reason != None:
            message += ", reason = %s" % self.reason
        return message


"""
prediction_maker.py
This source code is part of temp-monitoring program.
It contains the code to calculate and select the best hyperparameters for
every model that would be created for every vehicle plate.
All the hyperparameters, as the results, are saved inside '/prophet_folder'.
"""

import warnings
import json
import itertools
from pathlib import Path
from datetime import datetime

import pandas as pd
from prophet import Prophet
from matplotlib import pyplot as plt
from sklearn.metrics import r2_score
from prophet.diagnostics import cross_validation, performance_metrics
from prophet.serialize import model_to_json
from prophet.utilities import regressor_coefficients
import numpy as np
from configparser import ConfigParser

warnings.simplefilter('ignore')
parser = ConfigParser()
parser.read("config.ini")
my_path = Path.cwd()

class ProphetModel:
  """
  Receives a dataset and prepares the data before sending it to the Prophet
  algorithm to create the best model for every vehicle plate.

  Args:
    pd.Dataframe : dataframe where to run the model from.
  """
  def __init__(self, main_df):
    self.main_df = self.rename_features(main_df)
    self.create_folders()
    self.p_best_params = str(my_path)+parser.get("path_folder", "best_params")
    self.p_regrs_coef_path = str(my_path)+parser.get("path_folder", "regressors_coef")
    self.p_model_file = str(my_path)+parser.get("path_folder", "model_file")
    self.p_figures_folder = str(my_path)+parser.get("path_folder", "saved_figures")
    self.perf_metrics = str(my_path)+parser.get("path_folder", "perf_metrics")
    self.timestamp = datetime.now().strftime("%Y-%m-%d %H:%M")

  @staticmethod
  def create_folders():
    """
    This method creates all the the folders in /prophet_folder
    where the different files will be saved.
    """
    config_value_list = ["regressors_coef", "best_params",
                          "model_file", "perf_metrics", "saved_figures"]
    for value in config_value_list:
      value_path = parser.get("path_folder", value)
      value_path_folder = Path(value_path).parents[0]
      final_path = str(my_path)+str(value_path_folder)
      try:
        Path.mkdir(Path(final_path), parents=False, exist_ok=False)
        print(f"folder {value_path_folder} has been created.")
      except FileExistsError:
        pass

  def rename_features(self, main_df):
    """
    Renames the columns according to Prophet nomenclature.

    Args:
      pd.Dataframe: dataframe with columns without appropiate format for Prophet.

    Returns:
      pd.Dataframe : dataframe with columns renamed properly.
    """
    main_df.rename(columns={"temp1" : "y", "date": "ds"}, inplace=True)

    return main_df

  def run_model(self):
    """
    Detects if a model has been saved for each vehicle.
    If a model doesn't exits for a vehicle, it is generated with the best
    comnination of hyperpareters and saved.
    """
    # Iterates for every vehicle plate in the dataset.
    for veh_plate in self.main_df["vehicle_plate"].unique():
      if Path((self.p_model_file).format(veh_plate)).exists():
        print(f"Model for vehicle plate {veh_plate} already exists.")
        continue

      else:
        df_veh_plate = self.main_df[self.main_df["vehicle_plate"] == veh_plate]
        df_veh_plate.dropna(subset=["y", "temp2"], inplace=True)
        duration_plate_days = (df_veh_plate["ds"].iloc[-1] - df_veh_plate["ds"].iloc[0])

        # Dictionary of the values of the hyperparametrs to be used in the model.
        param_grid = {"changepoint_prior_scale": [0.001, 0.01, 0.1],
                      "changepoint_range": [0.75, 0.8, 0.85],
                      "daily_seasonality": [True, False],
                      "weekly_seasonality": [True, False],
                      }

        # Generates all possible combinations of hyperparameters.
        all_params = [dict(zip(param_grid.keys(), v)) for v in\
                           itertools.product(*param_grid.values())]

        rmses = []
        # Loop that runs through all the combinations of hyperparameters for each vehicle.
        df_full_metrics = pd.DataFrame()
        for params in all_params:
          model = Prophet(**params)

          # Adds regressors to the model.
          model.add_regressor("temp2")
          # model.add_regressor('ignition')
          # model.add_regressor('interval_time')
          # model.add_regressor('door1_status')
          # model.add_regressor('temp2_status')

          # Trains the model with the selected hyperparameters by vehcile.
          model.fit(df_veh_plate)

          # Cross validation. Parameters are scaled to the number of
          # entries in the dataframe.
          df_cv = cross_validation(model,
                                  initial = duration_plate_days*0.3,
                                  period=duration_plate_days*0.1,
                                  horizon=duration_plate_days*0.1,
                                  parallel="processes")

          df_perf = performance_metrics(df_cv, rolling_window=1).round(
                                                                    decimals=3)
          df_full_metrics = pd.concat([df_full_metrics, df_perf],
                                      ignore_index=True)  # nuevo

          # Creates a list that holds the rmse for each combination of hyperparemeters
          # of the model.
          rmses.append(df_perf["rmse"].values[0])

        # Saves the metrics of each parameter in a .csv
        tuning_results = pd.DataFrame(all_params)
        tuning_results = pd.concat([tuning_results, df_full_metrics], axis=1,
                                  ignore_index=True)
        tuning_results.insert(0, "timestamp", self.timestamp)
        columns = ["timestamp", "changepoint_prior_scale", "changepoint_range",
                  "daily_seasonality", "weekly_seasonality", "horizon","mse",
                  "rmse", "mae", "mape", "mdape", "smape", "coverage"]
        tuning_results.columns = columns
        tuning_results.to_csv(self.perf_metrics.format(veh_plate),
                              mode="a",)

        # Displays the results of the RMSE of all the combination of
        # hyperparameters in the terminal.
        print("="*70)
        print(tuning_results[["changepoint_prior_scale", "changepoint_range",
                  "daily_seasonality", "weekly_seasonality", "rmse"]])

        # Selects the combination of hyperparameters with the lowest RMSE score and
        # returns it to the terminal.
        best_params = all_params[np.argmin(rmses)]
        print("\nBest parameters for {vplate}:\n{params}".format(vplate=veh_plate,
                                                            params=best_params))
        print("="*70)

        # Saves the correlation coefficients of each regressor to a .txt
        reg_coef = regressor_coefficients(model)
        with open (self.p_regrs_coef_path.format(veh_plate), "w") as reg_txt:
          reg_txt.write(str(reg_coef))

        # Saves the best combination of hyperparameters in a .json
        with open((self.p_best_params).format(veh_plate), "w") as param_file:
          param_file.write(json.dumps(best_params))

        # Generates a new model using the best combination of hyperparameters.
        model = Prophet(changepoint_prior_scale=best_params["changepoint_prior_scale"],
                        changepoint_range=best_params["changepoint_range"],
                        daily_seasonality=best_params["daily_seasonality"],
                        weekly_seasonality=best_params["weekly_seasonality"],
                        )

        # Trains the model on the corresponding vehicle dataframe, creating predictions
        # and plotting the results.
        model.fit(df_veh_plate)

        # Saves the current model in a .json file. Each vehicle has its own file
        # which will be used in prediction_maker to precit the missing temperature data.
        with open((self.p_model_file).format(veh_plate), "w") as model_file:
          model_file.write(model_to_json(model))

        # Makes future predictions.
        future_periods = model.make_future_dataframe(periods=int(len(df_veh_plate)*0.2),
                                                    freq="0.3min")
        forecast = model.predict(future_periods)
        forecast = forecast[["ds", "yhat", "yhat_lower", "yhat_upper"]]

        # Combines the results of these predictions to the corresponding
        # vehicle dataframe.
        metric2 = pd.merge(forecast, self.main_df[["y", "ds"]], on="ds")
        metric2 = metric2[metric2["y"].notna()]
        r_sq_score = r2_score(metric2.y, metric2.yhat)

        # Plots the results of the model, including the predictions.
        fig = model.plot(forecast, uncertainty=False)
        fig.subplots_adjust(bottom=0.22, top=0.95)
        plt.xticks(rotation=90)
        plt.xlabel("Date")
        plt.ylabel("Temperature")
        plt.title(f"Fit & Prediction. Prophet. {veh_plate}. R2 = " + str("%.2f" % r_sq_score))
        plt.legend(["Real temp", "Predicted"])

        # save the figure in 'saved_figures' folder.
        plt.savefig(self.p_figures_folder.format(veh_plate))


# -*- coding: utf-8 -*-
import torch
import copy
import itertools
import pcode.utils.loss as loss
import torch.distributed as dist
from pcode.local_training.base_worker import BaseWorker
import pcode.create_dataset as create_dataset
from pcode.utils.stat_tracker import RuntimeTracker
import pcode.local_training.utils as utils
import pcode.create_optimizer as create_optimizer
import torch.nn.functional as F
import torch.nn as nn
from pcode.datasets.aug_data import aug
import torchvision.transforms as transforms

"""
Implementation of FedTHE (Ours)
    Test-Time Robust Personalization for Federated Learning
    https://arxiv.org/abs/2205.10920.
    When `is_finetune` is True, it becomes FedTHE+ (Ours).
"""

class THEWorker(BaseWorker):
    def __init__(self, conf, is_fine_tune=False):
        super(THEWorker, self).__init__(conf)
        self.conf = conf
        self.n_personalized_epochs = self.conf.n_personalized_epochs
        self.eval_dataset = self.conf.eval_dataset  # can be test_loader or val_loader
        self.is_in_childworker = True
        # test-time self-supervised aggregation
        self.num_head = 2
        self.THE_steps = self.conf.THE_steps
        self.agg_weight = torch.nn.Parameter(torch.rand((self.conf.batch_size, self.num_head)).cuda(), requires_grad=True)
        self.agg_weight.data.fill_(1 / self.num_head)
        self.alpha = self.conf.THE_alpha
        self.beta = self.conf.THE_beta

        self.is_tune_net = is_fine_tune
        self.is_rep_history_reused = self.conf.is_rep_history_reused

    def run(self):
        while True:
            self._listen_to_master()

            # receive global representation from server.
            self.global_rep = torch.zeros((self.conf.rep_len,))
            dist.broadcast(tensor=self.global_rep, src=0)
            dist.barrier()
            self.global_rep = self.global_rep.cuda()

            # check if we need to terminate the training or not.
            if self._terminate_by_early_stopping():
                return

            # receive the perform standard local training.
            self._recv_model_from_master()

            # create personal model and register hook.
            if not hasattr(self, "personal_head"):
                self.personal_head = nn.Linear(self.conf.rep_len, utils.get_num_classes(self.conf.data), bias=False)

            state = self._brm_train(model=self.model)
            params_to_send = copy.deepcopy(state["model"].state_dict())  # use deepcopy

            # personalization.
            p_state = self._personalized_train(model=self.model)

            # evaluate the personalized model.
            perf = self._evaluate_all_test_sets(p_state)

            # display the personalized perf and sync the model & perf.
            self._display_info(p_state, perf)

            # also send local rep
            perf.extend(self.local_rep.cpu().squeeze().tolist())
            self._send_model_to_master(params_to_send, perf)
            del state, p_state

            if self._terminate_by_complete_training():
                return

    def _brm_train(self, model):
        # change the criterion to balanced loss and do local training.
        self.criterion = loss.BalancedSoftmaxLoss(self._get_target_histogram())
        state = super(THEWorker, self).train(model)
        return state

    def _personalized_train(self, model):
        self.is_in_personalized_training = True
        self.n_local_epochs = self.n_personalized_epochs
        self.erm_criterion = nn.CrossEntropyLoss(reduction="mean")
        # define dataloader, optimizer, scheduler and tracker
        state = self._init_train_process(model=model)
        self.personal_head.to(self.graph.device)
        # we want to optimize personal head
        state["optimizer"] = create_optimizer.define_optimizer(
            self.conf, model=self.personal_head, optimizer_name=self.conf.optimizer, lr=self._get_round_lr()
        )
        # freeze the model, except the personal head
        state["model"].requires_grad_(False)
        self.personal_head.requires_grad_(True)
        self.rep_layer = utils.determine_hook(state["model"], name=self.conf.arch)
        self.rep_layer.register_forward_hook(utils.hook)
        while not self._is_finished_one_comm_round(state):
            self.local_rep = []
            self.per_class_rep = {i: [] for i in range(utils.get_num_classes(self.conf.data))}
            for _input, _target in state["train_loader"]:
                # load data
                with self.timer("load_data", epoch=state["scheduler"].epoch_):
                    data_batch = create_dataset.load_data_batch(
                        self.conf, _input, _target, is_training=True,
                    )

                # inference and get current performance.
                with self.timer("forward_pass", epoch=state["scheduler"].epoch_):
                    state["optimizer"].zero_grad()
                    g_out = state["model"](data_batch["input"])
                    p_out = self.personal_head(self.rep_layer.rep)
                    loss = self.erm_criterion(p_out, data_batch["target"])
                    agg_out = torch.stack([g_out, p_out], dim=1).mean(dim=1)
                    performance = self.metrics.evaluate(loss, agg_out, data_batch["target"])
                    state["tracker"].update_metrics(
                        [loss.item()] + performance, n_samples=data_batch["input"].size(0)
                    )
                    for i, label in enumerate(data_batch["target"]):
                        self.per_class_rep[label.item()].append(self.rep_layer.rep[i, :].unsqueeze(0))

                with self.timer("backward_pass", epoch=state["scheduler"].epoch_):
                    loss.backward()
                    state["optimizer"].step()
                    state["scheduler"].step()

                if self.conf.display_log:
                    self._display_logging(state)
                if self._is_diverge(state):
                    self._terminate_comm_round(state)
                    return state

            # refresh the logging cache at the end of each epoch.
            state["tracker"].reset()
            if self.logger.meet_cache_limit():
                self.logger.save_json()

        # terminate
        self._compute_prototype()
        self._terminate_comm_round(state)
        self.is_in_personalized_training = False
        return state

    def _compute_prototype(self):
        # compute the average representation of local training set.
        for (k, v) in self.per_class_rep.items():
            if len(v) != 0:
                self.local_rep.append(torch.cat(v).cuda())
        self.local_rep = torch.cat(self.local_rep).mean(dim=0).cuda()

    def _test_time_tune(self, state, data_batch, num_steps=3):
        # turn on model grads.
        state["model"].requires_grad_(True)
        self.personal_head.requires_grad_(True)
        # set optimizer.
        fe_optim = torch.optim.SGD(state["model"].parameters(), lr=0.0005)
        fe_optim.add_param_group({"params": self.personal_head.parameters()})
        g_pred, p_pred = [], []
        # do the unnormalize to ensure consistency.
        normalize, unnormalize = utils.get_normalization(self.conf.data)
        convert_img = transforms.Compose([unnormalize, transforms.ToPILImage()])
        agg_softmax = torch.nn.functional.softmax(self.agg_weight).detach()
        model_param = copy.deepcopy(state["model"].state_dict())
        p_head_param = copy.deepcopy(self.personal_head.state_dict())
        for i in range(data_batch["input"].shape[0]):
            image = convert_img(data_batch["input"][i])
            for _ in range(num_steps):
                # generate a batch of augmentations and minimize prediction entropy.
                inputs = [aug(image, normalize) for _ in range(16)]
                inputs = torch.stack(inputs).cuda()
                fe_optim.zero_grad()
                g_out = state["model"](inputs)
                p_out = self.personal_head(self.rep_layer.rep)
                agg_output = agg_softmax[i, 0] * g_out + agg_softmax[i, 1] * p_out
                loss, _ = utils.marginal_entropy(agg_output)
                loss.backward()
                fe_optim.step()
            with torch.no_grad():
                g_out = state["model"](data_batch["input"][i].unsqueeze(0).cuda())
                p_out = self.personal_head(self.rep_layer.rep)
                g_pred.append(g_out)
                p_pred.append(p_out)
            state["model"].load_state_dict(model_param)
            self.personal_head.load_state_dict(p_head_param)
        # turn off grads.
        state["model"].requires_grad_(False)
        self.personal_head.requires_grad_(False)
        return torch.cat(g_pred), torch.cat(p_pred)

    def _validate_training(self, state, dataset, num_epochs):
        self.is_in_personalized_training = True
        # dont requires gradients.
        state["model"].requires_grad_(False)
        self.personal_head.requires_grad_(False)
        state["model"].eval()
        tracker_te = RuntimeTracker(metrics_to_track=self.metrics.metric_names)
        # if enabled, then the test history is reused between test sets.
        if self.is_rep_history_reused:
            if not hasattr(self, "test_history"):
                self.test_history = None
        else:
            self.test_history = None

        for _input, _target in dataset:
            data_batch = create_dataset.load_data_batch(
                self.conf, _input, _target, is_training=False
            )
            with torch.no_grad():
                # update test history by exponential moving average.
                _ = state["model"](data_batch["input"])
                test_rep = self.rep_layer.rep.detach()
                test_history = None
                for i in range(test_rep.shape[0]):
                    if test_history is None and self.test_history is None:
                        test_history = [test_rep[0, :]]
                    elif test_history is None and self.test_history is not None:
                        test_history = [self.test_history[-1, :]]
                    else:
                        test_history.append(self.alpha * test_rep[i, :] + (1 - self.alpha) * test_history[-1])
                self.test_history = torch.stack(test_history)
                temperature = torch.hstack((torch.ones((test_rep.shape[0], 1)).cuda(), torch.ones((test_rep.shape[0], 1)).cuda()))

            self.agg_weight = torch.nn.Parameter(torch.tensor(temperature).cuda(), requires_grad=True)
            state["optimizer"] = torch.optim.Adam([self.agg_weight], lr=10*self._get_round_lr())

            self._calculate_samplewise_weight(state, data_batch, num_epochs)
            if self.is_tune_net:
                # test-timely tune the whole net.
                g_pred, p_pred = self._test_time_tune(state, data_batch, num_steps=3)

            # do inference for current batch
            with torch.no_grad():
                if self.is_tune_net:
                    self._multi_head_inference(data_batch, state["model"], tracker_te, g_pred, p_pred)
                else:
                    self._multi_head_inference(data_batch, state["model"], tracker_te)

        self.is_in_personalized_training = False
        self.agg_weight.data.fill_(1 / self.num_head)
        return tracker_te

    def _calculate_samplewise_weight(self, state, data_batch, num_epochs):
        # function that optimize the ensemble weights.
        g_out = state["model"](data_batch["input"])
        test_rep = self.rep_layer.rep
        p_out = self.personal_head(test_rep)
        grad_norm, loss_batch = [], []
        for _ in range(num_epochs):
            # normalize the aggregation weight by softmax
            agg_softmax = torch.nn.functional.softmax(self.agg_weight)
            agg_output = agg_softmax[:, 0].unsqueeze(1) * g_out.detach() \
                         + agg_softmax[:, 1].unsqueeze(1) * p_out.detach()
            # formulate test representation.
            test_rep = self.beta * self.rep_layer.rep + (1 - self.beta) * self.test_history
            p_feat_al = torch.norm((test_rep - self.local_rep), dim=1)
            g_feat_al = torch.norm((test_rep - self.global_rep), dim=1)
            cos = torch.nn.CosineSimilarity(dim=1, eps=1e-6)
            sim = cos(F.softmax(g_out).detach(), F.softmax(p_out).detach())
            # loss function based on prediction similarity, entropy minimization and feature alignment.
            loss = (-sim * (agg_output.softmax(1) * agg_output.log_softmax(1)).sum(1) + \
                    (1 - sim) * (agg_softmax[:, 0] * g_feat_al.detach() + agg_softmax[:, 1] * p_feat_al.detach())).mean(0)
            state["optimizer"].zero_grad()
            loss.backward()

            if torch.norm(self.agg_weight.grad) < 1e-5:
                break
            grad_norm.append(torch.norm(self.agg_weight.grad).item())
            loss_batch.append(loss.item())
            state["optimizer"].step()

    def _multi_head_inference(self, data_batch, model, tracker=None, g_pred=None, p_pred=None):
        # inference procedure for multi-head nets.
        agg_softmax = torch.nn.functional.softmax(self.agg_weight)
        if g_pred is not None and p_pred is not None:
            agg_output = agg_softmax[:, 0].unsqueeze(1) * g_pred \
                         + agg_softmax[:, 1].unsqueeze(1) * p_pred
        else:
            # do the forward pass and get the output.
            g_out = model(data_batch["input"])
            p_out = self.personal_head(self.rep_layer.rep)
            agg_output = agg_softmax[:, 0].unsqueeze(1) * g_out \
                         + agg_softmax[:, 1].unsqueeze(1) * p_out
        # evaluate the output and get the loss, performance.
        loss = self.criterion(agg_output, data_batch["target"])
        performance = self.metrics.evaluate(loss, agg_output, data_batch["target"])

        # update tracker.
        if tracker is not None:
            tracker.update_metrics(
                [loss.item()] + performance, n_samples=data_batch["input"].size(0)
            )

        return loss

    def _validate(self, state, dataset_name):
        # switch to evaluation mode.
        state["model"].eval()
        self.rep_layer = utils.determine_hook(state["model"], name=self.conf.arch)
        self.rep_layer.register_forward_hook(utils.hook)
        # test-time self-supervised aggregation
        tracker_te = self._validate_training(state, state[dataset_name], self.THE_steps)
        return tracker_te()

    def _get_target_histogram(self, display=True):
        local_data_loaders = self.fl_data_cls.intra_client_data_partition_and_create_dataloaders(
            localdata_id=self.client_id - 1,  # localdata_id starts from 0 while client_id starts from 1.
            other_ids=self._get_other_ids(),
            is_in_childworker=self.is_in_childworker,
            local_train_ratio=self.conf.local_train_ratio,
            batch_size=1,
            display_log=False,
        )
        hist = torch.zeros(utils.get_num_classes(self.conf.data))
        train_loader = local_data_loaders["train"]
        for _, _target in train_loader:
            hist[_target.item()] += 1
        if display:
            self.logger.log(
                f"\tWorker-{self.graph.worker_id} (client-{self.client_id}) training histogram is like {hist}"
            )

        return hist


#!/usr/bin/env python3

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

figsize = 2 * (0.7,)

radius = 1
dist = 0.65

font_size = 11

fig_dir = "../figures/diagrams/"

params = { "font.family" : "serif",
           "font.serif" : "Computer Modern",
           "text.usetex" : True,
           "font.size" : font_size }
plt.rcParams.update(params)

##################################################
# define circles

circles = "abc"

angles = [ 0, +2*np.pi/3, -2*np.pi/3 ]
def rotate(angle):
    if type(angle) is int:
        angle = angles[angle]
    if type(angle) is str:
        angle = angles[circles.index(angle)]
    return np.array([ [  np.cos(angle), -np.sin(angle) ],
                      [ +np.sin(angle),  np.cos(angle) ] ])
def vec(circle): return rotate(circle)[:,1]

circle_centers = { circle : vec(circle) * dist
                   for circle in circles }

##################################################
# define centers of regions

region_centers = { "abc" : np.zeros(2) }

# inner / outer intersections along the axis of each circle
int_term = np.sqrt(radius**2 - 3/4 * dist**2)
inner_ints = { circle : vec(circle) * ( +int_term - dist/2 )
               for circle in circles }
outer_ints = { circle : vec(circle) * ( -int_term - dist/2 )
               for circle in circles }

# innermost / outermost symmetric points
inner_mids = { circle : circle_centers[circle] - vec(circle) * radius
               for circle in circles }
outer_mids = { circle : circle_centers[circle] + vec(circle) * radius
               for circle in circles }

# centers of overlapping regions
region_mids = { "abc" : np.zeros(2) }
region_mids["a"] = ( outer_mids["a"] + inner_ints["a"] ) / 2
region_mids["b"] = ( outer_mids["b"] + inner_ints["b"] ) / 2
region_mids["c"] = ( outer_mids["c"] + inner_ints["c"] ) / 2

mid_bc_fst = np.mean([ outer_ints["a"], inner_ints["b"], inner_ints["c"] ], axis = 0)
mid_bc_snd = ( outer_ints["a"] + inner_mids["a"]  ) / 2
region_mids["bc"] = ( mid_bc_fst + mid_bc_snd ) / 2
region_mids["ab"] = rotate("c") @ region_mids["bc"]
region_mids["ac"] = rotate("b") @ region_mids["bc"]

##################################################
# define constructor for diagram

def place_dots(region, num_colors, markers = None, dot_sep = None,
               region_mids = region_mids, markersize = 3,
               default_marker = None):
    if type(num_colors) is int:
        num = num_colors
        colors = [ "k" for _ in range(num) ]
    else:
        num, colors = num_colors
        assert(len(colors) >= num)

    if default_marker is None:
        default_marker = "."
    if dot_sep == None:
        dot_sep = 1
    dot_sep *= radius / 3
    if markers is None:
        markers = default_marker * num
    if len(markers) < num:
        markers += default_marker * (num-len(markers))

    vecs = []

    if num == 1:
        vecs = [ np.zeros(2) ]

    elif num == 2:
        sep_vec = np.array([dot_sep,0])
        vecs = [ -sep_vec/2, +sep_vec/2 ]

    elif num == 3:
        sep_vec = np.array([0,dot_sep])
        vecs = [ rotate(jj) @ sep_vec / np.sqrt(3) for jj in range(3) ]

    elif num == 4:
        vecs = [ [ -dot_sep/2, +dot_sep/2 ],
                 [ +dot_sep/2, +dot_sep/2 ],
                 [ +dot_sep/2, -dot_sep/2 ],
                 [ -dot_sep/2, -dot_sep/2 ] ]

    elif num > 4:
        print("capability to place more than two dots not implemented")
        exit()

    point = region_mids[region]
    for vec, marker, color in zip(vecs, markers, colors):
        size = markersize
        markerfacecolor = "k"

        if marker == "o":
            fill_style = "none"
        else:
            fill_style = "full"

        if marker == ".":
            marker = "o"
        if marker == "+":
            size += 1

        pos = np.array(point) + np.array(vec)
        plt.plot(*pos, marker, color = color, markersize = size, fillstyle = fill_style)

def make_triple_diagram(region_dots, markers = {}, figsize = figsize,
                        dot_sep = None, pad = radius/20,
                        default_marker = None):
    fig = plt.figure(figsize = figsize)
    plt.axes().set_aspect("equal")

    # plot circles
    for center in circle_centers.values():
        circle = patches.Circle(center, radius, color = "k", fill = False)
        plt.gca().add_patch(circle)

    # set axis limits
    ymax = dist + radius
    ymin = -dist/2 - radius
    xmax = dist*np.sqrt(3)/2 + radius
    xmin = -xmax

    y_spread = ymax - ymin
    x_spread = xmax - xmin

    if y_spread > x_spread:
        xmin -= ( y_spread - x_spread ) / 2
        xmax += ( y_spread - x_spread ) / 2
    if x_spread > y_spread:
        ymin -= ( x_spread - y_spread ) / 2
        ymax += ( x_spread - y_spread ) / 2

    plt.ylim(ymin-pad, ymax+pad)
    plt.xlim(xmin-pad, xmax+pad)

    # plot dots
    for region, num in region_dots.items():
        place_dots(region, num, markers.get(region), dot_sep,
                   default_marker = default_marker)

    # trim figure and return handle
    plt.axis("off")
    plt.tight_layout(pad = 0)
    return fig

def make_double_diagram(region_dots, markers = {}, size_x = figsize[1],
                        dot_sep = None, pad = radius/20,
                        default_marker = None):
    dist = 0.5
    size_y = (radius+pad) / ( radius+3/4*dist*3/4+pad ) * size_x
    _figsize = ( size_x, size_y )
    fig = plt.figure(figsize = _figsize)
    plt.axes().set_aspect("equal")

    # plot circles
    centers = [ -dist, dist ]
    for center in centers:
        circle = patches.Circle([center,0], radius, color = "k", fill = False)
        plt.gca().add_patch(circle)

    # set axis limits
    xmax = dist + radius
    ymax = radius
    plt.xlim(-xmax-pad, xmax+pad)
    plt.ylim(-ymax-pad, ymax+pad)

    # plot dots
    region_mids = { "a" : ( -radius, 0),
                    "b" : ( +radius, 0),
                    "ab" : ( 0, 0) }
    for region, num in region_dots.items():
        place_dots(region, num, markers.get(region), dot_sep, region_mids,
                   default_marker = default_marker)

    # trim figure and return handle
    plt.axis("off")
    plt.tight_layout(pad = 0)
    return fig

def add_triple_labels(labels = None, pads = [ 0.3, 0.4, 0.4 ]):
    if labels is None: labels = [ "u", "v", "w" ]
    for circle, label, pad in zip(circles, labels, pads):
        pos = vec(circle) * ( dist + radius * ( 1 + pad ) )
        plt.text(*pos, f"${label}$",
                 horizontalalignment = "center", verticalalignment = "center")

def add_double_labels(labels = None, pads = [ 0.3, 0.3 ]):
    if labels is None: labels = [ "v", "w" ]
    for label, pad, direction in zip(labels, pads, [-1,+1]):
        plt.text(direction * dist, radius + pad, f"${label}$",
                 horizontalalignment = "center", verticalalignment = "center")

##################################################
# make diagrams

### general example diagrams

dots = { "a" : 1, "abc" : 1, "ac" : 1, "bc" : 2 }
make_triple_diagram(dots, figsize = (1,1))
add_triple_labels(["1","2","3"])
plt.tight_layout(pad = 0)
plt.savefig(fig_dir + "example_123.pdf")

make_triple_diagram(dots, figsize = (1,1))
add_triple_labels(["w_1","w_2","w_3"], pads = [ 0.25, 0.5, 0.55 ])
plt.tight_layout(pad = 0)
plt.savefig(fig_dir + "example_w123.pdf")

dots = { "a" : 4, "ab" : 1, "b" : 1 }
markers = { "a" : "..oo", "b" : "o" }
make_double_diagram(dots, markers)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_o.pdf")

dots = { "a" : 1, "abc" : 1, "ac" : 1, "b" : 1, "bc" : 1 }
markers = { "abc" : "x", "ac" : "x", "b" : "o" }
make_triple_diagram(dots, markers, figsize = (0.8,0.8))
add_triple_labels()
plt.tight_layout(pad = 0)
plt.savefig(fig_dir + "example_x.pdf")

plt.close("all")

### example diagrams for elimination

dots = { "a" : 1, "b" : 1, "ab" : 1 }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim.pdf")

make_double_diagram(dots, { "a" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_o.pdf")

make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_x.pdf")

dots = { "ab" : 2 }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_x_full.pdf")

dots = { "a" : 2, "b" : 3 }
make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_start.pdf")

dots = { "a" : 2, "b" : ( 3, [ "r", "m", "b" ]) }
make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_x.pdf")

dots = { "a" : 1, "b" : ( 2, [ "m", "b" ]), "ab" : ( 1, ["r"] ) }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_x_r.pdf")

dots = { "a" : 1, "b" : ( 2, [ "r", "b" ]), "ab" : ( 1, ["m"] ) }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_x_m.pdf")

dots = { "a" : 1, "b" : ( 2, [ "m", "r" ]), "ab" : ( 1, ["b"] ) }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_x_b.pdf")

dots = { "a" : 1, "b" : 2, "ab" : 1 }
make_double_diagram(dots)
add_double_labels()
plt.tight_layout(pad = 0.15)
plt.savefig(fig_dir + "example_elim_color_end.pdf")

plt.close("all")

### product of two single-body operators

for assignment in [ { "a" : 1, "b" : 1, "ab" : 0 },
                    { "ab" : 1 } ]:
    make_double_diagram(assignment)
    add_double_labels()
    plt.tight_layout(pad = 0.05)
    tag = assignment["ab"]
    plt.savefig(fig_dir + f"prod_b11_c{tag}.pdf")

plt.close("all")

dots = { "a" : 1, "b" : 1 }
make_double_diagram(dots, { "a" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b11_c0_o.pdf")

make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b11_c0_x.pdf")

make_double_diagram(dots, { "a" : "o", "b" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b11_c0_oo.pdf")

plt.close("all")

### product of three single-body operators

dots = { "a" : 1, "b" : 1, "c" : 1 }
make_triple_diagram(dots)
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b111_c0.pdf")

dots = { "a" : 1, "bc" : 1 }
make_triple_diagram(dots)
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b111_c11_u.pdf")

dots = { "b" : 1, "ac" : 1 }
make_triple_diagram(dots)
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b111_c11_v.pdf")

dots = { "c" : 1, "ab" : 1 }
make_triple_diagram(dots)
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b111_c11_w.pdf")

dots = { "abc" : 1 }
make_triple_diagram(dots)
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b111_c10.pdf")

plt.close("all")

### product of two two-body operators

for assignment in [ { "a" : 2, "b" : 2, "ab" : 0 },
                    { "a" : 1, "b" : 1, "ab" : 1 },
                    { "ab" : 2 } ]:
    make_double_diagram(assignment)
    add_double_labels()
    plt.tight_layout(pad = 0.05)
    tag = assignment["ab"]
    plt.savefig(fig_dir + f"prod_b22_c{tag}.pdf")

dots = { "ab" : 2 }
make_double_diagram(dots, { "ab" : "oo" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c2_oo.pdf")

dots = { "a" : 1, "b" : 1, "ab" : 1 }
make_double_diagram(dots, { "a" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c1_o.pdf")

make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c1_x.pdf")

make_double_diagram(dots, { "a" : "o", "b" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c1_oo.pdf")

make_double_diagram(dots, { "a" : "o", "b" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c1_ox.pdf")

make_double_diagram(dots, { "a" : "o", "b" : "o", "ab" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c1_ooo.pdf")

dots = { "a" : 2, "b" : 2 }
make_double_diagram(dots, { "a" : "o" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c0_o.pdf")

make_double_diagram(dots, { "a" : "oo" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c0_oo.pdf")

make_double_diagram(dots, { "a" : "ox" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c0_ox.pdf")

make_double_diagram(dots, { "a" : "x" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c0_x.pdf")

make_double_diagram(dots, { "a" : "oo", "b" : "oo" })
add_double_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + f"prod_b22_c0_oooo.pdf")

plt.close("all")

### product of three two-body operators

for marker, suffix in zip([ ".", "o" ], [ "", "_o" ]):

    # 6-point diagram
    make_triple_diagram({ "a" : 2, "b" : 2, "c" : 2 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c0" + suffix + ".pdf")

    plt.close("all")

    # 4-point diagrams
    make_triple_diagram({ "a" : 1, "b" : 1, "c" : 1, "abc" : 1 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c1" + suffix + ".pdf")

    make_triple_diagram({ "a" : 2, "b" : 1, "c" : 1, "bc" : 1 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c01" + suffix + ".pdf")

    plt.close("all")

    # 2-point diagrams
    make_triple_diagram({ "abc" : 2 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c2" + suffix + ".pdf")

    make_triple_diagram({ "a" : 1, "bc" : 1, "abc" : 1 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c11" + suffix + ".pdf")

    make_triple_diagram({ "a" : 2, "bc" : 2 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c02" + suffix + ".pdf")

    make_triple_diagram({ "ab" : 1, "ac" : 1, "b" : 1, "c" : 1 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c011" + suffix + ".pdf")

    # trace diagram
    make_triple_diagram({ "ab" : 1, "ac" : 1, "bc" : 1 }, default_marker = marker)
    plt.savefig(fig_dir + "prod_b222_c0111" + suffix + ".pdf")

    plt.close("all")

### example label assignments for unlabeled diagrams

make_triple_diagram({ "a" : 1, "b" : 1, "c" : 1, "abc" : 1 }, figsize = (0.9,0.7))
add_triple_labels()
plt.tight_layout(pad = 0.05)
plt.savefig(fig_dir + "prod_b222_c1_uvw.pdf")

for labels in [ ["u","v","w"], ["v","w","u"], ["w","u","v"] ]:
    make_triple_diagram({ "ab" : 1, "ac" : 1, "b" : 1, "c" : 1 }, figsize = (0.9,0.7))
    add_triple_labels(labels)
    plt.tight_layout(pad = 0)
    tag = "".join(labels)
    plt.savefig(fig_dir + f"prod_b222_c011_{tag}.pdf")


import httpx
from selectolax.parser import HTMLParser
from dataclasses import asdict, dataclass
from os import path, makedirs
import logging
from csv import DictWriter

# record general informations
logging.basicConfig(
        level=logging.INFO,
        filename="web_scraper.log",
        format="%(asctime)s - %(message)s",
        datefmt="%d-%b-%y %H%M"
        )

# Data container
@dataclass
class Manga:
    title: str
    price: str
    cover: str

# Make a directory to store data
def make_directory(directory):
    if not path.exists(directory):
        logging.info(f"Making directory: {directory}")
        makedirs(directory)
    else:
        logging.info(f"{directory} alredy exists")

# Make a request and get response
def get_http_request(page):
    url = f"https://j-pop.it/it/catalog/category/view/id/696/s/fumetti/?p={page}?product_list_dir=asc&product_list_order=name"
    headers = {
                "User-agent" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36"
                }

    response = httpx.get(url, headers=headers)

    try:
        response.raise_for_status()
    except httpx.HTTPStatusError as exc:
        logging.info(f"Error response {exc.response.status_code} while requesting {exc.request.url!r}.")
    except httpx.RequestError as exc:
        logging.info(f"An error occurred while requesting {exc.request.url!r}.")
    return response

def get_html_text(starting_page):
    r = get_http_request(starting_page)
    # Parse raw HTML
    return HTMLParser(r.text)

# Data parser
def product_info(html):
    info = []
    products = html.css("li.item.product.product-item")

    for product in products:
        p = Manga(
                title=product.css_first("a.product-item-link").text().strip(),
                price=product.css_first("span.price").text().replace("\xa0€", '€'),
                cover=product.css_first("img.img-fluid.product-image-photo").attributes["src"]
            )
        info.append(asdict(p))
    return info

# Make csv file
def make_csv(product):
    try:
        with open("products.csv", "a") as f:
            wr = DictWriter(f, fieldnames=["title", "price", "cover"])
            wr.writerows(product)
            logging.info("csv file successfully created")
    except OSError as e:
        logging.info(f"File already exists. Error {e}")

def main():
    # Loop over pages to extract data
    for i in range(1, 5):
        html = get_html_text(i)
        res = product_info(html)
        make_csv(res)

# Main driver
if __name__ == "__main__":
    # print(get_http_request(1))
    # print(get_html_text(1))
    main()


import tqdm

import numpy as np

import torch
import torch.nn as nn
import torch.optim as optim

# from ._models import CatBoosting
from ._models import rmse, RMSELoss
from sklearn.model_selection import train_test_split
from catboost import CatBoostRegressor, CatBoostClassifier, Pool
from xgboost import XGBRegressor, XGBClassifier
from lightgbm import LGBMRegressor, LGBMClassifier
from .utils import do_optuna,get_parameter

import json
import os

file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)) , 'config.json')
params_dict = {}

with open(file_path, 'r') as file:
    params_dict = json.load(file)

class BoostingModel:
    """
    [주의] 모델마다 train, predict 동작이 바뀌어야 하는 경우, class 도 나뉘어야 한다.
    [주의] CatBoostRegressor, XGBoostRegressor, LGBMRegressor 의 경우 train 과 predict 에 별 다른 동작이 없기 때문에 그냥 합쳐 둔다.
    """
    def __init__(self, args, data):

        super().__init__()

        self.criterion = RMSELoss()
        self.model_name = args.MODEL
        self.data = data

        self.my_model = CatBoostRegressor

        if( "LightGBMRegressor" == self.model_name ):
            print("[INFO] Let's go LGBMRegressor ! ")
            self.my_model = LGBMRegressor
        elif( "XGBRegressor" == self.model_name ):
            print("[INFO] Let's go XGBRegressor ! ")
            self.my_model = XGBRegressor
        elif( "CatBoostRegressor"== self.model_name ):
            print("[INFO] Let's go CatBoostRegressor ! ")
            self.my_model = CatBoostRegressor
        else :
            print("[WARN] Use Default Model ")

        self.best_params = do_optuna( self.data,
                   model = self.my_model,
                   model_name =  args.MODEL,
                   early_stopping_rounds = 35,
                   verbose = 100) if args.DO_OPTUNA == True else params_dict[args.MODEL]

        if( "CatBoostRegressor"== self.model_name ):
            self.best_params['cat_features'] =self.data['cat_features']

    def train(self):

        self.my_model = self.my_model(**self.best_params)
        if( "CatBoostRegressor"== self.model_name ):
            self.my_model.fit(self.data['X_train'],self.data['y_train'],\
                              eval_set=(self.data['X_valid'],self.data['y_valid']))
        else:
            self.my_model.fit(self.data['X_train'],self.data['y_train'],\
                              eval_set=[(self.data['X_train'],self.data['y_train']),(self.data['X_valid'],self.data['y_valid'])])

        preds = self.my_model.predict(self.data['X_valid'])
        preds = preds.squeeze(1) if 'Classifier' in self.model_name else preds

        print('RMSE : ', rmse(self.data['y_valid'], preds)) # Regressor
        # print('RMSE : ', rmse(self.data['y_valid'], preds.squeeze(1))) # Classifier
        #print('epoch:', epoch, 'validation: rmse:', rmse_score)

    def predict(self):

        preds = self.my_model.predict(self.data['test'])
        return preds

class CatBoostingModel:
    def __init__(self, args, data):
        super().__init__()

        self.criterion = RMSELoss()
        self.catmodels = {}
        self.data = data

        target_model = CatBoostRegressor

        self.best_params = do_optuna( self.data,
                   model = target_model,
                   model_name = 'CatBoostRegressor',
                   early_stopping_rounds = 35,
                   verbose = 100) if args.DO_OPTUNA == True else params_dict['CatBoostRegressor']

        self.best_params['cat_features'] =self.data['cat_features']

    def train(self):
        print(self.data['folds'])
        for fold in range(8):
            print(f'===================================={fold+1}============================================')
            train_idx, valid_idx = self.data['folds'][fold]
            X_train = self.data['train'].drop(['rating'],axis=1).iloc[train_idx]
            X_valid = self.data['train'].drop(['rating'],axis=1).iloc[valid_idx]
            y_train = self.data['train']['rating'][train_idx]
            y_valid = self.data['train']['rating'][valid_idx]

            cat = CatBoostRegressor( **self.best_params)
            cat.fit(X_train,y_train, eval_set=(X_valid, y_valid))
            self.catmodels[fold] = cat
            print(f'================================================================================\n\n')
            #print('epoch:', epoch, 'validation: rmse:', rmse_score)
        # print( **self.best_params)

    def predict(self):
        preds =np.zeros(self.data['test'].shape[0])
        for fold in range(8):
            a = self.catmodels[fold].predict(self.data['test']).transpose()
            preds += self.catmodels[fold].predict(self.data['test'])
        print(preds)
        return (preds/8).transpose()

class XGBModel:

    def __init__(self, args, data):
        super().__init__()

        self.criterion = RMSELoss()
        self.model =  XGBRegressor(n_estimators=2000, learning_rate=0.01, max_depth=5)
        # self.model =  XGBClassifier(n_estimators=2000, learning_rate=0.01, max_depth=5, num_feature=100)
        self.data = data

        # # direction : score 값을 최대 또는 최소로 하는 방향으로 지정
        # study = optuna.create_study(direction='minimize',sampler=TPESampler())

        # # n_trials : 시도 횟수 (미 입력시 Key interrupt가 있을 때까지 무한 반복)
        # study.optimize(lambda trial : objective(trial, self.data['X_train'], self.data['y_train'], 'XGB', 'reg'), n_trials=30)
        # # study.optimize(lambda trial : objective(trial, self.data['X_train'], self.data['y_train'], 'XGB', 'clf'), n_trials=30)
        # print('Best trial: score {},\nparams {}'.format(study.best_trial.value,study.best_trial.params))

        # # 하이퍼파라미터별 중요도를 확인할 수 있는 그래프
        # optuna.visualization.plot_param_importances(study)

        # # 하이퍼파라미터 최적화 과정을 확인
        # optuna.visualization.plot_optimization_history(study)

    def train(self):
        X_train, X_valid, y_train, y_valid = train_test_split(
                                                        self.data['train'].drop(['rating'], axis=1),
                                                        self.data['train']['rating'],
                                                        test_size=0.2,
                                                        random_state=42,
                                                        shuffle=True
                                                        )
        self.data['X_train'],  self.data['X_valid'],  self.data['y_train'],  self.data['y_valid'] = X_train, X_valid, y_train, y_valid

        self.model.fit(self.data['X_train'],self.data['y_train'])
        preds = self.model.predict(self.data['X_valid'])
        print('RMSE : ', rmse(self.data['y_valid'], preds)) # Regressor
        # print('RMSE : ', rmse(self.data['y_valid'], preds.squeeze(1))) # Classifier
        #print('epoch:', epoch, 'validation: rmse:', rmse_score)
        return

    def predict(self):
        preds = self.model.predict(self.data['test'])
        print(preds)
        return preds # Regressor
        # return preds.squeeze(1) # Classifier

class LGBMModel:

    def __init__(self, args, data):
        super().__init__()
        self.best_params = do_optuna( self.data,
                        model = self.my_model,
                        model_name =  args.MODEL,
                        early_stopping_rounds = 35,
                        verbose = 100) if args.DO_OPTUNA == True else params_dict[args.MODEL]
        self.criterion = RMSELoss()
        self.model =  LGBMRegressor(nthread=4,
                        n_estimators=1000,
                        learning_rate=0.02,
                        num_leaves=34,
                        colsample_bytree=0.94,
                        subsample=0.87,
                        max_depth=8,
                        reg_alpha=0.04,
                        reg_lambda=0.07,
                        min_split_gain=0.02,
                        min_child_weight=32,
                        silent=-1,
                        verbose=-1)
        self.data = data

    def train(self):
        X_train, X_valid, y_train, y_valid = train_test_split(
                                                        self.data['train'].drop(['rating'], axis=1),
                                                        self.data['train']['rating'],
                                                        test_size=0.2,
                                                        random_state=42,
                                                        shuffle=True
                                                        )
        self.data['X_train'],  self.data['X_valid'],  self.data['y_train'],  self.data['y_valid'] = X_train, X_valid, y_train, y_valid

        self.model.fit(self.data['X_train'],self.data['y_train'])
        preds = self.model.predict(self.data['X_valid'])
        print('RMSE : ', rmse(self.data['y_valid'], preds)) # Regressor
        # print('RMSE : ', rmse(self.data['y_valid'], preds.squeeze(1))) # Classifier
        #print('epoch:', epoch, 'validation: rmse:', rmse_score)
        return

    def predict(self):
        preds = self.model.predict(self.data['test'])
        print(preds)
        return preds # Regressor
        # return preds.squeeze(1) # Classifier