import json
import random
import re

from django.conf import settings
from django.db.models import Case
from django.db.models import F
from django.db.models import Value
from django.db.models import When
from django.db.models.functions import Concat

from db.raw_queries import get_monthly_products_by_main_category
from libs.aws.dynamodb import aws_dynamodb_etc_items
from libs.utils import get_image_ratio, request_ads
from models.keywords import Keyword
from models.products import SubCategory, MainCategory
from models.reviews import Review
from models.users import User
from resources.preprocess_category import get_category_id

class CategoryService:
    def get_product_categories(self, **kwargs):
        """
        제품 카테고리 리스트 ( 제품 검색 )
        """
        cursor = kwargs.get('cursor')
        limit = kwargs.get('limit')
        cursor = int(cursor or 1)
        offset = (cursor - 1) * limit

        query = kwargs.get('query')

        categories = SubCategory.objects.filter(is_display=True)

        if query:
            word_list = re.findall('([\w.]+)', query)

            ids = get_category_id(word_list)
            categories = categories.filter(id__in=ids)
            categories = categories.order_by(
                'name'
            ).distinct()

        categories = categories.annotate(
            sub_category_id=F('id'),
            sub_category_name=F('name'),
            main_category_id=F('main_category'),
            main_category_name=F('main_category__name'),
            main_category_is_new=F('main_category__is_new'),
            main_category_image=Case(
                When(main_category__file_name=None, then=None),
                default=Concat(Value(settings.CDN), 'main_category__file_dir', Value('/'), 'main_category__file_name')
            )
        )

        categories = categories[offset: offset + limit]

        return categories

    def get_all_product_categories(self, filter_format=False):
        """
        전제 제품 카테고리 리스트 ( 추전 제품 및 이달의 신제품 포함 )
        """
        categories = SubCategory.objects.filter(
            is_display=True,
            main_category__is_display=True
        )

        categories = categories.annotate(
            sub_category_id=F('id'),
            sub_category_name=F('name'),
            main_category_id=F('main_category'),
            main_category_name=F('main_category__name'),
            main_category_is_new=F('main_category__is_new'),
            main_category_image=Case(
                When(main_category__file_name=None, then=None),
                default=Concat(Value(settings.CDN), 'main_category__file_dir', Value('/'), 'main_category__file_name')
            )
        )

        categories = categories.order_by('main_category__seq')

        results = self._make_categories_response(categories.values(
            'sub_category_id', 'sub_category_name', 'is_new', 'seq',
            'main_category_id', 'main_category_name',
            'main_category_is_new', 'main_category_image'
        ), filter_format=filter_format)

        if not filter_format:
            # monthly
            for idx, category in enumerate(results):
                category['sub_categories'].sort(key=lambda x: x['seq'])

                products = get_monthly_products_by_main_category(category.get('id'))
                if len(products) > 0:
                    recommended_item = dict()
                    choice = random.choice(products)
                    if choice.get('banner_image'):
                        recommended_item['id'] = choice.get('id')
                        recommended_item['banner_image'] = choice.get('banner_image')
                        recommended_item['banner_image_720'] = choice.get('banner_image_720')
                        recommended_item['link_type'] = choice.get('link_type')
                        recommended_item['link_code'] = choice.get('link_code')

                        # 통합검색 인트로는 광고소재C가 있으면 링크 설정과 관계없이 광고링크로 연결됩니다.
                        recommended_item['is_custom'] = True
                        recommended_item['banner_ratio'] = get_image_ratio(
                            recommended_item['banner_image_720'], is_url=True
                        )
                        recommended_item['end_date'] = choice.get('end_date')
                        results[idx]['monthly'] = {
                            'type': 'banner',
                            'monthly_banner': recommended_item
                        }
                    else:
                        recommended_item['id'] = choice.get('id')
                        recommended_item['product_id'] = choice.get('product_id')
                        recommended_item['name'] = choice.get('product_name')
                        recommended_item['product_image'] = choice.get('product_image')
                        recommended_item['product_image_720'] = choice.get('product_image_720')

                        recommended_item['brand'] = dict()
                        recommended_item['brand']['brand_id'] = choice.get('brand_id')
                        recommended_item['brand']['name'] = choice.get('brand_name')

                        recommended_item['price'] = choice.get('price')
                        recommended_item['rating_avg'] = choice.get('rating_avg')
                        recommended_item['review_count'] = choice.get('review_count')
                        recommended_item['volume'] = choice.get('volume')
                        recommended_item['is_discontinue'] = choice.get('is_discontinue')

                        recommended_item['link_type'] = choice.get('link_type')
                        recommended_item['link_code'] = choice.get('link_code')
                        recommended_item['is_custom'] = False
                        recommended_item['banner_ratio'] = 0.249
                        recommended_item['end_date'] = choice.get('end_date')
                        results[idx]['monthly'] = {
                            'type': 'product',
                            'monthly_product': recommended_item
                        }

        return results

    def get_all_product_categories_by_dynamodb(self):
        results = aws_dynamodb_etc_items.get_categories()
        items = json.loads(results[0]['binary']['B'].decode())

        res = list()
        for item in items:
            # choice item (banner type c)
            if 'monthly' in item and item['monthly']:
                choice = random.choice(item['monthly'])

                if 'banner_image' in choice and choice['banner_image']:
                    item['monthly'] = {'type': 'banner', 'monthly_banner': choice}
                else:
                    item['monthly'] = {'type': 'product', 'monthly_product': choice}

            main = dict()
            main['id'] = item.get('id_first_category')
            main['name'] = item.get('first_category_text')
            main['image'] = item.get('first_category_image')
            main['is_new'] = True if item['is_new'] == "1" else False

            main['sub_categories'] = [{
                                          'id': second.get('id_second_category'),
                                          'name': second.get('second_category_text'),
                                          'is_new': True if second.get('is_new') == "1" else False
                                      } for second in item['second_category_list']]

            monthly = item.get('monthly')
            if monthly:
                main['monthly'] = dict()
                banner_type = monthly.get('type')
                main['monthly']['type'] = banner_type
                if banner_type == 'product':
                    product = monthly.get('monthly_product')
                    main['monthly']['monthly_product'] = dict()
                    main['monthly']['monthly_product']['id'] = product.get('id_monthly_product')
                    main['monthly']['monthly_product']['product_id'] = product.get('id_product')
                    main['monthly']['monthly_product']['name'] = product.get('product_title')
                    main['monthly']['monthly_product']['product_image'] = product.get('product_image')
                    main['monthly']['monthly_product']['product_image_720'] = product.get('product_thumbnail')
                    main['monthly']['monthly_product']['price'] = product.get('price')
                    main['monthly']['monthly_product']['volume'] = product.get('volume')
                    main['monthly']['monthly_product']['review_count'] = product.get('review_count')
                    main['monthly']['monthly_product']['rating_avg'] = product.get('rating_avg')
                    main['monthly']['monthly_product']['brand'] = dict()
                    main['monthly']['monthly_product']['brand']['brand_id'] = product.get('id_brand')
                    main['monthly']['monthly_product']['brand']['name'] = product.get('brand_title')

                    main['monthly']['monthly_product']['link_code'] = product.get('link_code')
                    main['monthly']['monthly_product']['link_type'] = product.get('link_type')
                    main['monthly']['monthly_product']['is_custom'] = product.get('is_category') == 'link'
                    main['monthly']['monthly_product']['banner_ratio'] = product.get('banner_ratio')
                    main['monthly']['monthly_product']['end_date'] = product.get('end_date')
                elif banner_type == 'banner':
                    banner = monthly.get('monthly_banner')
                    main['monthly']['monthly_banner'] = dict()
                    main['monthly']['monthly_banner']['id'] = banner.get('id_monthly_product')
                    main['monthly']['monthly_banner']['banner_image'] = banner.get('banner_image')
                    main['monthly']['monthly_banner']['banner_image_720'] = banner.get('banner_thumbnail')
                    main['monthly']['monthly_banner']['link_code'] = banner.get('link_code')
                    main['monthly']['monthly_banner']['link_type'] = banner.get('link_type')
                    main['monthly']['monthly_banner']['is_custom'] = banner.get('is_category') == 'link'
                    main['monthly']['monthly_banner']['banner_ratio'] = banner.get('banner_ratio')
                    main['monthly']['monthly_banner']['end_date'] = banner.get('end_date')

            res.append(main)

        return res

    def _make_categories_response(self, query_set, filter_format=True):
        """
        제품 카테고리 리스트를 응답 형태로 가공함
        """
        categories = list()
        if filter_format:
            categories.append({
                'id': 0,
                'name': '대분류 전체',
                'image': None,
                'is_new': False,
                'sub_categories': [
                    {'id': 0, 'name': '소분류 전체', 'seq': 0}
                ]
            })

        for row in query_set.iterator():
            b_main = True

            for category in categories:
                if row['main_category_id'] == category['id']:
                    new_sub = dict()
                    new_sub['id'] = row['sub_category_id']
                    new_sub['name'] = row['sub_category_name']
                    new_sub['seq'] = row['seq']
                    if not filter_format:
                        new_sub['is_new'] = row['is_new']

                    category['sub_categories'].append(new_sub)
                    b_main = False
                    break

            if b_main:
                new_main = dict()
                new_main['id'] = row['main_category_id']
                new_main['name'] = row['main_category_name']
                if not filter_format:
                    new_main['is_new'] = row['main_category_is_new']
                    new_main['image'] = row['main_category_image']

                new_main['sub_categories'] = list()
                if filter_format:
                    new_main['sub_categories'].append({
                        'id': 0,
                        'name': '소분류 전체',
                        'is_new': False,
                        'seq': 0
                    })

                new_sub = dict()
                new_sub['id'] = row['sub_category_id']
                new_sub['name'] = row['sub_category_name']
                new_sub['seq'] = row['seq']
                if not filter_format:
                    new_sub['is_new'] = row['is_new']

                new_main['sub_categories'].append(new_sub)
                categories.append(new_main)

        return categories

    def get_user_reviews_categories(self, user_id):
        """
        회원이 작성한 리뷰가 포함된 제품 카테고리 리스트
        리스트 필터의 데이터
        """
        query_set = Review.objects.filter(user_id=user_id, is_display=True, product__is_display=True)
        query_set = query_set.filter(
            product__categories__is_display=True,
            product__categories__main_category__is_display=True
        )
        query_set = query_set.annotate(
            sub_category_id=F('product__categories'),
            sub_category_name=F('product__categories__name'),
            seq=F('product__categories__seq'),
            main_category_id=F('product__categories__main_category'),
            main_category_name=F('product__categories__main_category__name')
        )

        query_set = query_set.order_by('product__categories__main_category__seq')
        query_set = query_set.distinct()

        results = self._make_categories_response(query_set.values(
            'sub_category_id', 'sub_category_name',
            'main_category_id', 'main_category_name', 'seq'
        ))

        for category in results:
            category['sub_categories'].sort(key=lambda x: x['seq'])

        return results

    def get_user_wishes_categories(self, user_id):
        """
        회원의 위시리스트의 제품이 포함된 제품 카테고리 리스트
        리스트 필터의 데이터
        """
        query_set = User.objects.get(id=user_id).wishes.visible()
        query_set = query_set.filter(
            categories__is_display=True,
            categories__main_category__is_display=True
        )
        query_set = query_set.annotate(
            sub_category_id=F('categories'),
            sub_category_name=F('categories__name'),
            seq=F('categories__seq'),
            main_category_id=F('categories__main_category'),
            main_category_name=F('categories__main_category__name')
        )
        query_set = query_set.order_by('categories__main_category__seq')
        query_set = query_set.distinct()

        results = self._make_categories_response(query_set.values(
            'sub_category_id', 'sub_category_name',
            'main_category_id', 'main_category_name', 'seq'
        ))

        for category in results:
            category['sub_categories'].sort(key=lambda x: x['seq'])

        return results

    def get_keywords_by_category_id(self, category_id):
        """
        키워드 리스트 ( sub_category id )
        """

        query_set = Keyword.objects.filter(
            subcategorykeywords__sub_category_id__exact=category_id
        )
        query_set = query_set.values(
            'id', 'name'
        ).distinct()

        return query_set

    def get_categoires_by_brand_id(self, brand_id):
        """
        특정 브랜드가 포함된 제품 카테고리 리스트
        """
        query_set = MainCategory.objects.filter(
            is_display=True,
            subcategory__is_display=True,
            subcategory__product__brand=brand_id
        ).prefetch_related(
            'subcategory_set'
        ).distinct()

        results = [
            {
                'id': 0,
                'name': "대분류 전체",
                'sub_categories': [
                    {
                        'id': 0,
                        'name': "소분류 전체"
                    }
                ]
            }
        ]
        for main in query_set:
            item = dict()
            item['id'] = main.id
            item['name'] = main.name
            item['sub_categories'] = [
                {
                    'id': 0,
                    'name': "소분류 전체"
                }
            ]
            for sub in main.subcategory_set.filter(is_display=True).all():
                sub_item = dict()
                sub_item['id'] = sub.id
                sub_item['name'] = sub.name
                item['sub_categories'].append(sub_item)

            results.append(item)

        return results

    def get_categories_by_store_id(self, store_id):
        """
        특정 스토어와 연관된 제품 카테고리 리스트
        """
        query_set = MainCategory.objects.filter(
            is_display=True,
            subcategory__is_display=True,
            subcategory__product__storesproducts__store=store_id
        ).prefetch_related(
            'subcategory_set'
        ).distinct()

        results = [
            {
                'id': 0,
                'name': "대분류 전체",
                'sub_categories': [
                    {
                        'id': 0,
                        'name': "소분류 전체"
                    }
                ]
            }
        ]
        for main in query_set:
            item = dict()
            item['id'] = main.id
            item['name'] = main.name
            item['sub_categories'] = [
                {
                    'id': 0,
                    'name': "소분류 전체"
                }
            ]
            for sub in main.subcategory_set.filter(is_display=True).all():
                sub_item = dict()
                sub_item['id'] = sub.id
                sub_item['name'] = sub.name
                item['sub_categories'].append(sub_item)

            results.append(item)

        return results

service = CategoryService()


#!/usr/bin/env python

'''Run clang-format on files in this repository

By default, runs on all files, but may pass specific files.
'''

from __future__ import absolute_import, division, print_function, unicode_literals

import argparse
import inspect
import fnmatch
import io
import os
import re
import stat
import sys
import platform
import time

from subprocess import check_call, check_output

if sys.version_info[0] < 3:
    # Python-2 check_output doesn't have encoding
    def check_output(*args, **kwargs):
        import subprocess
        kwargs.pop('encoding')
        return subprocess.check_output(*args, **kwargs)

THIS_FILE = os.path.normpath(os.path.abspath(inspect.getsourcefile(lambda: None)))
THIS_DIR = os.path.dirname(THIS_FILE)
# THIS_DIR = REPO_ROOT/.github
REPO_ROOT = os.path.dirname(THIS_DIR)

UPDATE_INTERVAL = .2

_last_update_len = 0
_on_update_line = False

def update_status(text):
    global _last_update_len
    global _on_update_line
    sys.stdout.write('\r')
    text_len = len(text)
    extra_chars = _last_update_len - text_len
    if extra_chars > 0:
        text += (' ' * extra_chars)
    sys.stdout.write(text)
    _last_update_len = text_len
    sys.stdout.flush()
    _on_update_line = True

def post_update_print(text):
    global _on_update_line
    if _on_update_line:
        print()
    print(text)
    _on_update_line = False

def regex_from_file(path, glob=False):
    with io.open(path, 'r') as f:
        patterns = f.read().splitlines()
    # ignore comment lines
    patterns = [x for x in patterns if x.strip() and not x.lstrip().startswith('#')]
    if glob:
        patterns = [fnmatch.translate(x) for x in patterns]
    regex = '({})'.format('|'.join(patterns))
    return re.compile(regex)

if platform.system() == "Windows" and sys.version_info >= (3, 6):
    import pathlib  # Python 3.6 is required for pathlib.Path
    def canonicalpath(path):
        path = os.path.abspath(os.path.realpath(os.path.normpath(os.path.normcase(path))))
        realpath = str(pathlib.Path(path).resolve())  # To get a properly cased path ie: from r'C:\WiNdOwS\SyStEm32\DeSkToP.iNi' get r'C:\Windows\System32\desktop.ini'
        if len(path) == len(realpath): # This is to avoid following symbolic links, there is still the possibility that they could be equal.
            path = realpath
        if os.path.isabs(path) and path[0].upper() != path[0]:
            path = path[0].upper() +path[1:] # path.capitalize()
        path = path.replace('\\', '/')
        return path
else:
    def canonicalpath(path):
        path = os.path.abspath(os.path.realpath(os.path.normpath(os.path.normcase(path))))
        return path.replace('\\', '/')

def run_clang_format(paths=(), verbose=False, commit=None):
    """Runs clang-format in-place on repo files

    Returns
    -------
    List[str]
        Files altered by clang-format
    """
    if not paths and not commit:
        paths = [REPO_ROOT]

    if commit:
        check_call(['git', 'checkout', commit], cwd=REPO_ROOT)
        text = check_output(
            ['git', 'diff-tree', '--no-commit-id', '--name-only', '-r',
             commit], cwd=REPO_ROOT, encoding=sys.stdout.encoding)
        commit_paths = text.splitlines()
        paths.extend(os.path.join(REPO_ROOT, p) for p in commit_paths)

    files = set()
    folders = set()

    include_path = os.path.join(REPO_ROOT, '.clang-format-include')
    include_regex = regex_from_file(include_path)

    ignore_path = os.path.join(REPO_ROOT, '.clang-format-ignore')
    ignore_regex = regex_from_file(ignore_path, glob=True)

    # tried to parse .gitignore with regex_from_file, but it has
    # too much special git syntax. Instead just using `git ls-files`
    # as a filter...
    git_files = check_output(['git', 'ls-files'], cwd=REPO_ROOT,
        encoding=sys.stdout.encoding)
    git_files = set(canonicalpath(x.strip()) for x in git_files.splitlines())

    def print_path(p):
        if p.startswith(REPO_ROOT):
                p = os.path.relpath(p, REPO_ROOT)
        return p

    def passes_filter(path):
        rel_path = os.path.relpath(path, REPO_ROOT)
        match_path = os.path.join('.', rel_path)
        # standardize on '/', because that's what's used in files,
        # and output by `git ls-files`
        match_path = match_path.replace('\\', '/')
        if not include_regex.search(match_path):
            return False
        if ignore_regex.pattern != '()' and ignore_regex.search(match_path):
            return False
        return True

    # parse the initial fed-in paths, which may be files or folders
    for path in paths:
        # Get the stat, so we only do one filesystem call, instead of
        # two for os.path.isfile() + os.path.isdir()
        try:
            st_mode = os.stat(path).st_mode
            if stat.S_ISDIR(st_mode):
                folders.add(path)
            elif stat.S_ISREG(st_mode):
                if canonicalpath(path) in git_files:
                    files.add(path)
        except Exception:
            print("Given path did not exist: {}".format(path))
            raise

    for folder in folders:
        # we already have list of potential files in git_files...
        # filter to ones in this folder
        folder = canonicalpath(folder) + '/'
        files.update(x for x in git_files if x.startswith(folder))

    # We apply filters even to fed-in paths... this is to aid
    # in use of globs on command line
    files = sorted(x for x in files if passes_filter(x))

    clang_format_executable = os.environ.get('CLANG_FORMAT_EXECUTABLE',
                                             'clang-format')
    if verbose:
        print("Running clang-format on {} files...".format(len(files)))
        last_update = time.time()

    def print_path(p):
        if p.startswith(REPO_ROOT):
                p = os.path.relpath(p, REPO_ROOT)
        return p

    altered = []
    for i, path in enumerate(files):
        if verbose:
            now = time.time()
            if now - last_update > UPDATE_INTERVAL:
                last_update = now
                update_status('File {}/{} ({:.1f}%) - {}'.format(
                            i + 1, len(files), (i + 1) / len(files) * 100,
                            print_path(path)))
        # Note - couldn't find a way to get clang-format to return whether
        # or not a file was altered with '-i' - so checking ourselves
        # Checking mtime is not foolproof, but is faster than reading file
        # and comparing, and probably good enough
        mtime_orig = os.path.getmtime(path)
        check_call([clang_format_executable, '-i', path])
        mtime_new = os.path.getmtime(path)
        if mtime_new != mtime_orig:
            post_update_print("File altered: {}".format(print_path(path)))
            altered.append(path)
    post_update_print("Done - altered {} files".format(len(altered)))
    return altered

def get_parser():
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('paths', nargs='*',
        help='Paths to run clang-format on; defaults to all files in repo')
    parser.add_argument('-v', '--verbose', action='store_true',
        help='Enable more output (ie, progress messages)')
    parser.add_argument('-c', '--commit',
        help='Git commit / revision / branch; will first check out that commit,'
            " then query it for it's list of affected files, to use as the files"
            ' to run clang-format on; if PATHS are also manually given, they are'
            ' appended')
    return parser

def main(raw_args=None):
    parser = get_parser()
    args = parser.parse_args(raw_args)
    try:
        altered = run_clang_format(paths=args.paths, verbose=args.verbose,
            commit=args.commit)
    except Exception:
        import traceback
        traceback.print_exc()
        return 1
    if altered:
        return 1
    return 0

if __name__ == '__main__':
    sys.exit(main())


import numpy as np

class Point:
    def __init__(self, i_x, i_y, i_h=0.0):
        self.x = i_x
        self.y = i_y
        self.h = i_h

    def distance_to(self, other):
        return np.sqrt((self.x - other.x)**2.0 + (self.y - other.y)**2.0)

class LineSegment:
    def __init__(self, i_a, i_b):
        self.a = i_a
        self.b = i_b

    def distance_to(self, point):
        if self.is_in_bounds(point):
            return self._distance_in_bounds(point)
        return min([point.distance_to(self.a), point.distance_to(self.b)])

    def _distance_in_bounds(self, point):
        a_b_x_diff = self.b.x - self.a.x
        a_b_y_diff = self.b.y - self.a.y
        left_numerator = a_b_x_diff * (self.a.y - point.y)
        right_numerator = (self.a.x - point.x) * a_b_y_diff
        numerator = abs(left_numerator - right_numerator)
        denominator = np.sqrt(a_b_x_diff**2.0 + a_b_y_diff**2.0)
        return numerator / denominator

    def is_in_bounds(self, point):
        if self.a.x == self.b.x:
            return is_between(point.y, self.a.y, self.b.y)
        if self.a.y == self.b.y:
            return is_between(point.x, self.a.x, self.b.x)
        slope = (self.b.y - self.a.y) / (self.b.x - self.a.x)
        inverse_slope = -1.0 / slope
        lower = inverse_slope * (point.x - self.a.x) + self.a.y
        upper = inverse_slope * (point.x - self.b.x) + self.b.y
        return is_between(point.y, lower, upper)

def is_between(value, left, right):
    return (value < right and value > left) or (value < left and value > right)

class Circle:
    def __init__(self, i_center, i_radius):
        self.center = i_center
        self.radius = i_radius

    def distance_to(self, point):
        return self.center.distance_to(point)

class Rectangle:
    def __init__(self, i_left_top, i_right_bot):
        self.left_top = i_left_top
        self.right_bot = i_right_bot

    def is_in_perimeter(self, point):
        return is_between(point.x, self.left_top.x, self.right_bot.x) and is_between(point.y, self.left_top.y, self.right_bot.y)

class LineSegmentLandmarker:
    def __init__(self, i_line_segments, i_exclusion_zone=Rectangle(Point(0, 0), Point(0, 0)), i_probability=1.0):
        self.line_segments = i_line_segments
        self.exclusion_zone = i_exclusion_zone
        self.probability = i_probability
        self.comp_probability = 1.0 - self.probability

    def correct_position(self, current_position, signal):
        if signal and not self.exclusion_zone.is_in_perimeter(current_position):
            self.line_segments.sort(
                key=lambda seg: seg.distance_to(current_position))
            closest_line_segment = self.line_segments[0]
            return self._weight(current_position, self._get_closest_point(current_position, closest_line_segment))
        return current_position

    def _get_closest_point(self, current_position, closest_line_segment):
        if closest_line_segment.is_in_bounds(current_position):
            if closest_line_segment.a.x == closest_line_segment.b.x:
                return Point(closest_line_segment.a.x, current_position.y, current_position.h)
            if closest_line_segment.a.y == closest_line_segment.b.y:
                return Point(current_position.x, closest_line_segment.a.y, current_position.h)
            slope = (closest_line_segment.b.y - closest_line_segment.a.y) / \
                (closest_line_segment.b.x - closest_line_segment.a.x)
            x = (slope**2.0 * closest_line_segment.a.x - slope * closest_line_segment.a.y +
                 current_position.x + slope * current_position.y) / (slope**2.0 + 1.0)
            y = slope * x - slope * closest_line_segment.a.x + closest_line_segment.a.y
            return Point(x, y)
        return closest_line_segment.a if closest_line_segment.a.distance_to(current_position) < closest_line_segment.b.distance_to(current_position) else closest_line_segment.b

    def _weight(self, current_position, proposed_position):
        return Point(current_position.x * self.comp_probability + proposed_position.x * self.probability, current_position.y * self.comp_probability + proposed_position.y * self.probability, current_position.h)

class GoalLandmarker:
    def __init__(self, i_goals, i_probability=1.0):
        self.goals = i_goals
        self.probability = i_probability
        self.comp_probability = 1.0 - self.probability

    def correct_position(self, current_position, signal):
        if signal:
            self.goals.sort(key=lambda goal: goal.distance_to(current_position))
            closest_goal = self.goals[0]
            return self._weight(current_position, self._get_closest_point(current_position, closest_goal))
        return current_position

    def _get_closest_point(self, current_position, closest_goal):
        x_diff = current_position.x - closest_goal.center.x
        y_diff = current_position.y - closest_goal.center.y
        x = closest_goal.center.x + closest_goal.radius * (x_diff / np.sqrt(x_diff**2.0 + y_diff**2.0))
        y = closest_goal.center.y + closest_goal.radius * (y_diff / np.sqrt(x_diff**2.0 + y_diff**2.0))
        h = np.arctan2(closest_goal.center.y - y, closest_goal.center.x - x)
        return Point(x, y, h)

    def _weight(self, current_position, proposed_position):
        return Point(current_position.x * self.comp_probability + proposed_position.x * self.probability, current_position.y * self.comp_probability + proposed_position.y * self.probability, current_position.h * self.comp_probability + proposed_position.h * self.probability)

a = Point(0.0, 0.0)
b = Point(10.0, 10.0)
c = Point(0.0, 10.0)
z = Point(-10.0, -5.0, 45.0)
exclude = Rectangle(Point(-9.0, -3.0), Point(-5.0, -7.0))
l1 = LineSegment(a, b)
l2 = LineSegment(a, c)
ls_landmarker = LineSegmentLandmarker([l1, l2], exclude, 0.9)
z = ls_landmarker.correct_position(z, False)
print(exclude.is_in_perimeter(z))
print(f"{z.x}, {z.y}, {z.h}")

p = Point(2.0, 2.0)
goal1 = Circle(Point(0.0, 0.0), 2.0)
goal2 = Circle(Point(5.0, 5.0), 1.0)
g_landmarker = GoalLandmarker([goal1, goal2])
p = g_landmarker.correct_position(p, True)
print(f"{p.x}, {p.y}, {p.h * 180.0 / 3.14159265}")


import time
import board

import neopixel

dot = neopixel.NeoPixel(board.NEOPIXEL, 1)

while True:
    print ("make it bubblegum!")
    dot.fill((255, 0, 255))
    time.sleep(1)
    print ("make it magenta!")
    dot.fill((255, 0, 102))
    time.sleep(1)


class Solution:
    def merge(self, intervals: List[List[int]]) -> List[List[int]]:
        merged = []
        intervals.sort()
        start = intervals[0][0]
        end = intervals[0][1]

        for i in range (1,len(intervals)):
            if intervals[i][0] <= end:
                end = max(intervals[i][1],end)

            else:
                merged.append([start,end])
                start = intervals[i][0]
                end = intervals[i][1]

        merged.append([start,end])

        return merged


import json
from random import choice

#  Config data
weekdays = [0, 1, 2, 3, 4]
weekends = [5, 6]

def add_pk():
    """Load to our json new pk (id)"""
    ll = []
    t = list(range(0, 455))
    for i in json.load(open('data1.json')):
        i['pk'] = choice(t)
        ll.append(i)
    json.dump(ll, open('data1.json', 'w'))

if __name__ == '__main__':
    add_pk()


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 14 14:24:30 2017

@author: ls
"""
from __future__ import print_function

import __builtin__
import socket
import sys
import threading
import traceback
import time

from termcolor import colored
from Src.Communication import pickler
from Src.Controller import controller as ctrlib

def print(*args, **kwargs):
    __builtin__.print(colored('Comm_Thread: ', 'red'), *args, **kwargs)

class CommunicationThread(threading.Thread):
    def __init__(self, cargo):
        """ """
        threading.Thread.__init__(self)
        self.cargo = cargo
        self.SOCK = None
        self.connection = None

        print('Starting server ...')
        self.SOCK = init_connection()

    def run(self):
        """ run the Communication """

        print('Waiting for connection')
        self.connection, client_adress = self.SOCK.accept()
        print('Sucessfull connected to ', client_adress)

        try:
            while self.cargo.state != 'EXIT':
                try:
                    self.get_tasks()
                except:
                    print('\n--caught exception! in Communictaion Thread--\n')
                    print("Unexpected error:\n", sys.exc_info()[0])
                    print(sys.exc_info()[1])
                    traceback.print_tb(sys.exc_info()[2])
                    print('\nBreaking the communication loop ...')
                    break
        finally:
            print('Exit the communication Thread ...')
            self.cargo.state = 'EXIT'
            graceful_exit(self.connection, self.SOCK)

        print('Communication Thread is done ...')

    def get_tasks(self):
        data_in_raw = self.connection.recv(4096)
        data_in_list = pickler.unpickle_data(data_in_raw)

        for data_in in data_in_list:
            if 'update' in data_in:
                self.send_back([self.cargo.rec, self.cargo.rec_u,
                                self.cargo.rec_r])

            if 'valve_meta_info' in data_in:
                valve_data = []
                for valve in self.cargo.valve:
                    valve_data.append(valve.name)
                PID_gains = []
                for c in self.cargo.controller:
                    PID_gains.append([c.Kp, c.Ti, c.Td])
                self.send_back([valve_data,
                                self.cargo.maxpressure,
                                self.cargo.maxctrout,
                                self.cargo.sampling_time,
                                PID_gains,
                                self.cargo.wcomm.pattern])

            if 'dvalve_meta_info' in data_in:
                dvalve_data = []
                for dvalve in self.cargo.dvalve:
                    dvalve_data.append(dvalve.name)
                self.send_back(dvalve_data)

            if 'change_state' in data_in:
                candidates = ['PAUSE', 'REFERENCE_TRACKING', 'EXIT',
                              'USER_CONTROL', 'USER_REFERENCE']
                new_state = None
                for candidate in candidates:
                    if candidate in data_in:
                        new_state = candidate
                        print('recieved task to change state to:', new_state)
                if new_state:
                    self.cargo.state = new_state
                    while not self.cargo.actual_state == new_state:
                        time.sleep(self.cargo.sampling_time)
                self.send_back(new_state)

            if 'set_valve' in data_in:
                valve_data = data_in[1]
                for key in valve_data:
                    self.cargo.pwm_task[key] = valve_data[key]

            if 'set_ref' in data_in:
                ref_data = data_in[1]
                for key in ref_data:
                    self.cargo.ref_task[key] = ref_data[key]

            if 'set_dvalve' in data_in:
                dvalve_data = data_in[1]
                for key in dvalve_data:
                    self.cargo.dvalve_task[key] = dvalve_data[key]

            if 'set_pidgain' in data_in:
                idx = data_in[1]
                gain_data = data_in[2]
                if isinstance(self.cargo.controller[idx],
                              ctrlib.PidController):
                    self.cargo.controller[idx].set_gain(gain_data)
                else:
                    raise NotImplementedError(
                        "Controller", self.cargo.controller[idx],
                        "doesn't support gain setting at runtime")
                c = self.cargo.controller[idx]
                gain = [c.Kp, c.Ti, c.Td]
                self.send_back(gain)

            if 'set_maxpressure' in data_in:
                maxpressure = data_in[1]
                if 10 > maxpressure > 0:
                    self.cargo.maxpressure = maxpressure
                    for sensor in self.cargo.sens:
                        sensor.set_maxpressure(maxpressure)
                self.send_back(self.cargo.maxpressure)

            if 'set_maxctrout' in data_in:
                maxctrout = data_in[1]
                if 1. > maxctrout > 0.:
                    self.cargo.maxctrout = maxctrout
                    for ctr in self.cargo.controller:
                        ctr.set_maxoutput(maxctrout)
                self.send_back(self.cargo.maxctrout)

            if 'set_tsampling' in data_in:
                tsampling = data_in[1]
                if 1. > tsampling > 0.:
                    self.cargo.sampling_time = tsampling
                self.send_back(self.cargo.sampling_time)

            if 'set_pattern' in data_in:
                pattern = data_in[1]
                self.cargo.wcomm.pattern = pattern
                self.send_back(self.cargo.wcomm.pattern)

            if 'set_walking' in data_in:
                state = data_in[1]
                self.cargo.wcomm.confirm = state
                self.send_back(self.cargo.wcomm.confirm)

    def send_back(self, data_out):
        data_out_raw = pickler.pickle_data(data_out)
        self.connection.sendall(data_out_raw)

def init_connection():
    HOST = None               # Symbolic name meaning all available interfaces
    PORT = 10000              # Arbitrary non-privileged port
    s = None
    for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC,
                                  socket.SOCK_STREAM, 0, socket.AI_PASSIVE):
        af, socktype, proto, canonname, sa = res
        try:
            s = socket.socket(af, socktype, proto)
            print('socket created with address family: ', af)
        except socket.error as msg:
            print('error while creating socket: ', msg)
            s = None
            continue
        try:
            s.bind(sa)
            s.listen(100)
            print('socket bounded with server address: ', sa)
        except socket.error as msg:
            s.close()
            s = None
            print('Error while sock.bind: ', msg)
            continue
        break
    if s is None:
        print('could not open socket')
        sys.exit(1)
    SOCK = s
    return SOCK

def graceful_exit(connection, SOCK):
    """Shutdown"""
    print('Closing Socket ...')
    SOCK.close()
    print('Closing Connection ...')
    connection.close()


__all__ = ["EulerBernoulliBeamComponent"]

import openmdao.api as om, matplotlib.pyplot as plt
import os

class EulerBernoulliBeamComponent(om.ExplicitComponent):
    def initialize(self):
        self.options.declare("beam_problem", types=object)
        self.options.declare("write_vtk", types=bool)

        self._iterations = 0
        self._history = {}
        self._history["mass"] = []
        self._history["stress"] = []

        # make a results folder
        self.vtk_folder = os.path.join(os.getcwd(), "vtk")
        if not (os.path.exists(self.vtk_folder)):
            os.mkdir(self.vtk_folder)

    def setup(self):
        """declare the inputs and outputs of the component"""
        beam_problem = self.options["beam_problem"]

        # register each of the thickness variables
        for key in beam_problem.var_dict:
            self.add_input(key, beam_problem.var_dict[key])

        # two different functions
        self.add_output("mass")
        self.add_output("stress")

    def compute(self, inputs, outputs):
        """compute the objective functions"""
        beam_problem = self.options["beam_problem"]
        write_vtk = self.options["write_vtk"]

        beam_problem.set_variables(inputs)
        beam_problem.solve_forward()

        mass = beam_problem.mass.real
        stress = beam_problem.stress.real

        outputs["mass"] = mass
        outputs["stress"] = stress

        # update function history
        self._history["mass"].append(mass)
        self._history["stress"].append(stress)
        self._iterations += 1

        # write to a vtk if desired
        beam_problem.write_vtk(path=self.vtk_folder, index=self._iterations)

        self._plot_history()

    def setup_partials(self):
        beam_problem = self.options["beam_problem"]
        for func in ["mass", "stress"]:
            for var in beam_problem.var_dict:
                self.declare_partials(func, var)

    def compute_partials(self, inputs, partials):
        beam_problem = self.options["beam_problem"]

        beam_problem.solve_adjoint()

        for var in beam_problem.var_dict:
            partials["mass", var] = beam_problem.dmass_dh[var].real
            partials["stress", var] = beam_problem.dstress_dh[var].real

    def _plot_history(self):
        plt.figure("")
        num_iterations = self._iterations
        iterations = [_ for _ in range(num_iterations)]
        plt.plot(iterations, self._history["mass"], label="mass")
        plt.plot(iterations, self._history["stress"], label="stress")
        plt.legend()
        plt.savefig("beam_opt_history.png")
        plt.close("all")


"""Implementation of Sum of nth Term Kata on Code Wars."""
from __future__ import division

def series_sum(n):
    """Return nth value in series."""
    if n <= 1:
        return "%.2f" % n
    count = 1
    for num in range(n - 1):
        n -= 1
        count += 1 / (3 * n + 1)
    return "%.2f" % count


from concurrent.futures import as_completed
from dataclasses import dataclass,field
from http import HTTPStatus
from logging import exception
from typing import Any,List
import requests
from requests_futures.sessions import FuturesSession
from requests.adapters import HTTPAdapter
from urllib3 import Retry

import time

'''
Mapping of boss IDs used in the logs to various other names for arcDPS outputs.

The key for the dictionary is the "Short Name" which is the abbreviation used for the singular encounter.

PrettyName = Displayed name of the encounter

FolderName = The arcDPS folder that holds the associated encounter.
             Note that multiple IDs may be in the same encounter.

IDs        = These are the ARC
'''
targetIdMap = {
    ###### Raids
    #### Wing 1
    'vg': {
        'PrettyName':  "Vale Guardian",
        'FolderNames': ["Vale Guardian"],
        'IDs':         [15438]
    },
    'gors': {
        'PrettyName':  "Gorseval",
        'FolderNames': ["Gorseval the Multifarious"],
        'IDs':         [15429]
    },
    'sab': {
        'PrettyName':  "Sabetha the Saboteur",
        'FolderNames': ["Sabetha the Saboteur"],
        'IDs':         [15375]
    },

    #### Wing 2
    'sloth': {
        'PrettyName':  "Slothasor",
        'FolderNames': ["Slothasor"],
        'IDs':         [16123]
    },
    'trio': {
        'PrettyName':  "Bandit Trio",
        'FolderNames': ["Berg", "Zane", "Narella"],
        'IDs':         [16088, 16137, 16125]
        # 0: Berg
        # 1: Zane
        # 2: Narella
    },
    'matt': {
        'PrettyName':  "Matthias Gabrel",
        'FolderNames': ["Matthias Gabrel"],
        'IDs':         [16115]
    },

    #### Wing 3
    'esc': {
        'PrettyName':  "Escort",
        'FolderNames': ["McLeod the Silent"],
        'IDs':         [16253]
    },
    'kc': {
        'PrettyName':  "Keep Construct",
        'FolderNames': ["Keep Construct"],
        'IDs':         [16235]
    },
    'tc': {
        'PrettyName':  "Twisted Castle",
        'FolderNames': ["Haunting Statue"],
        'IDs':         [16247]
    },
    'xera': {
        'PrettyName':  "Xera",
        'FolderNames': ["Xera"],
        'IDs':         [16246]
    },

    #### Wing 4
    'cairn': {
        'PrettyName':  "Cairn the Indomitable",
        'FolderNames': ["Cairn the Indomitable"],
        'IDs':         [17194]
    },
    'mo': {
        'PrettyName':  "Mursaat Overseer",
        'FolderNames': ["Mursaat Overseer"],
        'IDs':         [17172]
    },
    'sam': {
        'PrettyName':  "Samarog",
        'FolderNames': ["Samarog"],
        'IDs':         [17188]
    },
    'dei': {
        'PrettyName':  "Deimos",
        'FolderNames': ["Deimos"],
        'IDs':         [17154]
    },

    #### Wing 5
    'sh': {
        'PrettyName':  "Soulless Horror",
        'FolderNames': ["Soulless Horror"],
        'IDs':         [19767]
    },
    'rr': {
        'PrettyName': "Desmina Escort",
        'FolderNames': ["Desmina"],
        'IDs':         [19828]
    },
    'bk': {
        'PrettyName':  "Broken King",
        'FolderNames': ["Broken King"],
        'IDs':         [19691]
    },
    'se': {
        'PrettyName':  "Soul Eater",
        'FolderNames': ["Eater of Souls"],
        'IDs':         [19536]
    },
    'eyes': {
        'PrettyName':  "Eyes",
        'FolderNames': ["Eye of Judgment", "Eye of Fate"],
        'IDs':         [19651, 19844]
        # 0: Eye of Judgement
        # 1: Eye of Fate
    },
    'dhuum': {
        'PrettyName':  "Dhuum",
        'FolderNames': ["Dhuum"],
        'IDs':         [19450]
    },

    #### Wing 6
    'ca': {
        'PrettyName':  "Conjured Amalgamate",
        'FolderNames': ["Conjured Amalgamate"],
        'IDs':         [43974]
    },
    'twins': {
        'PrettyName':  "Twin Largos",
        'FolderNames': ["Nikare", "Kenut"],
        'IDs':         [21105, 21089]
    },
    'qadim': {
        'PrettyName':  "Qadim",
        'ShortName':  "qadim",
        'FolderNames': ["Qadim"],
        'IDs':         [20934, 21285, 21073, 21183, 20997]
        # 0: Qadim
        # 1: Hydra
        # 2: Destroyer
        # 3: Patriarch
        # 4: Matriarch
    },

    #### Wing 7
    'adina': {
        'PrettyName':  "Cardinal Adina",
        'FolderNames': ["Cardinal Adina"],
        'IDs':         [22006]
    },
    'sabir': {
        'PrettyName':  "Cardinal Sabir",
        'FolderNames': ["Cardinal Sabir"],
        'IDs':         [21964]
    },
    'qpeer': {
        'PrettyName':  "Qadim the Peerless",
        'FolderNames': ["Qadim the Peerless"],
        'IDs':         [22000]
    },

    ###### Fractals
    #### 98 CM
    'mama': {
        'PrettyName':  "M A M A",
        'FolderNames': ["MAMA"],
        'IDs':         [17021]
    },
    'siax': {
        'PrettyName':  "Siax the Corrupted",
        'FolderNames': ["Nightmare Oratuss"],
        'IDs':         [17028]
    },
    'enso': {
        'PrettyName':  "Ensolyss of the Endless Torment",
        'FolderNames': ["Ensolyss of the Endless Torment"],
        'IDs':         [16948]
    },

    #### 99 CM
    'skor': {
        'PrettyName': "Skorvald the Shattered",
        'FolderNames': ["Skorvald the Shattered"],
        'IDs':         [17632]
    },
    'arriv': {
        'PrettyName':  "Artsariiv",
        'FolderNames': ["Artsariiv"],
        'IDs':         [17949]
    },
    'arkk': {
        'PrettyName':  "Arkk",
        'FolderNames': ["Arkk"],
        'IDs':         [17759]
    },

    #### 100 CM
    'ai': {
        'PrettyName':  "Ai, Keeper of the Peak",
        'FolderNames': ["Sorrowful Spellcaster"],
        'IDs':         [23254, 20497]
    },

    ##### Strikes
    #### Icebrood Saga
    'ice': {
        'PrettyName':  "Icebrood Construct",
        'FolderNames': ["Icebrood Construct"],
        'IDs':         [22154]
    },
    'falln': {
        'PrettyName':  "The Voice and The Claw",
        'FolderNames': ["Voice of the Fallen", "Claw of the Fallen"],
        'IDs':         [22343, 22481, 22315]
        # 0: Voice of the Fallen
        # 1: Claw of the Fallen
        # 2: Voice and Claw
    },
    'frae': {
        'PrettyName':  "Fraenir of Jormag",
        'FolderNames': ["Fraenir of Jormag"],
        'IDs':         [22492, 22436]
        # 0: First phase
        # 1: Icebrood Construct Phase
    },
    'bone': {
        'PrettyName':  "Boneskinner",
        'FolderNames': ["Boneskinner"],
        'IDs':         [22521]
    },
    'whisp': {
        'PrettyName':  "Whisper of Jormag",
        'FolderNames': ["Whisper of Jormag"],
        'IDs':         [22711]
    },
    'varia': {
        'PrettyName':  "Cold War",
        'FolderNames': ["Varinia Stormsounder"],
        'IDs':         [22836]
    },

    #### End of Dragons
    'trin': {                                # Phase 1
        'PrettyName':  "Captain Mai Trin",
        'FolderNames': ["Captain Mai Trin"],
        'IDs':         [24033, 24768, 25247]
        # 0: Phase 1 - Mai Trin
        # 1: Phase 2 - Scarlet Briar, Normal Mode
        # 2: Phase 2 - Scarlet Briar, Challenge Mode
    },
    'ankka': {
        'PrettyName':  "Ankka",
        'FolderNames': ["Ankka"],
        'IDs':         [23957]
    },
    'li': {
        'PrettyName':  "Minister Li",
        'FolderNames': ["Minister Li"],
        'IDs':         [24485, 24266, 23612, 25259, 24660, 25271, 24261, 25236, 23618, 25242, 24254, 25280]
        # 0:  Minister Li - Normal Mode
        # 1:  Minister Li - Challenge Mode
        # 2:  Sniper - Normal Mode
        # 3:  Sniper - Challenge Mode
        # 4:  Mech Rider - Normal Mode
        # 5:  Mech Rider - Challenge Mode
        # 6:  Enforcer - Normal Mode
        # 7:  Enforcer - Chellenge Mode
        # 8:  Ritualist - Normal Mode
        # 9:  Ritualist - Challenge Mode
        # 10: Mindblade - Normal Mode
        # 11: Mindeblade - Challenge Mode
    },
    'void': {
        'PrettyName':  "Dragon Void",
        'FolderNames': ["The Dragonvoid"],
        'IDs':         [43488, 1378, 24375]
        # 0: Dragon Void 1
        # 1: Dragon Void 2
        # 2: Void Amalgamate 1
    },
    'olc': {
        'PrettyName': "Old Lions Court",
        'FolderNames': ["Prototype Vermilion"],
        'IDs': [25413, 25415, 25419, 25414, 25416, 25423]
        # 0: Prototype Vermilion
        # 1: Prototype Arsenite
        # 2: Prototype Indigo
        # 3: Prototype Vermilion - Challenge Mode
        # 4: Prototpye Arsenite - Challenge Mode
        # 5: Prototype Indigo - Challenge Mode
    },

    # Holiday Missions
    'frezi': {
        'PrettyName': "Freezie",
        'FolderNames': ["Freezie"],
        'IDs':         [21333]
    }
}

'''
Buff ID of Emboldened
'''
emboldenedID = 68087

@dataclass
class dpsReportObjEtvc():
    version: str = ''
    bossId: int = -1

@dataclass
class dpsReportObjPlayer():
    displayName: str
    charName: str
    profession: int
    eliteSpec: int

@dataclass
class dpsReportObjEncounter():
    uniqueId: str = ''
    success: bool = False
    duration: int = -1
    compDps: int = -1
    numberOfPlayers: int = -1
    numberOfGroups: int = -1
    bossId: int = -1
    boss: str = ''
    isCm: bool = False
    gw2Build: int = -1
    jsonAvailable: bool = False

    # This field isn't in the API, but is placed here for convenience and efficiency. It should be used
    # to store the resulting json (IE: the raw EliteInsights output) for this encounter if fetched
    json: dict = None

    # This field isn't in the API, but placed here for convenience. The duration in the encounter object
    # super off, so you typically need to parse the actual JSON to determine the proper time. In the case
    # that this gets filled from another method, this allows us to shortcut needing to load a JSON
    accurateDuration: int = None

@dataclass
class dpsReportObj():
    id: str = None
    permalink: str = ''
    uploadTime: int = -1
    encounterTime: int = -1
    generator: str = ''
    generatorId: int = -1
    generatorVersion: int = -1
    language: str = ''
    languageId: int = -1
    etvc: dpsReportObjEtvc = field(default_factory=dpsReportObjEtvc)
    players: List[dpsReportObjPlayer] = field(default_factory=list)
    encounter: dpsReportObjEncounter = field(default_factory=dpsReportObjEncounter)

    def __post_init__(self):
        if (self.id is None):
            # Remove the first part of the URL
            idStr = self.permalink.removeprefix('https://dps.report/')

            suffixLoc = idStr.rfind('_')
            self.id = idStr[:suffixLoc]

class dpsReport():
    def __init__(self, token:str=None):
        self.baseUrl = 'https://dps.report/'

        # User Settings
        self.token = token

        # This sets the maximum retries the underlying requests library will attempt for any error
        self.maxRetries = 3

        # From others testing it seems that 4 is around what the dpsReport servers will take safely
        self.maxThreads = 1

        self.retry = Retry(total=5,
                           backoff_factor=10,
                           respect_retry_after_header=True,
                           status_forcelist=[HTTPStatus.REQUEST_TIMEOUT,        # 408
                                              HTTPStatus.CONFLICT,              # 409
                                              HTTPStatus.TOO_MANY_REQUESTS,     # 429
                                              HTTPStatus.INTERNAL_SERVER_ERROR, # 500
                                              HTTPStatus.BAD_GATEWAY,           # 502
                                              HTTPStatus.SERVICE_UNAVAILABLE,   # 503
                                              HTTPStatus.GATEWAY_TIMEOUT])      # 504
        # Create a requests session since the dpsReport server occasionally fails, especially if we pack
        # too many requests in a row. This allows us to automatically retry with the library
        self.session = FuturesSession(max_workers=self.maxThreads)
        self.session.mount('https://', adapter=HTTPAdapter(max_retries=self.retry))

    def jsonToObject(self, json):
        ##### Parse Players
        playerObjs = []
        for player in json['players'].values():
            p = dpsReportObjPlayer(
                displayName = player['display_name'],
                charName = player['character_name'],
                profession = player['profession'],
                eliteSpec = player['elite_spec']
            )
            playerObjs.append(p)

        ##### Parse ETVC
        etvcObj = dpsReportObjEtvc(
            version = json['evtc']['version'],
            bossId  = json['evtc']['bossId']
        )

        ##### Parse Encounter

        # isCm parameter can be null, so set it to a valid boolean value
        if (json['encounter']['isCm'] is None):
            isCm = False
        else:
            isCm = json['encounter']['isCm']

        encounterObj = dpsReportObjEncounter(
            uniqueId = json['encounter']['uniqueId'],
            success = json['encounter']['success'],
            duration = json['encounter']['duration'],
            compDps = json['encounter']['compDps'],
            numberOfPlayers = json['encounter']['numberOfPlayers'],
            numberOfGroups = json['encounter']['numberOfGroups'],
            bossId = json['encounter']['bossId'],
            boss = json['encounter']['boss'],
            isCm = isCm,
            gw2Build = json['encounter']['gw2Build'],
            jsonAvailable = json['encounter']['jsonAvailable']
        )

        ##### Parse Main Object
        rtnObj = dpsReportObj(
            id = json['id'],
            permalink = json['permalink'],
            uploadTime = json['uploadTime'],
            encounterTime = json['encounterTime'],
            generator = json['generator'],
            generatorId = json['generatorId'],
            generatorVersion = json['generatorVersion'],
            language = json['language'],
            languageId = json['languageId'],
            etvc = etvcObj,
            players = playerObjs,
            encounter = encounterObj
        )

        return rtnObj

    def uploadLogs(self, logs:list[str]) -> list[tuple[str, dpsReportObj]]:
        """ Uploads a log. Optionally attaches a userToken to it for tracking
        """

        # dps.report recommends we always set this so the response is JSON formatted
        params = {'json': 1}

        # Attach the userToken if provided
        if (self.token is not None):
            params['userToken'] = self.token

        # Rate trackers
        startTime = time.perf_counter()

        # Response List
        futures = []

        # Open logfile and transmit it
        for l in logs:
            logFile = open(l, mode='rb')
            future = self.session.post(self.baseUrl + 'uploadContent', params=params, files={'file': logFile})
            future.origFile = l
            future.logFile = logFile
            futures.append(future)
            print('Queued {:s}'.format(l))

        # Collect the finished logs as we get them
        uploadedLogs = []
        for future in as_completed(futures):
            r = future.result()
            print('Finished {:s}'.format(future.origFile))

            # Close the file handle
            future.logFile.close()

            # Check what the error was
            if (r.status_code != requests.codes['ok']):
                print('Log {:s} got an error code {}'.format(future.origFile, r.status_code))
                uploadedLogs.append((future.origFile, None))
                continue

            uploadedLogs.append((future.origFile, self.jsonToObject(r.json())))

        endTime = time.perf_counter()
        print('Upload Total Time: {}'.format(endTime - startTime))

        return uploadedLogs

    def getUploadMetaData(self, identifier:str, isId:bool=False) -> dpsReportObj:
        """ Gets a previous encounter's meta data
            The identifier can either be the ID or the permalink, both are fairly similar.
            By default, the function assumes the identifier is the permalink. To treat it
            as an ID, you must set isId to true.
        """
        # Select the right identifier
        if (isId):
            params = {'id': identifier}
        else:
            params = {'permalink': identifier}

        # Get the data
        retryCnt = 0
        while (retryCnt < self.maxRetries):
            # Queue the Request
            future = self.session.get(self.baseUrl + 'getUploadMetadata', params=params)

            # Wait for response
            r = future.result()

            # Make sure we were successful
            r.raise_for_status()

            # Sometimes the JSON gets corrupted, in which case we should try again
            try:
                respJson = r.json()
            except:
                retryCnt+=1
                continue

            return self.jsonToObject(respJson)

    def getUploadMetaDatas(self, identifiers:list[str], isId:bool=False) -> list[dpsReportObj]:
        """ Gets a previous encounter's meta data. Similar to getUploadMetaData, but
            takes a list of logs to grab instead of a single one.
            The identifiers can either be the ID or the permalink, both are fairly similar.
            By default, the function assumes the identifier is the permalink. To treat it
            as an ID, you must set idId to true. All identifiers must be the same type.
        """

        startTime = time.perf_counter()

        # Loop until the requestsList is empty. Ideally the list gets emptied the first time around,
        # but incase the JSON gets corrupted, we will try again
        requestsList = identifiers
        resultsList = []
        while (requestsList):
            # Response List
            futures = []

            # Queue up the requests
            for request in requestsList:
                # Select the right identifier
                if (isId):
                    params = {'id': request}
                else:
                    params = {'permalink': request}

                future = self.session.get(self.baseUrl + 'getUploadMetadata', params=params)
                future.id = request
                futures.append(future)
                print('Queued {:s}'.format(request))

            # Clear the request list
            requestsList = []

            # Collect the completed tasks
            for future in as_completed(futures):
                r = future.result()

                try:
                    respJson = r.json()
                except:
                    requestsList.append(future.id)
                    print('JSON malformed, trying again for {:s}'.format(future.id))
                    continue

                resultsList.append(self.jsonToObject(respJson))
                print('Finished {:s}'.format(future.id))

        endTime = time.perf_counter()
        print('Fetch Metadata Total Time: {}'.format(endTime - startTime))
        return resultsList

    def getJson(self, id:str=None, link:str=None) -> dict:
        """ Gets a previous encounter's raw data
        """
        # Make sure we got at least one identifier
        if (id is not None):
            params = {'id': id}
        elif (link is not None):
            params = {'permalink': link}
        else:
            raise ValueError('Must pass either ID or Permalink to lookup metadata')

        # Get the data
        retryCnt = 0
        while (retryCnt < self.maxRetries):
            # Queue Request
            future = self.session.get(self.baseUrl + 'getJson', params=params)

            # Wait for response
            r = future.result()

            # Make sure we were successful
            r.raise_for_status()

            # Sometimes the JSON gets corrupted, in which case we should try again
            try:
                respJson = r.json()
            except:
                retryCnt+=1
                continue

            # This is Elite Insights output, so we just pass this as it
            return respJson

    def getJsons(self, logs:list[dpsReportObj]):
        """ Given a list of dpsReportObjs, fill in their JSON field with the EI raw JSON
        """
        startTime = time.perf_counter()

        # Loop until the requestsList is empty. Ideally the list gets emptied the first time around,
        # but incase the JSON gets corrupted, we will try again
        requestsList = logs
        while (requestsList):
            # Response List
            futures = []

            # Queue up the requests
            for request in requestsList:
                params = {'permalink': request.permalink}
                future = self.session.get(self.baseUrl + 'getJson', params=params)
                future.obj = request
                futures.append(future)
                print('Queued {:s}'.format(request.permalink))

            # Clear the request list
            requestsList = []

            # Collect the completed tasks
            for future in as_completed(futures):
                r = future.result()

                try:
                    respJson = r.json()
                    future.obj.encounter.json = respJson
                    print('Finished {:s}'.format(future.obj.permalink))
                except:
                    requestsList.append(future.obj)
                    print('JSON malformed, trying again for {:s}'.format(future.obj.permalink))

        endTime = time.perf_counter()
        print('Fetch JSON Total Time: {}'.format(endTime - startTime))

    def getUploads(self, page:int=1) ->list[dpsReportObj]:
        """ Returns previous logs
        """
        # Queue the request
        params = {'userToken':self.token, 'page':page}
        future = self.session.get(self.baseUrl + 'getUploads', params=params)

        # Wait for a response
        r = future.result()

        # Make sure we were successful
        r.raise_for_status()

        # The returned value is an array of results, so we need to break each one up
        data = r.json()

        rtnObjs = []
        for encounter in data['uploads']:
            rtnObjs.append(self.jsonToObject(encounter))

        return rtnObjs

    def getUserToken(self) -> str:
        """ Gets a User Token from DPS.report. Since this generates uniquely if you don't
            have a cookie, which we don't, this will always return a new token
        """
        # Queue the request
        future = self.session.get(self.baseUrl + 'getUserToken')

        # Wait for a response
        r = future.result()

        # Make sure we were successful
        r.raise_for_status()

        # Store token in self
        self.token = r.text

        return(r.text)

class dpsReportIds():
    '''
    A static class of utility functions to help with look up into the mapping table
    '''

    @staticmethod
    def folderNameToShortName(folderName:str) -> str:
        ''' Given a folder name, look up the corresponding short name.

            Will raise a KeyError is the folder name doesn't match
        '''

        # Iterate through all IDs in the map to find all folder name matches
        for (shortName, values) in targetIdMap.items():
            if (folderName in values['FolderNames']):
                return shortName
        else:
            raise KeyError('Folder Name {:s} does not match any known IDs'.format(folderName))

    @staticmethod
    def idToShortName(id:int) -> str:
        ''' Given a boss ID, will look up the shor name it is a part of.

            Will raise a KeyError if the ID isn't associated with any short name
        '''

        for (shortName, values) in targetIdMap.items():
            for i in values['IDs']:
                if (id == i):
                    return shortName
        else:
            raise KeyError('ID {:d} does not match any known IDs'.format(id))

    @staticmethod
    def shortNameToIds(shortName:str) -> List[int]:
        ''' Given a short name, look up the corresponding boss IDs. Since some folders contain multiple IDs,
            this function returns a list of IDs

            Will raise a KeyError if the short name doesn't match
        '''

        if (shortName in targetIdMap.keys()):
            return targetIdMap[shortName]['IDs']
        else:
            raise KeyError('Short Name {:s} not known'.format(shortName))

    @staticmethod
    def shortNameToPrettyName(shortName:str) -> str:
        ''' Given a short name, look up the pretty name. There should only be one matching pretty name to short
            name.

            Will raise a KeyError if the short name doesn't match
        '''

        if (shortName in targetIdMap.keys()):
            return targetIdMap[shortName]['PrettyName']
        else:
            raise KeyError('Short Name {:s} not known'.format(shortName))


#!/usr/bin/env python3
# --------------
# Load modules
# --------------
from bs4 import BeautifulSoup
from collections import defaultdict
from random import choice
from time import sleep
import urllib.request
import csv
import logging

# --------------------------------
# Scraping and parsing functions
# --------------------------------
def get_url(url):
    user_agents = [
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25',
      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/600.1.17 (KHTML, like Gecko) Version/7.1 Safari/537.85.10',
      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.104 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
    ]

    agent = choice(user_agents)
    wait = choice(range(10, 17))

    response = urllib.request.Request(url, headers={'User-Agent': agent})
    handler = urllib.request.urlopen(response).read()

    sleep(wait)

    return(handler)

def parse_chapter(url):
    print("Parsing {0}".format(url))

    try:
        soup = BeautifulSoup(get_url(url))
    except:
        logging.warning("Error with {0}".format(url))
        print("Error. Moving on.")
        return(None)

    content_section = soup.select('.ContentCss')
    content_clean = [x.text.replace('\u3000', '') for x in content_section][0]

    content = {}
    content['content'] = content_clean.split('\n')
    content['url'] = url

    split_url = url.split('/')[-1].replace('.html', '').split('_')
    content['book'] = ' '.join(split_url[0:-1])
    content['chapter'] = split_url[-1]

    return(content)

def convert_to_long(chapter):
    long_results = defaultdict(dict)
    entry_num = -1

    for x in chapter['content']:
        entry_num += 1
        long_results[entry_num]['book'] = chapter['book']
        long_results[entry_num]['chapter'] = chapter['chapter']
        long_results[entry_num]['paragraph'] = entry_num + 1
        long_results[entry_num]['text'] = x

    return(long_results)

# --------------------------------
# Build a list of URLs to scrape
# --------------------------------
url_base = 'http://www.readfreeonline.net/OnlineBooks/'
specific_books = ['Harry_Potter_and_the_Sorcerers_Stone/Harry_Potter_and_the_Sorcerers_Stone_CHAPTER.html',
    'Harry_Potter_and_The_Chamber_Of_Secrets/Harry_Potter_and_The_Chamber_Of_Secrets_CHAPTER.html',
    'Harry_Potter_and_the_Prisoner_of_Azkaban/Harry_Potter_and_the_Prisoner_of_Azkaban_CHAPTER.html',
    'Harry_Potter_and_the_Goblet_of_Fire/Harry_Potter_and_the_Goblet_of_Fire_CHAPTER.html',
    'Harry_Potter_and_the_Order_of_the_Phoenix/Harry_Potter_and_the_Order_of_the_Phoenix_CHAPTER.html',
    'Harry_Potter_and_the_Half-Blood_Prince/Harry_Potter_and_the_Half-Blood_Prince_CHAPTER.html',
    'Harry_Potter_and_the_Deathly_Hallows/Harry_Potter_and_the_Deathly_Hallows_CHAPTER.html']

chapters = [17, 19, 22, 37, 38, 30, 37]

urls_to_parse = []

for i in range(0, len(specific_books)):
    for j in range(1, chapters[i]+1):
        final_url = url_base + specific_books[i].replace('CHAPTER', str(j))
        urls_to_parse.append(final_url)

print(urls_to_parse.index('http://www.readfreeonline.net/OnlineBooks/Harry_Potter_and_the_Prisoner_of_Azkaban/Harry_Potter_and_the_Prisoner_of_Azkaban_20.html'))

# ---------------------------------------
# Scrape all those URLs and save to CSV
# ---------------------------------------
# # Set up log
# logging.basicConfig(filename='errors.log', filemode='w', level=logging.DEBUG,
#                     format='%(levelname)s %(asctime)s: %(message)s')
# logging.captureWarnings(True)

# csv_started = False
# fieldnames = ['book', 'chapter', 'paragraph', 'text']

# for url in urls_to_parse[0:2]:
#     contents = convert_to_long(parse_chapter(url))

#     for key, value in contents.items():
#         w = csv.DictWriter(open('hp.csv', 'a'), fieldnames)
#         if csv_started is False:
#             w.writeheader()
#             csv_started = True
#         w.writerow(value)


# Modified from Rico Sennrich

import codecs

class BPE(object):

  def __init__(self, codes, separator='@@'):
    with codecs.open(codes, encoding='utf-8') as codes:
      self.bpe_codes = [tuple(item.split()) for item in codes]
    # some hacking to deal with duplicates (only consider first instance)
    self.bpe_codes = dict([(code,i) for (i,code) in reversed(list(enumerate(self.bpe_codes)))])
    self.cache     = {}
    self.separator = separator

  def segment(self, sentence):
    """segment single sentence (whitespace-tokenized string) with BPE encoding"""

    output = []
    for word in sentence:
      new_word = self.encode(word, self.bpe_codes)

      for item in new_word[:-1]:
        output.append(item + self.separator)
      output.append(new_word[-1])

    return output

  def encode(self, orig, bpe_codes):
    """Encode word based on list of BPE merge operations, which are applied consecutively"""

    if orig in self.cache:
      return self.cache[orig]

    word = tuple(orig) + ('</w>',)
    pairs = get_pairs(word)

    while True:
      bigram = min(pairs, key = lambda pair: bpe_codes.get(pair, float('inf')))
      if bigram not in bpe_codes:
        break
      first, second = bigram
      new_word = []
      i = 0
      while i < len(word):
        try:
          j = word.index(first, i)
          new_word.extend(word[i:j])
          i = j
        except:
          new_word.extend(word[i:])
          break

        if word[i] == first and i < len(word)-1 and word[i+1] == second:
          new_word.append(first+second)
          i += 2
        else:
          new_word.append(word[i])
          i += 1
      new_word = tuple(new_word)
      word = new_word
      if len(word) == 1:
        break
      else:
        pairs = get_pairs(word)

    # don't print end-of-word symbols
    if word[-1] == '</w>':
      word = word[:-1]
    elif word[-1].endswith('</w>'):
      word = word[:-1] + (word[-1].replace('</w>',''),)

    self.cache[orig] = word
    return word

def get_pairs(word):
  """Return set of symbol pairs in a word.
  word is represented as tuple of symbols (symbols being variable-length strings)
  """
  pairs = set()
  prev_char = word[0]
  for char in word[1:]:
    pairs.add((prev_char, char))
    prev_char = char
  return pairs

def codec_from_file(self, codes, separator="@@"):
  return BPE(codes, separator)


from flask import Flask, render_template, request, session, url_for, redirect, jsonify
from werkzeug.utils import secure_filename
from flask_sqlalchemy import SQLAlchemy
import pandas as pd
import io
import eda_draw
import os
import time
import requests
import json
from celery import Celery
import asynchronous
import numpy as np
from imblearn.over_sampling import SMOTE

app = Flask(__name__)
app.secret_key = "super secret key"

# database
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data.db'
db = SQLAlchemy(app)

# eda plots save folder
EDA_FOLDER = os.path.join('static', 'eda')
app.config['UPLOAD_FOLDER'] = EDA_FOLDER

class Dataset(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    filename = db.Column(db.String(200), nullable=False)
    classification = db.Column(db.Integer, default=0)

# write json
def write_json(dictionary, filename='instance/database.json'):
    with open(filename, "w") as outfile:
        json.dump(dictionary, outfile)

    print("successfully wrote", dictionary, "to", filename)

# read json
def read_json(filename='instance/database.json'):
    with open(filename, 'r') as openfile:
        dictionary = json.load(openfile)

    return dictionary

# update json
def update_json(key, value, filename='instance/database.json'):
    dictionary = read_json(filename)
    dictionary[key] = value
    write_json(dictionary, filename)
    print("successfully updated dictionary")

# filter out columns
def filtercolumn(datasetname,columns, label):
    # reading from pandas
    df = pd.read_csv(datasetname)
    df = df[columns + [label]]
    return df

# get the value of something in json
def get_json(key, filename='instance/database.json'):
    dictionary = read_json(filename)
    return dictionary[key]

@app.route('/')
def main():
    return render_template("index.html")

@app.route('/results_csv', methods=['GET']) # call linux to create results.csv files
def create_results_csv():
    r = requests.get(url = 'http://localhost:5000/results_csv')

    return jsonify( r.json() )

@app.route('/download_models', methods=['POST'])
def download_models():
    if request.method == 'POST':
        downloaded = []

        for model_name in request.files.keys():
            model = request.files[model_name]
            model.save('downloads/'+model.filename)
            downloaded.append[model_name]
            print(model_name, "saved")

        return jsonify({'downloaded':downloaded})

@app.route('/full_trained_models')
def full_trained_models():
    """request linux server to send fully trained models"""
    r = requests.get('http://127.0.0.1:5000/upload_models')

    return 'response from training server: ' + str(r.text)

@app.route('/upload', methods = ['GET', 'POST'])
def upload_file():
    save_dir = 'D:/Dataset2/'

    if request.method == 'POST':
        f = request.files['file']
        # filename = save_dir + secure_filename(f.filename)
        filename = save_dir + f.filename
        f.save(filename)

        # asynchronous task
        asynchronous.upload_to_training_server.delay(filename)

        # add dataset to db
        database = {}
        database['dataset_filename'] = filename
        write_json(database)
        return redirect(url_for('.display_data'))

        # try:
        #     db.session.add(new_dataset)
        #     db.session.commit()
        #     return redirect(url_for('.display_data'))
        # except:
        #     return "There was an issue adding dataset to db"

@app.route('/eda', methods = ['GET', 'POST'])
def eda():
    if request.method == 'POST':
        columns=request.form.getlist('columns')
        label=request.form['label']

        # add label to db
        update_json('label',label)

        # prevent label from appearing twice
        if label in columns:
            columns.remove(label)

        # save columns into json file
        update_json("features", columns)

        # get dataset from db
        # current_dataset = Dataset.query.order_by(Dataset.id.desc()).first()
        current_dataset = read_json()
        dataset_filename = current_dataset['dataset_filename']

        df = filtercolumn(dataset_filename, columns,label)

        # reading from pandas
        # df = pd.read_csv(dataset_filename)

        # unique classes for label
        classes = list(df[label].unique()).sort()
        update_json('classes',classes)

        binary_classification = False
        if df[label].nunique() == 2:
            binary_classification = True

        update_json('binary_classification',binary_classification)

        # filter out columns
        # df = df[columns + [label]]
        describe = df.describe().applymap('{:,.2f}'.format)
        operation = list(describe.index)
        describe.insert(loc=0, column='', value=operation)

        # df to use for feature selection
        # df_feature = df_feature[columns]
        feature_dict = {"Features":columns}
        df_feature = pd.DataFrame.from_dict(feature_dict)

        # clear eda folder first
        for file in os.listdir('static/eda'):
            os.remove('static/eda/'+file)

        # draw and save the graphs
        eda_draw.draw_all(df, label)

        # distplot filenames
        distplots = [i for i in os.listdir('static/eda') if 'distplot' in i]
        temp = []
        for plot in distplots: # add full filepath
            temp.append(os.path.join(app.config['UPLOAD_FOLDER'], plot))
        distplots = temp

        return render_template("eda.html",
                                columns=request.form.getlist('columns'),
                                label=request.form['label'],

                                # df.describe
                                column_names=describe.columns.values,
                                row_data=list(describe.values.tolist()),
                                zip=zip,
                                link_column='',

                                correlation_plot=os.path.join(app.config['UPLOAD_FOLDER'], 'correlation.png'),

                                distplots = distplots,

                                # metadata_df
                                df_feature_column_names=df_feature.columns.values,
                                df_feature_row_data=list(df_feature.values.tolist()),

                                )

@app.route('/permutation', methods=['GET'])
def permutation():
    r = requests.get(url = 'http://127.0.0.1:5000/permutation')

    permutations = r.json()

    json_file = read_json()
    features = json_file['features']

    for key in permutations:
        value_list, label_list = eda_draw.get_sorted_permutation(permutations[key], features)
        eda_draw.draw_permutation_importance(value_list, label_list, f"{key}.png")

    print('\n')
    print("permutation importance", permutations)

    return jsonify(permutations)

@app.route('/predictions', methods=['GET'])
def draw_predictions():
    r = requests.get(url = 'http://127.0.0.1:5000/predictions')
    predictions_dict = r.json()

    binary_classification = get_json('binary_classification')
    dataframes_list = eda_draw.generate_insights(predictions_dict, binary_classification, get_json('classes'))

    return jsonify({'dataframes_list':dataframes_list,'binary_classification':binary_classification})

@app.route('/train', methods = ['GET', 'POST'])
def train():
    if request.method == 'POST':
        columns=request.form.getlist('columns')
        time_left_for_this_task=request.form['time_left_for_this_task']
        per_run_time_limit=request.form['per_run_time_limit']
        test_set_size = request.form['test_set_size']
        cv = request.form['cv_folds']

        # sending post to the training server
        payload = {}

        # get dataset from db
        # dataset_filename = Dataset.query.order_by(Dataset.id.desc()).first().filename.split('/')[-1]
        current_dataset = read_json()
        dataset_filename = current_dataset['dataset_filename'].split('/')[-1]
        payload['dataset_filename'] = dataset_filename

        payload['columns'] = columns
        payload['time_left_for_this_task'] = time_left_for_this_task
        payload['per_run_time_limit'] = per_run_time_limit
        payload['test_set_size'] = test_set_size

        # label = Dataset.query.order_by(Dataset.id.desc()).first().label
        current_dataset = read_json()
        label = current_dataset['label']

        return render_template('train.html',
                                columns=json.dumps(columns),
                                time_left_for_this_task=time_left_for_this_task,
                                per_run_time_limit=per_run_time_limit,
                                dataset_filename=dataset_filename,
                                label=label,
                                test_set_size=test_set_size,
                                cv=cv,
                                # train_job=train_job
                                payload=payload
        )

@app.route('/documentation')
def documentation():
    return render_template('documentation.html')

@app.route('/train_model', methods=['GET'])
def train_model():
    if request.method == 'GET':
        print(request.args)

        columns=request.args['columns']
        time_left_for_this_task=request.args['time_left_for_this_task']
        per_run_time_limit=request.args['per_run_time_limit']
        test_set_size = request.args['test_set_size']

        payload = {}

        # get dataset from db
        # dataset_filename = Dataset.query.order_by(Dataset.id.desc()).first().filename.split('/')[-1]
        current_dataset = read_json()
        dataset_filename = current_dataset['dataset_filename'].split('/')[-1]

        payload['dataset_filename'] = dataset_filename
        payload['test_set_size'] = test_set_size
        payload['columns'] = columns
        payload['time_left_for_this_task'] = time_left_for_this_task
        payload['per_run_time_limit'] = per_run_time_limit

        # return payload

        print("payload from windows", payload)
        res = requests.post('http://127.0.0.1:5000/train_model', params=request.args)
        return jsonify(res.text)

@app.route('/display_data')
def display_data():
    # get dataset from db
    # current_dataset = Dataset.query.order_by(Dataset.id.desc()).first()
    current_dataset = read_json()
    dataset_filename = current_dataset['dataset_filename']

    # reading from pandas
    # df = pd.read_csv(current_dataset.filename)
    df = pd.read_csv(dataset_filename)

    # metadata df
    buf = io.StringIO()
    df.info(buf=buf)
    s = buf.getvalue()
    lines = [line.split() for line in s.splitlines()[3:-2]]
    metadata_df = pd.DataFrame(lines[2:], columns=lines[0])

    # prevent displaying entire dataset
    df = df.head()
    return render_template("display_data.html",
                            column_names=df.columns.values,
                            row_data=list(df.values.tolist()),
                            zip=zip,
                            # filename=current_dataset.filename.split('/')[-1],
                            filename=dataset_filename.split('/')[-1],

                            # metadata_df
                            metadata_column_names=metadata_df.columns.values,
                            metadata_row_data=list(metadata_df.values.tolist()),
                            )

@app.route('/augmentation', methods=['POST'])
def augmentation():
    #dataset from eda page (user determine lable and deselect unrelated columns)
    columns=request.form.getlist('columns') # get columns
    label=request.form['targetcolumn'] #get labels
    update_json('label',label) # add label to db
    if label in columns: # prevent label from appearing twice
        columns.remove(label)
    current_dataset = Dataset.query.order_by(Dataset.id.desc()).first()# get dataset from db
    current_dataset = read_json()
    dataset_filename = current_dataset['dataset_filename']

    df = filtercolumn(dataset_filename, columns, label) # final dataset ready for improvment

    # get the desired number of rows for augmentation from user
    # desired_rows = 500
    desired_rows = request.form['desired_row']
    desired_rows_int = int(desired_rows)
    # Data augmentation
    augmented_data = df.sample(n=desired_rows_int - len(df), replace=True) # Duplicate the existing rows randomly to increase the number of rows
    augmented_df = pd.concat([df, augmented_data], axis=0) # Concatenate the original dataset with the augmented rows
    augmented_df = augmented_df.sample(frac=1).reset_index(drop=True) # Shuffle the rows to introduce diversity

    save_dir = 'datasets/'
    dataset_filename = current_dataset['dataset_filename'].split('/')[-1]
    new_dataset_filename = dataset_filename.replace('.csv',"")+"_augmented.csv"
    new_ds_fullpath = save_dir+new_dataset_filename
    augmented_df.to_csv(new_ds_fullpath, index=False)
    save_dir='D:/Dataset2/'
    new_ds_fullpath = save_dir+new_dataset_filename
    augmented_df.to_csv(new_ds_fullpath, index=False)

    update_json('dataset_filename',new_ds_fullpath)

    # show the result on improve.html
    print("Original dataset shape:", df.shape)
    print("Augmented dataset shape:", augmented_df.shape)

    test_set_size= request.form['test_set_size']
    columns=request.form.getlist('columns')
    time_left_for_this_task=request.form['time_left_for_this_task']
    per_run_time_limit=request.form['per_run_time_limit']
    cv_folds=request.form['cv_folds']
    class_distribution = df[label].value_counts()
    return render_template('improve.html',
                            augmented= augmented_df.shape,
                            class_distribution=class_distribution,
                            test_set_size=test_set_size,
                            columns=columns,
                            time_left_for_this_task=time_left_for_this_task,
                            per_run_time_limit=per_run_time_limit,
                            cv_folds=cv_folds
                           )

@app.route('/smote', methods=['POST'])
def smote():
    #dataset from eda page (user determine lable and deselect unrelated columns)
    columns=request.form.getlist('columns') # get columns
    label=request.form['targetcolumn'] #get labels
    update_json('label',label) # add label to db
    if label in columns: # prevent label from appearing twice
        columns.remove(label)
    current_dataset = Dataset.query.order_by(Dataset.id.desc()).first()# get dataset from db
    current_dataset = read_json()
    dataset_filename = current_dataset['dataset_filename']

    df = filtercolumn(dataset_filename, columns, label) # final dataset ready for improvment

    df = df.fillna(df.mean())

    # Data Improvment
    # Assuming your features are stored in 'X' and the target variable in 'y'
    X = df.drop(label, axis=1)
    y = df[label]

    # get the desired proportion of the minority class for oversampling
    # desired_proportion = 3
    desired_proportion = request.form['desired_proportion']
    desired_proportion_int = int(desired_proportion)
    # Calculate the number of samples needed for the minority class
    min_class_samples = int(desired_proportion_int * len(y[y == 1]))
    # Create an instance of SMOTE with the desired sampling strategy
    smote = SMOTE(sampling_strategy={0: len(y[y == 0]), 1: min_class_samples}, random_state=42)
    # Perform oversampling using SMOTE
    X_resampled, y_resampled = smote.fit_resample(X, y)

    # Check the new sample size and class distribution
    total_samples = len(X_resampled)
    class_distribution = y_resampled.value_counts()

    print("Total samples after oversampling:", total_samples)
    print("Class distribution after oversampling:")
    print(class_distribution)

    # Create a new DataFrame with the oversampled data
    smote_df = pd.concat([X_resampled, y_resampled], axis=1)

    save_dir = 'datasets/'
    dataset_filename = current_dataset['dataset_filename'].split('/')[-1]
    new_dataset_filename = dataset_filename.replace('.csv',"")+"_smote.csv"
    new_ds_fullpath = save_dir+new_dataset_filename
    smote_df.to_csv(new_ds_fullpath, index=False)
    save_dir='D:/Dataset2/'
    new_ds_fullpath = save_dir+new_dataset_filename
    smote_df.to_csv(new_ds_fullpath, index=False)

    update_json('dataset_filename',new_ds_fullpath)

    test_set_size= request.form['test_set_size']
    columns=request.form.getlist('columns')
    time_left_for_this_task=request.form['time_left_for_this_task']
    per_run_time_limit=request.form['per_run_time_limit']
    cv_folds=request.form['cv_folds']

    return render_template("improve.html",
                            totalsample= total_samples,
                            class_distribution= class_distribution,
                            augmented= smote_df.shape,
                            test_set_size=test_set_size,
                            columns=columns,
                            time_left_for_this_task=time_left_for_this_task,
                            per_run_time_limit=per_run_time_limit,
                            cv_folds=cv_folds
                            )

@app.route('/train_after_improve', methods=['POST'])
def train_after_improve():

    columns=eval(request.form.get('columns'))
    time_left_for_this_task=request.form['time_left_for_this_task']
    per_run_time_limit=request.form['per_run_time_limit']
    test_set_size = request.form['test_set_size']
    cv = request.form['cv_folds']

    # sending post to the training server
    payload = {}

    # get dataset from db
    # dataset_filename = Dataset.query.order_by(Dataset.id.desc()).first().filename.split('/')[-1]
    current_dataset = read_json()
    dataset_filename = current_dataset['dataset_filename'].split('/')[-1]
    payload['dataset_filename'] = dataset_filename

    payload['columns'] = columns
    payload['time_left_for_this_task'] = time_left_for_this_task
    payload['per_run_time_limit'] = per_run_time_limit
    payload['test_set_size'] = test_set_size

    # label = Dataset.query.order_by(Dataset.id.desc()).first().label
    # current_dataset = read_json()
    label = current_dataset['label']

    return render_template('train.html',
                            columns=json.dumps(columns),
                            time_left_for_this_task=time_left_for_this_task,
                            per_run_time_limit=per_run_time_limit,
                            dataset_filename=dataset_filename,
                            label=label,
                            test_set_size=test_set_size,
                            cv=cv,
                            # train_job=train_job
                            payload=payload
)

if __name__ == '__main__':
    app.run(debug = True, port=3000)


'''
Plot sims with expliict parameter values. For implicit, see run_scenarios.py.
'''

import numpy as np
import sciris as sc
import covasim as cv
import create_sim as cs
import pandas as pd
print(cs.__file__)

do_save = True
baseline_file = 'results/kc-baseline-nov17.msim'

T = sc.tic()

indices    = np.arange(8)
ncpus      = min(len(indices), 12)
from_cache = 0

# Load data
tsdf = pd.read_excel(cs.epi_data_file)
ctdf = pd.read_excel(cs.contact_tracing_file, sheet_name='final')
ctdf['date'] = pd.to_datetime(ctdf['date']).dt.date
tsdf['date'] = pd.to_datetime(tsdf['date']).dt.date

projpars = sc.objdict(
        trace_time   = {k:1   for k in 'hswcl'},
        i_factor     = {k:0.1 for k in 'hswcl'},
        q_factor     = {k:0.2 for k in 'hswcl'},
        rel_severe   = 1.0, # Change to 0.5 to match hospitalizations, but disrupts optimization
        death_change = 0.3,
        test_delay   = 1,
        trace_probs  = sc.mergedicts({k:0.6 for k in 'hsl'}, {'w':0.05}, {'c': 0.0}),
)

kwargs = dict(projpars=projpars, from_cache=from_cache, do_shrink=0)

if __name__ == '__main__':

    # Settings
    if ncpus>1:
        sims = sc.parallelize(cs.run_sim, iterarg=indices, ncpus=ncpus, kwargs=kwargs)
    else:
        sims = [cs.run_sim(index, **kwargs) for index in indices]

    msim = cv.MultiSim(sims=sims)
    msim.median()
    sim = msim.base_sim
    res = msim.results
    plot_msim = True
    if plot_msim:
        msim.plot(to_plot='overview', fig_args={'figsize':(38,19)})

    if do_save:
        msim.save(baseline_file)

    sc.toc(T)


import pandas as pd

df = pd.read_csv("cast.csv")
num_rows = len(df)
mid = int(num_rows / 2)
df1 = df[:mid]
df2 = df[mid:]
df1.to_csv("cast1.csv", index=False)
df2.to_csv("cast2.csv", index=False)

print(df1.head())
print(df2.head())


# utils/keypath.py

import copy

''' Utility functions for accessing elements of json-like object by keypath.'''

def search(obj, keypath, container=False, raise_keyerror=False):
    ''' Returns the element from a json-like object by keypath.

    Examples:
    ```
    >>> obj = {
        "a": {
            "b": {
                "3": 2,
                "43": 30,
                "c": [],
                "d": ['red', 'buggy', 'bumpers'],
            }
        }
    }
    >>> keypath_search(obj, 'a.b.d.1')
    'buggy'
    >>> keypath_search(obj, 'a.b.d.1', True)
    ['red', 'buggy', 'bumpers']
    >>> keypath_search(obj, 'a.b.d.1', True)[1] = 'foobar'
    >>> obj
    {'a': {'b': {'3': 2, '43': 30, 'c': [], 'd': ['red', 'foobar', 'bumpers']}}}
    ```

    Params:
        obj: The json-like object (nested dicts and/or lists)
        keypath: string. A period separated list of keys.
        container: bool. If set to True, the container of the one but last level
            is returned so this function can be used to update the input.
        raise_keyerror: bool. If set to True and no object found at the given
            keypath, a KeyError will be raised. Otherwise None is returned.

    Returns:
        The object found at the given keypath.
    '''
    keys = keypath.split('.')
    prev = None
    current = obj
    for key in keys:
        prev = current
        if isinstance(current, list):
            current = current[int(key)]
        elif isinstance(current, dict):
            if key in current:
                current = current[key]
            else:
                if raise_keyerror:
                    raise KeyError('No object found at {}'.format(keypath))
                else:
                    return prev if container else None
        else:
            raise ValueError('Can not get object at keypath "{}" from json-like object: Unknown object type.'.format(keypath))
    return prev if container else current

def replace(obj, keypath, repl, inline=True):
    ''' Replaces an element at a given keypath in a json-like object.

    Params:
        obj: The json-like object (nested dicts and/or lists)
        keypath: string. A period separated list of keys.
        repl: callable. The current element will be passed as an argument.
        inline: bool. If set to False the replace will be carried out on a
            copy of the obj.

    Returns:
        True if the replacement was made.
    '''
    if not inline:
        obj = copy.deepcopy(obj)
    container = search(obj, keypath, container=True)
    last_key = keypath.split('.')[-1]
    if isinstance(container, list):
        last_key = int(last_key)
        container[last_key] = repl(container[last_key])
    else:
        if last_key in container:
            container[last_key] = repl(container[last_key])
    return obj