"""
Asynchronous API for data scraping
"""

from __future__ import annotations

import traceback
import warnings
from typing import Type, TypeVar, Union

from playwright.async_api import Page, Route, TimeoutError, async_playwright
from pydantic import ValidationError
from tiktokapipy import TikTokAPIError, TikTokAPIWarning
from tiktokapipy.api import TikTokAPI
from tiktokapipy.models.challenge import Challenge
from tiktokapipy.models.raw_data import (
    ChallengePage,
    PrimaryResponseType,
    SentToLoginResponse,
    UserResponse,
    VideoPage,
)
from tiktokapipy.models.user import User, user_link
from tiktokapipy.models.video import Video, is_mobile_share_link
from tiktokapipy.util.queries import get_challenge_detail_async, get_video_detail_async

_DataModelT = TypeVar("_DataModelT", bound=PrimaryResponseType, covariant=True)
"""
Generic used for data scraping.
"""

class AsyncTikTokAPI(TikTokAPI):
    """Asynchronous API used to scrape data from TikTok"""

    def __enter__(self):
        raise TikTokAPIError("Must use async context manager with AsyncTikTokAPI")

    async def __aenter__(self) -> AsyncTikTokAPI:
        self._playwright = await async_playwright().start()
        self._browser = await self.playwright.chromium.launch(
            headless=self.headless, **self.kwargs
        )

        context_kwargs = self.context_kwargs
        context_kwargs.update(self.playwright.devices["Desktop Edge"])

        self._context = await self.browser.new_context(**context_kwargs)
        self.context.set_default_navigation_timeout(self.navigation_timeout)

        return self

    async def __aexit__(self, exc_type, exc_val, exc_tb):
        await self.context.close()
        await self.browser.close()
        await self.playwright.stop()

    async def _scrape_data(
        self,
        link: str,
        data_model: Type[_DataModelT],
    ) -> _DataModelT:
        for _ in range(self.navigation_retries + 1):
            await self.context.clear_cookies()
            page: Page = await self._context.new_page()
            await page.add_init_script(
                """
if (navigator.webdriver === false) {
    // Post Chrome 89.0.4339.0 and already good
} else if (navigator.webdriver === undefined) {
    // Pre Chrome 89.0.4339.0 and already good
} else {
    // Pre Chrome 88.0.4291.0 and needs patching
    delete Object.getPrototypeOf(navigator).webdriver
}
            """
            )

            async def ignore_scripts(route: Route):
                if route.request.resource_type == "script":
                    return await route.abort()
                return await route.continue_()

            await page.route("**/*", ignore_scripts)
            try:
                await page.goto(link, wait_until=None)
                await page.wait_for_selector("#SIGI_STATE", state="attached")
                content = await page.content()

                data = content.split(
                    '<script id="SIGI_STATE" type="application/json">'
                )[1].split("</script>")[0]

                if "LoginContextModule" in data:
                    warnings.warn(
                        "Redirected to a login page. Trying again...",
                        category=TikTokAPIWarning,
                        stacklevel=2,
                    )
                    sent_to_login = SentToLoginResponse.model_validate_json(data)
                    await page.goto(
                        sent_to_login.login_context_module.redirect_url, wait_until=None
                    )
                    await page.wait_for_selector("#SIGI_STATE", state="attached")
                    content = await page.content()
                    data = content.split(
                        '<script id="SIGI_STATE" type="application/json">'
                    )[1].split("</script>")[0]

                await page.close()

                extracted = self._extract_and_dump_data(data, data_model)
            except (ValidationError, IndexError) as e:
                traceback.print_exception(type(e), e, e.__traceback__)
                await page.close()
                continue
            except TimeoutError:
                warnings.warn(
                    "Reached navigation timeout. Retrying...",
                    category=TikTokAPIWarning,
                    stacklevel=2,
                )
                await page.close()
                continue
            break
        else:
            raise TikTokAPIError(
                f"Data scraping unable to complete in {self.navigation_timeout / 1000}s "
                f"(retries: {self.navigation_retries})"
            )

        return extracted

    async def challenge(
        self, challenge_name: str, *, video_limit: int = -1
    ) -> Challenge:
        response = ChallengePage.model_validate(
            await get_challenge_detail_async(challenge_name, self.context)
        )
        challenge = self._extract_challenge_from_response(response)
        challenge.videos.limit(video_limit)
        return challenge

    async def user(self, user: str, *, video_limit: int = -1) -> User:
        link = user_link(user)
        response = await self._scrape_data(
            link,
            UserResponse,
        )
        user = self._extract_user_from_response(response)
        user.videos.limit(video_limit)
        return user

    async def video(
        self,
        link_or_id: Union[int, str],
    ) -> Video:
        if isinstance(link_or_id, str):
            if is_mobile_share_link(link_or_id):
                await self.context.clear_cookies()
                page: Page = await self.context.new_page()
                await page.add_init_script(
                    """
    if (navigator.webdriver === false) {
        // Post Chrome 89.0.4339.0 and already good
    } else if (navigator.webdriver === undefined) {
        // Pre Chrome 89.0.4339.0 and already good
    } else {
        // Pre Chrome 88.0.4291.0 and needs patching
        delete Object.getPrototypeOf(navigator).webdriver
    }
                """
                )

                async def ignore_scripts(route: Route):
                    if route.request.resource_type == "script":
                        return await route.abort()
                    return await route.continue_()

                await page.route("**/*", ignore_scripts)
                await page.goto(link_or_id, wait_until=None)
                await page.wait_for_selector("#SIGI_STATE", state="attached")

                link_or_id = page.url

                await page.close()
            video_id = link_or_id.split("/")[-1].split("?")[0]
        else:
            video_id = link_or_id

        response = VideoPage.model_validate(
            await get_video_detail_async(video_id, self.context)
        )
        return self._extract_video_from_response(response)

__all__ = ["AsyncTikTokAPI"]


# coding=utf-8
# @Author   : zpchcbd HG team
# @Time     : 2021-08-25 23:46

from spider.thirdLib.public import *
from spider.thirdLib import BaseThird

class Chinaz(BaseThird):
    """
    chinaz third spider
    """
    def __init__(self, domain):
        super().__init__()
        self.domain = domain
        self.addr = 'https://api.sublist3r.com/search.php?domain={}'
        self.source = 'chinaz'
        self.api = config.chinazApi

    async def spider(self):
        print('Load chinaz api ...')
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(url=self.addr.format(self.api, self.domain), headers=self.headers,
                                       verify_ssl=False,
                                       timeout=self.reqTimeout) as response:
                    if response is not None:
                        text = await response.text()
                        if text != 'null':
                            for subdomain in eval(text):
                                self.resList.append(subdomain)
                        else:
                            print('chinaz API No Subdomains.')
        except Exception as e:
            print('[-] curl chinaz.com api error, the error is {}'.format(e.args))

        self.resList = list(set(self.resList))
        print('[{}] {}'.format(len(self.resList), self.resList))
        return self.resList

# async def do(domain):
#     chinaz = Chinaz(domain)
#     res = await chinaz.spider()
#     return res

if __name__ == '__main__':
    pass
    # loop = asyncio.get_event_loop()
    # res = loop.run_until_complete(do('zjhu.edu.cn'))


import json
import functools
import subprocess

from univention.lib.i18n import Translation
from univention.management.console.base import Base
from univention.management.console.error import UMC_Error, UnprocessableEntity
from univention.management.console.config import ucr

from univention.management.console.modules.decorators import sanitize, simple_response, file_upload, allow_get_request
from univention.management.console.modules.sanitizers import StringSanitizer, DictSanitizer, EmailSanitizer, ValidationError, MultiValidationError

from univention.googleapps.auth import GappsAuth, SCOPE, GoogleAppError, AuthenticationError, AuthenticationErrorRetry, SSLError
from univention.googleapps.listener import GoogleAppsListener
from univention.googleapps.handler import ForbiddenError

_ = Translation('univention-management-console-module-googleapps').translate

def sanitize_body(sanizer):  # TODO: move into UMC core
	def _decorator(function):
		@functools.wraps(function)
		def _decorated(self, request, *args, **kwargs):
			try:
				sanizer.sanitize('request.body', {'request.body': request.body})
			except MultiValidationError as exc:
				raise UnprocessableEntity(str(exc), exc.result())
			except ValidationError as exc:
				raise UnprocessableEntity(str(exc), {exc.name: str(exc)})
			return function(self, request, *args, **kwargs)
		return _decorated
	return _decorator

def progress(component=None, message=None, percentage=None, errors=None, critical=None, finished=False, **kwargs):
	return dict(
		component=component,
		message=message,
		percentage=percentage,
		errors=errors or [],
		critical=critical,
		finished=finished,
		**kwargs
	)

class Instance(Base):

	@simple_response
	def query(self):
		fqdn = '%s.%s' % (ucr['hostname'], ucr['domainname'])
		sso_fqdn = ucr.get('ucs/server/sso/fqdn', 'ucs-sso.%s' % (ucr['domainname'],))
		return {
			'initialized': GappsAuth.is_initialized(),
			'sign-in-url': 'https://%s/simplesamlphp/saml2/idp/SSOService.php' % (sso_fqdn,),
			'sign-out-url': 'https://%s/simplesamlphp/saml2/idp/SingleLogoutService.php?ReturnTo=/univention/' % (sso_fqdn,),
			'change-password-url': 'https://%s/univention/management/' % (fqdn,),
		}

	@allow_get_request
	def certificate(self, request):
		with open(ucr['saml/idp/certificate/certificate'], 'rb') as fd:
			self.finished(request.id, fd.read(), mimetype='application/octet-stream')

	@file_upload
	@sanitize(DictSanitizer(dict(
		tmpfile=StringSanitizer(required=True)
	), required=True))
	@sanitize_body(DictSanitizer(dict(
		email=EmailSanitizer(required=True),
		domain=StringSanitizer(required=True),
	), required=True))
	def upload(self, request):
		GappsAuth.uninitialize()
		with open(request.options[0]['tmpfile']) as fd:
			try:
				data = json.load(fd)
			except ValueError:
				raise UMC_Error(_('The uploaded file is not a JSON credentials file.'))
			try:
				GappsAuth.store_credentials(data, request.body['email'], domain=request.body['domain'])
			except GoogleAppError as exc:
				raise UMC_Error(str(exc))
		self.finished(request.id, {
			'client_id': data['client_id'],
			'scope': ','.join(SCOPE),
			# 'serviceaccounts_link': 'https://console.developers.google.com/permissions/serviceaccounts?project=%s' % (urllib.quote(data['project_id']),)
		}, message=_('The credentials have been successfully uploaded.'))

	@simple_response
	def state(self):
		if not GappsAuth.is_initialized():
			raise UMC_Error(_('The configuration of Google Apps for Work is not yet complete.'))
		try:
			ol = GoogleAppsListener(None, {}, {})
			ol.gh.list_users(projection="basic")
			try:
				subprocess.call(["service", "univention-directory-listener", "restart"])
			except (EnvironmentError,):
				pass
			return progress(finished=True)
		except AuthenticationErrorRetry:
			return progress(message=_('Waiting for Google directory to authorize the connection.'))
		except (AuthenticationError, ForbiddenError, SSLError) as exc:
			raise UMC_Error(str(exc))


import profile
from pyexpat import model
from rest_framework import serializers
from youonline_social_app.serializers.post_serializers import DefaultProfileSerializer, PostSerializer
from youonline_social_app.serializers.utility_serializers import LanguageSerializer
from . models import *
from django.conf import settings

# Blog Category Serializer
class BlogCategorySerializer(serializers.ModelSerializer):
    class Meta:
        model = BlogCategory
        fields = '__all__'

# Blog Serializer
class BlogSerializer(serializers.ModelSerializer):

    class Meta:
        model = Blog
        fields = '__all__'

# Blog Watched Serializer
class BlogWatchedSerializer(serializers.ModelSerializer):
    class Meta:
        model=BlogWatched
        fields='__all__'

# Get Blog Serializer
class GetBlogSerializer(serializers.ModelSerializer):
    total_comments = serializers.SerializerMethodField()
    featured_image = serializers.SerializerMethodField()
    language = LanguageSerializer(read_only=True)
    rtl = serializers.SerializerMethodField()
    category = BlogCategorySerializer(read_only=True)
    profile = DefaultProfileSerializer(read_only=True)

    class Meta:
        model = Blog
        fields = ['id', 'category', 'profile', 'language', 'post', 'title', 'slug',
                    'description', 'posted_date', 'view_count', 'shared_count',
                    'is_promoted', 'is_deleted', 'created_at',
                    'updated_at', 'is_compressed', 'featured_image', 'total_comments', 'rtl']

    def get_featured_image(self, obj):
        request = self.context.get('request')
        try:
            blog_media = BlogMedia.objects.get(blog=obj)
            if blog_media.featured_image:
                return f"{settings.S3_BUCKET_LINK}{blog_media.featured_image}"
            else:
                return None
        except Exception as e:
            print(e)
            return None

    def get_total_comments(self, obj):
        request=self.context.get('request')
        if obj.post:
            total_comment=obj.post.comments_count
            return total_comment
        else:
            None

    def get_rtl(self, obj):

        try:
            rtl = Language.objects.get(id=obj.language.id)
            if rtl.rtl == True:
                rtl=True
            else:
                rtl=False
            return rtl
        except Exception as e:
            pass

class BlogAuthorSerializer(serializers.ModelSerializer):
    class Meta:
        model = BlogAuthor
        fields = '__all__'

class GetBlogAuthorSerializer(serializers.ModelSerializer):
    profile = DefaultProfileSerializer(read_only=True)
    resume =serializers.SerializerMethodField()

    def get_resume(self, obj):
        if obj.resume:
            resume_url = obj.resume
            print(f"{settings.S3_BUCKET_LINK}{resume_url}")
            return f"{settings.S3_BUCKET_LINK}{resume_url}"
        else:
            return None
    class Meta:
        model = BlogAuthor
        fields = ['id', 'profile', 'first_name', 'last_name', 'description', 'resume' , 'language', 'country', 'is_deleted', 'created_at', 'updated_at']


from flask import jsonify, request, Flask
from flask_cors import CORS

from backend.fetchers.fetch_categories import fetch_categories, fetch_subcategories
from backend.fetchers.fetch_cities import fetch_cities
from backend.fetchers.fetch_hotels import fetch_hotels
from backend.fetchers.fetch_pois import fetch_pois
from backend.fetchers.fetch_routes import fetch_routes

from urllib.parse import parse_qs

app = Flask(__name__)
CORS(app)

@app.route('/api/cities', methods=['GET'])
def get_locations():
    # Retrieve the list of available locations from your data source
    cities = fetch_cities()
    print("here")

    # Return the locations as a response
    return jsonify(cities)

@app.route('/')
def hello_world():
    return 'Hello, World!'

@app.route('/api/hotels', methods=['GET'])
def get_hotels():
    city = request.args.get('city')
    if city:
        hotels = fetch_hotels(city)
    else:
        hotels = []

    # Return the locations as a response
    return jsonify(hotels)

@app.route('/api/pois', methods=['GET'])
def get_pois():
    categories = request.args.get('categories')
    categories = categories.split(",")
    city = request.args.get('city')
    pois = fetch_pois(city, categories)

    return jsonify(pois)

@app.route('/api/categories', methods=['GET'])
def get_categories():
    city = request.args.get('city')
    categories = fetch_categories(city)

    return jsonify(categories)

@app.route('/api/subcategories', methods=['GET'])
def get_subcategories():
    city = request.args.get('city')
    subcategories = fetch_subcategories(city)

    return jsonify(subcategories)

@app.route('/api/routes', methods=['GET'])
def get_routes():
    params = parse_qs(request.args.get('params'))
    city = params.get('city')[0]
    hotel = params.get('hotel')[0]
    categories = params.get('categories')[0].split(",")
    compulsory_pois = params.get('compulsory_pois', [])
    if compulsory_pois:
        compulsory_pois = compulsory_pois[0].split(",")
    num_days = int(params.get('days')[0])
    max_distance = int(params.get('max_distance')[0])
    max_locations = int(params.get('max_locations')[0])

    routes, distance = fetch_routes(city, hotel, categories, compulsory_pois, num_days, max_distance, max_locations)
    result = {
        'routes': routes,
        'distance': distance
    }

    return result

#app.run(debug=True)


import logging

import torch
import numpy
from fedstellar.learning.aggregators.aggregator import Aggregator

class Krum(Aggregator):
    """
    Krum [Peva Blanchard et al., 2017]
    Paper: https://papers.nips.cc/paper/2017/hash/f4b9ec30ad9f68f89b29639786cb62ef-Abstract.html
    """

    def __init__(self, node_name="unknown", config=None):
        super().__init__(node_name, config)
        self.config = config
        self.role = self.config.participant["device_args"]["role"]
        logging.info("[Krum] My config is {}".format(self.config))

    def aggregate(self, models):
        """
        Krum selects one of the m local models that is similar to other models
        as the global model, the euclidean distance between two local models is used.

        Args:
            models: Dictionary with the models (node: model,num_samples).
        """
        # Check if there are models to aggregate
        if len(models) == 0:
            logging.error(
                "[Krum] Trying to aggregate models when there is no models"
            )
            return None

        models = list(models.values())

        # Total Samples
        total_samples = sum([y for _, y in models])

        # Create a Zero Model
        accum = (models[-1][0]).copy()
        for layer in accum:
            accum[layer] = torch.zeros_like(accum[layer])

        # Add weighteds models
        logging.info("[Krum.aggregate] Aggregating models: num={}".format(len(models)))

        # Create model distance list
        total_models = len(models)
        distance_list = [0 for i in range(0, total_models)]

        # Calculate the L2 Norm between xi and xj
        min_index = 0
        min_distance_sum = float('inf')

        for i in range(0, total_models):
            m1, _ = models[i]
            for j in range(0, total_models):
                m2, _ = models[j]
                distance = 0
                if i == j:
                    distance = 0
                else:
                    for layer in m1:
                        l1 = m1[layer]
                        # l1 = l1.view(len(l1), 1)

                        l2 = m2[layer]
                        # l2 = l2.view(len(l2), 1)
                        distance += numpy.linalg.norm(l1 - l2)
                distance_list[i] += distance

            if min_distance_sum > distance_list[i]:
                min_distance_sum = distance_list[i]
                min_index = i

        # Assign the model with min distance with others as the aggregated model
        m, _ = models[min_index]
        for layer in m:
            accum[layer] = accum[layer] + m[layer]

        return accum


# 1부터 주어진 횟수까지 2를 곱한 값(들)을 출력하시오.
# 입력 하나 / 출력 띄어서 횟수만큼 나옴.

# 23.01.31 정호님께 배운대로 해보자!

# int로 정수를 input() 받는다.
# 빈 리스트가 필요할 것 같다. (횟수만큼 곱해야하니까.)
# 2의 ^n 이니까, 횟수를 반복해주면 되는데,,, range를 써서 하나씩 넣고 그걸 제곱으로 하면 되지 않을까.
#리스트 출력.

num = int(input())   #정수 input() 받기.

lst =[]             # 빈 리스트 만들기.

for i in range(0, num+1):   # 입력된 정수의 범위만큼 하나씩 i 에 할당.
    i = 2**i                # 그 i를 2**i 로 2의 i 제곱을 하나씩 해줌.
    lst.append(i)           # 그 값을 이제 빈 리스트에 하나씩 append 로 더해줌.

print(*lst)  #*lst 는 언패킹 해줘서 리스트로 감싼 걸 빼줌.


"""
    Author: Negar Golestani
    Created: August 2023
"""

import os
import torch
import torch.nn as nn
import pandas as pd
from torch.utils.data import DataLoader
from torch.optim.lr_scheduler import StepLR
import torch.optim as optim
import time

from .loss import*
from .networks import*

####################################################################################################################################
class baseModel(nn.Module):
    def __init__(self, opt, version=None):
        super().__init__()
        # Checkpoints directory
        self.save_dir = os.path.join(opt.checkpoints_rootdir, opt.name)   # save all the checkpoints to checkpoints_rootdir
        if version is not None: self.save_dir = os.path.join(self.save_dir, str(version))
        if not os.path.exists(self.save_dir): os.makedirs(self.save_dir)

        # Initialization
        opt_dict = vars(opt)
        self.set_criterions(**opt_dict)
        self.set_network(**opt_dict)
        self.set_optimizer(**opt_dict)

        # Parameters
        self.epoch = 0
        self.tr_cases = []
        torch.cuda.empty_cache()
    # ----------------------------------------------------------------------------------------------------
    def __call__(self, test_data, verbose=True, criterions=['mle'], calculate_theta=True, **kwargs):
        start_time = time.time()
        self.set_inputs(test_data)                                              # set inputs

        if calculate_theta:
            self.forward(isTraining=False, **kwargs)                            # forward pass

        et = time.time() - start_time                                           # execution time
        evalRes = self.eval(criterions=criterions)                              # calculate eval results
        evalRes['et'] = et

        # Display Results
        if verbose:
            label = test_data['label'] if 'label' in test_data else ''
            self.print_results(label, evalRes)

        # Output data from batch
        output_data = dict()
        for key, val in self.batch.items():
            try: output_data[key] = val.squeeze(0)
            except: output_data[key] = val

        # torch.cuda.empty_cache()
        return output_data, evalRes
    # ====================================================================================================
    #                                       PLACEHOLDERS
    # ====================================================================================================
    def set_network(self, *args, **kwargs):
        ''' Placeholder '''
        pass
    # ----------------------------------------------------------------------------------------------------
    def set_inputs(self, *args, **kwargs):
        ''' Placeholder '''
        pass
    # ----------------------------------------------------------------------------------------------------
    def forward(self, *args, **kwargs):
        ''' Placeholder '''
        pass
    # ----------------------------------------------------------------------------------------------------
    def get_evalRes(self, *args, **kwargs):
        ''' Placeholder '''
        pass
    # ====================================================================================================
    # ====================================================================================================
    def set_criterions(self, loss='mse', geometric_model='affine', grid_size=120, device='cpu', **kwargs):
        ''' Placeholder '''
        self.tnfGridLoss = TransformedGridLoss(geometric_model=geometric_model, grid_size=grid_size, device=device)
        self.MSE = MSE(device=device)
        self.SSIM = SSIM(device=device)
        self.MI = MI(device=device)
        self.DICE = Dice(device=device)
        self.MLE = MLE(device=device)
        self.HD = HD(device=device)
        self.HD95 = HD95(device=device)
        self.NCC = NCC(device=device)
        self.MMD = MMD(device=device)
        self.GANloss = nn.BCELoss()
        self.LOSS = WeightedLoss(loss)
    # ----------------------------------------------------------------------------------------------------
    def set_optimizer(self, lr=1e-4, step_size=1, gamma=.95, **kwargs):
        self.optimizer = optim.Adam(self.net.parameters(), lr=lr)
        self.scheduler = StepLR(self.optimizer, step_size=step_size, gamma=gamma)
    # ----------------------------------------------------------------------------------------------------
    def backward(self):
        self.optimizer.zero_grad()                                          # set gradients to zero
        self.evalRes_tensor['loss'].backward(retain_graph=True)             # calculate gradients
        self.optimizer.step()                                               # update weights
    # ----------------------------------------------------------------------------------------------------
    def eval(self, criterions=[]):
        self.evalRes_tensor = self.get_evalRes(criterions)

        try: self.evalRes_tensor['loss'] = self.LOSS( self.evalRes_tensor )
        except: self.evalRes_tensor['loss'] = None

        # evalRes (numpy)
        evalRes_npy = dict()
        for key, val in  self.evalRes_tensor.items():
            try:  val = val.data.cpu().numpy()
            except: pass

            if val is not None: evalRes_npy[key] = val

        return evalRes_npy
    # ----------------------------------------------------------------------------------------------------
    def run_singleEpoch(self, dataset, isTraining=False, num_workers=4, batch_size=1, shuffle=False, criterions=[]):
        criterions_ = [*criterions, *self.LOSS.metrics]                             # make sure all criterions for loss are considered to be calculated
        evalRes_epoch = pd.DataFrame()                                              # epoch eval

        for batch in DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers):
            self.set_inputs(batch)
            self.forward(isTraining=isTraining)                                     # forward pass
            evalRes_batch = self.eval(criterions=criterions_)                       # calculate eval results
            if isTraining: self.backward()                                          # get gradients and update network weights
            evalRes_epoch = evalRes_epoch.append(evalRes_batch, ignore_index=True)  # Batch evaluation

        evalRes_epoch = evalRes_epoch.mean().to_dict()                              # Results (epoch evaluation)
        # torch.cuda.empty_cache()
        return evalRes_epoch
    # ----------------------------------------------------------------------------------------------------
    def train(self, train_dataset, verbose=True, **kwargs):
        evalRes_epoch = self.run_singleEpoch(train_dataset, isTraining=True, **kwargs)
        self.scheduler.step() # Update scheduler and epoch
        self.epoch += 1
        self.tr_cases = list( set(self.tr_cases).union(set(train_dataset.get_cases())) )

        # Display Results
        if verbose:
            print( f'Epoch {self.epoch}' )
            self.print_results('Train', evalRes_epoch)

        return evalRes_epoch
    # ----------------------------------------------------------------------------------------------------
    def validate(self, test_dataset, verbose=True, **kwargs):
        evalRes_epoch = self.run_singleEpoch( test_dataset, isTraining=False, **kwargs)
        if verbose: self.print_results('Test', evalRes_epoch)   # Display Results

        return evalRes_epoch
    # ----------------------------------------------------------------------------------------------------
    def save_states(self, filename='net'):
        save_filename = f"{filename}.pth"
        save_path = os.path.join(self.save_dir, save_filename)
        state = {
            'net_state_dict': self.net.state_dict(),
            'optimizer_state_dict': self.optimizer.state_dict(),
            'epoch': self.epoch,
            'tr_cases': self.tr_cases   }

        torch.save(state, save_path)
    # ----------------------------------------------------------------------------------------------------
    def load_states(self, filename='net'):
        load_path = os.path.join(self.save_dir, f"{filename}.pth")

        if os.path.exists(load_path):
            checkpoint = torch.load(load_path)
            self.net.load_state_dict( checkpoint['net_state_dict'] )
            self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
            self.epoch = checkpoint['epoch']
            self.tr_cases = checkpoint['tr_cases']
    # ----------------------------------------------------------------------------------------------------
    def print_network(self):
        print('---------- Networks initialized -------------')
        num_params = 0
        for param in self.net.parameters(): num_params += param.numel()
        print(self.net)
        print('Network Total number of parameters : %.3f M' % (num_params / 1e6))
        print('-----------------------------------------------')
    # ----------------------------------------------------------------------------------------------------
    def print_results(self, header, evalRes, N=8):
        txt = "\t" + header + ' '*(N-len(header)) + ':'
        txt_last = ''

        for key, val in evalRes.items():
            if key == 'mle_list': txt_last = f"{key}={val}"
            else: txt += f"\t {key}={val:.6f}"

        print( f"{txt} \t {txt_last}")
####################################################################################################################################


class Solution:
    def subsetsWithDup(self, nums):
        nums.sort()
        subsets = [[]]
        endIdx = 0

        for i in range(len(nums)):
            startIdx = 0
            if i > 0 and nums[i] == nums[i-1]:
                startIdx = endIdx + 1

            endIdx = len(subsets) - 1
            for j in range(startIdx, endIdx+1):
                subset = subsets[j][:]
                subset.append(nums[i])
                subsets.append(subset)

        return subsets


import os
import syslog
import fcntl

def flag2mode(flags):
    md = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'}
    m = md[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)]

    if flags | os.O_APPEND:
        m = m.replace('w', 'a', 1)

    return m

class File(object):

    def __init__(self, fuse, path, flags, *mode):
        self.fuse = fuse

        syslog.syslog('self: %s' % self)
        syslog.syslog('Fuse: %s' % fuse)
        syslog.syslog('path: %s' % path)

        if (fuse.files.has_key(path)):
            path = fuse.files[path]

        self.file = os.fdopen(os.open("." + path, flags, *mode),
                              flag2mode(flags))
        self.fd = self.file.fileno()

    def read(self, length, offset):
        syslog.syslog('Function read(%i, %i) called' % (length, offset))
        self.file.seek(offset)
        return self.file.read(length)

    def write(self, buf, offset):
        syslog.syslog('Function write() called')
        self.file.seek(offset)
        self.file.write(buf)
        return len(buf)

    def release(self, flags):
        syslog.syslog('Function release(%s) called' % (flags))
        self.file.close()

    def _fflush(self):
        syslog.syslog('Function fflush() called')
        if 'w' in self.file.mode or 'a' in self.file.mode:
            self.file.flush()

    def fsync(self, isfsyncfile):
        syslog.syslog('Function fsync(%b) called' % (isfsyncfile))
        self._fflush()
        if isfsyncfile and hasattr(os, 'fdatasync'):
            os.fdatasync(self.fd)
        else:
            os.fsync(self.fd)

    def flush(self):
        syslog.syslog('Function flush() called')
        self._fflush()
        # cf. xmp_flush() in fusexmp_fh.c
        os.close(os.dup(self.fd))

    def fgetattr(self):
        syslog.syslog('Function fgetattr() called')
        return os.fstat(self.fd)

    def ftruncate(self, len):
        syslog.syslog('Function ftruncate() called')
        self.file.truncate(len)

    def lock(self, cmd, owner, **kw):
        syslog.syslog('Function lock(%s, %s, %s) called' % (cmd, owner, kw))
        # The code here is much rather just a demonstration of the locking
        # API than something which actually was seen to be useful.

        # Advisory file locking is pretty messy in Unix, and the Python
        # interface to this doesn't make it better.
        # We can't do fcntl(2)/F_GETLK from Python in a platfrom independent
        # way. The following implementation *might* work under Linux.
        #
        # if cmd == fcntl.F_GETLK:
        #     import struct
        #
        #     lockdata = struct.pack('hhQQi', kw['l_type'], os.SEEK_SET,
        #                            kw['l_start'], kw['l_len'], kw['l_pid'])
        #     ld2 = fcntl.fcntl(self.fd, fcntl.F_GETLK, lockdata)
        #     flockfields = ('l_type', 'l_whence', 'l_start', 'l_len', 'l_pid')
        #     uld2 = struct.unpack('hhQQi', ld2)
        #     res = {}
        #     for i in xrange(len(uld2)):
        #          res[flockfields[i]] = uld2[i]
        #
        #     return fuse.Flock(**res)

        # Convert fcntl-ish lock parameters to Python's weird
        # lockf(3)/flock(2) medley locking API...
        op = { fcntl.F_UNLCK : fcntl.LOCK_UN,
               fcntl.F_RDLCK : fcntl.LOCK_SH,
               fcntl.F_WRLCK : fcntl.LOCK_EX }[kw['l_type']]
        if cmd == fcntl.F_GETLK:
            return -EOPNOTSUPP
        elif cmd == fcntl.F_SETLK:
            if op != fcntl.LOCK_UN:
                op |= fcntl.LOCK_NB
        elif cmd == fcntl.F_SETLKW:
            pass
        else:
            return -EINVAL

        fcntl.lockf(self.fd, op, kw['l_start'], kw['l_len'])


from django.urls import path
from django.contrib.auth import views as auth_views
from . import views

urlpatterns = [
    path('', views.home, name='phish-home'),
    path('assigned/', views.assigned, name='phish-assigned'),
    path('created/', views.created, name='phish-created'),
    path('logout/', auth_views.LogoutView.as_view(template_name='home/logout.html', next_page='phish-login'), name='phish-logout'),
    path('login/', auth_views.LoginView.as_view(template_name='home/login.html', redirect_field_name='phish-home'), name='phish-login'),
    path('edit/', views.edit, name='phish-edit'),
    path('upload/', views.upload, name='phish-upload'),
    path('register/', views.register, name='phish-register'),
    path('suggested/', views.suggested, name='phish-suggested'),
    path('post/<int:postId>/', views.post, name='phish-post'),
    path('email/', views.email, name='phish-email'), # For testing purposes
]


import dash
import dash_core_components as dcc
import dash_html_components as html
import pandas as pd
import plotly.graph_objs as go
from dash.dependencies import Input, Output

external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']

app = dash.Dash(__name__, external_stylesheets=external_stylesheets)

app.layout = html.Div([
    html.H2("U.S. National Parks",
            style={"text-align": "center"}),
    dcc.Graph(
        id='map',
        style={
            "margin": "0px calc(100% / 8)",
        }
    ),
    html.Div([
        html.Label('Year Selector', style={"text-align": "center"}),
        dcc.Slider(
            id='year-slider',
            min=2009,
            max=2018,
            value=2018,
            marks={str(year): str(year) for year in range(2009, 2019)},
            step=None
        )
    ], style={
        "width": "50%",
        "margin": "0 auto",
    }),
    html.Div([
        html.Label('Statistic Selector', style={"text-align": "center"}),
        dcc.Dropdown(
            id="stat_selector",
            options=[
                {'label': 'Size of Park', 'value': 'Gross Area Acres'},
                {'label': u'Number of Visitors', 'value': 'Value'},
            ],
            value='Gross Area Acres',
            style={
                "label": "Dropdown",
                "width": "80%",
                "margin": "0 auto",
            })
    ], style={
        "width": "50%",
        "margin": "0 auto",
        "padding-top": "30px"
    }),

])

@app.callback(
    Output(component_id='map', component_property='figure'),
    [Input(component_id='year-slider', component_property='value'),
     Input(component_id='stat_selector', component_property='value')]
)
def update_output_div(year_selector, stat_selector):
    df = pd.read_csv(f'./data/merged_data/merged_data_{year_selector}.csv', thousands=',')

    if stat_selector == "Value":
        size = df[stat_selector] / 10000
        color_label = "Visitors"
    else:
        size = df[stat_selector] / 1000
        color_label = stat_selector

    df['text'] = "Park: " + df['ParkName'] + "\n" + color_label + ": " + df[stat_selector].astype(str)

    return {
        'data': [go.Scattergeo(
            locationmode='USA-states',
            lon=df['longitude'],
            lat=df['latitude'],
            text=df['text'],
            marker=dict(
                size=size,
                color=df[stat_selector],
                line_color='rgb(40,40,40)',
                line_width=0.5,
                sizemode='area',
                colorbar=dict(
                    title=color_label
                ),
                colorscale="Hot"
            ),
        )],
        'layout': go.Layout(
            geo_scope='usa',
            hovermode='closest',
            height=700,
            autosize=True,
            margin=go.layout.Margin(
                l=0,
                r=50,
                b=100,
                t=100,
                pad=4
            ),
        )
    }

if __name__ == '__main__':
    app.run_server(debug=True)


import numpy as np
import matplotlib.pyplot as plt
from math import *

DENSITY = 7800   # Kg/m^3
THERMAL_COND = 55 # W/m-K
SPECIFIC_HEAT = 450 # J/Kg-K
LENGTH = 0.6 # m
TIME =1000# secs

h = 1e5# W/m-K
NUM_PTS_X = 100
NUM_PTS_T = 10000
DELTA_X = LENGTH/NUM_PTS_X
DELTA_T = TIME/NUM_PTS_T

SIGMA = (THERMAL_COND*DELTA_T)/(DENSITY*SPECIFIC_HEAT*DELTA_X**2)
BETA = DELTA_T/(DENSITY*SPECIFIC_HEAT)
T_amb = 293.00 # K

ALPHA = (h*DELTA_X)/THERMAL_COND

def initialTemp (x = 0):

    f= 293.00
    return f

def heatSource (x):

    q = 2e7*exp(-1000*(x-0.2)**2)

    return q

def tridiag_solver(a, b, c, f):
    """Implementing Thomas Algorithm"""

    n = len(f)
    a_dash = np.zeros(n)
    c_dash = np.zeros(n-1)
    y = np.zeros(n)
    u = np.zeros(n)
    a_dash[0] = a[0]

    for i in range(1,n):
        a_dash[i] = a[i] - (c[i-1]*b[i-1]/a_dash[i-1])

    for i in range(0,n-1):
        c_dash[i] = c[i]/a_dash[i]

    y[0] = f[0]

    for i in range(1,n):
        y[i] = f[i] - c_dash[i-1]*y[i-1]

    u[n-1] = y[n-1]/a_dash[n-1]

    for i in range(n-2,-1,-1):
        u[i] = (y[i]-b[i]*u[i+1])/a_dash[i]

    return u

def meshing():

    spaceMeshLocations = np.zeros(NUM_PTS_X+1)
    timeMeshLocations = np.zeros(NUM_PTS_T+1)

    for i in range(0,NUM_PTS_X+1):
        spaceMeshLocations[i] = i*DELTA_X

    for i in range(0,NUM_PTS_T+1):
        timeMeshLocations[i] = i*DELTA_T

    return spaceMeshLocations, timeMeshLocations

def plottemp(u,spaceLocation, u_time, timeLocation, flux):

    plt.figure(1)
    plt.plot(spaceLocation,u[0], 'r')
    plt.plot(spaceLocation,u[1], 'g')
    plt.plot(spaceLocation, u[2], 'b')
    plt.plot(spaceLocation, u[3], '--r')
    plt.plot(spaceLocation, u[4], '--g')

    plt.xlabel('Length (m)')
    plt.ylabel('Tempreature (K)')
    plt.title('Length ($x$)  vs. Tempreature ($u$)')
    label0 = 't = 0 secs'
    label1 = 't = 10 secs'
    label2 = 't = 100 secs'
    label3 = 't = 500 secs'
    label4 = 't = 1000 secs'
    label5 = 'x = 0.2 m'

    plt.gca().legend((label0,label1,label2, label3, label4), loc = 'upper right')
    plt.ylim(280,1800)
    plt.xlim(0,0.6)
    plt.show()

    plt.figure(2)
    plt.plot(timeLocation,u_time,'-r')
    plt.xlabel('Time (secs)')
    plt.ylabel('Tempreature (K)')
    plt.title('Time ($t$)  vs. Tempreature ($u$) at $x = 0.2$ m')
    plt.xlim(0, 1000)

    plt.show()

    plt.figure(3)
    plt.plot(spaceLocation, flux[0,:], 'r')
    plt.plot(spaceLocation, flux[1,:], 'g')
    plt.plot(spaceLocation, flux[2,:], 'b')
    plt.plot(spaceLocation, flux[3,:], '--r')
    plt.plot(spaceLocation, flux[4,:], '--g')

    plt.xlabel('Length (m)')
    plt.ylabel('Flux ($W/m^2$)')
    plt.title('Flux  vs. Tempreature')
    label0 = 't = 0 secs'
    label1 = 't = 10 secs'
    label2 = 't = 100 secs'
    label3 = 't = 500 secs'
    label4 = 't = 1000 secs'

    plt.gca().legend((label0, label1, label2, label3, label4), loc='upper right')
    # plt.ylim(280, 1800)
    plt.xlim(0, 0.6)
    plt.show()

def flux(u_hist, spaceMeshLocations):
    """Calculating the flux"""

    FLUX = np.zeros((len(u_time),len(spaceMeshLocations)))
    n = len(spaceMeshLocations)
    flux = np.zeros(n)

    for i in range(0,len(u_hist)):

        u_temp = u_hist[i]

        flux[0] = - THERMAL_COND*((u_temp[1]-(1-ALPHA)*u_temp[0]-ALPHA*T_amb)/(2*DELTA_X))

        for j in range(1,n-1):

            flux[j] = - THERMAL_COND*((u_temp[j+1]-u_temp[j-1])/(2*DELTA_X))

        flux[n-1] = - THERMAL_COND*(((1-ALPHA)*u_temp[n-1] + ALPHA*T_amb-u_temp[n-2])/(2*DELTA_X))

        FLUX[i,:] = flux

    return FLUX

def solver_BTCS(spaceMeshLocations, timeMeshLocations):

    """Solving using BTCS"""

    n = len(spaceMeshLocations)

    b = np.full(n-1,-SIGMA)
    c = np.full(n-1, -SIGMA)
    a = np.zeros(n)

    a[0] = (1+SIGMA+SIGMA*ALPHA)
    a[n-1] = (1+SIGMA+SIGMA*ALPHA)
    a[1:n-1] = (1+2*SIGMA)

    heatSource_term = np.zeros(n)

    for i in range(0, len(spaceMeshLocations)):
        heatSource_term[i] = heatSource(spaceMeshLocations[i])*BETA

    BC_term = np.zeros(n)

    BC_term[0] = SIGMA*ALPHA*T_amb
    BC_term[n-1] = SIGMA*ALPHA*T_amb

    u_hist = []

    const_value = np.full(n,0.2)

    difference = spaceMeshLocations-const_value

    loc_index = np.argwhere(difference >= 0)
    index = loc_index[0]

    u_initial = np.full(n,initialTemp())
    u_initial[0] = (u_initial[1]+ALPHA*T_amb)/(1+ALPHA)
    u_initial[n-1] = (u_initial[n-2] + ALPHA * T_amb) / (1 + ALPHA)

    u_time = np.zeros(len(timeMeshLocations))

    u_time[0] = u_initial[index]

    u_hist.append(u_initial)

    u_curr = u_initial+heatSource_term+BC_term

    for i in range(1,len(timeMeshLocations)):

        u_next = tridiag_solver(a,b,c, u_curr)
        u_time[i] = u_next[index]

        u_curr = u_next+heatSource_term+BC_term

        if timeMeshLocations[i] == 10:

            u_hist.append(u_next)

        elif timeMeshLocations[i] == 100:

            u_hist.append(u_next)

        elif timeMeshLocations[i] == 500:

            u_hist.append(u_next)

        elif timeMeshLocations[i] == 1000:

            u_hist.append(u_next)

    return u_hist, u_time

spaceMeshLocations, timeMeshLocations = meshing()

u_history, u_time = solver_BTCS(spaceMeshLocations, timeMeshLocations)

heat_flux = flux(u_history,spaceMeshLocations)

plottemp(u_history,spaceMeshLocations, u_time, timeMeshLocations, heat_flux)


import requests
import pandas as pd
from bs4 import BeautifulSoup

page = requests.get("https://www.navegg.com/segmentos/arvore-segmentos/?demo")
soup = BeautifulSoup(page.content, "html.parser")

segments = soup.find_all("div", class_ = "box-tree")
result_df = pd.DataFrame(columns = ['segment', 'type', 'sub_type', 'navegg_id', 'dv360_id', 'verizon_id', 'mediamath_id'])

for segment in segments:

    segment_id = segment.get("id")
    segment_div = soup.find("div", id = segment.get("id"))

    for sub_type in segment_div.find_all("h3"):
        segment_type = sub_type.text
        for sibling in sub_type.find_next_sibling():
            if sibling.name == "li":
                sibling_array = sibling.text.replace("ID", ":").replace("Navegg","").replace("DV360","").replace("Verizon","").replace("MediaMath","").replace(" ","").replace("::", ":").split(":")
                tmp = pd.DataFrame({'segment': segment_id,
                            'type': segment_type,
                            'sub_type': sibling_array[0],
                            'navegg_id': sibling_array[1],
                            'dv360_id': sibling_array[2],
                            'verizon_id': sibling_array[3],
                           'mediamath_id': sibling_array[4]}, index=[0])

                result_df = result_df.append(tmp)

result_df.reset_index(drop = True).to_csv("navegg_tree.csv")


# Processamento de Imagens Digitais - PPGCC 2018

import cv2
import random
import os, sys
import argparse
import operator
import numpy as np
import scipy.spatial as sp
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

from tqdm import tqdm
from skimage import feature
from mtcnn.mtcnn import MTCNN

numOfClasses = 60
numOfSamplesPerClass = 4

# Função utilizada para dividir uma imagem de entrada
# em 4 sub-imagens;
def getSubImages(subImage):
    height, width = subImage.shape

    height = height // 2
    width = width // 2

    subImages = []
    subImages.append(subImage[:height, :width])
    subImages.append(subImage[:height, width:])
    subImages.append(subImage[height:, :width])
    subImages.append(subImage[height:, width:])
    return subImages

# Função utilizada para obter todas as imagens do dataset.
# (76 indivíduos (homens), apenas as imagens de 1 - 4);
def getDataset():
    dataset = []
    coloredDataset = []
    for i in range(1, numOfClasses + 1):
        for j in range(1, 5):
            #imgFilename = 'new_m-{0:0=3d}-{1}.bmp'.format(i, j)
            imgFilename = 'm-{0:0=3d}-{1}.bmp'.format(i, j)
            imgContent = cv2.imread(imgFilename)
            coloredDataset.append(imgContent)
            imgContent = cv2.cvtColor(imgContent, cv2.COLOR_BGR2GRAY)
            dataset.append(imgContent)
    return dataset, coloredDataset

# Função utilizada para obter todas as imagens de teste do dataset.
# (76 indivíduos (homens), apenas as imagens de 14 - 17);
def getTestDataset():
    testDataset = []
    testColoredDataset = []

    for i in range(1, numOfClasses + 1):
        #randomSampleNumber = random.randint(14, 17)
        #imgFilename = 'new_m-{0:0=3d}-{1}.bmp'.format(i, 14)
        imgFilename = 'm-{0:0=3d}-{1}.bmp'.format(i, 14)
        imgContent = cv2.imread(imgFilename)
        testColoredDataset.append(imgContent)
        imgContent = cv2.cvtColor(imgContent, cv2.COLOR_BGR2GRAY)
        testDataset.append(imgContent)
    return testDataset, testColoredDataset

# Função utilizada para calcular a distância de uma amostra ("base")
# em relação à todas as outras (com a func. euclidiana);
def getRelativeDistances(base, descriptors):
    relativeDistances = {}

    for i in range(0, descriptors.shape[0]):
        if i == base: continue
        relativeDistances[i] = cv2.compareHist(np.float32(descriptors[i]), np.float32(descriptors[base]), cv2.HISTCMP_CHISQR)
        #relativeDistances[i] = sp.distance.cityblock(descriptors[i], descriptors[base])
        #relativeDistances[i] = sp.distance.euclidean(descriptors[i], descriptors[base])

    return relativeDistances

# Análogo ao método acima, porém leva em consideração os histogramas das amostras de teste
# para efetuar o cálculo do CMC posteriormente;
def getRelativeDistancesCMC(base, testHistograms, descriptors):
    relativeDistances = {}

    for i in range(0, descriptors.shape[0]):
        #relativeDistances[i] = cv2.compareHist(np.float32(descriptors[i]), np.float32(testHistograms[base]), cv2.HISTCMP_CHISQR)
        relativeDistances[i] = sp.distance.cityblock(descriptors[i], testHistograms[base])
        #relativeDistances[i] = sp.distance.euclidean(descriptors[i], testHistograms[base])
    return relativeDistances

# Função utilizada para calcular o vetor de PR de uma amostra ("i")
# levando em consideração seu vetor de distâncias relativas calculado
# anteriormente;
def getDescriptorPR(i, sortedRelativeDistances):
    global numOfSamplesPerClass

    currentDescriptorClass = i // numOfSamplesPerClass

    retrievedInClass = 0
    totalRetrieved = 0
    totalInClass = numOfSamplesPerClass - 1

    currentDescriptorPR = [0] * (numOfSamplesPerClass - 1)
    for descriptorNumber, relativeDistance in sortedRelativeDistances:
        totalRetrieved += 1
        if (descriptorNumber // numOfSamplesPerClass) == currentDescriptorClass:
            retrievedInClass += 1
            currentDescriptorPR[retrievedInClass - 1] = retrievedInClass / totalRetrieved
    return currentDescriptorPR

if __name__ == '__main__':
    descriptors = []

    # Obtém as imagens do dataset.
    # (Apenas homens, 76 indivíduos, sub-imagens de 1 à 4 (luz ambiente));
    dataset, coloredDataset = getDataset()
    detector = MTCNN()

    errCount = 0

    # Para cada sub-imagem de um mesmo indivíduo,
    # cria um processo para calcular o respectivo histograma LBP;
    print('Calculating each sample\'s LBP histogram...')
    for i in tqdm(range(0, numOfClasses * numOfSamplesPerClass)):
        # Divide cada amostra em 4 sub-imagens para calcular os respectivos histogramas LBP.
        # (Lembrando que eles são concatenados posteriormente em um único histograma);

        # Para a abordagem de 4 Sub-imagens, descomentar a linha abaixo.
        #subImages = getSubImages(dataset[i])

        result = detector.detect_faces(coloredDataset[i])

        try:
            box = result[0]['box']
            l_eye = result[0]['keypoints']['left_eye']
            r_eye = result[0]['keypoints']['right_eye']
        except Exception as e:
            print((i // numOfSamplesPerClass) + 1)
            print((i % numOfSamplesPerClass) + 1)
            continue

        #subImages = getSubImages(dataset[i][box[0]:box[0] + box[2] + 1,box[1]:box[1] + box[3] + 1])

        concatenatedHistograms = []
        '''
        # Para a abordagem de 4 Sub-imagens, descomentar este bloco.
        for subImage in subImages:
            # Calcula o LBP da sub-imagem, utilizando o método NRI UNIFORM.
            # São levados em consideração 8 vizinhos e um raio de distância 2.
            # (NRI UNIFORM: Non-Rotational Invariant Uniform);
            lbp = np.float32(feature.local_binary_pattern(subImage, 8, 2, method='nri_uniform'))

            #lbp_bins = int(lbp.max())
            #histogram = np.histogram(lbp.ravel(), bins=lbp_bins, range=(0, lbp_bins), density=True)

            _, histogram = np.unique(lbp, return_counts=True)
            histogram = np.true_divide(histogram, np.max(histogram))

            concatenatedHistograms.extend(histogram)
        '''

        # Para a abordagem de Close-up, descomentar este bloco.
        lbp = np.float32(feature.local_binary_pattern(dataset[i][box[0]:box[0] + box[2] + 1,box[1]:box[1] + box[3] + 1], 8, 2, method='nri_uniform'))
        _, histogram = np.unique(lbp, return_counts=True)
        histogram = np.true_divide(histogram, np.max(histogram))

        concatenatedHistograms.extend(histogram)

        '''
        # Para a abordagem com descritores dos olhos, descomentar este bloco.
        left_eye_lbp = np.float32(feature.local_binary_pattern(dataset[i][l_eye[0] - 50:l_eye[0] + 50,l_eye[1] - 50:l_eye[1] + 50], 8, 2, method='default'))
        right_eye_lbp = np.float32(feature.local_binary_pattern(dataset[i][r_eye[0] - 50:r_eye[0] + 50,r_eye[1] - 50:r_eye[1] + 50], 8, 2, method='default'))

        le_bins = int(left_eye_lbp.max())
        re_bins = int(right_eye_lbp.max())

        le_histogram = np.histogram(left_eye_lbp.ravel(), bins=le_bins, range=(0, le_bins), density=True)
        re_histogram = np.histogram(right_eye_lbp.ravel(), bins=re_bins, range=(0, re_bins), density=True)

        avg_histogram = (le_histogram[0] + re_histogram[0]) / 2

        #concatenatedHistograms.extend(le_histogram[0])
        #concatenatedHistograms.extend(re_histogram[0])
        #concatenatedHistograms.extend(avg_histogram)
        '''

        descriptors.append(concatenatedHistograms)

    descriptors = np.array(descriptors)
    print('Final descriptor dimensions:', descriptors.shape)

    # Realiza o cálculo do vetor PR de cada uma das amostras e,
    # consequentemente, calcula a média deles.
    print('Calculating each sample\'s Precision-Recall (PR) array...')
    descriptorsPR = []
    for i in tqdm(range(0, descriptors.shape[0])):
        relativeDistances = getRelativeDistances(i, descriptors)
        sortedRelativeDistances = sorted(relativeDistances.items(), key=operator.itemgetter(1))
        currentDescriptorPR = getDescriptorPR(i, sortedRelativeDistances)
        descriptorsPR.append(currentDescriptorPR)

    descriptorsPR = np.array(descriptorsPR)
    descriptorsPR_mean = np.mean(descriptorsPR, axis=0)
    print('Precision-Recall descriptor dimensions:', descriptorsPR.shape)

    print('{}| Average Precision x Recall:'.format('LBP'))
    print(descriptorsPR_mean)

    # A partir deste ponto, a execução é bem semelhante ao código escrito até aqui.
    # A ideia é gerar um conjunto de teste, calcular os histogramas destas amostras de teste e
    # compará-los com os histogramas mais próximos de cada uma delas, calculando o CMC.

    testHistograms = []
    testRelativeDistances = []
    testDataset, testColoredDataset = getTestDataset()

    print('Calculating the CMC Curve based on test samples...')
    for i, testSample in enumerate(testDataset):
        # Para a abordagem de 4 Sub-imagens, descomentar a linha abaixo.
        #subImages = getSubImages(testSample)

        result = detector.detect_faces(testColoredDataset[i])

        try:
            l_eye = result[0]['keypoints']['left_eye']
            r_eye = result[0]['keypoints']['right_eye']
        except Exception as e:
            print(i)
            continue

        concatenatedHistograms = []
        '''
        # Para a abordagem de 4 Sub-imagens, descomentar este bloco.
        for subImage in subImages:
            lbp = np.float32(feature.local_binary_pattern(subImage, 8, 2, method='nri_uniform'))
            _, histogram = np.unique(lbp, return_counts=True)
            concatenatedHistograms.extend(histogram)
        '''

        # Para a abordagem de Close-up, descomentar este bloco.
        lbp = np.float32(feature.local_binary_pattern(testSample[box[0]:box[0] + box[2] + 1,box[1]:box[1] + box[3] + 1], 8, 2, method='nri_uniform'))
        _, histogram = np.unique(lbp, return_counts=True)
        concatenatedHistograms.extend(histogram)

        '''
        # Para a abordagem com descritores dos olhos, descomentar este bloco.
        left_eye_lbp = np.float32(feature.local_binary_pattern(testDataset[i][l_eye[0] - 30:l_eye[0] + 30,l_eye[1] - 25:l_eye[1] + 25], 8, 2, method='default'))
        right_eye_lbp = np.float32(feature.local_binary_pattern(testDataset[i][r_eye[0] - 30:r_eye[0] + 30,r_eye[1] - 25:r_eye[1] + 25], 8, 2, method='default'))

        le_bins = int(left_eye_lbp.max())
        re_bins = int(right_eye_lbp.max())

        le_histogram = np.histogram(left_eye_lbp.ravel(), bins=le_bins, range=(0, le_bins), density=True)
        re_histogram = np.histogram(right_eye_lbp.ravel(), bins=re_bins, range=(0, re_bins), density=True)
        avg_histogram = (le_histogram[0] + re_histogram[0]) / 2

        #concatenatedHistograms.extend(le_histogram[0])
        #concatenatedHistograms.extend(re_histogram[0])
        concatenatedHistograms.extend(avg_histogram)
        '''

        testHistograms.append(concatenatedHistograms)

    testHistograms = np.array(testHistograms)

    for i in tqdm(range(0, testHistograms.shape[0])):
        relativeDistances = getRelativeDistancesCMC(i, testHistograms, descriptors)
        sortedRelativeDistances = sorted(relativeDistances.items(), key=operator.itemgetter(1))
        testRelativeDistances.append(sortedRelativeDistances)

    # O trecho de código abaixo calcula efetivamente a curva CMC.
    # Trata-se de um loop que fica iterando repetitivamente sobre as amostras de teste,
    # até que todas elas tenham finalmente encontrado o histograma (de mesma classe) mais próximo.

    isLooping = True

    cmcRankings = []
    cmcCurrentRanking = 1
    cmcControl = [False] * numOfClasses
    while isLooping:
        trueCounter = 0
        for i in range(0, numOfClasses):
            if cmcControl[i]:
                trueCounter += 1
                continue

            descriptorNumber, relativeDistance = testRelativeDistances[i][cmcCurrentRanking - 1]
            if (descriptorNumber // numOfSamplesPerClass) == i:
                cmcControl[i] = True
                trueCounter += 1

        if trueCounter == numOfClasses:
            isLooping = False
            cmcRankings.append(1.0)
        else:
            cmcCurrentRanking += 1
            cmcRankings.append(trueCounter / numOfClasses)

    print('{}| CMC Curve:'.format('LBP'))
    print('Total Rankings: {}'.format(cmcCurrentRanking))
    print(cmcRankings)


class NhGallery :

    def __init__ ( self, title: str, link: str, tags: list ) :
        self.title = title
        self.link = link
        self.tags = tags
        self.lang = NhGallery.__extract_lang_from_tags( tags )

    @staticmethod
    def __extract_lang_from_tags ( tags: list ) -> str :
        if "29963" in tags :
            return "Chinese"
        if "12227" in tags :
            return "English"
        if "6346" in tags :
            return "Japanese"
        return "Unknown Language"

    def get_reply_form ( self, index = None ) -> str :

        # if given index, show index in reply message
        reply = "" if index is None else "- {0}\n".format( index )

        # convert gallery info into reply form
        reply += """+ 標題: {0}\n""".format( self.title )
        reply += """+ 語言: {0}\n""".format( self.lang )
        reply += """+ 連結: {0}\n\n""".format( self.link )

        return reply


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 13 12:44:46 2018

@author: neal8
"""

import requests
import pandas as pd
from dateutil.parser import parse

def fb_crawler(token,fb_id,n):  #n為總共要爬幾則貼文
    t=0
    columns = ['發文時間', '文章內容', '分享內容', '留言數', '分享數','發文類型' ,
               'likes','love','haha','wow','sad','angry','total','留言']
    posts = []
    res = requests.get('https://graph.facebook.com/v2.9/{}/posts?limit=20&access_token={}'.format(fb_id, token),headers={'Connection':'close'})
    while 'paging' in res.json():

        for post in res.json()['data']:
            try:
                p_res = requests.get(
                        'https://graph.facebook.com/v2.9/{}?'.format(post['id']) +
                        'fields=comments.limit(0).summary(true),likes.limit(0).summary(true),shares,reactions.type(LOVE).limit(0).summary(total_count).as(reactions_love),reactions.type(HAHA).limit(0).summary(total_count).as(reactions_haha),reactions.type(WOW).limit(0).summary(total_count).as(reactions_wow),reactions.type(SAD).limit(0).summary(total_count).as(reactions_sad),reactions.type(ANGRY).limit(0).summary(total_count).as(reactions_angry),reactions.type(NONE).limit(0).summary(total_count).as(reactions_none),type&' +
                        'access_token={}'.format(token),headers={'Connection':'close'})
    # 留言數
                if 'comments' in p_res.json():
                    comments = p_res.json()['comments']['summary'].get('total_count')
                else:
                    comments = 0
    # 按讚數
                if 'likes' in p_res.json():
                    likes = p_res.json()['likes']['summary'].get('total_count')
                else:
                    likes = 0
    # 分享數
                if 'shares' in p_res.json():
                    shares = p_res.json()['shares'].get('count')
                else:
                    shares = 0

                if 'reactions_love' in p_res.json():
                    love = p_res.json()['reactions_love']['summary'].get('total_count')
                else:
                    love = 0

                if 'reactions_haha' in p_res.json():
                    haha = p_res.json()['reactions_haha']['summary'].get('total_count')
                else:
                    haha = 0

                if 'reactions_wow' in p_res.json():
                    wow = p_res.json()['reactions_wow']['summary'].get('total_count')
                else:
                    wow = 0

                if 'reactions_sad' in p_res.json():
                    sad = p_res.json()['reactions_sad']['summary'].get('total_count')
                else:
                    sad = 0

                if 'reactions_angry' in p_res.json():
                    angry = p_res.json()['reactions_angry']['summary'].get('total_count')
                else:
                    angry = 0

                if 'reactions_none' in p_res.json():
                    total = p_res.json()['reactions_none']['summary'].get('total_count')
                else:
                    total = 0

                if 'type' in p_res.json():
                    ty = p_res.json().get('type')
                else:
                    ty='none'

                comm=fb_crawler_comments_sec(token,post['id'])

                posts.append([(str(parse(post['created_time']).date()))+" "+(str(parse(post['created_time']).time())),
                      post.get('message'),
                      post.get('story'),
                      comments,
                      shares,
                      ty,
                      likes,
                      love,
                      haha,
                      wow,
                      sad,
                      angry,
                      total,
                      comm])

            except ConnectionError:
                print(post['id']+' post got ConnectionError error')

            t=t+1
            print(t)
            if t>2700:
                break
        if t>2700:
            break

        if 'next' in res.json()['paging']:
            res = requests.get(res.json()['paging']['next'],headers={'Connection':'close'})

        else:
            break
    df = pd.DataFrame(posts, columns=columns)
    df.to_csv('M:\\playstation0429.csv',index=False)

def fb_crawler_comments_sec(token,fb_post_id):
    comment = []
    com_res = requests.get('https://graph.facebook.com/'+fb_post_id+'/comments?access_token='+token,headers={'Connection':'close'})
    while 'paging' in com_res.json():

        for com in com_res.json()['data']:

            comment.append(com.get('message'))
        if 'next' in com_res.json()['paging']:
            try:
                com_res = requests.get(com_res.json()['paging']['next'],headers={'Connection':'close'})
            except ConnectionError:
                print(fb_post_id+'  got ConnectionError error')
        else:
            break
    return comment

a='EAAaFYjWejrIBAApkrz07vldcq96CmFcR5OaebEPmB7zJv8JFz5QyeGehg4rzhO5gumC2cpte7nkZASm5X3eZBwb5ZBiHIwaDroGudvxowsMZCr0sL8xfANZCjeZANoqzmA1SkRnHqWBVugx2eOQ0ZB5YmIJLfOdLKoZD'    #長期權杖
b='14104316802'

fb_crawler(a,b,5)


#!/usr/bin/env python3

import argparse
import os
import sys

from environs import Env

from src.service.consumer_service import consumer_service
from src.service.logging_service import logging_service
from src.service.producer_service import producer_service

def main():
    """ Derived from https://github.com/aiven/aiven-examples/blob/master/kafka/python/main.py """

    parser = argparse.ArgumentParser()
    parser.add_argument('--kafka-service-uri', help="Kafka Service URI in the form host:port",
                        required=True)
    parser.add_argument('--db-service-uri', help="PostgreSQL Database service URI in the form host:port. "
                                                 "Can be left blank if only running the producer.",
                        required=True)
    parser.add_argument('--ca-path', help="Path to project CA certificate",
                        required=True)
    parser.add_argument('--key-path', help="Path to the Kafka Access Key (obtained from Aiven Console)",
                        required=True)
    parser.add_argument('--cert-path', help="Path to the Kafka Certificate Key (obtained from Aiven Console)",
                        required=True)

    parser.add_argument('--consumer', action='store_true', default=False, help="Run Kafka consumer for the "
                                                                               "processing monitor")
    parser.add_argument('--producer', action='store_true', default=False, help="Run Kafka producer for the "
                                                                               "processing monitor")
    parser.add_argument('--logger', action='store_true', default=False,
                        help="Run logger that prints out recent updates to the DB.")

    env = Env()
    env.read_env()
    validate_env(env)

    args = parser.parse_args()
    validate_args(args)

    if args.producer:
        producer_service(
            args.kafka_service_uri,
            args.ca_path,
            args.cert_path,
            args.key_path,
            env.str("WEBSITE_MONITOR_AVAILABILITY_TOPIC"),
            env.int("PING_INTERVAL_SECONDS"))

    elif args.consumer:
        consumer_service(args.kafka_service_uri,
                         args.ca_path,
                         args.cert_path,
                         args.key_path,
                         env.str("WEBSITE_MONITOR_AVAILABILITY_TOPIC"),
                         env.str("WEBSITE_MONITOR_TABLE_NAME"),
                         args.db_service_uri)
    elif args.logger:
        logging_service(args.db_service_uri, env.str("WEBSITE_MONITOR_TABLE_NAME"))

def validate_env(env):
    for env_param in ("WEBSITE_MONITOR_AVAILABILITY_TOPIC", "PING_INTERVAL_SECONDS", "WEBSITE_MONITOR_TABLE_NAME"):
        try:
            env.str(env_param)
        except:
            fail(f"Failed to retrieve env_param :: {env_param} :: from env file.\n"
                 f"You can set these to your liking in the env file.")

def validate_args(args):
    for path_option in ("ca_path", "key_path", "cert_path"):
        path = getattr(args, path_option)
        if not os.path.isfile(path):
            fail(f"Failed to open --{path_option.replace('_', '-')} at path: {path}.\n"
                 f"You can retrieve these details from Overview tab in the Aiven Console")
    if args.producer and args.consumer:
        fail("--producer and --consumer are mutually exclusive")
    elif not args.producer and not args.consumer and not args.logger:
        fail("--producer, --logger or --consumer are required")

def fail(message):
    print(message, file=sys.stderr)
    exit(1)

if __name__ == '__main__':
    main()


def return_pitch_class_set(pitches):
    """
    >>> import mason
    >>> mason.return_pitch_class_set([15, 10, 8, -5, 3, 11])
    [0, 1, 3, 4, 8]

    """
    pitches = [i % 12 for i in pitches]
    pitches = set(pitches)
    print(pitches)
    pitches = sorted(pitches)
    print(pitches)
    pitches = list(pitches)
    print(pitches)
    if pitches[1] - pitches[0] > pitches[-1] - pitches[-2]:
        pitches.reverse()
        intervals = [pitches[0] - i for i in pitches]
    else:
        intervals = [i - pitches[0] for i in pitches]
    root = 0
    pitches = [root + interval for interval in intervals]
    return pitches


import sys
import re
import csv

def readDict(filename):
    f = open(filename)
    a = f.readlines()
    dic = dict()
    for i in range(len(a)):
        text = re.sub('\n','',a[i])
        dic[text] = i
    return dic

def createOutput(y,filename):

    output = []
    catDict = readDict(filename)
    v = catDict.keys()
    v.sort()
    v.insert(0,'id')
    #print v
    output.append(v)
    count = 0
    for t in y:
        body = []
        body.append(count)
        for i in range(1,len(v)):
            #print t, catDict[v[i]]
            if t == catDict[v[i]]:
                #print t, v[i], catDict[v[i]]
                body.append(1)
                #print body
            else: body.append(0)
        #print 'body', body
        count+=1
        output.append(body)
    #print output
    return output

#t = createOutput([0,3,5])
#print t

def writeOutputToCSV(nameOfFile,y,dic):
    output = createOutput(y,dic)
    resultFile = open(nameOfFile,'wb')
    wr = csv.writer(resultFile,dialect = 'excel')
    wr.writerows(output)

#writeOutputToCSV('test.csv',[0,1,3],'dictionary.txt')


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 27 14:00:59 2019

@author: DELL
"""

import random

def isEqual(arr1,arr2):
    if((arr1 is None and arr2 is not None) or (arr1 is not None and arr2 is None )):
        return False
    if(arr1 is None and arr2 is None):
        return True
    arr1Len = len(arr1)
    arr2Len = len(arr2)
    if( arr1Len != arr2Len):
        return False
    for i in range(arr1Len):
        if(arr1[i] != arr2[i]):
            return False
    return True

def ArrayCompare(arrFunction,testTime = 500 ,maxSize = 50,maxValue = 100):
    for i in range(0,testTime):
        testArr1 = random.sample(range(0,testTime),maxSize)
        testArr2 = testArr1.copy()
        testArr1 = arrFunction(testArr1)
        (testArr2).sort()
        if not isEqual(testArr1,testArr2):
            print('the function is not rigit!')
    print('Congratulations, the function is correct!')


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow.contrib.ignite.python.ops.igfs_ops  # pylint: disable=unused-import
from tensorflow.python.platform import gfile
from tensorflow.python.platform import test

class IGFSTest(test.TestCase):
  """The Apache Ignite servers have to setup before the test and tear down

     after the test manually. The docker engine has to be installed.

     To setup Apache Ignite servers:
     $ bash start_ignite.sh

     To tear down Apache Ignite servers:
     $ bash stop_ignite.sh
  """

  def test_create_file(self):
    """Test create file.

    """
    # Setup and check preconditions.
    file_name = "igfs:///test_create_file/1"
    self.assertFalse(gfile.Exists(file_name))
    # Create file.
    with gfile.Open(file_name, mode="w") as w:
      w.write("")
    # Check that file was created.
    self.assertTrue(gfile.Exists(file_name))

  def test_write_read_file(self):
    """Test write/read file.

    """
    # Setup and check preconditions.
    file_name = "igfs:///test_write_read_file/1"
    rows = 10000
    self.assertFalse(gfile.Exists(file_name))
    # Write data.
    with gfile.Open(file_name, mode="w") as w:
      for i in range(rows):
        w.write("This is row\n")
    # Read data.
    with gfile.Open(file_name, mode="r") as r:
      lines = r.readlines()
    # Check that data is equal.
    self.assertEqual(rows, len(lines))
    for i in range(rows):
      self.assertEqual("This is row\n", lines[i])

  def test_delete_recursively(self):
    """Test delete recursively.

    """
    # Setup and check preconditions.
    dir_name = "igfs:///test_delete_recursively/"
    file_name = "igfs:///test_delete_recursively/1"
    self.assertFalse(gfile.Exists(dir_name))
    self.assertFalse(gfile.Exists(file_name))
    gfile.MkDir(dir_name)
    with gfile.Open(file_name, mode="w") as w:
      w.write("")
    self.assertTrue(gfile.Exists(dir_name))
    self.assertTrue(gfile.Exists(file_name))
    # Delete directory recursively.
    gfile.DeleteRecursively(dir_name)
    # Check that directory was deleted.
    self.assertFalse(gfile.Exists(dir_name))
    self.assertFalse(gfile.Exists(file_name))

  def test_copy(self):
    """Test copy.

    """
    # Setup and check preconditions.
    src_file_name = "igfs:///test_copy/1"
    dst_file_name = "igfs:///test_copy/2"
    self.assertFalse(gfile.Exists(src_file_name))
    self.assertFalse(gfile.Exists(dst_file_name))
    with gfile.Open(src_file_name, mode="w") as w:
      w.write("42")
    self.assertTrue(gfile.Exists(src_file_name))
    self.assertFalse(gfile.Exists(dst_file_name))
    # Copy file.
    gfile.Copy(src_file_name, dst_file_name)
    # Check that files are identical.
    self.assertTrue(gfile.Exists(src_file_name))
    self.assertTrue(gfile.Exists(dst_file_name))
    with gfile.Open(dst_file_name, mode="r") as r:
      data = r.read()
    self.assertEqual("42", data)

  def test_is_directory(self):
    """Test is directory.

    """
    # Setup and check preconditions.
    dir_name = "igfs:///test_is_directory/1"
    file_name = "igfs:///test_is_directory/2"
    with gfile.Open(file_name, mode="w") as w:
      w.write("")
    gfile.MkDir(dir_name)
    # Check that directory is a directory.
    self.assertTrue(gfile.IsDirectory(dir_name))
    # Check that file is not a directory.
    self.assertFalse(gfile.IsDirectory(file_name))

  def test_list_directory(self):
    """Test list directory.

    """
    # Setup and check preconditions.
    dir_name = "igfs:///test_list_directory/"
    file_names = [
        "igfs:///test_list_directory/1", "igfs:///test_list_directory/2/3"
    ]
    ch_dir_names = [
        "igfs:///test_list_directory/4",
    ]
    for file_name in file_names:
      with gfile.Open(file_name, mode="w") as w:
        w.write("")
    for ch_dir_name in ch_dir_names:
      gfile.MkDir(ch_dir_name)
    ls_expected_result = file_names + ch_dir_names
    # Get list of files in directory.
    ls_result = gfile.ListDirectory(dir_name)
    # Check that list of files is correct.
    self.assertEqual(len(ls_expected_result), len(ls_result))
    for e in ["1", "2", "4"]:
      self.assertTrue(e in ls_result)

  def test_make_dirs(self):
    """Test make dirs.

    """
    # Setup and check preconditions.
    dir_name = "igfs:///test_make_dirs/"
    self.assertFalse(gfile.Exists(dir_name))
    # Make directory.
    gfile.MkDir(dir_name)
    # Check that directory was created.
    self.assertTrue(gfile.Exists(dir_name))

  def test_remove(self):
    """Test remove.

    """
    # Setup and check preconditions.
    file_name = "igfs:///test_remove/1"
    self.assertFalse(gfile.Exists(file_name))
    with gfile.Open(file_name, mode="w") as w:
      w.write("")
    self.assertTrue(gfile.Exists(file_name))
    # Remove file.
    gfile.Remove(file_name)
    # Check that file was removed.
    self.assertFalse(gfile.Exists(file_name))

  def test_rename_file(self):
    """Test rename file.

    """
    # Setup and check preconditions.
    src_file_name = "igfs:///test_rename_file/1"
    dst_file_name = "igfs:///test_rename_file/2"
    with gfile.Open(src_file_name, mode="w") as w:
      w.write("42")
    self.assertTrue(gfile.Exists(src_file_name))
    # Rename file.
    gfile.Rename(src_file_name, dst_file_name)
    # Check that only new name of file is available.
    self.assertFalse(gfile.Exists(src_file_name))
    self.assertTrue(gfile.Exists(dst_file_name))
    with gfile.Open(dst_file_name, mode="r") as r:
      data = r.read()
    self.assertEqual("42", data)

  def test_rename_dir(self):
    """Test rename dir.

    """
    # Setup and check preconditions.
    src_dir_name = "igfs:///test_rename_dir/1"
    dst_dir_name = "igfs:///test_rename_dir/2"
    gfile.MkDir(src_dir_name)
    # Rename directory.
    gfile.Rename(src_dir_name, dst_dir_name)
    # Check that only new name of directory is available.
    self.assertFalse(gfile.Exists(src_dir_name))
    self.assertTrue(gfile.Exists(dst_dir_name))
    self.assertTrue(gfile.IsDirectory(dst_dir_name))

if __name__ == "__main__":
  test.main()


import lxml.etree as ET
import pandas as pd
from nltk.tokenize import word_tokenize
import numpy as np
import argparse
import os.path

parser = argparse.ArgumentParser()
parser.add_argument("--output_file", default=None, type=str, required=True)
parser.add_argument("--start_in", default=0, type=int, required=False)
args = parser.parse_args()

data = pd.read_csv('../corpus.csv')

if os.path.isfile(args.output_file):
	print('This file already exists!')
	exit()

with open(args.output_file, 'w') as myfile:
	# # for each instance, open the text part
	for i, instance in enumerate(data.text[args.start_in:]):
		#instance = instance.lower()
		index = i + args.start_in
		print('\n')
		print('##########################################')
		#print('\n')
		print('IDENTIFICADOR:',index)
		print(str(data.user_id[index]))
		print('##########################################')
		print('\n')
		print(instance)
		print('\n')

		# check the twit first
		tweet = ET.Element('tweet', id=str(data.user_id[index]))
		tweet_info = input('If the tweet has a *PROBLEM* type it (OL/NC/NV). Otherwise press ENTER. ')
		print('\n')
		if tweet_info not in ['OL','NC','NV', '']:
			tweet_info = input('That was not a valid option. Type OL/NC/NV or press ENTER. ')
		print('\n')
		#print(tweet_info)
		if tweet_info != "":
			tweet.set('problem', tweet_info)
			mydata = ET.tostring(tweet, encoding='unicode')#.decode("latin1") pretty_print=True,
			#print(mydata)
			myfile.write(mydata+'\n')
			myfile.flush()
			continue

		sentence = ET.SubElement(tweet, 'sentence')

		tokenised = word_tokenize(instance)
		for i, word in enumerate(tokenised):
			print(i, '\t', word)
		print('\n')
		print(instance)
		print('\n')
		sentence_info = input('Where is the first *VERB* of the tweet? (Type the position). If there is no other type ENTER. ')
		while sentence_info != "":
			main_verb = tokenised[int(sentence_info)]
			#grupverb = ET.SubElement(sentence, 'grup.verb')
			verb = ET.SubElement(sentence, 'v')
			verb.set('verb', main_verb.lower())
			verb.set('token_id', sentence_info)
			verb_sense = input("What's the verb sense?")
			verb.set('sense', verb_sense)

			print('\n')
			argument_info = input('What *ARGUMENT* does the verb "{}" have? If there is no other press ENTER. '.format(main_verb))
			print('\n')
			if argument_info not in ['arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'argM', 'argL', '']:
				argument_info = input("That was not a valid option. Type an argument or press ENTER. Options: ['arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'argM', 'argL'] ")

			while argument_info != "":
				argument = ET.SubElement(verb, 'sn')
				argument.set('arg', argument_info)

				print('\n')
				for i, word in enumerate(tokenised):
					print(i, '\t', word)
				print('\n')
				print(instance)
				print('\n')
				components = input('What *WORDS* are part of "{}"? (Type the positions coma separated): '.format(argument_info))
				for w in components.split(','):
					word = ET.SubElement(argument, 'word')
					word.set('word', tokenised[int(w)])
					word.set('token_id', w)
				#tokenised.pop(components.split(','))
				print('\n')
				print(instance)

				print('\n')
				argument_info = input('What other *ARGUMENT* does the verb "{}" have? If there is no other press ENTER. '.format(main_verb))
				print('\n')
				if argument_info not in ['arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'argM', 'argL', '']:
					argument_info = input("That was not a valid option. Type an argument or press ENTER. Options: ['arg0', 'arg1', 'arg2', 'arg3', 'arg4', 'argM', 'argL'] ")
			print('\n')
			for i, word in enumerate(tokenised):
				print(i, '\t', word)
			print('\n')
			print(instance)
			print('\n')
			sentence_info = input('Where is the next *VERB* of the tweet? (Type the position). If there is no other type ENTER. ')
			if sentence_info != "":
				new_sentence = input('Does it belong to a *NEW SENTENCE*? (y/n) ')
				if new_sentence == 'y':
					sentence = ET.SubElement(tweet, 'sentence')
			#tweet = ET.indent(tweet)
		mydata = ET.tostring(tweet, encoding='unicode')
		myfile.write(mydata+'\n')
		myfile.flush()

myfile.close()


from enum import Enum

# *``"GEOMETRY"``,
# *``"POINT"``,
# *``"LINESTRING"``,
# *``"POLYGON"``,
# *``"MULTIPOINT"``,
# *``"MULTILINESTRING"``,
# *``"MULTIPOLYGON"``,
# *``"GEOMETRYCOLLECTION"``,
# *``"CURVE"``,
# *``None``.
class GeoType(str, Enum):
    POINT = "POINT"
    Line = "LINESTRING"
    POLYGON = "POLYGON"
    MULTIPOINT = "MULTIPOINT"
    MULTILINESTRING = "MULTILINESTRING"
    MULTIPOLYGON = "MULTIPOLYGON"

class FieldType(str, Enum):
    INT = "Integer"
    FLOAT = "Float"
    STRING = "String"
    DATETIME = "DateTime"

class WorkspaceType(str, Enum):
    PUBLIC = "public"
    PRIVATE = "private"
    SHARE = "share"

class LayerType(str, Enum):
    RASTER = "raster"
    FEATURE = "feature"

class StoreType(str, Enum):
    Private = "private"
    Public = "public"
    Share = "share"


from datetime import date, datetime
from django.db.models import Q
from rest_framework import viewsets, status
from rest_framework.response import Response
from .models import Finance
from .serializers import FinanceSerializer

class FinanceViewSet(viewsets.ModelViewSet):
    """API to manage Finance objects"""
    serializer_class = FinanceSerializer
    queryset = Finance.objects.all().order_by('due_date')

    def get_queryset(self):
        queryset = super().get_queryset()

        if self.request.GET.get('type') is None:
            return queryset

        queryset = queryset.filter(concept__type=self.request.GET.get('type'))

        start_date = self.request.GET.get('start_date')
        end_date = self.request.GET.get('end_date')

        if (start_date is None):
            current_month = datetime.now().month
            current_year = datetime.now().year
            return queryset.filter(Q(due_date__year__gte=current_year) & Q(due_date__month__range=(current_month, current_month)))

        if (end_date):
            return queryset.filter(due_date__range=(start_date, end_date))

        return queryset.filter(due_date__gte=start_date)

    def partial_update(self, request, *args, **kwargs):
        try:
            finance = Finance.objects.get(pk=kwargs['pk'])
            if finance.paid == False:
                finance.paid = True
                finance.pay_date = date.today()
            else:
                finance.paid = False
                finance.pay_date = None
            finance.save()
            return Response(FinanceSerializer(finance).data, status=status.HTTP_200_OK)
        except Finance.DoesNotExist:
            message = 'Finance not found'
            return Response({'message': message}, status=status.HTTP_404_NOT_FOUND)


# Write a Python program to implement Graph using Adjacency list along with the following function:
# a. To count number of vertices and edges present in a graph.
# b. To find the adjacent vertices of a given vertex.
# c. To search a node in a given graph.

class AdjNode:
    def __init__(self, value):
        self.vertex = value
        self.next = None

class Graph:
    def __init__(self, num):
        self.V = num
        self.graph = [None] * self.V

    def addEdge(self, s, d):
        node = AdjNode(d)
        node.next = self.graph[s]
        self.graph[s] = node

        node = AdjNode(s)
        node.next = self.graph[d]
        self.graph[d] = node

    def printGraph(self):
        for i in range(self.V):
            print("\nVertex", i, end="")
            temp = self.graph[i]
            while temp:
                print(" -> {}".format(temp.vertex), end="")
                temp = temp.next

if __name__ == "__main__":
    n = int(input("Enter Number of Vertices: "))
    m = int(input("Enter Number of Edges: "))
    graph = Graph(n)
    for i in range(m):
        u, v = map(int, input("Add Edge Between: ").split())
        graph.addEdge(u, v)
    graph.printGraph()

"""
    0-----1
   /  \   |      6
  3    \  |    /  \
  |     \ |   7----5
  4 ---- 2

"""

# OUTPUT
# Enter Number of Vertices: 8
# Enter Number of Edges: 9
# Add Edge Between: 0 1
# Add Edge Between: 0 2
# Add Edge Between: 0 3
# Add Edge Between: 1 2
# Add Edge Between: 2 4
# Add Edge Between: 3 4
# Add Edge Between: 5 6
# Add Edge Between: 5 7
# Add Edge Between: 6 7

# Vertex 0 -> 3 -> 2 -> 1
# Vertex 1 -> 2 -> 0
# Vertex 2 -> 4 -> 1 -> 0
# Vertex 3 -> 4 -> 0
# Vertex 4 -> 3 -> 2
# Vertex 5 -> 7 -> 6
# Vertex 6 -> 7 -> 5
# Vertex 7 -> 6 -> 5


import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from PyQt5 import uic
from Kiwoom import *
import time
#ui에 연결
form_class = uic.loadUiType("pytrader.ui")[0]

class MyWindow(QMainWindow, form_class):
    def __init__(self):
        super().__init__()
        self.setupUi(self)

        self.trade_stocks_done = False
        #키움에 연결
        self.kiwoom = Kiwoom()
        self.kiwoom.comm_connect()
        # 현시간 확인
        self.timer = QTimer(self)
        self.timer.start(1000)
        self.timer.timeout.connect(self.timeout)
        #타이머 2 10초후 연결
        self.timer2 = QTimer(self)
        self.timer2.start(1000 *10)
        self.timer2.timeout.connect(self.timeout2)
        #계좌정보 얻기 (지금은 모의계좌)
        accouns_num = int(self.kiwoom.get_login_info("ACCOUNT_CNT"))
        accounts = self.kiwoom.get_login_info("ACCNO")
        #계좌번호는 ;로 분리되어있어서 ; 기준으로 분류해서 저장
        accounts_list = accounts.split(';')[0:accouns_num]
        self.w_account.addItems(accounts_list)
        #연결 사실 잘 모르겠음 아마 접속하는 버튼 누르는거임
        self.w_event_code.textChanged.connect(self.code_changed)
        self.w_orderbutton.clicked.connect(self.send_order)
        self.w_lookup.clicked.connect(self.check_balance)

        self.load_buy_sell_list()
    # 파일 buy리스트와 sell리스트를 들고와서 변수에 값 저장
    def trade_stocks(self):
        hoga_lookup = {'지정가': "00", '시장가': "03"}

        f = open("buy_list.txt", 'rt')
        buy_list = f.readlines()
        f.close()

        f = open("sell_list.txt", 'rt')
        sell_list = f.readlines()
        f.close()

        # account
        account = self.w_account.currentText()

        # buy list 에서 살 주식 주문
        for row_data in buy_list:
            split_row_data = row_data.split(';')
            hoga = split_row_data[2]
            code = split_row_data[1]
            num = split_row_data[3]
            price = split_row_data[4]

            if split_row_data[-1].rstrip() == '매수전':
                self.kiwoom.send_order("send_order_req", "0101", account, 1, code, num, price, hoga_lookup[hoga], "")

        # sell list 팔 주식 주문
        for row_data in sell_list:
            split_row_data = row_data.split(';')
            hoga = split_row_data[2]
            code = split_row_data[1]
            num = split_row_data[3]
            price = split_row_data[4]

            if split_row_data[-1].rstrip() == '매도전':
                self.kiwoom.send_order("send_order_req", "0101", account, 2, code, num, price, hoga_lookup[hoga], "")

        # buy list 삿을경우 매수전을 주문완료로
        for i, row_data in enumerate(buy_list):
            buy_list[i] = buy_list[i].replace("매수전", "주문완료")

        # file update 사야할 목록 업데이트
        f = open("buy_list.txt", 'wt')
        for row_data in buy_list:
            f.write(row_data)
        f.close()

        # sell list 팔았을 경우 주문완료로
        for i, row_data in enumerate(sell_list):
            sell_list[i] = sell_list[i].replace("매도전", "주문완료")

        # file update 팔아야할 목록 업데이트 이는 pymon에서 들고올거
        f = open("sell_list.txt", 'wt')
        for row_data in sell_list:
            f.write(row_data)
        f.close()

    # 산 주식의 리스트를 가져온다
    def load_buy_sell_list(self):
        f = open("buy_list.txt", 'rt')
        buy_list = f.readlines()
        f.close()

        f = open("sell_list.txt", 'rt')
        sell_list = f.readlines()
        f.close()

        row_count = len(buy_list) + len(sell_list)
        self.tableWidget_4.setRowCount(row_count)

        # buy list
        for j in range(len(buy_list)):
            row_data = buy_list[j]
            split_row_data = row_data.split(';')
            split_row_data[1] = self.kiwoom.get_master_code_name(split_row_data[1].rsplit())

            for i in range(len(split_row_data)):
                item = QTableWidgetItem(split_row_data[i].rstrip())
                item.setTextAlignment(Qt.AlignVCenter | Qt.AlignCenter)
                self.tableWidget_4.setItem(j, i, item)

        # sell list
        for j in range(len(sell_list)):
            row_data = sell_list[j]
            split_row_data = row_data.split(';')
            split_row_data[1] = self.kiwoom.get_master_code_name(split_row_data[1].rstrip())

            for i in range(len(split_row_data)):
                item = QTableWidgetItem(split_row_data[i].rstrip())
                item.setTextAlignment(Qt.AlignVCenter | Qt.AlignCenter)
                self.tableWidget_4.setItem(len(buy_list) + j, i, item)

        self.tableWidget_4.resizeRowsToContents()
    #이건 주식 코드를 치면 코드의 이름을 받아온다.
    def code_changed(self):
        code = self.w_event_code.text()
        name = self.kiwoom.get_master_code_name(code)
        self.w_event_name.setText(name)
    #주문하는창
    def send_order(self):
        order_type_lookup = {'신규매수': 1, '신규매도': 2, '매수취소': 3, '매도취소': 4}
        hoga_lookup = {'지정가': "00", '시장가': "03"}

        account = self.w_account.currentText()
        order_type = self.w_order.currentText()
        code = self.w_event_code.text()
        hoga = self.w_kinds.currentText()
        num = self.w_quantity.value()
        price = self.w_price.value()

        self.kiwoom.send_order("send_order_req", "0101", account, order_type_lookup[order_type], code, num, price, hoga_lookup[hoga], "")
    #시간이 9시가 되면 자동으로 사게하는 함수
    def timeout(self):
        market_start_time = QTime(9, 0, 0)
        current_time = QTime.currentTime()
        #현재시간이 마켓이 시작했고, trade_stocks_done이라는 변수는 맨처음에 false로 저장해놓는데 첫번째로 시작했으면 trade_stocks를 실행하는 코드
        if current_time > market_start_time and self.trade_stocks_done is False:
            self.trade_stocks()
            self.trade_stocks_done = True

        text_time = current_time.toString("hh:mm:ss")
        time_msg = "현재시간: " + text_time
        #연결상태
        state = self.kiwoom.get_connect_state()
        if state == 1:
            state_msg = "서버 연결 중"
        else:
            state_msg = "서버 미 연결 중"

        self.statusbar.showMessage(state_msg + " | " + time_msg)

    def timeout2(self):
        if self.checkBox.isChecked():
            self.check_balance()
    #이름은 잔고확인인데 계좌에서 데이터 뽑아온다 (이는 opw00018이다.)
    def check_balance(self):
        self.kiwoom.reset_opw00018_output()
        account_number = self.kiwoom.get_login_info("ACCNO")
        account_number = account_number.split(';')[0]

        self.kiwoom.set_input_value("계좌번호", account_number)
        self.kiwoom.comm_rq_data("opw00018_req", "opw00018", 0, "2000")

        while self.kiwoom.remained_data:
            time.sleep(0.2)
            self.kiwoom.set_input_value("계좌번호", account_number)
            self.kiwoom.comm_rq_data("opw00018_req", "opw00018", 2, "2000")

        # opw00001
        self.kiwoom.set_input_value("계좌번호", account_number)
        self.kiwoom.comm_rq_data("opw00001_req", "opw00001", 0, "2000")

        # balance
        item = QTableWidgetItem(self.kiwoom.d2_deposit)
        item.setTextAlignment(Qt.AlignVCenter | Qt.AlignRight)
        self.tableWidget.setItem(0, 0, item)

        for i in range(1, 6):
            item = QTableWidgetItem(self.kiwoom.opw00018_output['single'][i - 1])
            item.setTextAlignment(Qt.AlignVCenter | Qt.AlignRight)
            self.tableWidget.setItem(0, i, item)

        self.tableWidget.resizeRowsToContents()

        # Item list
        item_count = len(self.kiwoom.opw00018_output['multi'])
        self.tableWidget_2.setRowCount(item_count)

        for j in range(item_count):
            row = self.kiwoom.opw00018_output['multi'][j]
            for i in range(len(row)):
                item = QTableWidgetItem(row[i])
                item.setTextAlignment(Qt.AlignVCenter | Qt.AlignRight)
                self.tableWidget_2.setItem(j, i, item)

        self.tableWidget_2.resizeRowsToContents()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    myWindow = MyWindow()
    myWindow.show()
    app.exec_()


# pylint: disable=global-statement
# pylint: disable=unused-import

from tkinter import *
from tkinter import ttk
import tkinter as tk
from search_function import *

# Global variable for dark mode toggle status.
DARK_MODE_TOGGLE = 0

# function that takes the passed frame and controller variable and adds the navigation bar
# to the top of the frame
def display_nav_bar(frame, controller):
    """
    Helper function to display the nav bar.
    """
    # defines nav_bar a new canvas on the passed frame
    nav_bar = Canvas(frame)  ##FDFDFD
    search_canvas = Canvas(frame)

    # places nav bar at the top of the frame
    nav_bar.place(relx=0, relheight=0.08, relwidth=1)

    # these variables influence the placement and appearance of the menu option buttons
    # they've been centralised into variables for ease of modification
    # used for the width of the buttons
    btn_width = 0.11
    # used to track the button num (used for spacing on the right side of the nav_bar equally)
    btn_num = 1
    # defines the buttons that will appear in the navigation bar
    button_home = ttk.Button(frame, text="Home", style="Accent.TButton",
                             command=lambda: [search_leave(search_canvas),
                             controller.show_frame("StartPage")])
    button_vectors = ttk.Button(frame, text="Attack Vectors", style="Accent.TButton",
                                command=lambda: [search_leave(search_canvas),
                                controller.show_frame("VectorsPage")])
    button_about = ttk.Button(frame, text="About", style="Accent.TButton",
                              command=lambda: [search_leave(search_canvas),
                              controller.show_frame("AboutPage")])
    button_tools = ttk.Button(frame, text="Tools", style="Accent.TButton",
                              command=lambda: [search_leave(search_canvas),
                              controller.show_frame("ToolsPage")])
    button_walkthroughs = ttk.Button(frame, text="Walkthroughs", style="Accent.TButton",
                                     command=lambda: [search_leave(search_canvas),
                                     controller.show_frame("WalkthroughClass")])
    button_references = ttk.Button(frame, text="References", style="Accent.TButton",
                                   command=lambda: [search_leave(search_canvas),
                                   controller.show_frame("ReferencesPage")])
    # Dark mode switch global to ensure state is retained.
    global DARK_MODE_TOGGLE
    DARK_MODE_TOGGLE = 0
    dark_mode_switch = ttk.Checkbutton(frame, text="Dark mode", style="Switch.TCheckbutton",
                                       command=lambda: change_theme(controller),
                                       variable=DARK_MODE_TOGGLE)

    # this code block places all the buttons in the correct order
    # btn_num is used to correctly space out the buttons on the right side of the nav_bar
    # and must be incremented every time a button is added to the right side of the nav bar
    button_home.place(rely=0.01, relx=0.01, relheight=0.06, relwidth=0.055)
    button_references.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01,
    relheight=0.06,
                            relwidth=btn_width)

    btn_num += 1
    button_walkthroughs.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01,
    relheight=0.06,
                              relwidth=btn_width)

    btn_num += 1
    button_tools.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01, relheight=0.06,
    relwidth=btn_width)

    btn_num += 1
    button_vectors.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01, relheight=0.06,
    relwidth=btn_width)

    btn_num += 1
    button_about.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01, relheight=0.06,
    relwidth=btn_width)
    btn_num += 1

    dark_mode_switch.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num*0.01,relheight=0.06,
                            relwidth=btn_width)
    btn_num += 1

    stringvar = tk.StringVar(frame)
    search_field = Entry(frame, textvariable=stringvar, text="Search")
    search_field.place(rely=0.01, relx=1 - btn_width * btn_num - btn_num * 0.01, relheight=0.06,
                      relwidth=btn_width)

    #whenever the user has released a key press (e.g. any changes to the searchbar)...
    #do search event
    search_field.bind("<KeyRelease>",lambda event,
                      search_field=search_field: search_event(search_field,frame,search_canvas,
                                                               controller))
    search_field.bind("<Return>",lambda event,
                      search_field = search_field: build_search(search_field, search_canvas,
                                                                controller))

def change_theme(controller):
    """
    Helper function to change the theme of the application.
    """
    # NOTE: The theme's real name is azure-<mode>
    if controller.tk.call("ttk::style", "theme", "use") == "azure-dark":
        # Set light theme
        controller.tk.call("set_theme", "light")
    else:
        # Set dark theme
        controller.tk.call("set_theme", "dark")


from ex30 import load_mecab

if __name__ == '__main__':
    filename = 'neko.txt.mecab'
    tokens = load_mecab(filename)
    norms:list[str] = list()
    for token in tokens:
        if token['pos'] == '名詞':
            norms.append(token['surface'])
        else:
            if len(norms) > 1:
                print(''.join(norms))
                norms = list()