import os
import posixpath
import errno
import json
import resource
import sys
import shutil
import textwrap
import urllib.parse
import urllib.request
import warnings
import logging

# External modules
import click
import yaml
# We import botocore here so we can catch when the user tries to
# access AWS without having their credentials configured and provide
# a friendly error message. Apart from that, flintrock.py should
# not really know anything about EC2 or boto since that is delegated
# to ec2.py.
import botocore

# Flintrock modules
from . import ec2
from .exceptions import (
    UsageError,
    UnsupportedProviderError,
    NothingToDo,
    Error)
from flintrock import __version__
from .util import spark_hadoop_build_version
from .services import HDFS, Spark  # TODO: Remove this dependency.

FROZEN = getattr(sys, 'frozen', False)

if FROZEN:
    THIS_DIR = sys._MEIPASS
else:
    THIS_DIR = os.path.dirname(os.path.realpath(__file__))

logger = logging.getLogger('flintrock.flintrock')

def format_message(*, message: str, indent: int=4, wrap: int=70):
    """
    Format a lengthy message for printing to screen.
    """
    return textwrap.indent(
        textwrap.fill(
            textwrap.dedent(text=message),
            width=wrap),
        prefix=' ' * indent)

def option_name_to_variable_name(option: str):
    """
    Convert an option name like `--ec2-user` to the Python name it gets mapped to,
    like `ec2_user`.
    """
    return option.replace('--', '', 1).replace('-', '_')

def variable_name_to_option_name(variable: str):
    """
    Convert a variable name like `ec2_user` to the Click option name it gets mapped to,
    like `--ec2-user`.
    """
    return '--' + variable.replace('_', '-')

def option_requires(
        *,
        option: str,
        conditional_value=None,
        requires_all: list=[],
        requires_any: list=[],
        scope: dict):
    """
    Raise an exception if an option's requirements are not met.

    The option's requirements are checked only if the option has a "truthy" value
    (i.e. it's not a "falsy" value like '', None, or False), and if its value is
    equal to conditional_value, if conditional_value is not None.

    requires_all: Every option in this list must be defined.
    requires_any: At least one option in this list must be defined.

    This function looks for values by converting the option names to their
    corresponding variable names (e.g. --option-a becomes option_a) and looking them
    up in the provided scope.
    """
    option_value = scope[option_name_to_variable_name(option)]

    if option_value and \
            (conditional_value is None or option_value == conditional_value):
        if requires_all:
            for required_option in requires_all:
                required_name = option_name_to_variable_name(required_option)
                if required_name not in scope or not scope[required_name]:
                    raise UsageError(
                        "Error: Missing option \"{missing_option}\" is required by "
                        "\"{option}{space}{conditional_value}\"."
                        .format(
                            missing_option=required_option,
                            option=option,
                            space=' ' if conditional_value is not None else '',
                            conditional_value=conditional_value if conditional_value is not None else ''))
        if requires_any:
            for required_option in requires_any:
                required_name = option_name_to_variable_name(required_option)
                if required_name in scope and scope[required_name] is not None:
                    break
            else:
                raise UsageError(
                    "Error: \"{option}{space}{conditional_value}\" requires at least "
                    "one of the following options to be set: {at_least}"
                    .format(
                        option=option,
                        space=' ' if conditional_value is not None else '',
                        conditional_value=conditional_value if conditional_value is not None else '',
                        at_least=', '.join(['"' + ra + '"' for ra in requires_any])))

def mutually_exclusive(*, options: list, scope: dict):
    """
    Raise an exception if more than one of the provided options is specified.

    This function looks for values by converting the option names to their
    corresponding variable names (e.g. --option-a becomes option_a) and looking them
    up in the provided scope.
    """
    mutually_exclusive_names = [option_name_to_variable_name(o) for o in options]

    used_options = set()
    for name, value in scope.items():
        if name in mutually_exclusive_names and scope[name]:  # is not None:
            used_options.add(name)

    if len(used_options) > 1:
        bad_option1 = used_options.pop()
        bad_option2 = used_options.pop()
        raise UsageError(
            "Error: \"{option1}\" and \"{option2}\" are mutually exclusive.\n"
            "  {option1}: {value1}\n"
            "  {option2}: {value2}"
            .format(
                option1=variable_name_to_option_name(bad_option1),
                value1=scope[bad_option1],
                option2=variable_name_to_option_name(bad_option2),
                value2=scope[bad_option2]))

def get_config_file() -> str:
    """
    Get the path to Flintrock's default configuration file.
    """
    config_dir = click.get_app_dir(app_name='Flintrock')
    config_file = os.path.join(config_dir, 'config.yaml')
    return config_file

def configure_log(debug: bool):
    root_logger = logging.getLogger('flintrock')
    handler = logging.StreamHandler(sys.stdout)
    handler.setLevel(logging.DEBUG)
    if debug:
        root_logger.setLevel(logging.DEBUG)
        handler.setFormatter(logging.Formatter('%(asctime)s - flintrock.%(module)-9s - %(levelname)-5s - %(message)s'))
    else:
        root_logger.setLevel(logging.INFO)
        handler.setFormatter(logging.Formatter('%(message)s'))
    root_logger.addHandler(handler)

def build_hdfs_download_url(ctx, param, value):
    hdfs_version = ctx.params['hdfs_version']
    if value.endswith('.gz') or value.endswith('.tgz'):
        logger.warning(
            "Hadoop download source appears to point to a file, not a directory. "
            "Flintrock will not try to determine the correct file to download based on "
            "the Hadoop version."
        )
        hdfs_download_url = value
    else:
        hdfs_download_url = (value.rstrip('/') + '/hadoop-{v}.tar.gz')
    return hdfs_download_url.format(v=hdfs_version)

def build_spark_download_url(ctx, param, value):
    spark_version = ctx.params['spark_version']
    hadoop_version = ctx.params['hdfs_version']
    hadoop_build_version = spark_hadoop_build_version(hadoop_version)

    # Starting in Spark 3.3.0, the build artifact naming scheme changed a bit.
    # Instead of 'hadoop3.2', for example, that part now reads 'hadoop3'.
    if spark_version:
        spark_version_tuple = tuple(map(int, spark_version.split('.')))
        if spark_version_tuple >= (3, 3, 0):
            hadoop_build_version = hadoop_build_version.split('.')[0]

    if value.endswith('.gz') or value.endswith('.tgz'):
        logger.warning(
            "Spark download source appears to point to a file, not a directory. "
            "Flintrock will not try to determine the correct file to download based on "
            "the Spark and Hadoop versions."
        )
        spark_download_url = value
    else:
        spark_download_url = (value.rstrip('/') + '/spark-{v}-bin-{hv}.tgz')
    return spark_download_url.format(
        v=spark_version,
        hv=hadoop_build_version,
    )

def validate_download_source(url):
    if 'spark' in url:
        software = 'Spark'
    elif 'hadoop' in url:
        software = 'Hadoop'
    else:
        software = 'software'

    parsed_url = urllib.parse.urlparse(url)

    if parsed_url.netloc == 'www.apache.org' and parsed_url.path == '/dyn/closer.lua':
        logger.warning(
            "Warning: "
            "Downloading {software} from an Apache mirror. Apache mirrors are "
            "often slow and unreliable, and typically only serve the most recent releases. "
            "We strongly recommend you specify a custom download source. "
            "For more background on this issue, please see: https://github.com/nchammas/flintrock/issues/238"
            .format(
                software=software,
            )
        )
        try:
            urllib.request.urlopen(url)
        except urllib.error.HTTPError as e:
            raise Error(
                "Error: Could not access {software} download. Maybe try a more recent release?\n"
                "  - Automatically redirected to: {url}\n"
                "  - HTTP error: {code}"
                .format(
                    software=software,
                    url=e.url,
                    code=e.code,
                )
            )

@click.group()
@click.option(
    '--config',
    help="Path to a Flintrock configuration file.",
    default=get_config_file())
@click.option('--provider', default='ec2', type=click.Choice(['ec2']))
@click.version_option(version=__version__)
# TODO: implement some solution like in https://github.com/pallets/click/issues/108
@click.option('--debug/--no-debug', default=False, help="Show debug information.")
@click.pass_context
def cli(cli_context, config, provider, debug):
    """
    Flintrock

    A command-line tool for launching Apache Spark clusters.
    """
    cli_context.obj['provider'] = provider

    if os.path.isfile(config):
        with open(config) as f:
            config_raw = yaml.safe_load(f)
            debug = config_raw.get('debug') or debug
            config_map = config_to_click(normalize_keys(config_raw))

        cli_context.default_map = config_map
    else:
        if config != get_config_file():
            raise FileNotFoundError(errno.ENOENT, 'No such file', config)
    configure_log(debug=debug)

@cli.command()
@click.argument('cluster-name')
@click.option('--num-slaves', type=click.IntRange(min=1), required=True)
@click.option('--java-version', type=click.IntRange(min=8), default=11)
@click.option('--install-hdfs/--no-install-hdfs', default=False)
@click.option('--hdfs-version', default='3.3.4')
@click.option('--hdfs-download-source',
              help=(
                  "URL to download Hadoop from. If an S3 URL, Flintrock will use the "
                  "AWS CLI from the cluster nodes to download it. "
                  "Flintrock will append the appropriate file name to the end "
                  "of the URL based on the Apache release file names here: "
                  "https://dist.apache.org/repos/dist/release/hadoop/common/"
              ),
              default='https://www.apache.org/dyn/closer.lua?action=download&filename=hadoop/common/hadoop-{v}/',
              show_default=True,
              callback=build_hdfs_download_url)
@click.option('--install-spark/--no-install-spark', default=True)
@click.option('--spark-executor-instances', default=1,
              help="How many executor instances per worker.")
@click.option('--spark-version',
              # Don't set a default here because it will conflict with
              # the config file if the git commit is set.
              # See: https://github.com/nchammas/flintrock/issues/190
              # default=,
              help="Spark release version to install.")
@click.option('--spark-download-source',
              help=(
                  "URL to download Spark from. If an S3 URL, Flintrock will use the "
                  "AWS CLI from the cluster nodes to download it. "
                  "Flintrock will append the appropriate file "
                  "name to the end of the URL based on the selected Hadoop version and "
                  "Apache release file names here: "
                  "https://dist.apache.org/repos/dist/release/spark/"
              ),
              default='https://www.apache.org/dyn/closer.lua?action=download&filename=spark/spark-{v}/',
              show_default=True,
              callback=build_spark_download_url)
@click.option('--spark-git-commit',
              help="Git commit to build Spark from. "
                   "Set to 'latest' to build Spark from the latest commit on the "
                   "repository's default branch.")
@click.option('--spark-git-repository',
              help="Git repository to clone Spark from.",
              default='https://github.com/apache/spark',
              show_default=True)
@click.option('--assume-yes/--no-assume-yes', default=False)
@click.option('--ec2-key-name')
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-instance-type', default='m5.medium', show_default=True)
@click.option('--ec2-region', default='us-east-1', show_default=True)
# We set some of these defaults to empty strings because of boto3's parameter validation.
# See: https://github.com/boto/boto3/issues/400
@click.option('--ec2-availability-zone', default='')
@click.option('--ec2-ami')
@click.option('--ec2-user')
@click.option('--ec2-security-group', 'ec2_security_groups',
              multiple=True,
              help="Additional security groups names to assign to the instances. "
                   "You can specify this option multiple times.")
@click.option('--ec2-spot-price', type=float)
@click.option('--ec2-spot-request-duration', default='7d',
              help="Duration a spot request is valid (e.g. 3d 2h 1m).")
@click.option('--ec2-min-root-ebs-size-gb', type=int, default=30)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--ec2-subnet-id', default='')
@click.option('--ec2-instance-profile-name', default='')
@click.option('--ec2-placement-group', default='')
@click.option('--ec2-tenancy', default='default')
@click.option('--ec2-ebs-optimized/--no-ec2-ebs-optimized', default=False)
@click.option('--ec2-instance-initiated-shutdown-behavior', default='stop',
              type=click.Choice(['stop', 'terminate']))
@click.option('--ec2-user-data',
              type=click.File(mode='r', encoding='utf-8'),
              help="Path to EC2 user data script that will run on instance launch.")
@click.option('--ec2-tag', 'ec2_tags',
              callback=ec2.cli_validate_tags,
              multiple=True,
              help="Additional tags (e.g. 'Key,Value') to assign to the instances. "
                   "You can specify this option multiple times.")
@click.option('--ec2-authorize-access-from',
              callback=ec2.cli_validate_ec2_authorize_access,
              multiple=True,
              help=(
                  "Authorize cluster access from a specific source (e.g. on a private "
                  "network). The source can be a) a plain IP address, b) an IP "
                  "address in CIDR notation, or c) an EC2 Security Group ID. "
                  "Using this option disables automatic detection of client's public IP "
                  "address."
              ))
@click.pass_context
def launch(
        cli_context,
        cluster_name,
        num_slaves,
        java_version,
        install_hdfs,
        hdfs_version,
        hdfs_download_source,
        install_spark,
        spark_executor_instances,
        spark_version,
        spark_git_commit,
        spark_git_repository,
        spark_download_source,
        assume_yes,
        ec2_key_name,
        ec2_identity_file,
        ec2_instance_type,
        ec2_region,
        ec2_availability_zone,
        ec2_ami,
        ec2_user,
        ec2_security_groups,
        ec2_spot_price,
        ec2_spot_request_duration,
        ec2_min_root_ebs_size_gb,
        ec2_vpc_id,
        ec2_subnet_id,
        ec2_instance_profile_name,
        ec2_placement_group,
        ec2_tenancy,
        ec2_ebs_optimized,
        ec2_instance_initiated_shutdown_behavior,
        ec2_user_data,
        ec2_tags,
        ec2_authorize_access_from):
    """
    Launch a new cluster.
    """
    provider = cli_context.obj['provider']
    services = []

    option_requires(
        option='--install-hdfs',
        requires_all=['--hdfs-version'],
        scope=locals())
    option_requires(
        option='--install-spark',
        requires_any=[
            '--spark-version',
            '--spark-git-commit'],
        scope=locals())
    mutually_exclusive(
        options=[
            '--spark-version',
            '--spark-git-commit'],
        scope=locals())
    option_requires(
        option='--install-spark',
        requires_all=[
            '--hdfs-version'],
        scope=locals())
    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-key-name',
            '--ec2-identity-file',
            '--ec2-instance-type',
            '--ec2-region',
            '--ec2-ami',
            '--ec2-user'],
        scope=locals())
    # The subnet is required for non-default VPCs because EC2 does not
    # support user-defined default subnets.
    # See: https://forums.aws.amazon.com/thread.jspa?messageID=707417
    #      https://github.com/mitchellh/packer/issues/1935#issuecomment-111235752
    option_requires(
        option='--ec2-vpc-id',
        requires_all=['--ec2-subnet-id'],
        scope=locals())

    check_external_dependency('ssh-keygen')

    if install_hdfs:
        validate_download_source(hdfs_download_source)
        hdfs = HDFS(
            version=hdfs_version,
            download_source=hdfs_download_source,
        )
        services += [hdfs]
    if install_spark:
        if spark_version:
            validate_download_source(spark_download_source)
            spark = Spark(
                spark_executor_instances=spark_executor_instances,
                version=spark_version,
                hadoop_version=hdfs_version,
                download_source=spark_download_source,
            )
        elif spark_git_commit:
            logger.warning(
                "Warning: Building Spark takes a long time. "
                "e.g. 15-20 minutes on an m5.xlarge instance on EC2.")
            if spark_git_commit == 'latest':
                spark_git_commit = get_latest_commit(spark_git_repository)
                logger.info("Building Spark at latest commit: {c}".format(c=spark_git_commit))
            spark = Spark(
                spark_executor_instances=spark_executor_instances,
                git_commit=spark_git_commit,
                git_repository=spark_git_repository,
                hadoop_version=hdfs_version,
            )
        services += [spark]

    if provider == 'ec2':
        cluster = ec2.launch(
            cluster_name=cluster_name,
            num_slaves=num_slaves,
            java_version=java_version,
            services=services,
            assume_yes=assume_yes,
            key_name=ec2_key_name,
            identity_file=ec2_identity_file,
            instance_type=ec2_instance_type,
            region=ec2_region,
            availability_zone=ec2_availability_zone,
            ami=ec2_ami,
            user=ec2_user,
            security_groups=ec2_security_groups,
            spot_price=ec2_spot_price,
            spot_request_duration=ec2_spot_request_duration,
            min_root_ebs_size_gb=ec2_min_root_ebs_size_gb,
            vpc_id=ec2_vpc_id,
            subnet_id=ec2_subnet_id,
            instance_profile_name=ec2_instance_profile_name,
            placement_group=ec2_placement_group,
            tenancy=ec2_tenancy,
            ebs_optimized=ec2_ebs_optimized,
            instance_initiated_shutdown_behavior=ec2_instance_initiated_shutdown_behavior,
            user_data=ec2_user_data,
            tags=ec2_tags,
            ec2_authorize_access_from=ec2_authorize_access_from)
    else:
        raise UnsupportedProviderError(provider)

    print("Cluster master: {}".format(cluster.master_host))
    print("Login with: flintrock login {}".format(cluster.name))

def get_latest_commit(github_repository: str):
    """
    Get the latest commit on the default branch of a repository hosted on GitHub.
    """
    parsed_url = urllib.parse.urlparse(github_repository)
    repo_domain, repo_path = parsed_url.netloc, parsed_url.path.strip('/')

    if repo_domain != 'github.com':
        raise UsageError(
            "Error: Getting the latest commit is only supported "
            "for repositories hosted on GitHub. "
            "Provided repository domain was: {d}".format(d=repo_domain))

    url = "https://api.github.com/repos/{rp}/commits".format(rp=repo_path)
    try:
        with urllib.request.urlopen(url) as response:
            result = json.loads(response.read().decode('utf-8'))
            return result[0]['sha']
    except Exception as e:
        raise Exception(
            "Could not get latest commit for repository: {r}"
            .format(r=repo_path)) from e

@cli.command()
@click.argument('cluster-name')
@click.option('--assume-yes/--no-assume-yes', default=False)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.pass_context
def destroy(cli_context, cluster_name, assume_yes, ec2_region, ec2_vpc_id):
    """
    Destroy a cluster.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=['--ec2-region'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
    else:
        raise UnsupportedProviderError(provider)

    if not assume_yes:
        cluster.print()
        click.confirm(
            text="Are you sure you want to destroy this cluster?",
            abort=True)

    logger.info("Destroying {c}...".format(c=cluster.name))
    cluster.destroy()

@cli.command()
@click.argument('cluster-name', required=False)
@click.option('--master-hostname-only', is_flag=True, default=False)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.pass_context
def describe(
        cli_context,
        cluster_name,
        master_hostname_only,
        ec2_region,
        ec2_vpc_id):
    """
    Describe an existing cluster.

    Leave out the cluster name to find all Flintrock-managed clusters.

    The output of this command is both human- and machine-friendly. Full cluster
    descriptions are output in YAML.
    """
    provider = cli_context.obj['provider']
    search_area = ""

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=['--ec2-region'],
        scope=locals())

    if cluster_name:
        cluster_names = [cluster_name]
    else:
        cluster_names = []

    if provider == 'ec2':
        search_area = "in region {r}".format(r=ec2_region)
        clusters = ec2.get_clusters(
            cluster_names=cluster_names,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
    else:
        raise UnsupportedProviderError(provider)

    if cluster_name:
        cluster = clusters[0]
        if master_hostname_only:
            logger.info(cluster.master_host)
        else:
            cluster.print()
    else:
        if master_hostname_only:
            for cluster in sorted(clusters, key=lambda x: x.name):
                logger.info("{}: {}".format(cluster.name, cluster.master_host))
        else:
            logger.info("Found {n} cluster{s}{space}{search_area}.".format(
                n=len(clusters),
                s='' if len(clusters) == 1 else 's',
                space=' ' if search_area else '',
                search_area=search_area))
            if clusters:
                logger.info('---')
                for cluster in sorted(clusters, key=lambda x: x.name):
                    cluster.print()

# TODO: Provide different command or option for going straight to Spark Shell. (?)
@cli.command()
@click.argument('cluster-name')
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
# TODO: Move identity-file to global, non-provider-specific option. (?)
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-user')
@click.pass_context
def login(cli_context, cluster_name, ec2_region, ec2_vpc_id, ec2_identity_file, ec2_user):
    """
    Login to the master of an existing cluster.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-identity-file',
            '--ec2-user'],
        scope=locals())

    check_external_dependency('ssh')

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
    else:
        raise UnsupportedProviderError(provider)

    # TODO: Check that master up first and error out cleanly if not
    #       via ClusterInvalidState.
    cluster.login(user=user, identity_file=identity_file)

@cli.command()
@click.argument('cluster-name')
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
# TODO: Move identity-file to global, non-provider-specific option. (?)
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-user')
@click.pass_context
def start(cli_context, cluster_name, ec2_region, ec2_vpc_id, ec2_identity_file, ec2_user):
    """
    Start an existing, stopped cluster.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-identity-file',
            '--ec2-user'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
    else:
        raise UnsupportedProviderError(provider)

    cluster.start_check()
    logger.info("Starting {c}...".format(c=cluster_name))
    cluster.start(user=user, identity_file=identity_file)

@cli.command()
@click.argument('cluster-name')
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--assume-yes/--no-assume-yes', default=False)
@click.pass_context
def stop(cli_context, cluster_name, ec2_region, ec2_vpc_id, assume_yes):
    """
    Stop an existing, running cluster.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=['--ec2-region'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
    else:
        raise UnsupportedProviderError(provider)

    cluster.stop_check()

    if not assume_yes:
        cluster.print()
        click.confirm(
            text="Are you sure you want to stop this cluster?",
            abort=True)

    logger.info("Stopping {c}...".format(c=cluster_name))
    cluster.stop()
    logger.info("{c} is now stopped.".format(c=cluster_name))

@cli.command(name='add-slaves')
@click.argument('cluster-name')
@click.option('--num-slaves', type=click.IntRange(min=1), required=True)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-user')
@click.option('--ec2-spot-price', type=float)
@click.option('--ec2-spot-request-duration', default='7d',
              help="Duration a spot request is valid (e.g. 3d 2h 1m).")
@click.option('--ec2-min-root-ebs-size-gb', type=int, default=30)
@click.option('--assume-yes/--no-assume-yes', default=False)
@click.option('--ec2-tag', 'ec2_tags',
              callback=ec2.cli_validate_tags,
              multiple=True,
              help="Additional tags (e.g. 'Key,Value') to assign to the instances. "
                   "You can specify this option multiple times.")
@click.pass_context
def add_slaves(
        cli_context,
        cluster_name,
        num_slaves,
        ec2_region,
        ec2_vpc_id,
        ec2_identity_file,
        ec2_user,
        ec2_spot_price,
        ec2_spot_request_duration,
        ec2_min_root_ebs_size_gb,
        ec2_tags,
        assume_yes):
    """
    Add slaves to an existing cluster.

    Flintrock will configure new slaves based on information queried
    automatically from the master.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-identity-file',
            '--ec2-user'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
        provider_options = {
            'min_root_ebs_size_gb': ec2_min_root_ebs_size_gb,
            'spot_price': ec2_spot_price,
            'spot_request_duration': ec2_spot_request_duration,
            'tags': ec2_tags
        }
    else:
        raise UnsupportedProviderError(provider)

    if cluster.num_masters == 0:
        raise Error(
            "Cannot add slaves to cluster '{c}' since it does not "
            "appear to have a master."
            .format(
                c=cluster_name))

    cluster.load_manifest(
        user=user,
        identity_file=identity_file)
    cluster.add_slaves_check()

    if provider == 'ec2':
        cluster.add_slaves(
            user=user,
            identity_file=identity_file,
            num_slaves=num_slaves,
            assume_yes=assume_yes,
            **provider_options)

@cli.command(name='remove-slaves')
@click.argument('cluster-name')
@click.option('--num-slaves', type=click.IntRange(min=1), required=True)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--ec2-user')
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--assume-yes/--no-assume-yes', default=False)
@click.pass_context
def remove_slaves(
        cli_context,
        cluster_name,
        num_slaves,
        ec2_region,
        ec2_vpc_id,
        ec2_user,
        ec2_identity_file,
        assume_yes):
    """
    Remove slaves from an existing cluster.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-user',
            '--ec2-identity-file'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
    else:
        raise UnsupportedProviderError(provider)

    if num_slaves > cluster.num_slaves:
        logger.warning(
            "Warning: Cluster has {c} slave{cs}. "
            "You asked to remove {n} slave{ns}."
            .format(
                c=cluster.num_slaves,
                cs='' if cluster.num_slaves == 1 else 's',
                n=num_slaves,
                ns='' if num_slaves == 1 else 's'))
        num_slaves = cluster.num_slaves

    if not assume_yes:
        cluster.print()
        click.confirm(
            text=("Are you sure you want to remove {n} slave{s} from this cluster?"
                  .format(
                      n=num_slaves,
                      s='' if num_slaves == 1 else 's')),
            abort=True)

    logger.info("Removing {n} slave{s}..."
                .format(
                    n=num_slaves,
                    s='' if num_slaves == 1 else 's'))
    cluster.remove_slaves(
        user=user,
        identity_file=identity_file,
        num_slaves=num_slaves)

@cli.command(name='run-command')
@click.argument('cluster-name')
@click.argument('command', nargs=-1)
@click.option('--master-only', help="Run on the master only.", is_flag=True)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-user')
@click.pass_context
def run_command(
        cli_context,
        cluster_name,
        command,
        master_only,
        ec2_region,
        ec2_vpc_id,
        ec2_identity_file,
        ec2_user):
    """
    Run a shell command on a cluster.

    Examples:

        flintrock run-command my-cluster 'touch /tmp/flintrock'
        flintrock run-command my-cluster -- yum install -y package

    Flintrock will return a non-zero code if any of the cluster nodes raises an error
    while running the command.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-identity-file',
            '--ec2-user'],
        scope=locals())

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
    else:
        raise UnsupportedProviderError(provider)

    cluster.run_command_check()

    logger.info("Running command on {target}...".format(
        target="master only" if master_only else "cluster"))

    cluster.run_command(
        command=command,
        master_only=master_only,
        user=user,
        identity_file=identity_file)

@cli.command(name='copy-file')
@click.argument('cluster-name')
@click.argument('local_path', type=click.Path(exists=True, dir_okay=False))
@click.argument('remote_path', type=click.Path())
@click.option('--master-only', help="Copy to the master only.", is_flag=True)
@click.option('--ec2-region', default='us-east-1', show_default=True)
@click.option('--ec2-vpc-id', default='', help="Leave empty for default VPC.")
@click.option('--ec2-identity-file',
              type=click.Path(exists=True, dir_okay=False),
              help="Path to SSH .pem file for accessing nodes.")
@click.option('--ec2-user')
@click.option('--assume-yes/--no-assume-yes', default=False, help="Prompt before large uploads.")
@click.pass_context
def copy_file(
        cli_context,
        cluster_name,
        local_path,
        remote_path,
        master_only,
        ec2_region,
        ec2_vpc_id,
        ec2_identity_file,
        ec2_user,
        assume_yes):
    """
    Copy a local file up to a cluster.

    This will copy the file to the same path on each node of the cluster.

    Examples:

        flintrock copy-file my-cluster /tmp/file.102.txt /tmp/file.txt
        flintrock copy-file my-cluster /tmp/spark-defaults.conf /tmp/

    Flintrock will return a non-zero code if any of the cluster nodes raises an error.
    """
    provider = cli_context.obj['provider']

    option_requires(
        option='--provider',
        conditional_value='ec2',
        requires_all=[
            '--ec2-region',
            '--ec2-identity-file',
            '--ec2-user'],
        scope=locals())

    # We assume POSIX for the remote path since Flintrock
    # only supports clusters running CentOS / Amazon Linux.
    if not posixpath.basename(remote_path):
        remote_path = posixpath.join(remote_path, os.path.basename(local_path))

    if provider == 'ec2':
        cluster = ec2.get_cluster(
            cluster_name=cluster_name,
            region=ec2_region,
            vpc_id=ec2_vpc_id)
        user = ec2_user
        identity_file = ec2_identity_file
    else:
        raise UnsupportedProviderError(provider)

    cluster.copy_file_check()

    if not assume_yes and not master_only:
        file_size_bytes = os.path.getsize(local_path)
        num_nodes = len(cluster.slave_ips) + 1  # TODO: cluster.num_nodes
        total_size_bytes = file_size_bytes * num_nodes

        if total_size_bytes > 10 ** 6:
            logger.warning("WARNING:")
            logger.warning(
                format_message(
                    message="""\
                        You are trying to upload {total_size} bytes ({size} bytes x {count}
                        nodes in {cluster}). Depending on your upload bandwidth, this may take
                        a long time.
                        You may be better off uploading this file to a storage service like
                        Amazon S3 and downloading it from there to the cluster using
                        `flintrock run-command ...`.
                        """.format(
                            size=file_size_bytes,
                            count=num_nodes,
                            cluster=cluster_name,
                            total_size=total_size_bytes),
                    wrap=60))
            click.confirm(
                text="Are you sure you want to continue?",
                default=True,
                abort=True)

    logger.info("Copying file to {target}...".format(
        target="master only" if master_only else "cluster"))

    cluster.copy_file(
        local_path=local_path,
        remote_path=remote_path,
        master_only=master_only,
        user=user,
        identity_file=identity_file)

def normalize_keys(obj):
    """
    Used to map keys from config files to Python parameter names.
    """
    if type(obj) != dict:
        return obj
    else:
        return {k.replace('-', '_'): normalize_keys(v) for k, v in obj.items()}

def config_to_click(config: dict) -> dict:
    """
    Convert a dictionary of configurations loaded from a Flintrock config file
    to a dictionary that Click can use to set default options.
    """
    service_configs = {}

    if 'services' in config:
        for service in config['services']:
            if config['services'][service]:
                service_configs.update(
                    {service + '_' + k: v for (k, v) in config['services'][service].items()})

    ec2_configs = {
        'ec2_' + k: v for (k, v) in config['providers']['ec2'].items()}

    click_map = {
        'launch': dict(
            list(config['launch'].items())
            + list(ec2_configs.items())
            + list(service_configs.items())),
        'describe': ec2_configs,
        'destroy': ec2_configs,
        'login': ec2_configs,
        'start': ec2_configs,
        'stop': ec2_configs,
        'add-slaves': ec2_configs,
        'remove-slaves': ec2_configs,
        'run-command': ec2_configs,
        'copy-file': ec2_configs,
    }

    return click_map

@cli.command()
@click.option('--locate', is_flag=True, default=False,
              help="Don't open an editor. "
              "Just open the folder containing the configuration file.")
@click.pass_context
def configure(cli_context, locate):
    """
    Configure Flintrock's defaults.

    This will open Flintrock's configuration file in your default YAML editor so
    you can set your defaults.
    """
    config_file = get_config_file()

    if not os.path.isfile(config_file):
        logger.info("Initializing config file from template...")
        os.makedirs(os.path.dirname(config_file), exist_ok=True)
        shutil.copyfile(
            src=os.path.join(THIS_DIR, 'config.yaml.template'),
            dst=config_file)
        os.chmod(config_file, mode=0o644)

    ret = click.launch(config_file, locate=locate)

    if ret != 0:
        raise Error(
            "Flintrock could not launch an application to {action} "
            "the config file at '{location}'. You may want to manually "
            "find and edit this file."
            .format(
                action="locate" if locate else "edit",
                location=config_file
            )
        )

def flintrock_is_in_development_mode() -> bool:
    """
    Check if Flintrock was installed in development mode.

    Use this function to toggle behavior that only Flintrock developers should
    see.
    """
    # This esoteric technique was pulled from pip.
    # See: https://github.com/pypa/pip/pull/3258/files#diff-ab583908279e865537dec218246edcfcR310
    for path_item in sys.path:
        egg_link = os.path.join(path_item, 'Flintrock.egg-link')
        if os.path.isfile(egg_link):
            return True
    else:
        return False

def set_open_files_limit(desired_limit):
    """
    On POSIX systems, set the open files limit to the desired number, unless
    it is already equal to or higher than that.

    Setting a high limit enables Flintrock to launch or interact with really
    large clusters.

    Background discussion: https://github.com/nchammas/flintrock/issues/81
    """
    soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)

    if soft_limit < desired_limit:
        if desired_limit > hard_limit:
            warnings.warn(
                "Flintrock cannot set the open files limit to {desired} "
                "because the OS hard limit is {hard}. Going with {hard}. "
                "You may have problems launching or interacting with "
                "really large clusters."
                .format(
                    desired=desired_limit,
                    hard=hard_limit),
                category=RuntimeWarning,
                stacklevel=2)
        resource.setrlimit(
            resource.RLIMIT_NOFILE,
            (min(desired_limit, hard_limit), hard_limit))

def check_external_dependency(executable_name: str):
    if shutil.which(executable_name) is None:
        raise Error(
            "Error: Flintrock could not find '{executable}' on your PATH. "
            "Flintrock needs this executable to carry out the operation you "
            "requested. Please install it and try again."
            .format(
                executable=executable_name
            )
        )

def main() -> int:
    # Starting in Python 3.7, deprecation warnings are shown by default. We
    # don't want to show these to end-users.
    # See: https://docs.python.org/3/library/warnings.html#default-warning-filter
    if not flintrock_is_in_development_mode():
        warnings.simplefilter(action='ignore', category=DeprecationWarning)

    set_open_files_limit(4096)

    try:
        try:
            # We pass in obj so we can add attributes to it, like provider, which
            # get shared by all commands.
            # See: http://click.pocoo.org/6/api/#click.Context
            cli(obj={})
        except botocore.exceptions.NoCredentialsError:
            raise Error(
                "Flintrock could not find your AWS credentials. "
                "You can fix this by providing your credentials "
                "via environment variables or by creating a shared "
                "credentials file.\n"
                "For more information see:\n"
                "  * https://boto3.readthedocs.io/en/latest/guide/configuration.html#environment-variables\n"
                "  * https://boto3.readthedocs.io/en/latest/guide/configuration.html#shared-credentials-file"
            )
    except NothingToDo as e:
        print(e)
        return 0
    except UsageError as e:
        print(e, file=sys.stderr)
        return 2
    except Error as e:
        print(e, file=sys.stderr)
        return 1


import numpy as np
import cv2
import mss
import os

LABELS_PATH = os.path.dirname(__file__) + '/coco.names'
LABELS = open(LABELS_PATH).read().strip().split("\n")
COLORS = np.random.randint(0, 255, size=(len(LABELS), 3), dtype="uint8")

class Frame:
    def __init__(self, source='webcam', screen_size=(800, 640)):
        self.source_name = source
        self.screen_size = (int(screen_size.split('x')[0]),
                            int(screen_size.split('x')[1])) if isinstance(screen_size, str) else screen_size
        self.source = cv2.VideoCapture(0) if source == 'webcam' else mss.mss()

    def get_frame(self):
        if self.source_name == 'webcam':
            _, image = self.source.read()
        elif self.source_name == 'screen':
            monitor = {"top": 0, "left": 0,
                       "width": self.screen_size[0], "height": self.screen_size[1]}
            image = np.array(self.source.grab(monitor))
            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
        return image


import requests
from twilio.rest import Client
import os
import time

url = 'http://api.vk.com/method/'
token = os.environ['sms_token']
account_sid = os.environ['account_sid']
auth_token = os.environ['auth_token']

def get_json(user_id):
    method = url + 'users.get'
    data = {
        'user_ids': user_id,
        'fields': 'online',
        'access_token' : token,
        'v' : '5.103',
    }

    r = requests.get(method, data)
    return r.json()

def sms_sender(mesage):
    client = Client(account_sid, auth_token)
    message = client.messages.create(
        body= mesage,
        from_='+19367553922',
        to='+79298405593'
        )

    return message.sid

def get_status():
    r = get_json('zyoma')
    status = r['response'][0]['online']
    return status

def main():
    while True:
        status = get_status()
        if status == 1:
            sms_sender(mesage='User online!')
            break

        time.sleep(5)

if __name__ == '__main__':
    main()


import solution

class Solution(solution.Solution):
    def solve(self, test_input=None):
        return self.smallerNumbersThanCurrent(test_input)

    def smallerNumbersThanCurrent(self, nums):
        """
        :type nums: List[int]
        :rtype: List[int]
        """
        num_set = {}

        # LeetCode does not support list.copy()
        # c_nums = list.copy(nums)
        c_nums = []
        for i in nums:
            c_nums.append(i)

        list.sort(c_nums)
        for i in range(len(c_nums)):
            if c_nums[i] not in num_set:
                num_set[c_nums[i]] = i

        result = []
        for i in nums:
            result.append(num_set[i])
        return result


from django.conf import settings
from django.conf.urls import include, url  # noqa
from django.contrib import admin
from django.views.generic import TemplateView

import django_js_reverse.views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^jsreverse/$', django_js_reverse.views.urls_js, name='js_reverse'),

    url(r'^$', TemplateView.as_view(template_name='base.html'), name='home'),
    url(r'', include('battles.urls', namespace='battles')),
    url(r'', include('users.urls')),

    url(r'social/', include('social_django.urls', namespace='social')),
    url(r'^api-auth/', include('rest_framework.urls')),
    url(r'api/', include('battles.endpoints_urls', namespace='api_battles')),

]

if settings.DEBUG:
    import debug_toolbar
    urlpatterns = [
        url(r'^__debug__/', include(debug_toolbar.urls)),
    ] + urlpatterns


"""
target_df 생성 (users에 없는 user_id 추가)
: 매개변수에는 dev_users_list / test_users_list 와 users.
"""
def target_df_generator(target_users_list, users):
    print('1. target user의 dataframe 생성 중')
    target_df = users[users['user_id'].isin(target_users_list)]

    for target_user in target_users_list:
        if (target_user in target_df['user_id'].tolist()) == False:
            new_df = pd.DataFrame({'following_list':[[]], 'user_id':[target_user], 'keyword_list':[[]]})
            target_df = target_df.append(new_df)

    print('target user의 dataframe 생성 완료')
    return target_df

"""
전체 기간동안 target user가 읽은 article 저장
"""
def target_read_article(target_df, read):
    print('2. 전체 기간동안 target user가 읽은 article 저장 중\n')

    target_read_article = []
    for idx in target_df['user_id'].values.tolist(): # user_id한개씩 루프
        read_list = read[read['user_id']==idx]
        read_list = read_list[read_list['article_id'].str.startswith('@')]['article_id'].tolist() #해당 user가 읽은 글 목록
        target_read_article.append(read_list)

    #target_df['read'] = read_list
    target_df['read'] = target_read_article # 읽은 글

    print('전체 기간동안 target user가 읽은 article 저장 완료\n')
    return target_df

"""
최근 2주동안 target user가 읽은 article저장
"""
def target_recent_article(target_df, read, from_dt, to_dt):
    print('3. 최근 2주 동안 target user가 읽은 article 저장 중\n')

    target_recent_article = []
    partial_read = read[(read.date >= from_dt) & (read.date <= to_dt)]

    for idx in target_df['user_id'].values.tolist(): # user_id한개씩 루프
        read_list = partial_read[partial_read['user_id']==idx]
        read_list = read_list[read_list['article_id'].str.startswith('@')]['article_id'].tolist()
        target_recent_article.append(read_list)
    target_df['recent'] = target_recent_article

    print('최근 2주 동안 target user가 읽은 article 저장 완료\n')
    return target_df

    """
target user가 본 '구독 작가 글의 cnt' 저장
"""
# mode: read / recent
def target_read_following(target_df, mode):
    print('4. target user가 본 following의 빈도수 저장 중\n')

    target_read_following = []
    for idx in target_df['user_id'].values.tolist():
        # following_frequency: read_list 중에서 해당 작가(f_id)의 글의 빈도수
        following_frequency = {}

        # read_list: 타겟 사용자가 전체기간/최근 동안 읽은 글 리스트
        read_list = target_df[target_df['user_id']==idx][mode].values[0][:]

        if len(read_list) > 0:
            # read_series: 읽은 글 리스트 -> Series
            read_series = pd.Series(read_list)

            # f_list: 타겟 사용자가 구독하는 작가 리스트
            f_list = target_df[target_df['user_id']==idx]['following_list'].values[0][:]
            for i in range(len(f_list)):
                f_list[i] = f_list[i] + '_'

            # read_series[read_series.str.startswith(f_id)].tolist()
            # 읽은 글 중에 구독하는 작가의 글을 출력
            for f_id in f_list:
                # frequency: 구독하는 작가 글 중 읽은 글 갯수
                frequency = len(read_series[read_series.str.startswith(f_id)].tolist())
                if frequency > 0:
                    # 그 구독하는 작가명 : frequency 형태로 저장
                    following_frequency[f_id[:-1]]=frequency

        target_read_following.append(following_frequency)

    target_df[mode+'_following'] = target_read_following

    print('target user가 본 following의 빈도수 저장 완료\n')
    return target_df

    """
target user가 본 '매거진 cnt' 저장
"""
# mode: read / recent
def target_read_magazine(target_df, metadata, mode):
    print('5. target user가 본 magazine의 빈도수 저장 중\n')
    target_read_magazine = []

    for idx in target_df['user_id'].values.tolist():
        # read_list: 타겟 사용자가 전체기간/최근 동안 읽은 글 리스트
        read_list = target_df[target_df['user_id']==idx][mode].values[0][:]
        # magazine_list: metadata에서 타겟 사용자가 읽은 글 레코드(id)의 magazine_id를 뽑아냄
        magazine_list = metadata[metadata['id'].isin(read_list)]['magazine_id'].tolist()

        # magazine_id 빈도수 dictionary로 저장
        magazine_frequency = Counter(magazine_list)
        del magazine_frequency[0]
        target_read_magazine.append(magazine_frequency)

    target_df[mode+'_magazine'] = target_read_magazine

    print('target user가 본 magazine의 빈도수 저장 완료\n')
    return target_df

    """
target user가 본 글의 태그 cnt저장
--> 읽은 글의 태그와 같은 태그를 가진 글을 추천해주기 위함
"""
# mode: read / recent
def target_read_tag(target_df, metadata, mode):
    print('6. target user가 본 글의 태그 빈도수 저장 중\n')

    target_read_tag = []
    keyword_list = {}
    for idx in target_df['user_id'].values.tolist():
        # read_list: 타겟 사용자가 전체기간/최근 동안 읽은 글 리스트
        read_list = target_df[target_df['user_id']==idx][mode].values[0][:]
        # metadata에서 타겟 사용자가 읽은 글 레코드(id)의 keyword_list(검색키워드)
        # 각 타겟 사용자가 일정 기간 동안 읽은 글의 태그('article_tag_list') 합쳐서 저장
        keyword_list = metadata[metadata['id'].isin(read_list)]['article_tag_list'].tolist()
        read_tag = sum(keyword_list, [])

        # 각 타겟 사용자가 일정 기간 동안 읽은 글의 태그들의 빈도수 dictionary로 저장
        frequency = Counter(read_tag)
        target_read_tag.append(frequency)

    target_df[mode+'_tag'] = target_read_tag

    print('target user가 본 글의 태그 빈도수 저장 완료\n')
    return target_df

    """
target user의 tag에서 빈도수가 높은 상위 top_N개를 관심 키워드로 저장
--> 타겟 유저의 관심사를 알고 추천해주기 위함.
"""
# mode: read / recent
# top_N: 태그 중 빈도수 높은 top_N개의 interest를 저장.
def target_read_interest(target_df, top_N, mode):
    print('7. target user 관심 태그 상위',top_N,'개 저장 중\n')
    target_read_interest = []
    # read_tag에서 빈도수가 높은 상위 top_N개의 키워드 저장
    for idx in target_df['user_id'].values.tolist():
        interest = []
        # rt: 읽은 글의 태그 정보
        rt = target_df[target_df['user_id']==idx][mode+'_tag'].values[0]
        # sorted_rt: rt를 순서대로 정렬(내림차순)
        sorted_rt = sorted(rt.items(), key=lambda x: x[1], reverse=True)

        for i in range(len(sorted_rt[:top_N])):
            interest.append(sorted_rt[:top_N][i][0])
        target_read_interest.append(interest)

    target_df[mode+'_interest'] = target_read_interest

    print('target user 관심 태그 상위',top_N,'개 저장 완료\n')
    return target_df

    ## target_df 'behavior' 전처리 (target user의 글 소비 성향 저장)
def target_read_behavior(target_df, metadata, mode):
    print('8. target user 글 소비 성향 저장 중\n')

    ratio_list = []

    if mode == 'recent':
    # 최근 기간 -> pop: recent_view가 상위 20%인 글 / reg: 2019.02.15 ~ 2019.03.01 동안 발행된 글
        pop_id = metadata[metadata['recent_view'] > metadata['recent_view'].quantile(0.80)]['id'].tolist()
        reg_id = metadata[(metadata['reg_ts'] >= get_unix_time(20190215)) & (metadata['reg_ts'] < get_unix_time(20190301))]['id'].tolist()
    else:
    # 전체 기간 -> pop: view가 상위 20%인 글 / reg: 2018.09.15 ~ 2019.03.01 동안 발행된 글
        pop_id = metadata[metadata['view'] > metadata['view'].quantile(0.80)]['id'].tolist()
        reg_id = metadata[(metadata['reg_ts'] >= get_unix_time(20180915)) & (metadata['reg_ts'] < get_unix_time(20190301))]['id'].tolist()

    for idx in target_df['user_id'].values.tolist():
        read_list = target_df[target_df['user_id']==idx][mode].values[0][:]

        fr_dic = target_df[target_df['user_id']==idx][mode+'_following'].values[0]
        mr_dic = target_df[target_df['user_id']==idx][mode+'_magazine'].values[0]
        pop = pd.Series(read_list)[pd.Series(read_list).isin(pop_id)].tolist()
        reg = pd.Series(read_list)[pd.Series(read_list).isin(reg_id)].tolist()

        # f_ratio -> target user가 본 article 중에서 following의 비율
        # m_ratio -> target user가 본 article 중에서 magazine의 비율
        # p_ratio -> target user가 본 article 중에서 인기가 많은 글('recent_view' or 'view'가 상위 20%)의 비율
        # r_ratio -> target user가 본 article 중에서 최근 발행된 글('reg_ts'가 '190215' or '180915' 이후)의 비율
        f_ratio, m_ratio, p_ratio, r_ratio = 0, 0, 0 ,0

        if len(read_list) >= 1:
            f_ratio = round(sum(fr_dic.values())/len(read_list), 2)
            m_ratio = round(sum(mr_dic.values())/len(read_list), 2)
            p_ratio = round(len(pop)/len(read_list), 2)
            r_ratio = round(len(reg)/len(read_list), 2)

        ratio_list.append([f_ratio, m_ratio, p_ratio, r_ratio])

    ratio_df = pd.DataFrame(ratio_list, columns=[mode+'_f_ratio', mode+'_m_ratio', mode+'_p_ratio', mode+'_r_ratio'])
    target_df.index = list(range(len(target_df)))
    target_df = pd.concat([target_df, ratio_df], axis=1)

    print('8. target user 글 소비 성향 저장 중\n')
    return target_df

    """
타겟 사용자 데이터프레임 전처리
"""
def target_users_preprocessing(target_users_list, users, metadata, read, from_dt, to_dt, top_N, mode):
    target_df = target_df_generator(target_users_list, users)
    target_df = target_df.reset_index() # id 리셋
    del target_df['index']
    target_df = target_read_article(target_df, read)
    target_df = target_recent_article(target_df, read, from_dt, to_dt)
    target_read_following(target_df, mode)
    target_df = target_read_magazine(target_df, metadata, mode)
    target_df = target_read_tag(target_df, metadata, mode)
    target_df = target_read_interest(target_df, top_N, mode)
    target_df = target_read_behavior(target_df, metadata, mode)
    return target_df


from distutils.errors import CompileError
from traceback import print_tb
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
import torch
import transformers
from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
from transformers import BertConfig, BertForSequenceClassification, BertTokenizer, pipeline, AutoTokenizer, AutoModelForSequenceClassification
import tensorflow as tf
import pandas as pd
import numpy as np
from sklearn.svm import SVC
import sys

from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from feature_engineering import refuting_features, polarity_features, hand_features, gen_or_load_feats
from feature_engineering import word_overlap_features
from transformers.modeling_utils import load_state_dict
from utils.dataset import DataSet
from utils.generate_test_splits import kfold_split, get_stances_for_folds
from utils.score import report_score, LABELS, score_submission
from utils.system import parse_params, check_version

from transformers import BertModel, AdamW, get_linear_schedule_with_warmup
import seaborn as sns
from pylab import rcParams
from matplotlib import rc
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, classification_report
from collections import defaultdict
from textwrap import wrap

from torch import device, nn
from torch.utils.data import Dataset, DataLoader
import torch.nn.functional as F
from tensorflow.python.client import device_lib
import time

PRE_TRAINED_MODEL_NAME = 'bert-base-uncased'
MODEL = 'bert-base-multilingual-uncased'
transformers.logging.set_verbosity_error()

class GPReviewDataset(Dataset):

  def __init__(self, headlines, articles, targets, tokenizer, max_len):
    self.headline = headlines
    self.article = articles
    self.target = targets
    self.tokenizer = tokenizer
    self.max_len = max_len

  def __len__(self):
    return len(self.headline)

  def __getitem__(self, item):
    headline = str(self.headline[item])
    article = str(self.article[item])
    target = self.target[item]

    encoding = self.tokenizer(
      headline,
      article,
      add_special_tokens=True,
      max_length=self.max_len,
      padding='max_length',
      truncation=True,
      return_token_type_ids=False,
      return_attention_mask=True,
      return_tensors='pt',
      verbose=False
    )

    return {
      'headline_text': headline,
      'article_text': article,
      'input_ids': encoding['input_ids'].flatten(),
      'attention_mask': encoding['attention_mask'].flatten(),
      'targets': torch.tensor(target, dtype=torch.float)
    }

def create_data_loader(df, tokenizer, max_len, batch_size):
  ds = GPReviewDataset(
    headlines = df['Headline'].to_numpy(),
    articles = df['article'].to_numpy(),
    targets = df['Stance'].to_numpy(),
    tokenizer = tokenizer,
    max_len = max_len
  )

  return DataLoader(
    dataset=ds,
    batch_size=batch_size,
    num_workers=4
  )

class SentimentClassifier(nn.Module):

  def __init__(self, n_classes):
    super(SentimentClassifier, self).__init__()
    self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)

    self.out = nn.Linear(self.bert.config.hidden_size, n_classes)

  def forward(self, input_ids, attention_mask):
    _, pooled_output = self.bert(
      input_ids=input_ids.to(device),
      attention_mask=attention_mask.to(device),
      return_dict=False,
    )

    return self.out(pooled_output)

def get_predictions(model, data_loader):
  model = model.eval()

  headlines_texts = []
  articles_texts = []
  predictions = []
  prediction_probs = []
  real_values = []

  with torch.no_grad():
    for d in data_loader:

      headlines = d["headline_text"],
      articles = d['article_text']
      input_ids = d["input_ids"].to(device)
      attention_mask = d["attention_mask"].to(device)
      targets = d["targets"].to(device)

      outputs = model(
        input_ids=input_ids,
        attention_mask=attention_mask
      )

      _, preds = torch.max(outputs, dim=1)
      probs = F.softmax(outputs, dim=1)

      headlines_texts.extend(headlines)
      articles_texts.extend(articles)
      predictions.extend(preds)
      prediction_probs.extend(probs)
      real_values.extend(targets)

  predictions = torch.stack(predictions).cpu()
  prediction_probs = torch.stack(prediction_probs).cpu()
  real_values = torch.stack(real_values).cpu()

  return headlines_texts, articles_texts, predictions, prediction_probs, real_values

def show_confusion_matrix(confusion_matrix):
  hmap = sns.heatmap(confusion_matrix, annot=True, fmt="d", cmap="Blues")
  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')
  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')
  plt.ylabel('True sentiment')
  plt.xlabel('Predicted sentiment')
  plt.show()

if __name__ == "__main__":
  RANDOM_SEED = 43
  np.random.seed(RANDOM_SEED)
  torch.manual_seed(RANDOM_SEED)
  device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
  print(f'RUNNING ON DEVICE: {device}')

  # LOAD THE TOKENIZER AND MODEL
  model = SentimentClassifier(4)
  model.load_state_dict(torch.load('model/acc_best_model_state.bin'))
  model = model.to(device)
  tokenizer = BertTokenizer('model/acc_best_model_tokenizer.bin', do_lower_case=True, local_files_only=True)

# GET THE COMPETITION DATASET
  Competition_dataset = DataSet("competition_test")
  Competition_dataset_stances_frame = pd.DataFrame(Competition_dataset.stances)
  Competition_dataset_articles_frame = pd.DataFrame(list(Competition_dataset.articles.items()),columns = ['Body ID','article'])
  Competition_dataset_frame = Competition_dataset_stances_frame.merge(Competition_dataset_articles_frame[["Body ID","article"]], on="Body ID", how="left")
  Competition_dataset_frame.loc[Competition_dataset_frame["Stance"] == "unrelated", "Stance"] = 0
  Competition_dataset_frame.loc[Competition_dataset_frame["Stance"] == "agree", "Stance"] = 1
  Competition_dataset_frame.loc[Competition_dataset_frame["Stance"] == "discuss", "Stance"] = 2
  Competition_dataset_frame.loc[Competition_dataset_frame["Stance"] == "disagree", "Stance"] = 3
  # Competition_dataset_frame = Competition_dataset_frame.iloc[:5]

# CREATE A DATALOADER WITH THE COMPETITION DATASET AND THE TOKENIZER
  start_time = time.time()

  test_data_loader = create_data_loader(Competition_dataset_frame, tokenizer, 500, 1)
  y_headline_text, y_article_text, y_pred, y_pred_probs, y_test = get_predictions(
      model,
      test_data_loader
  )

  print("--- %s minutes ---" % round((time.time() - start_time)/60, 2))

  class_names = ['class 0', 'class 1', 'class 2', 'class 3']
  labels=[0,1,2,3]
  print(classification_report(y_test, y_pred, labels=labels, zero_division=0))

  dy = pd.DataFrame(y_pred.numpy())
  dx = pd.DataFrame(y_test.numpy())
  print(dy)
  print(dx)

  cm = confusion_matrix(dx, dy, labels=labels)
  df_cm = pd.DataFrame(cm)
  show_confusion_matrix(df_cm)

  actual_stance = Competition_dataset_frame['Stance'].values.tolist()
  df = Competition_dataset_frame
  df['Stance'] = y_pred
  predicted_stance = df['Stance'].values.tolist()
  df.loc[df["Stance"] == 0, "Stance"] = "unrelated"
  df.loc[df["Stance"] == 1, "Stance"] = "agree"
  df.loc[df["Stance"] == 2, "Stance"] = "discuss"
  df.loc[df["Stance"] == 3, "Stance"] = "disagree"
  df.reset_index(drop=True, inplace=True)
  df = df[["Headline", "Body ID", "Stance"]]
  df.to_csv('answer.csv', index=False, encoding='utf-8')

  report_score([LABELS[e] for e in actual_stance],[LABELS[e] for e in predicted_stance])


from http import HTTPStatus

import django.test
import django.urls

__all__ = []

class StaticURLTests(django.test.TestCase):
    def test_homepage_endpoint(self):
        response = django.test.Client().get(
            django.urls.reverse("homepage:home"),
        )
        self.assertEqual(response.status_code, HTTPStatus.OK)

    def test_coffee_endpoint(self):
        response = django.test.Client().get(
            django.urls.reverse("homepage:coffee"),
        )
        self.assertIn("Я чайник", response.content.decode("utf-8"))
        self.assertEqual(response.status_code, HTTPStatus.IM_A_TEAPOT)


from django.db import models
import auto_prefetch
from django_resized import ResizedImageField
from folio.utils.media import MediaHelper
from folio.utils.choices import PortfolioChoices
from folio.utils.models import  NameBaseModel, ExperienceAndSchoolModel
from django.utils.text import slugify
from django.urls import reverse
from ckeditor.fields import RichTextField
from taggit.managers import TaggableManager
from django.utils.html import mark_safe
from user.models import UserModel

# Create your models here.

class Experiences(ExperienceAndSchoolModel):
    name_of_place = models.CharField(max_length=200 )
    work_as = models.CharField(max_length=200 )

    class Meta:
        verbose_name = 'Exprience'
        verbose_name_plural = 'Experiences'

class Schools(ExperienceAndSchoolModel):
    name_of_school = models.CharField(max_length=200 )
    course = models.CharField(max_length=200 )
    place = models.CharField(max_length=200 )

    class Meta:
        verbose_name = 'School'
        verbose_name_plural = 'Schools'

class AboutMeModel(NameBaseModel):

    email = models.EmailField(null=True)
    phone = models.BigIntegerField(null=True, blank=True)
    location = models.CharField(max_length=200, null=True)
    thumbnail = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image")
    resume_photo = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)

class ServicesModel(NameBaseModel):
    i_class = models.CharField(max_length=200)
    description2 = models.TextField(null=True)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    def get_absolute_url(self):
        return reverse('servicedetail',kwargs={'slug':self.slug})

    class Meta:
        verbose_name = 'Service'
        verbose_name_plural = 'Services'

class ServiceRatingModel(NameBaseModel):
    pesentage = models.IntegerField()

class NewsModel(NameBaseModel):
    thumbnail = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image")

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

class Tags(models.Model):
    name = models.CharField( max_length=50)
    def __str__(self):
        return self.name

    class Meta:
        verbose_name = 'Tag'
        verbose_name_plural = 'Tags'

class Images(models.Model):
    files = models.FileField(upload_to=MediaHelper.get_image_upload_path)
    potfolio = auto_prefetch.ForeignKey( "PotfolioModel", on_delete=models.CASCADE, null=True, blank=True)
    news = auto_prefetch.ForeignKey( NewsModel, on_delete=models.CASCADE,  null=True, blank=True)
    blogs = auto_prefetch.ForeignKey( 'PortfolioBlogModel', on_delete=models.CASCADE,  null=True, blank=True)

    def image_tag(self):
        return mark_safe(''%(self.files))
        # return u'' % escape(MediaHelper.get_image_upload_path)
    image_tag.short_description = 'Image'

    class Meta:
        verbose_name = 'Image'
        verbose_name_plural = 'Images'

class PotfolioModel(NameBaseModel):
    client = models.CharField(max_length=200)
    firm_name = models.CharField(max_length=200)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)
    category = TaggableManager()
    potfoliochoices = models.CharField(choices=PortfolioChoices.choices, max_length=20, default=PortfolioChoices.Backend)
    live_website = models.CharField(max_length=200, null=True, blank=True)
    date_created = models.DateTimeField(null=True, blank=True)

    def get_potfolio_url(self):
        return reverse('servicedetail',kwargs={'slug':self.slug})

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

    class Meta:
        verbose_name = 'Portfolio'
        verbose_name_plural = 'Portfolios'

class PotfolioQuestionModel(models.Model):
    questions = models.CharField(max_length=200, null=True)
    answer = RichTextField(null=True)

class CategoryModel(models.Model):
    name = models.CharField(max_length=200)
    slug = models.SlugField(null=True,blank=True)

    def __str__(self):
        return self.name

class PortfolioBlogModel(NameBaseModel):
    category = models.ForeignKey(CategoryModel, on_delete=models.CASCADE)
    image = ResizedImageField(upload_to=MediaHelper.get_image_upload_path, verbose_name="Image", null=True, blank=True)
    tags = TaggableManager()

    def save(self, *args, **kwargs):
        if not self.slug and self.name:
            self.slug = slugify(self.name)
        return super().save(*args, **kwargs)

class CommentsModel(models.Model):
    author = models.ForeignKey(UserModel, on_delete=models.CASCADE)
    post = models.ForeignKey(PortfolioBlogModel,  on_delete=models.CASCADE)
    body = models.TextField(help_text='Add a comment')
    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)
    active = models.BooleanField(default=True)

    def __str__(self):
        return f'Comment by {self.author} on {self.post}'


# . - один любой символ кроме \n
# ? - 0 или 1 вхождение шаблона слева
# + - 1 и более вхождений
# * - 0 и более вхождений или повторений
# \w - любая цифра или буква
# \W - все кроме
# \d - любая цифра
# \D - все кром цифр
# \s - любой "пробельный символ"
# \S - все кроме
# \b - граница слова
# [...] - символы в скобках
# [^...] - любые символы, кроме тех, что в скобках
# \ - экранирование
# ^ $ - начало или конец слова
# {n, m} - от и до вхождений
# a | b - а или b
# () - группы
# \t, \n, \r - символы табуляции, новой строки и возврата

import re

s = "hello"

comp = re.compile('(.)+')
print(comp.search(s).group(1))

print(r'\\')


#06-007.py

import re
pattern = re.compile(r'abc')
mc = re.search(pattern, '123abcdabc')
if mc :
    print('{}~{}에 {}존재'.format(mc.start(), mc.end(), mc.group()))
else:
    print('문자열에 패턴이 존재하지 않음')

#mc = pattern.search('123abcd')
#mc = re.search(r'abc', '123abcd')


# ========================
# Panda3d - panda3d_gpu.py
# ========================

# Panda3d imports.
from panda3d.core import NodePath, ClockObject, Filename, Texture
from panda3d.core import Shader, ShaderAttrib, PNMImage
from panda3d.core import LVector3i

'''# Local imports.
from etc import _path'''

# Basic Timer object for local script invocation.

class TimeIt:

    def __init__(self, msg=""):
        self.__msg = msg
        self.__clock = ClockObject()
    def __enter__(self):
        self.__start_dt = self.__clock.getRealTime()
        return self
    def __exit__(self, *e_info):
        self.__dur = round(self.__clock.getRealTime()-self.__start_dt, 3)
        if self.__msg:
            print()
            print("{}:  {}".format(self.__msg, self.__dur, 3))
            for attr in self.__dict__:
                if attr.startswith("_TimeIt__"): continue
                print("  {}:  {}".format(attr, round(self.__dict__[attr], 3)))
        self.total_time = self.__dur

class GPU_Image:

    lib_path = "/e/dev/solex/gpu/gpu_image_lib.glsl"

    def __init__(self, ref_img,
                       workgroup_size = LVector3i(32,32,1),
                       img_format = Texture.FRgba8,
                       print_times = False):

        self.workgroup_size = workgroup_size
        self.img_format = img_format
        self.x_size = ref_img.getXSize()
        self.y_size = ref_img.getYSize()
        self.z_size = 1
        self.prepare_time = 0
        self.process_time = 0
        self.extract_time = 0

        self.__NP = NodePath("gpu")
        self.__gsg = base.win.getGsg()
        self.__ref_tex = self.__get_Texture(ref_img)
        self.__LINES = self.__Setup()
        self.__print_times = print_times

    def __enter__(self):
        self.process_time = 0
        self.extract_time = 0
        return self

    def __exit__(self, *e_info):
        if self.__print_times:
            total_time = round(self.prepare_time+self.process_time+self.extract_time, 3)
            prep_time = round(self.prepare_time, 3)
            proc_time = round(self.process_time, 3)
            extr_time = round(self.extract_time, 3)
            print()
            print("GPU_Image total time: {}".format(total_time))
            print("  prepare: {}  ({}%)".format(prep_time, round(prep_time/total_time*100),2))
            print("  process: {}  ({}%)".format(proc_time, round(proc_time/total_time*100),2))
            print("  extract: {}  ({}%)".format(extr_time, round(extr_time/total_time*100),2))

    def __get_Texture(self, ref_img):
        # Convert ref_img into texture.
        with TimeIt() as prep_timer:
            ref_tex = Texture()
            # Ensure ref image has an alpha channel.
            if not ref_img.hasAlpha():
                ref_img.addAlpha()
                ref_img.alphaFill(1.0)
            # Load tex and set format
            ref_tex.load(ref_img)
            ref_tex.setFormat(self.img_format)
        self.prepare_time += round(prep_timer.total_time, 3)
        return ref_tex

    def __Setup(self):
        """Prepares GPU_Image obj to receive python calls to
        the shader library."""

        # Open the shader as a file and get lines so we
        # can extract some setup info from it.
        shader_os_path = Filename(self.lib_path).toOsLongName()
        with open(shader_os_path, "r") as shader_file:
            lines = list(shader_file.readlines())

        # Extract lib function names.
        for line in lines:
            # Each function within the image_lib is defined within the confines
            # of an "#ifdef/#endif" block; the name we need immediately follows
            # the "#ifdef" keyword. This name gets mapped directly to "self"
            # as an alias for "self.__Call" so that the user can simply call
            # the shader function as though it were a regular method of "self".
            if line.startswith("#ifdef"):
                func_name = line.split(" ")[1].strip()

                # Setup callback that redirects to self.__Call each time
                # "self.<func_name> is called, passing along arguments;
                # return the modified image (as a Texture object).
                def call(func_name=func_name, **kwargs):
                    mod_tex = self.__Call(str(func_name), **kwargs)
                    return mod_tex

                # Map "func_name" directly to "self".
                self.__dict__[func_name] = call

        # Add workgroup size layout declaration.
        wg = self.workgroup_size
        wg_str = "layout (local_size_x={}, local_size_y={}) in;\n"
        wg_line = wg_str.format(wg.x, wg.y)
        lines.insert(8, wg_line)
        return lines

    def __Call(self, func_name, **kwargs):
        """Receive python call and redirect request to relevant
        function in image shader library; return modified image."""

        # Copy self.__Lines (need to keep orig for further calls) and
        # add "#define" statement to top to trigger compilation of
        # relevant "#ifdef/def" function block in shader.

        lines = list(self.__LINES)
        lines.insert(2, "#define {}".format(func_name))

        # Assemble lines into shader str and compile.
        shader_str = "".join(lines)
        self.__NP.setShader(Shader.makeCompute(Shader.SL_GLSL, shader_str))

        # Set block size from workgroup size.
        block_x = int(self.x_size/self.workgroup_size.x)
        block_y = int(self.y_size/self.workgroup_size.y)
        block_z = int(self.z_size/self.workgroup_size.z)
        block_size = LVector3i(block_x,block_y,block_z)

        # Create mod_tex for GPU.
        with TimeIt() as prep_timer:
            mod_img = PNMImage(self.x_size, self.y_size, 4)
            mod_tex = Texture()
            mod_tex.load(mod_img)
            mod_tex.setMinfilter(Texture.FTLinear)
            mod_tex.setFormat(self.img_format)
        self.prepare_time += prep_timer.total_time

        # Pass textures to shader.
        self.__NP.setShaderInput("ref_tex", self.__ref_tex)
        self.__NP.setShaderInput("mod_tex", mod_tex)

        # Set any additional required inputs for this function.
        for input_name, input_val in list(kwargs.items()):
            if type(input_val) == PNMImage:
                input_val = self.__get_Texture(input_val)
            self.__NP.setShaderInput(input_name, input_val)

        # Call function in shader library.
        shader_attrib = self.__NP.getAttrib(ShaderAttrib)
        with TimeIt() as proc_timer:
            base.graphicsEngine.dispatch_compute(block_size, shader_attrib, self.__gsg)
        self.process_time += proc_timer.total_time

        # Extract modified texture from GPU.
        with TimeIt() as extract_timer:
            base.graphicsEngine.extractTextureData(mod_tex, self.__gsg)
        self.extract_time += extract_timer.total_time
        return mod_tex


import numpy as np
from collections import defaultdict
from scipy import sparse

def trans_matrice(L,D,vois):
    I,J,V=[],[],[]
    for i in range(len(L)):
        for l in vois[L[i]]:
            j=D[l]
            if j>=0:
                I.append(i)
                J.append(j)
                V.append(1./len(vois[L[i]]))
    I=np.array(I)
    J=np.array(J)
    V=np.array(V)
    return(sparse.csc_matrix((V,(J,I)),shape=(len(L),len(L))))
import sys

n=int(sys.argv[1])
K=int(sys.argv[2])

vois=defaultdict(list)
filename = 'percLeo1000/cl_sel1000/perc_voisins_L_1000_'+str(K)+'.dat'
Tm = np.loadtxt(filename)[:]
print(Tm)
for k in range(len(Tm)):
    Tm[k]=[int(Tm[k][i]) for i in range(2)]
for i in range(len(Tm)):
  vois[Tm[i][0]].append(Tm[i][1])

t=100000
L=np.zeros(t)
Err=np.zeros(t)

for r in range(1,10**4+1):
    p_k=[]
    pos_list=[]
    filename = 'percLeo1000/perc/data_n'+str(n)+'_'+str(K)+'/sites'+str(r)+'.dat'
    Tm = np.loadtxt(filename)[:]
    for a in Tm:
        pos_list.append(a)
    filename = 'percLeo1000/perc/data_n'+str(n)+'_'+str(K)+'/end'+str(r)+'.dat'
    c1 = np.loadtxt(filename)
    i0=pos_list.index(c1)
    def moins_un():
       return(-1)
    dic=defaultdict(moins_un)
    for i in range(n):
       dic[pos_list[i]]=i
    M=trans_matrice(pos_list,dic,vois)
    P=np.zeros(n)
    P[i0]=1
    p_k=[1]
    S=0
    for k in range(t):
        P=M.dot(P)
        p_k.append(np.sum(P))
    p_k=np.array(p_k)
    p_k=p_k[:-1]-p_k[1:]
    L+=p_k
    Err+=p_k**2
    if r%10**2==0:
      np.save('percolation/'+str(n)+'/Probabilities_n'+str(n)+', '+str(K),[L,Err])


import os
import json
import pytz
import logging
from flask_socketio import SocketIO, emit
from flask import (Flask, render_template, request, jsonify)
from pymongo import MongoClient
from threading import Thread, Event
from datetime import datetime
from convxai.utils import *

logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)

app = Flask(__name__)
app.config["SECRET_KEY"] = "liaopi6u123sdfakjb23sd"
socketio = SocketIO(app, logger=True, engineio_logger=True,
                    async_mode="threading")
mongo = get_mongo_connection()
thread = Thread()
thread_stop_event = Event()
task_mapping = {}

# ########################################
# # Set up paths to save log files
# ########################################
# system_config = parse_system_config_file()
# logFileName = "log_" + datetime.now().astimezone(pytz.timezone('US/Eastern')
#                                                  ).strftime("%m%d%Y_%H%M%S") + ".txt"
# logFile = open(os.path.join(
#     system_config['system']['logfilePath'], logFileName), "a")

########################################
# Threading and MongoDB
########################################

def init_threading() -> None:
    """
    Initiate the thread to monitor interaction and save data into mongoDB database.
    """
    global thread
    if not thread.is_alive():
        thread = Thread(
            name='mongo-monitoring',
            target=mongo_monitoring,
        )
        thread.start()
        logging.info("Starting Mongo Monitoring Thread")

def mongo_monitoring():
    """
    Set up mongoDB monitoring.
    """
    global task_mapping
    pipeline = [{
        "$match": {
            "$and": [
                {"operationType": "insert"},
                {"fullDocument.role": "agent"},
                {"fullDocument.reply_to": {"$exists": True}}
            ]
        }
    }]
    with mongo.message.watch(pipeline) as stream:
        for insert_change in stream:
            source_message_id = insert_change["fullDocument"]["reply_to"]
            response_text = insert_change["fullDocument"]["text"]
            message_type = insert_change["fullDocument"]["payload"]["message_type"]
            if message_type == "task":
                socketio.emit(
                    "task_response",
                    {"text": response_text},
                    to=task_mapping[source_message_id],
                    namespace="/connection"
                )
            elif message_type == "conv":
                writingIndex = insert_change["fullDocument"]["payload"]["writingIndex"]
                socketio.emit(
                    "conv_response",
                    {"text": response_text, "writingIndex": writingIndex},
                    to=task_mapping[source_message_id],
                    namespace="/connection"
                )
            del task_mapping[source_message_id],

########################################
# Flask Implementation
########################################

def get_data():
    """
    Get the request data.
    """
    data = json.loads(str(request.data, encoding='utf-8'))
    return data

@app.route("/")
def index():
    return render_template("user_interface.html")

@app.route("/init_conversation", methods=["POST"])
def init_conversation():
    """
    Initiate the conversation.
    """
    body = get_data()
    res = mongo.message.insert_one({
        "text": body["text"],
        "role": "user",
        "init": True,
        "time": datetime.now(),
        "conversation_id": body.get("conversation_id"),
    })
    return jsonify({"text": "hello world", "conversation_id": str(res.inserted_id)})

@app.route("/reset")
def reset():
    """
    Reset the interface.
    """
    data = json.dumps({"text": "[RESET]"})
    return jsonify({"text": "RESET"})

########################################
# SocketIO Implementation
########################################

@socketio.on('connect', namespace="/connection")
def test_connect():
    emit('connected', {'data': 'Connected'})
    init_threading()

@socketio.on('disconnect', namespace="/connection")
def test_disconnect():
    logging.info('Client disconnected')

@socketio.on("interact", namespace="/connection")
def interact_socket(body):
    """
    Interaction between the web server and interface via socketIO.
    """
    result = mongo.message.insert_one({
        "text": body["text"],
        "role": "user",
        "init": False,
        "time": datetime.now(),
        "conversation_id": body["conversation_id"],
        "payload": {
            "message_type": body["message_type"],
            "writing_model": body.get("writing_model", None),
        },
        "note": "socket",
        "user_id": body.get("user_id", ""),
    })

    # log text editing results into mongo
    mongo.log.insert_one({
        "user_id": body.get("user_id", ""),
        "time": datetime.now(),
        "text": body["text"],
        "type": body["message_type"],
        "writing_model": body.get("writing_model", None),
        "conversation_id": body["conversation_id"],
    })

    ### Write logs into files ###
    task_mapping[result.inserted_id] = request.sid

    responseTime = datetime.now().astimezone(
        pytz.timezone('US/Eastern')).strftime("%m-%d-%Y %H:%M:%S")
    responseText = body["text"]
    responseMessageType = body["message_type"]
    responseWritingModel = body.get("writing_model", None)

    # logEntry = responseTime + "  Text: " + responseText + "  \n\t\tMessageType:" + \
    #     responseMessageType + "  \n\t\tWritingModel: " + responseWritingModel + "\n"
    # logFile.write(logEntry+"\n")
    # logFile.close()
    # logging.info("Written to file" + logEntry)

@socketio.on("save", namespace="/connection")
def save(body):
    # log text editing results into mongo
    res = mongo.log.insert_one({
        "user_id": body.get("user_id", ""),
        "time": datetime.now(),
        "text": body["text"],
        "event_type": body.get("mode", "auto-save"),
    })

    if res:
        emit('save', {"status": "success", "mode": body.get("mode", "auto-save")})

########################################
# Run Flask and SocketIO.
########################################
def run_flask_socketio():
    """
    Run Flask and Websocket.
    """
    socketio.run(app, host="0.0.0.0", port=8080, debug=False)

    # for debugging
    # socketio.run(app, host="0.0.0.0", port=8001, debug=True)

if __name__ == "__main__":
    init_threading()
    run_flask_socketio()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2020/11/9 0:05
# @Author  : SELF-T-YY
# @Site    :
# @File    : classical_sample_ambiguous_shortestPath.py
# @Software: PyCharm

import json
import networkx as nx
import sys
import numpy

fileWritePath = r'../data_forSystem/ieee_visC/IV_ambiguousBetweennessData.json'
fileOriDataPath = r'../data_forSystem/ieee_visC/IVxy.json'
sampleNameList = ['ISRW', 'TIES', 'SRW', 'RES', 'RJ', 'RNS']
sampleRateList = ['5', '10', '15', '20', '25', '30', '35', '40']

with open(fileOriDataPath) as f:
    oriData = json.load(f)
    tempEdgesList = oriData['edges']
    edgesList = [[_['source'], _['target']] for _ in tempEdgesList]
    tempNodesList = oriData['nodes']
    nodesList = [_['id'] for _ in tempNodesList]

    G = nx.Graph()
    G.add_edges_from(edgesList)
    oriShortestPathData = {}
    # 计算原始数据的平均最短路径
    for node in nodesList:
        length = dict(nx.single_target_shortest_path_length(G, node))
        sum = 0
        for _ in length:
            sum += length[_]
        oriShortestPathData[node] = sum / len(length)

    ambiguousData = {}
    for sampleName in sampleNameList:
        tempAmbiguousData_addName_addRate = {}
        for rate in sampleRateList:
            filePath = r'../data_forSystem/ieee_visC/forceData/IV_forceData_' + sampleName + '_rate_' + rate + '.json'
            print(filePath)
            sampleRate = 'rate-' + rate
            with open(filePath) as f1:
                sampleData = json.load(f1)
                tempEdgesList = sampleData['edges']
                edgesList = [[_['source'], _['target']] for _ in tempEdgesList]

                tempNodesList = sampleData['nodes']
                nodesList = [_['id'] for _ in tempNodesList]

                G = nx.Graph()
                G.add_edges_from(edgesList)
                shortestPath = {}
                for node in nodesList:
                    if not G.has_node(node):
                        continue
                    length = dict(nx.single_target_shortest_path_length(G, node))
                    sum = 0
                    for _ in length:
                        sum += length[_]
                    shortestPath[node] = sum/len(length)

                # 计算歧义
                tempAmbiguousData = {_: oriShortestPathData[_] - shortestPath[_] for _ in
                                     shortestPath
                                     if _ in oriShortestPathData}

                # 归一化
                maxVal = 0
                minVal = 100000000
                for _ in tempAmbiguousData:
                    maxVal = max(maxVal, tempAmbiguousData[_])
                    minVal = min(minVal, tempAmbiguousData[_])

                tempAmbiguousData = {_: (tempAmbiguousData[_] - minVal) / (maxVal - minVal) for _ in tempAmbiguousData}

                tempAmbiguousData_addName_addRate[sampleRate] = tempAmbiguousData
            ambiguousData[sampleName] = tempAmbiguousData_addName_addRate

fw = open(fileWritePath, 'w+')
fw.write(json.dumps(ambiguousData))
fw.close()


# definimos una función para solicitar la carga del string
def carga():
    return input("Ingrese un string: ")

# definimos una función para comprobar el total de vocales
def comprobar(string):
    cant=0
    lista=["a","e","i","o","u"]
    for x in range(len(string)):
        for k in range(len(lista)):
            if string[x] == lista[k]:
                cant=cant+1
    print("El total de vocales es ",cant)

# bloque principal
string1=carga()
string2=carga()
string3=carga()
print(string1)
comprobar(string1)
print(string2)
comprobar(string2)
print(string3)
comprobar(string3)


import datetime
import os
import platform

class Clock:

    def __init__(self):
        self._stopwatch_counter_num = 10800
        self._running = None
        self._alarm_minute = None
        self._alarm_hour = None
        self._seconds = None
        self._minutes = None
        self._hour = None
        self._alarm_time = None
        self._time = None
        self._date = None

    def get_time(self):
        self._time = datetime.datetime.now().strftime("%H:%M:%S")
        self._hour, self._minutes, self._seconds = self._time.split(':')
        return self._time

    def get_date(self):
        self._date = datetime.datetime.now().strftime("%d/%m/%Y")
        return self._date

    def clock_loop(self):
        while True:
            self.get_time()

    def print_time(self):
        self.get_time()
        self.get_date()
        print(self._time, self._date)

    def alarm(self, alarm_time):
        self._alarm_time = alarm_time
        self._alarm_hour, self._alarm_minute = self._alarm_time.split(':')
        self.alarm_loop()

    def alarm_check(self):
        if self._hour == self._alarm_hour and self._minutes == self._alarm_minute:
            self.beep()
            return True
        else:
            return False

    @staticmethod
    def beep():
        for i in range(3):
            if platform.system() == 'Windows':
                winsound.Beep(5000, 1000)
            elif platform.system() == 'Darwin':
                os.system('say Time is Up')
            elif platform.system() == 'Linux':
                os.system('beep -f 5000')

    def alarm_loop(self):
        while self.alarm_check() is False:
            self.get_time()
            self.alarm_check()

    def stopwatch(self, running):
        self._running = running
        while self._running:
            tt = datetime.datetime.fromtimestamp(self._stopwatch_counter_num)
            self._stopwatch_counter_num += 1
            return tt.strftime("%H:%M:%S")

    def reset_stopwatch(self):
        self._stopwatch_counter_num = 10800


#nested dictionary...............
people={1:{'name':'john','age':78,'sex':'Male'},2:{'name':'marry','age':68,'sex':'Female'}}
print(people)

print(people[1]['name'])
print(people[2]['sex'])

#Append...................
people[3]={}
people[3]['name']='Simran'
people[3]['age']=22
people[3]['sex']='female'
people[3]['married']='NO'

people[4]={'name':'Piru','age':39,'sex':'Female'}

#DEL.............

del people[3]['married']
del people[4]
print(people)
print(people[3])

#Traverse.............

for p_id,p_info in people.items():
   print("Person id",p_id)
   for key in p_info:
      print( key, " :",p_info[key])