#!/usr/bin/env python3
# logging.basicConfig(level=logging.DEBUG)
import argparse
import json
import logging
import math
import os
import re
import socket
import stat
import traceback
from collections import deque, namedtuple, UserDict, defaultdict
from concurrent import futures
from datetime import datetime, timezone
from functools import lru_cache
from http.client import HTTPConnection
from sys import argv
from urllib import request
from urllib.error import HTTPError, URLError
from urllib.request import AbstractHTTPHandler, HTTPHandler, HTTPSHandler, OpenerDirector, HTTPRedirectHandler, \
    Request, HTTPBasicAuthHandler

logger = logging.getLogger()
__author__ = 'Tim Laurence'
__copyright__ = "Copyright 2019"
__credits__ = ['Tim Laurence']
__license__ = "GPL"
__version__ = "2.2.2"

'''
nrpe compatible check for docker containers.

Requires Python 3

Note: I really would have preferred to have used requests for all the network connections but that would have added a
dependency.
'''

DEFAULT_SOCKET = '/var/run/docker.sock'
DEFAULT_TIMEOUT = 10.0
DEFAULT_PORT = 2375
DEFAULT_MEMORY_UNITS = 'B'
DEFAULT_HEADERS = [('Accept', 'application/vnd.docker.distribution.manifest.v2+json')]
DEFAULT_PUBLIC_REGISTRY = 'registry-1.docker.io'

# The second value is the power to raise the base to.
UNIT_ADJUSTMENTS_TEMPLATE = {
    '%': 0,
    'B': 0,
    'KB': 1,
    'MB': 2,
    'GB': 3,
    'TB': 4
}
unit_adjustments = None

# Reduce message to a single OK unless a checks fail.
no_ok = False

# Suppress performance data reporting
no_performance = False

OK_RC = 0
WARNING_RC = 1
CRITICAL_RC = 2
UNKNOWN_RC = 3

# These hold the final results
rc = -1
messages = []
performance_data = []

ImageName = namedtuple('ImageName', "registry name tag full_name")

class ThresholdSpec(UserDict):
    def __init__(self, warn, crit, units=''):
        super().__init__(warn=warn, crit=crit, units=units)

    def __getattr__(self, item):
        if item in ('warn', 'crit', 'units'):
            return self.data[item]
        return super().__getattr__(item)

# How much threading can we do? We are generally not CPU bound so I am using this a worse case cap
DEFAULT_PARALLELISM = 10

# Holds list of all threads
threads = []

# This is used during testing
DISABLE_THREADING = False

# Hacked up urllib to handle sockets
#############################################################################################
# Docker runs a http connection over a socket. http.client is knows how to deal with these
# but lacks some niceties. Urllib wraps that and makes up for some of the deficiencies but
# cannot fix the fact http.client can't read from socket files. In order to take advantage of
# urllib and http.client's  capabilities the class below tweaks HttpConnection and passes it
# to urllib registering for socket:// connections

# This is all side effect so excluding coverage
class SocketFileHandler(AbstractHTTPHandler):
    class SocketFileToHttpConnectionAdaptor(HTTPConnection): # pragma: no cover
        def __init__(self, socket_file, timeout=DEFAULT_TIMEOUT):
            super().__init__(host='', port=0, timeout=timeout)
            self.socket_file = socket_file

        def connect(self):
            self.sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, fileno=None)
            self.sock.settimeout(self.timeout)
            self.sock.connect(self.socket_file)

    def socket_open(self, req):
        socket_file, path = req.selector.split(':', 1)
        req.host = socket_file
        req.selector = path
        return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)

# Tokens are not cached because I expect the callers to cache the responses
class Oauth2TokenAuthHandler(HTTPBasicAuthHandler):
    auth_failure_tracker = defaultdict(int)

    def http_response(self, request, response):
        code, hdrs = response.code, response.headers

        www_authenticate_header = response.headers.get('www-authenticate', None)
        if code == 401 and www_authenticate_header:
            scheme = www_authenticate_header.split()[0]
            if scheme.lower() == 'bearer':
                return self.process_oauth2(request, response, www_authenticate_header)

        return response

    https_response = http_response

    @staticmethod
    def _get_outh2_token(www_authenticate_header):
        auth_fields = dict(re.findall(r"""(?:(?P<key>[^ ,=]+)="([^"]+)")""", www_authenticate_header))

        auth_url = "{realm}?scope={scope}&service={service}".format(
            realm=auth_fields['realm'],
            scope=auth_fields['scope'],
            service=auth_fields['service'],
        )
        token_request = Request(auth_url)
        token_request.add_header("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
        token_response = request.urlopen(token_request)
        return process_urllib_response(token_response)['token']

    def process_oauth2(self, request, response, www_authenticate_header):

        # This keeps infinite auth loops from happening
        full_url = request.full_url
        self.auth_failure_tracker[full_url] += 1
        if self.auth_failure_tracker[full_url] > 1:
            raise HTTPError(full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url),
                            response.headers, response)

        auth_token = self._get_outh2_token(www_authenticate_header)

        request.add_unredirected_header('Authorization', 'Bearer ' + auth_token)
        return self.parent.open(request, timeout=request.timeout)

better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPSHandler())
better_urllib_get.add_handler(HTTPRedirectHandler())
better_urllib_get.add_handler(SocketFileHandler())
better_urllib_get.add_handler(Oauth2TokenAuthHandler())

class RegistryError(Exception):
    def __init__(self, response):
        self.response_obj = response

# Util functions
#############################################################################################
def parse_thresholds(spec, include_units=True, units_required=True):
    """
    Given a spec string break it up into ':' separated chunks. Convert strings to ints as it makes sense

    :param spec: The threshold specification being parsed
    :param include_units: Specifies that units should be processed and returned if present
    :param units_required: Mark spec as invalid if the units are missing.
    :return: A list containing the thresholds in order of warn, crit, and units(if included and present)
    """
    parts = deque(spec.split(':'))
    if not all(parts):
        raise ValueError("Blanks are not allowed in a threshold specification: {}".format(spec))

    # Warn
    warn = int(parts.popleft())
    # Crit
    crit = int(parts.popleft())

    units = ''
    if include_units:
        if len(parts):
            # units
            units = parts.popleft()
        elif units_required:
            raise ValueError("Missing units in {}".format(spec))

    if len(parts) != 0:
        raise ValueError("Too many threshold specifiers in {}".format(spec))

    return ThresholdSpec(warn=warn, crit=crit, units=units)

def pretty_time(seconds):
    remainder = seconds
    result = []
    if remainder > 24 * 60 * 60:
        days, remainder = divmod(remainder, 24 * 60 * 60)
        result.append("{}d".format(int(days)))
    if remainder > 60 * 60:
        hours, remainder = divmod(remainder, 60 * 60)
        result.append("{}h".format(int(hours)))
    if remainder > 60:
        minutes, remainder = divmod(remainder, 60)
        result.append("{}min".format(int(minutes)))
    result.append("{}s".format(int(remainder)))
    return result

def evaluate_numeric_thresholds(container, value, thresholds, name, short_name,
                                min=None, max=None, greater_than=True):
    rounder = lambda x: round(x, 2)

    INTEGER_UNITS = ['B', '%', '']

    # Some units don't have decimal places
    rounded_value = int(value) if thresholds.units in INTEGER_UNITS else rounder(value)

    perf_string = "{container}_{short_name}={value}{units};{warn};{crit}".format(
        container=container,
        short_name=short_name,
        value=rounded_value,
        **thresholds)
    if min is not None:
        rounded_min = math.floor(min) if thresholds.units in INTEGER_UNITS else rounder(min)
        perf_string += ';{}'.format(rounded_min)
        if max is not None:
            rounded_max = math.ceil(max) if thresholds.units in INTEGER_UNITS else rounder(max)
            perf_string += ';{}'.format(rounded_max)

    global performance_data
    performance_data.append(perf_string)

    if thresholds.units == 's':
        nice_time = ' '.join(pretty_time(rounded_value)[:2])
        results_str = "{} {} is {}".format(container, name, nice_time)
    else:
        results_str = "{} {} is {}{}".format(container, name, rounded_value, thresholds.units)

    if greater_than:
        comparator = lambda value, threshold: value >= threshold
    else:
        comparator = lambda value, threshold: value <= threshold

    if comparator(value, thresholds.crit):
        critical(results_str)
    elif comparator(value, thresholds.warn):
        warning(results_str)
    else:
        ok(results_str)

@lru_cache(maxsize=None)
def get_url(url):
    logger.debug("get_url: {}".format(url))
    response = better_urllib_get.open(url, timeout=timeout)
    logger.debug("get_url: {} {}".format(url, response.status))
    return process_urllib_response(response), response.status

def process_urllib_response(response):
    response_bytes = response.read()
    body = response_bytes.decode('utf-8')
    # logger.debug("BODY: {}".format(body))
    return json.loads(body)

def get_container_info(name):
    content, _ = get_url(daemon + '/containers/{container}/json'.format(container=name))
    return content

def get_image_info(name):
    content, _ = get_url(daemon + '/images/{image}/json'.format(image=name))
    return content

def get_state(container):
    return get_container_info(container)['State']

def get_stats(container):
    content, _ = get_url(daemon + '/containers/{container}/stats?stream=0'.format(container=container))
    return content

def get_ps_name(name_list):
    # Pick the name that starts with a '/' but doesn't contain a '/' and return that value
    for name in name_list:
        if '/' not in name[1:] and name[0] == '/':
            return name[1:]
    else:
        raise NameError("Error when trying to identify 'ps' name in {}".format(name_list))

def get_containers(names, require_present):
    containers_list, _ = get_url(daemon + '/containers/json?all=1')

    all_container_names = set(get_ps_name(x['Names']) for x in containers_list)

    if 'all' in names:
        return all_container_names

    filtered = set()
    for matcher in names:
        found = False
        for candidate in all_container_names:
            if re.match("^{}$".format(matcher), candidate):
                filtered.add(candidate)
                found = True
        # If we don't find a container that matches out regex
        if require_present and not found:
            critical("No containers match {}".format(matcher))

    return filtered

def get_container_image_id(container):
    # find registry and tag
    inspection = get_container_info(container)
    return inspection['Image']

def get_container_image_urls(container):
    inspection = get_container_info(container)
    image_id = inspection['Image']
    image_info = get_image_info(image_id)
    return image_info['RepoTags']

def normalize_image_name_to_manifest_url(image_name, insecure_registries):
    parsed_url = parse_image_name(image_name)

    lower_insecure = [reg.lower() for reg in insecure_registries]

    # Registry query url
    scheme = 'http' if parsed_url.registry.lower() in lower_insecure else 'https'
    url = '{scheme}://{registry}/v2/{image_name}/manifests/{image_tag}'.format(scheme=scheme,
                                                                               registry=parsed_url.registry,
                                                                               image_name=parsed_url.name,
                                                                               image_tag=parsed_url.tag)
    return url, parsed_url.registry

# Auth servers seem picky about being hit too hard. Can't figure out why. ;)
# As result it is best to single thread this check
# This is based on https://docs.docker.com/registry/spec/auth/token/#requesting-a-token
def get_digest_from_registry(url):
    logger.debug("get_digest_from_registry")
    # query registry
    # TODO: Handle logging in if needed
    registry_info, status_code = get_url(url=url)

    if status_code != 200:
        raise RegistryError(response=registry_info)
    return registry_info['config'].get('digest', None)

def set_rc(new_rc):
    global rc
    rc = new_rc if new_rc > rc else rc

def ok(message):
    set_rc(OK_RC)
    messages.append('OK: ' + message)

def warning(message):
    set_rc(WARNING_RC)
    messages.append('WARNING: ' + message)

def critical(message):
    set_rc(CRITICAL_RC)
    messages.append('CRITICAL: ' + message)

def unknown(message):
    set_rc(UNKNOWN_RC)
    messages.append('UNKNOWN: ' + message)

def require_running(name):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            container_state = get_state(container)
            state = normalize_state(container_state)
            if state.lower() == "running":
                func(container, *args, **kwargs)
            else:
                # container is not running, can't perform check
                critical('{container} is not "running", cannot check {check}"'.format(container=container,
                                                                                check=name))

        return wrapper

    return inner_decorator

def multithread_execution(disable_threading=DISABLE_THREADING):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            if DISABLE_THREADING:
                func(container, *args, **kwargs)
            else:
                threads.append(parallel_executor.submit(func, container, *args, **kwargs))

        return wrapper

    return inner_decorator

def singlethread_execution(disable_threading=DISABLE_THREADING):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            if DISABLE_THREADING:
                func(container, *args, **kwargs)
            else:
                threads.append(serial_executor.submit(func, container, *args, **kwargs))

        return wrapper

    return inner_decorator

def parse_image_name(image_name):
    """
    Parses image names into their constituent parts.
    :param image_name:
    :return: ImageName
    """

    # These are based on information found here
    #   https://docs.docker.com/engine/reference/commandline/tag/#extended-description
    #   https://github.com/docker/distribution/blob/master/reference/regexp.go
    host_segment_re = '[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?'
    hostname_re = r'({host_segment}\.)+{host_segment}'.format(host_segment=host_segment_re)
    registry_re = r'((?P<registry>({hostname_re}(:\d+)?|{host_segment_re}:\d+))/)'.format(
        host_segment_re=host_segment_re, hostname_re=hostname_re)
    name_component_ends_re = '[a-z0-9]'
    name_component_middle_re = '[a-z0-9._-]'  # Ignoring spec limit of two _
    name_component_re = '({end}{middle}*{end}|{end})'.format(end=name_component_ends_re,
                                                             middle=name_component_middle_re)
    image_name_re = "(?P<image_name>({name_component}/)*{name_component})".format(name_component=name_component_re)
    image_tag_re = '(?P<image_tag>[a-zA-Z0-9_][a-zA-Z0-9_.-]*)'
    full_re = '^{registry}?{image_name}(:{image_tag})?$'.format(registry=registry_re, image_name=image_name_re,
                                                                image_tag=image_tag_re)
    parsed = re.match(full_re, image_name)

    registry = parsed.group('registry') if parsed.group('registry') else DEFAULT_PUBLIC_REGISTRY

    image_name = parsed.group('image_name')
    image_name = image_name if '/' in image_name or registry != DEFAULT_PUBLIC_REGISTRY else 'library/' + image_name

    image_tag = parsed.group('image_tag')
    image_tag = image_tag if image_tag else 'latest'

    full_image_name = "{registry}/{image_name}:{image_tag}".format(
        registry=registry,
        image_name=image_name,
        image_tag=image_tag)

    return ImageName(registry=registry, name=image_name, tag=image_tag, full_name=full_image_name)

def normalize_state(status_info):
    # Ugh, docker used to report state in as silly way then they figured out how to do it better.
    # This tries the simpler new way and if that doesn't work fails back to the old way

    # On new docker engines the status holds whatever the current state is, running, stopped, paused, etc.
    if "Status" in status_info:
        return status_info['Status']

    status = 'Exited'
    if status_info["Restarting"]:
        status = 'Restarting'
    elif status_info["Paused"]:
        status = 'Paused'
    elif status_info["Dead"]:
        status = 'Dead'
    elif status_info["Running"]:
        return "Running"
    return status

# Checks
#############################################################################################

@multithread_execution()
@require_running(name='memory')
def check_memory(container, thresholds):
    if not thresholds.units in unit_adjustments:
        unknown("Memory units must be one of  {}".format(list(unit_adjustments.keys())))
        return

    inspection = get_stats(container)

    # Subtracting cache to match what `docker stats` does.
    adjusted_usage = inspection['memory_stats']['usage'] - inspection['memory_stats']['stats']['total_cache']
    if thresholds.units == '%':
        max = 100
        usage = int(100 * adjusted_usage / inspection['memory_stats']['limit'])
    else:
        max = inspection['memory_stats']['limit'] / unit_adjustments[thresholds.units]
        usage = adjusted_usage / unit_adjustments[thresholds.units]

    evaluate_numeric_thresholds(container=container, value=usage, thresholds=thresholds, name='memory',
                                short_name='mem', min=0, max=max)

@multithread_execution()
def check_status(container, desired_state):
    normized_desired_state = desired_state.lower()
    normalized_state = normalize_state(get_state(container)).lower()
    if normized_desired_state != normalized_state:
        critical("{} state is not {}".format(container, desired_state))
        return
    ok("{} status is {}".format(container, desired_state))

@multithread_execution()
@require_running('health')
def check_health(container):
    state = get_state(container)
    if "Health" in state and "Status" in state["Health"]:
        health = state["Health"]["Status"]
        message = "{} is {}".format(container, health)
        if health == 'healthy':
            ok(message)
        elif health == 'unhealthy':
            critical(message)
        else:
            unknown(message)
    else:
        unknown('{} has no health check data'.format(container))

@multithread_execution()
@require_running('uptime')
def check_uptime(container, thresholds):
    inspection = get_container_info(container)['State']['StartedAt']
    only_secs = inspection[0:19]
    start = datetime.strptime(only_secs, "%Y-%m-%dT%H:%M:%S")
    start = start.replace(tzinfo=timezone.utc)
    now = datetime.now(timezone.utc)
    uptime = (now - start).total_seconds()

    graph_padding = 2
    thresholds.units = 's'
    evaluate_numeric_thresholds(container=container, value=uptime, thresholds=thresholds, name='uptime',
                                short_name='up', min=0, max=graph_padding, greater_than=False)

@multithread_execution()
def check_image_age(container, thresholds):
    container_image = get_container_info(container)['Image']
    image_created = get_image_info(container_image)['Created']
    only_secs = image_created[0:19]
    start = datetime.strptime(only_secs, "%Y-%m-%dT%H:%M:%S")
    start = start.replace(tzinfo=timezone.utc)
    now = datetime.now(timezone.utc)
    image_age = (now - start).days

    graph_padding = 2
    thresholds.units = 'd'
    evaluate_numeric_thresholds(container=container, value=image_age, thresholds=thresholds, name='image_age',
                                short_name='age', min=0, max=graph_padding, greater_than=True)

@multithread_execution()
@require_running('restarts')
def check_restarts(container, thresholds):
    inspection = get_container_info(container)

    restarts = int(inspection['RestartCount'])
    graph_padding = 2
    evaluate_numeric_thresholds(container=container, value=restarts, thresholds=thresholds, name='restarts',
                                short_name='re', min=0, max=graph_padding)

@singlethread_execution()
def check_version(container, insecure_registries):
    image_id = get_container_image_id(container)
    logger.debug("Local container image ID: {}".format(image_id))
    if image_id is None:
        unknown('Checksum missing for "{}", try doing a pull'.format(container))
        return

    image_urls = get_container_image_urls(container=container)
    if len(image_urls) > 1:
        unknown('"{}" has multiple tags/names. Unsure which one to use to check the version.'.format(container))
        return
    elif len(image_urls) == 0:
        unknown('"{}" has last no repository tag. Is this anywhere else?'.format(container))
        return

    url, registry = normalize_image_name_to_manifest_url(image_urls[0], insecure_registries)
    logger.debug("Looking up image digest here {}".format(url))
    try:
        registry_hash = get_digest_from_registry(url)
    except URLError as e:
        if hasattr(e.reason, 'reason') and e.reason.reason == 'UNKNOWN_PROTOCOL':
            unknown(
                "TLS error connecting to registry {} for {}, should you use the '--insecure-registry' flag?" \
                    .format(registry, container))
            return
        elif hasattr(e.reason, 'strerror') and e.reason.strerror == 'nodename nor servname provided, or not known':
            unknown(
                "Cannot reach registry for {} at {}".format(container, url))
            return
        else:
            raise e
    except RegistryError as e:
        unknown("Cannot check version, couldn't retrieve digest for {} while checking {}.".format(container, url))
        return
    logger.debug("Image digests, local={} remote={}".format(image_id, registry_hash))
    if registry_hash == image_id:
        ok("{}'s version matches registry".format(container))
        return
    critical("{}'s version does not match registry".format(container))

def calculate_cpu_capacity_precentage(info, stats):
    host_config = info['HostConfig']

    if 'online_cpus' in stats['cpu_stats']:
        num_cpus = stats['cpu_stats']['online_cpus']
    else:
        num_cpus = len(stats['cpu_stats']['cpu_usage']['percpu_usage'])

    # Identify limit system being used
    # --cpus
    if 'NanoCpus' in host_config and host_config['NanoCpus'] != 0:
        period = 1000000000
        quota = host_config['NanoCpus']
    # --cpu-quota
    elif 'CpuQuota' in host_config and host_config['CpuQuota'] != 0:
        period = 100000 if host_config['CpuPeriod'] == 0 else host_config['CpuPeriod']
        quota = host_config['CpuQuota']
    # unlimited
    else:
        period = 1
        quota = num_cpus

    if period * num_cpus < quota:
        # This handles the case where the quota is actually bigger than amount available by all the cpus.
        available_limit_ratio = 1
    else:
        available_limit_ratio = (period * num_cpus) / quota

    cpu_delta = stats['cpu_stats']['cpu_usage']['total_usage'] - stats['precpu_stats']['cpu_usage']['total_usage']
    system_delta = stats['cpu_stats']['system_cpu_usage'] - stats['precpu_stats']['system_cpu_usage']
    usage = (cpu_delta / system_delta) * available_limit_ratio
    usage = round(usage * 100, 0)
    return usage

@multithread_execution()
@require_running('cpu')
def check_cpu(container, thresholds):
    info = get_container_info(container)

    stats = get_stats(container=container)

    usage = calculate_cpu_capacity_precentage(info=info, stats=stats)

    max = 100
    thresholds.units = '%'
    evaluate_numeric_thresholds(container=container, value=usage, thresholds=thresholds, name='cpu', short_name='cpu',
                                min=0, max=max)

def process_args(args):
    parser = argparse.ArgumentParser(description='Check docker containers.')

    # Connect to local socket or ip address
    connection_group = parser.add_mutually_exclusive_group()
    connection_group.add_argument('--connection',
                                  dest='connection',
                                  action='store',
                                  default=DEFAULT_SOCKET,
                                  type=str,
                                  metavar='[/<path to>/docker.socket|<ip/host address>:<port>]',
                                  help='Where to find docker daemon socket. (default: %(default)s)')

    connection_group.add_argument('--secure-connection',
                                  dest='secure_connection',
                                  action='store',
                                  type=str,
                                  metavar='[<ip/host address>:<port>]',
                                  help='Where to find TLS protected docker daemon socket.')

    base_group = parser.add_mutually_exclusive_group()
    base_group.add_argument('--binary_units',
                            dest='units_base',
                            action='store_const',
                            const=1024,
                            help='Use a base of 1024 when doing calculations of KB, MB, GB, & TB (This is default)')

    base_group.add_argument('--decimal_units',
                            dest='units_base',
                            action='store_const',
                            const=1000,
                            help='Use a base of 1000 when doing calculations of KB, MB, GB, & TB')
    parser.set_defaults(units_base=1024)

    # Connection timeout
    parser.add_argument('--timeout',
                        dest='timeout',
                        action='store',
                        type=float,
                        default=DEFAULT_TIMEOUT,
                        help='Connection timeout in seconds. (default: %(default)s)')

    # Container name
    parser.add_argument('--containers',
                        dest='containers',
                        action='store',
                        nargs='+',
                        type=str,
                        default=['all'],
                        help='One or more RegEx that match the names of the container(s) to check. If omitted all containers are checked. (default: %(default)s)')

    # Container name
    parser.add_argument('--present',
                        dest='present',
                        default=False,
                        action='store_true',
                        help='Modifies --containers so that each RegEx must match at least one container.')

    # Threads
    parser.add_argument('--threads',
                        dest='threads',
                        default=DEFAULT_PARALLELISM,
                        action='store',
                        type=int,
                        help='This + 1 is the maximum number of concurent threads/network connections. (default: %(default)s)')

    # CPU
    parser.add_argument('--cpu',
                        dest='cpu',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Check cpu usage percentage taking into account any limits.')

    # Memory
    parser.add_argument('--memory',
                        dest='memory',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT:UNITS',
                        help='Check memory usage taking into account any limits. Valid values for units are %%,B,KB,MB,GB.')

    # State
    parser.add_argument('--status',
                        dest='status',
                        action='store',
                        type=str,
                        help='Desired container status (running, exited, etc).')

    # Health
    parser.add_argument('--health',
                        dest='health',
                        default=None,
                        action='store_true',
                        help="Check container's health check status")

    # Age
    parser.add_argument('--uptime',
                        dest='uptime',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Minimum container uptime in seconds. Use when infrequent crashes are tolerated.')

    # Image Age
    parser.add_argument('--image-age',
                        dest='image_age',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Maximum image age in days.')

    # Version
    parser.add_argument('--version',
                        dest='version',
                        default=None,
                        action='store_true',
                        help='Check if the running images are the same version as those in the registry. Useful for finding stale images. Does not support login.')

    # Version
    parser.add_argument('--insecure-registries',
                        dest='insecure_registries',
                        action='store',
                        nargs='+',
                        type=str,
                        default=[],
                        help='List of registries to connect to with http(no TLS). Useful when using "--version" with images from insecure registries.')

    # Restart
    parser.add_argument('--restarts',
                        dest='restarts',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Container restart thresholds.')

    # no-ok
    parser.add_argument('--no-ok',
                        dest='no_ok',
                        action='store_true',
                        help='Make output terse suppressing OK messages. If all checks are OK return a single OK.')

    # no-performance
    parser.add_argument('--no-performance',
                        dest='no_performance',
                        action='store_true',
                        help='Suppress performance data. Reduces output when performance data is not being used.')

    parser.add_argument('-V', action='version', version='%(prog)s {}'.format(__version__))

    if len(args) == 0:
        parser.print_help()

    parsed_args = parser.parse_args(args=args)

    global timeout
    timeout = parsed_args.timeout

    global daemon
    global connection_type
    if parsed_args.secure_connection:
        daemon = 'https://' + parsed_args.secure_connection
        connection_type = 'https'
    elif parsed_args.connection:
        if parsed_args.connection[0] == '/':
            daemon = 'socket://' + parsed_args.connection + ':'
            connection_type = 'socket'
        else:
            daemon = 'http://' + parsed_args.connection
            connection_type = 'http'

    return parsed_args

def no_checks_present(parsed_args):
    # Look for all functions whose name starts with 'check_'
    checks = [key[6:] for key in globals().keys() if key.startswith('check_')]
    # Act like --present is a check though it is not implemented like one
    return all(getattr(parsed_args, check) is None for check in checks) and not parsed_args.present

def socketfile_permissions_failure(parsed_args):
    if connection_type == 'socket':
        return not (os.path.exists(parsed_args.connection)
                    and stat.S_ISSOCK(os.stat(parsed_args.connection).st_mode)
                    and os.access(parsed_args.connection, os.R_OK)
                    and os.access(parsed_args.connection, os.W_OK))
    else:
        return False

def print_results():
    if no_ok:
        # Remove all the "OK"s
        filtered_messages = [message for message in messages if not message.startswith('OK: ')]
        if len(filtered_messages) == 0:
            messages_concat = 'OK'
        else:
            messages_concat = '; '.join(filtered_messages)

    else:
        messages_concat = '; '.join(messages)

    if no_performance or len(performance_data) == 0:
        print(messages_concat)
    else:
        perfdata_concat = ' '.join(performance_data)
        print(messages_concat + '|' + perfdata_concat)

def perform_checks(raw_args):
    args = process_args(raw_args)

    global parallel_executor
    parallel_executor = futures.ThreadPoolExecutor(max_workers=args.threads)
    global serial_executor
    serial_executor = futures.ThreadPoolExecutor(max_workers=1)

    global unit_adjustments
    unit_adjustments = {key: args.units_base ** value for key, value in UNIT_ADJUSTMENTS_TEMPLATE.items()}

    global no_ok
    no_ok = args.no_ok

    global no_performance
    no_performance = args.no_ok

    if socketfile_permissions_failure(args):
        unknown("Cannot access docker socket file. User ID={}, socket file={}".format(os.getuid(), args.connection))
        return

    if args.containers == ["all"] and args.present:
        unknown("You can not use --present without --containers")
        return

    if no_checks_present(args):
        unknown("No checks specified.")
        return

    # Here is where all the work happens
    #############################################################################################
    containers = get_containers(args.containers, args.present)

    if len(containers) == 0 and not args.present:
        unknown("No containers names found matching criteria")
        return

    for container in containers:

        # Check status
        if args.status:
            check_status(container, args.status)

        # Check version
        if args.version:
            check_version(container, args.insecure_registries)

        # below are checks that require a 'running' status

        # Check status
        if args.health:
            check_health(container)

        # Check cpu usage
        if args.cpu:
            check_cpu(container, parse_thresholds(args.cpu, units_required=False))

        # Check memory usage
        if args.memory:
            check_memory(container, parse_thresholds(args.memory, units_required=False))

        # Check uptime
        if args.uptime:
            check_uptime(container, parse_thresholds(args.uptime, include_units=False))

        # Check image age
        if args.image_age:
            check_image_age(container, parse_thresholds(args.image_age, include_units=False))

        # Check restart count
        if args.restarts:
            check_restarts(container, parse_thresholds(args.restarts, include_units=False))

def main():
    try:
        perform_checks(argv[1:])

        # get results to let exceptions in threads bubble out
        [x.result() for x in futures.as_completed(threads)]

    except Exception as e:
        traceback.print_exc()
        unknown("Exception raised during check': {}".format(repr(e)))
    print_results()
    exit(rc)

if __name__ == '__main__':
    main()


import gspread
from oauth2client.service_account import ServiceAccountCredentials

# python-app@my-project-202123.iam.gserviceaccount.com

# use creds to create a client to interact with the Google Drive API
scope = ['https://spreadsheets.google.com/feeds',
         'https://www.googleapis.com/auth/drive']
creds = ServiceAccountCredentials.from_json_keyfile_name('Project_278c0e241b49.json', scope)
client = gspread.authorize(creds)

# Find a workbook by name and open the first sheet
# Make sure you use the right name here.
sheet = client.open("Floating_Button_Results").sheet1

# Extract and print all of the values
# list_of_hashes = sheets[0].get_all_records()
sheet.update_cell(1,1,'qqqqqq')
print(sheet.get_all_values())


import tkinter as tk
from tkinter import filedialog
import cv2
import pandas as pd
from PIL import Image, ImageTk
import os
from tkinter import filedialog, ttk  # ttk is imported for Treeview

class VideoPlayer:
    def __init__(self, window, window_title):
        self.window = window
        self.window.title(window_title)

        # Set window to full-screen mode
        self.is_full_screen = True
        self.window.attributes('-fullscreen', self.is_full_screen)

        # Bind the ESC key to the toggle_full_screen method
        self.window.bind('<Escape>', self.toggle_full_screen)

        # Bind mouse click event on the main window
        self.window.bind("<Button-1>", self.on_window_click)

        # Main frame to hold the video and log side by side
        self.main_frame = tk.Frame(window)
        self.main_frame.pack(fill=tk.BOTH, expand=True)

        # Frame for the Treeview
        self.log_frame = tk.Frame(self.main_frame)
        self.log_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)

        # Treeview for displaying log data
        self.log_view = ttk.Treeview(self.log_frame, columns=("Index", "Event", "Frame"), show="headings")
        self.log_view.column("Index", width=40, anchor=tk.CENTER)
        self.log_view.column("Event", anchor=tk.CENTER)
        self.log_view.column("Frame", anchor=tk.CENTER)
        self.log_view.heading("Index", text="Index")
        self.log_view.heading("Event", text="Event")
        self.log_view.heading("Frame", text="Frame")
        self.log_view.pack(side=tk.LEFT, fill=tk.BOTH, expand=True)

        # Adding vertical scrollbar for the Treeview
        self.tree_scroll = tk.Scrollbar(self.log_frame, orient="vertical", command=self.log_view.yview)
        self.tree_scroll.pack(side='right', fill='y')

        # Configure the Treeview to use the scrollbar
        self.log_view.configure(yscrollcommand=self.tree_scroll.set)

        # Canvas for video playback
        self.canvas = tk.Canvas(self.main_frame, width=1600, height=900)
        self.canvas.pack(side=tk.RIGHT, fill=tk.BOTH, expand=True)

        # Frame number display
        self.frame_number = tk.Entry(window)
        self.frame_number.pack()

        # Bind Enter key in the frame number entry to jump_to_frame method
        self.frame_number.bind("<Return>", self.jump_to_frame)

        # Jump to frame button
        self.btn_jump = tk.Button(window, text="Jump to Frame", command=self.jump_to_frame)
        self.btn_jump.pack()

        # Slider for frame navigation
        self.slider = tk.Scale(window, from_=0, to=100, orient=tk.HORIZONTAL, length=1900, command=self.slider_used)
        self.slider.pack(fill=tk.X)

        # Event selection using radio buttons
        self.event_var = tk.StringVar()
        self.event_options = {
            "0: Pinprick": "0",
            "1: 0.07g": "1",
            "2: 0.4g": "2",
            "3: 2g": "3",
            "4: Cold water": "4",
            "5: Room temp": "5",
            "6: Hot water": "6"
        }
        self.radio_frame = tk.Frame(window)
        self.radio_frame.pack()
        for (text, value) in self.event_options.items():
            tk.Radiobutton(self.radio_frame, text=text, variable=self.event_var, value=value).pack(side=tk.LEFT)

        # Log button
        self.btn_log = tk.Button(window, text="Log Event", command=self.log_event)
        self.btn_log.pack()

        # Load video button
        self.btn_load = tk.Button(window, text="Load Video", command=self.load_video)
        self.btn_load.pack()

        # Frame navigation buttons
        self.btn_backward_1800 = tk.Button(window, text="Backward 1800 frames", command=lambda: self.jump_frames(-1800))
        self.btn_backward_1800.pack(side=tk.LEFT)

        self.btn_backward_900 = tk.Button(window, text="Backward 900 frames", command=lambda: self.jump_frames(-900))
        self.btn_backward_900.pack(side=tk.LEFT)

        self.btn_backward_100 = tk.Button(window, text="Backward 100 frames", command=lambda: self.jump_frames(-100))
        self.btn_backward_100.pack(side=tk.LEFT)

        self.btn_backward_50 = tk.Button(window, text="Backward 50 frames", command=lambda: self.jump_frames(-50))
        self.btn_backward_50.pack(side=tk.LEFT)

        self.btn_backward_10 = tk.Button(window, text="Backward 10 frames", command=lambda: self.jump_frames(-10))
        self.btn_backward_10.pack(side=tk.LEFT)

        self.btn_backward_1 = tk.Button(window, text="Backward 1 frames", command=lambda: self.jump_frames(-1))
        self.btn_backward_1.pack(side=tk.LEFT)

        self.btn_forward_1 = tk.Button(window, text="Forward 1 frames", command=lambda: self.jump_frames(1))
        self.btn_forward_1.pack(side=tk.LEFT)

        self.btn_forward_10 = tk.Button(window, text="Forward 10 frames", command=lambda: self.jump_frames(10))
        self.btn_forward_10.pack(side=tk.LEFT)

        self.btn_forward_50 = tk.Button(window, text="Forward 50 frames", command=lambda: self.jump_frames(50))
        self.btn_forward_50.pack(side=tk.LEFT)

        self.btn_forward_100 = tk.Button(window, text="Forward 100 frames", command=lambda: self.jump_frames(100))
        self.btn_forward_100.pack(side=tk.LEFT)

        self.btn_forward_900 = tk.Button(window, text="Forward 900 frames", command=lambda: self.jump_frames(900))
        self.btn_forward_900.pack(side=tk.LEFT)

        self.btn_forward_1800 = tk.Button(window, text="Forward 1800 frames", command=lambda: self.jump_frames(1800))
        self.btn_forward_1800.pack(side=tk.LEFT)

        self.video_source = None
        self.vid = None
        self.log_data = []

    def load_video(self):
        self.video_source = filedialog.askopenfilename()
        if self.video_source:
            self.vid = cv2.VideoCapture(self.video_source)
            total_frames = int(self.vid.get(cv2.CAP_PROP_FRAME_COUNT))
            self.slider.configure(to=total_frames - 1)
            self.update()

    def slider_used(self, event):
        if self.vid:
            frame_no = int(self.slider.get())
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.update_frame_number(frame_no)
            self.update()

    def jump_frames(self, frame_count):
        if self.vid:
            frame_no = int(self.frame_number.get() or 0) + frame_count
            frame_no = max(0, min(frame_no, self.vid.get(cv2.CAP_PROP_FRAME_COUNT) - 1))
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.slider.set(frame_no)
            self.update_frame_number(frame_no)
            self.update()

    def log_event(self):
        if self.vid and self.event_var.get():
            current_frame = int(self.vid.get(cv2.CAP_PROP_POS_FRAMES))
            event_text = [key for key, value in self.event_options.items() if value == self.event_var.get()][0]
            event_index = len(self.log_data) + 1  # Index for the new event
            self.log_data.append([event_index, event_text, current_frame])
            print(f"Logged Event: {event_text}, Frame: {current_frame}")

            # Insert log data into Treeview with index
            new_entry = self.log_view.insert('', 'end', values=(event_index, event_text, current_frame))

            # Ensure the new entry is visible
            self.log_view.see(new_entry)

            df = pd.DataFrame(self.log_data, columns=['Index', 'Event', 'Frame'])
            df.to_excel(os.path.splitext(self.video_source)[0] + "_log.xlsx", index=False)

    def update_frame_number(self, frame_no):
        self.frame_number.delete(0, tk.END)
        self.frame_number.insert(0, str(frame_no))

    def update(self):
        if self.vid:
            ret, frame = self.vid.read()
            if ret:
                frame = self.resize_frame(frame, 1600, 900)
                self.photo = ImageTk.PhotoImage(image=Image.fromarray(frame))
                self.canvas.create_image(0, 0, image=self.photo, anchor=tk.NW)

    def resize_frame(self, frame, width, height):
        (h, w) = frame.shape[:2]
        r = width / float(w)
        dim = (width, int(h * r))
        resized = cv2.resize(frame, dim, interpolation=cv2.INTER_AREA)
        return cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)

    def __del__(self):
        if self.vid:
            self.vid.release()

    def toggle_full_screen(self, event=None):
        self.is_full_screen = not self.is_full_screen
        self.window.attributes('-fullscreen', self.is_full_screen)
        if not self.is_full_screen:
            self.window.state('zoomed')  # This maximizes the window

    def on_window_click(self, event):
        """Handle mouse click events on the window to remove focus from Entry widget."""
        # Check if the click is outside the Entry widget
        if event.widget != self.frame_number:
            self.window.focus_set()

    def jump_to_frame(self, event=None):
        # event parameter added to handle the Enter key event
        if self.vid:
            frame_no = int(self.frame_number.get())
            frame_no = max(0, min(frame_no, self.vid.get(cv2.CAP_PROP_FRAME_COUNT) - 1))
            self.vid.set(cv2.CAP_PROP_POS_FRAMES, frame_no)
            self.slider.set(frame_no)
            self.update()
            self.window.focus_set()

# Create a window and pass it to the video player
root = tk.Tk()
VideoPlayer(root, "Tkinter and OpenCV with Event Logger")
root.mainloop()


"""Show data labels"""
import gin

from explain.actions.utils import get_parse_filter_text

@gin.configurable
def show_labels_operation(conversation, parse_text, i, n_features_to_show=float("+inf"), **kwargs):
    """Generates text that shows labels."""
    y_values = conversation.temp_dataset.contents['y']
    intro_text = get_parse_filter_text(conversation)

    if len(y_values) == 0:
        return "There are no instances in the data that meet this description.", 0
    if len(y_values) == 1:
        label = y_values.item()
        label_text = conversation.get_class_name_from_label(label)
        return_string = f"{intro_text} the label is {label_text}."
    else:
        return_string = f"{intro_text} the labels are:

"
        for index, label in zip(list(y_values.index), y_values):
            label_text = conversation.get_class_name_from_label(label)
            return_string += f"id {index} is labeled {label_text}
"

    return return_string, 1


#!/usr/bin/env python
# -*- encoding: utf-8 -*-

'使用元类'

__author__ = 'Darren Lu'

# type()

# 动态语言和静态语言最大的不同，就是函数和类的定义，不是编译时定义的，而是运行时动态创建的。


import os
from tempfile import TemporaryFile
from xlwt import Workbook
from .sheet import Sheet

class Excel(object):
    def __init__(self):
        self.book = Workbook(encoding='utf-8')
        self.sheets = []

    def add_sheet(self, sheet):
        if not isinstance(sheet, Sheet):
            raise Exception('sheet need be instance Sheet')

        self.sheets.append(sheet)
        sheet._set_book(self.book)

    def save(self, file_name, dir=''):
        if not '.xls' in file_name:
            file_name += '.xls'

        file = dir + file_name
        self.create_dirs_if_not_existed(file)

        self.book.save(TemporaryFile())
        self.book.save(file)

    @staticmethod
    def create_dirs_if_not_existed(filename):
        if not os.path.exists(os.path.dirname(filename)):
            try:
                os.makedirs(os.path.dirname(filename))
            except OSError as exc:
                raise Exception()


import json

def read_data_json(filename):
    with open(filename, 'r') as f:
        return json.load(f)

new_data = {}
sentences = []
equ_dict = {}
for elem in read_data_json("../../data/train23k_processed.json"):
    sentence = elem['text'].strip().split(' ')
    equation = elem['target_norm_post_template'][2:]  # .strip().split(' ')
    for equ_e in equation:
        if equ_e not in equ_dict:
            equ_dict[equ_e] = 1
    sentences.append(sentence)
    for elem in sentence:
        new_data[elem] = new_data.get(elem, 0) + 1
token_list = ['PAD_token', 'UNK_token', 'END_token']
ext_list = ['PAD_token', 'END_token']
for k, v in sorted(new_data.items()):
    token_list.append(k)

for equ_k in sorted(equ_dict.keys()):
    ext_list.append(equ_k)
print("encode_len:", len(token_list), "decode_len:", len(ext_list))
print("de:", ext_list)


#!/usr/bin/python
# -*- coding: UTF-8 -*-
# 获取男生身高、财产和帅气值
height = int(input("请输入身高(厘米)："))
property = int(input("请输入财富值(万)："))
looks = int(input("请输入帅气度(平方厘米)："))
# 根据条件不同判断是否嫁
if height > 180 and property > 1000 and looks > 500:
    print("我一定要嫁给他")
elif height > 180 or property > 1000 or looks > 500:
    print("嫁吧，比上不足，比下有余。")
elif height <= 180 and property <= 1000 and looks <= 500:
    print("不嫁！")

03_条件判断语句练习5.py


import random

rock = '🪨'

paper = '📄'

scissors = '✂'

#Write your code below this line 👇
figures = [rock, paper, scissors]
user_choise = int(input("What do you choose? Type 0 for Rock, 1 for Paper or 2 for Scissors."))
if user_choise >= 3 or user_choise < 0:
    print("Incorrect input")
    exit(0)
print(f"Your choise is: \n{figures[user_choise]}")

computer_choise = random.randint(0, 2)
print(f"Computer choise is: \n{figures[computer_choise]}")

if computer_choise == 2 and user_choise == 0:
    print("You Win!")
elif computer_choise == 0 and user_choise == 2:
    print("You Loose!")
elif computer_choise > user_choise:
    print("You Loose!")
elif computer_choise < user_choise:
    print("You Win!")
elif computer_choise == user_choise:
    print("It's a Draw")


import numpy as np

class FNN:
    def __init__(self, num_inputs, num_outputs, hidden_layer_sizes):
        self.num_inputs = num_inputs
        self.num_outputs = num_outputs
        self.hidden_layer_sizes = hidden_layer_sizes

        self.W1 = np.random.randn(num_inputs, hidden_layer_sizes[0]) * 0.01
        self.b1 = np.zeros((1, hidden_layer_sizes[0]))
        self.W2 = np.random.randn(hidden_layer_sizes[0], hidden_layer_sizes[1]) * 0.01
        self.b2 = np.zeros((1, hidden_layer_sizes[1]))
        self.W3 = np.random.randn(hidden_layer_sizes[1], num_outputs) * 0.01
        self.b3 = np.zeros((1, num_outputs))

    def softmax(self, X):
        exp_scores = np.exp(X - np.max(X, axis=1, keepdims=True))
        return exp_scores / np.sum(exp_scores, axis=1, keepdims=True)

    def forward_propagation(self, X):
        Z1 = np.dot(X, self.W1) + self.b1
        A1 = np.maximum(0, Z1)
        Z2 = np.dot(A1, self.W2) + self.b2
        A2 = np.maximum(0, Z2)
        Z3 = np.dot(A2, self.W3) + self.b3
        y_pred = self.softmax(Z3)
        cache = (X, Z1, A1, Z2, A2, Z3)
        return y_pred, cache

    def categorical_cross_entropy(self, y_true, y_pred):
        num_samples = y_true.shape[0]
        loss = -np.sum(y_true * np.log(y_pred)) / num_samples
        return loss

    def backward_propagation(self, X, y_true, y_pred, cache, learning_rate):
        (X, Z1, A1, Z2, A2, Z3) = cache
        num_samples = X.shape[0]

        dZ3 = (y_pred - y_true) / num_samples
        dW3 = np.dot(A2.T, dZ3)
        db3 = np.sum(dZ3, axis=0, keepdims=True)
        dA2 = np.dot(dZ3, self.W3.T)
        dZ2 = dA2 * (Z2 > 0)
        dW2 = np.dot(A1.T, dZ2)
        db2 = np.sum(dZ2, axis=0, keepdims=True)
        dA1 = np.dot(dZ2, self.W2.T)
        dZ1 = dA1 * (Z1 > 0)
        dW1 = np.dot(X.T, dZ1)
        db1 = np.sum(dZ1, axis=0, keepdims=True)

        # Update the weights and biases
        self.W1 -= learning_rate * dW1
        self.b1 -= learning_rate * db1
        self.W2 -= learning_rate * dW2
        self.b2 -= learning_rate * db2
        self.W3 -= learning_rate * dW3
        self.b3 -= learning_rate * db3

    def train(self, X_train, y_train, num_epochs=1000, learning_rate=0.01):
        for epoch in range(num_epochs):
            # Forward propagation
            y_pred, cache = self.forward_propagation(X_train)

            # Compute the loss
        loss = self.categorical_cross_entropy(y_train, y_pred)

        # Backward propagation
        self.backward_propagation(X_train, y_train, y_pred, cache, learning_rate)

        if epoch % 100 == 0:
            print(f"Epoch {epoch} Loss: {loss:.4f}")

def predict(self, X):
        y_pred, _ = self.forward_propagation(X)
        if self.threshold is not None:
            y_pred = np.where(y_pred >= self.threshold, 1, 0)
        return y_pred


import os
import argparse
import tensorflow as tf
from datetime import datetime
from object_detection import model_lib
from object_detection import model_hparams
from object_detection.protos import pipeline_pb2

def tensorflow_shutup(verbose=False):
    """
    Make Tensorflow less verbose
    """
    try:
        # noinspection PyPackageRequirements
        if not verbose:
            #import os
            from tensorflow.compat.v1 import logging
            logging.set_verbosity(logging.ERROR)
            os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

        # Monkey patching deprecation utils to shut it up! Maybe good idea to disable this once after upgrade
        # noinspection PyUnusedLocal
        def deprecated(date, instructions, warn_once=True):
            def deprecated_wrapper(func):
                return func
            return deprecated_wrapper

        from tensorflow.python.util import deprecation
        deprecation.deprecated = deprecated

    except ImportError:
        pass

def find_latest_checkpoint(log_dir, modelId):
    from glob import glob

    log_dir = os.path.normpath(log_dir)
    runs = glob(f"{log_dir}/{modelId}*")

    if not runs:
        return None

    runs.sort()
    latest_run = runs[-1]

    checkpoint = tf.train.get_checkpoint_state(latest_run)
    return checkpoint.model_checkpoint_path if checkpoint else None

parser = argparse.ArgumentParser(description="Train model from pipeline configs")
parser.add_argument('-o', '--out_dir', required=True,
                    help="Path to output model directory where event and checkpoint files will be written.")
parser.add_argument('-p', '--pipeline_config', required=True, help="Path to pipeline config file.")
parser.add_argument('--num_train_steps', required=False, default=None, type=int,
                    help="Number of training steps (overrides pipeline).")
parser.add_argument('--max_checkpoints', required=False, default=None, type=int,
                    help="Maximum checkpoints to save. Default: no limit")
parser.add_argument('--checkpoint_steps', required=False, default=1000, type=int,
                    help="Save checkpoint at every x steps. Default: every 1000 steps")
parser.add_argument('--sample_1_of_n_eval_examples', required=False, default=1, type=int,
                    help="Will sample one of every n eval input examples, where n is provided.")
parser.add_argument('-l', '--logs', required=False, help="Path to logs directory")
parser.add_argument('-v', '--verbose', required=False, action='store_true')

args = parser.parse_args()

tensorflow_shutup(args.verbose)

timestamp = datetime.now().strftime("-%Y%m%d-%H%M%S-%f")
modelId = os.path.basename(os.path.normpath(args.out_dir))
out_dir = args.out_dir+timestamp
log_dir = args.logs if args.logs else out_dir

config_proto = tf.ConfigProto()
config_proto.gpu_options.allow_growth = True
config = tf.estimator.RunConfig(model_dir=out_dir, session_config=config_proto, log_step_count_steps=1,
                                save_checkpoints_steps=args.checkpoint_steps, keep_checkpoint_max=args.max_checkpoints)

pipeline_overrides = None
latest_checkpoint = find_latest_checkpoint(log_dir, modelId)
if latest_checkpoint:
    pipeline_overrides = pipeline_pb2.TrainEvalPipelineConfig()
    pipeline_overrides.train_config.fine_tune_checkpoint = latest_checkpoint

train_and_eval_dict = model_lib.create_estimator_and_inputs(
    run_config=config,
    pipeline_config_path=args.pipeline_config,
    config_override=pipeline_overrides,
    train_steps=args.num_train_steps,
    sample_1_of_n_eval_examples=args.sample_1_of_n_eval_examples,
    hparams=model_hparams.create_hparams(None),
    save_final_config=True)

estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
eval_input_fns = train_and_eval_dict['eval_input_fns']
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
predict_input_fn = train_and_eval_dict['predict_input_fn']
train_steps = train_and_eval_dict['train_steps']

train_spec, eval_specs = model_lib.create_train_and_eval_specs(
        train_input_fn,
        eval_input_fns,
        eval_on_train_input_fn,
        predict_input_fn,
        train_steps,
        eval_on_train_data=False)

tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])

# Log that training completed
with open(os.path.join(log_dir, "completed.log"), 'a') as f:
    f.write(f"{modelId},{timestamp}\n")


import FWCore.ParameterSet.Config as cms

from PhysicsTools.PatAlgos.tools.metTools import *

def applyMETParameters(process, isMC):
    process.patMETs.addGenMET = cms.bool(False)
    addTcMET(process, 'TC')
    addPfMET(process, 'PF')
    process.patMETsPF.metSource = cms.InputTag("pfMet")

    process.load("JetMETCorrections.Type1MET.pfMETCorrections_cff")
    process.load("JetMETCorrections.Type1MET.pfMETCorrectionType0_cfi")

    process.pfJetMETcorr.offsetCorrLabel = cms.string("ak5PFL1Fastjet")
    if isMC:
        process.pfJetMETcorr.jetCorrLabel = cms.string("ak5PFL1FastL2L3")
    else:
        process.pfJetMETcorr.jetCorrLabel = cms.string("ak5PFL1FastL2L3Residual")

    process.pfType1CorrectedMet.applyType0Corrections = cms.bool(False)
    process.pfType1CorrectedMet.srcType1Corrections = cms.VInputTag(
        cms.InputTag('pfMETcorrType0'),
        cms.InputTag('pfJetMETcorr', 'type1')
    )

    return


import numpy as np

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.font_manager as fm
from scipy.special import comb

matplotlib.rcParams['mathtext.fontset'] = 'stix'
matplotlib.rcParams['font.family'] = 'STIXGeneral'
plt.rcParams["figure.figsize"] = [5,3]

# Import Files
init_state = "UpZero"
Larray = np.arange(4, 11, 2)
T = 1
J = [0.0001, 0.001, 0.01, 0.1, 1.0]
V = 3
hz = 16
alpha = [0.50, 3.00]
kick = 0.00 #0.01
num_J = len(J)
num_alpha = len(alpha)
n_points = num_J * num_alpha
numL = len(Larray)
iter2 = 20480
steps = 10000

m = 0

filename = np.empty(( numL, num_J, num_alpha), dtype=object)
files = np.empty(( numL, num_J, num_alpha), dtype=object)
data = np.empty(( numL, num_J, num_alpha), dtype=object)

sigmaz = np.empty(( numL, num_J, num_alpha), dtype=object)
Z = np.empty(( numL, num_J, num_alpha), dtype=object)

for i in range(num_J):
    for j in range(num_alpha):
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])
            n_iter = iter2 * 2 ** (1 - L / 2)
            #dim_Sz0 = comb(L, L / 2)

#            filename[q, i, j] = 'data/eigen/quasienergies_Swap_LR_Sz0_nspin%d_period%.2f_n_disorder%d_Jxy%.5f_Vzz%.2f_hz%.2f_alpha%.3f_kick%.2f.txt' % (
#                L, T, n_iter, J[i], V, hz, alpha[j], kick)
            filename[ q, i, j] = "data/dynamics/sigmaz_Swap_LR_%s_nspin%d_steps%d_period%.2f_n_disorder%d_J%.5f_V%.2f_hz%.2f_kick%.3f_alpha%.2f.txt" % (
                init_state, L, steps, T, n_iter, J[i], V, hz, kick, alpha[j])

            print(filename[ q, i, j])
            files[ q, i, j] = np.genfromtxt(filename[ q, i, j], skip_header=8)
            print(files[ q, i, j])
            data[ q, i, j] = files[ q, i, j]
            print(data[ q, i, j][0, :])

for i in range(num_J):
    for j in range(num_alpha):
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])
            n_iter = iter2 * 2 ** (1 - L / 2)
            #dim_Sz0 = comb(L, L / 2)
            sigmaz[ q, i, j] = data[ q, i, j][:, 1:L+1]
            Z[ q, i, j] = (np.sign(sigmaz[ q, i,j][0,0::2] - sigmaz[q,i,j][0,1::2]) * (sigmaz[q,i,j][:,0::2] - sigmaz[q,i,j][:,1::2])).sum( axis=1 ) / L
            print(Z[ q,i,j][0:4],"\n")

X = np.empty((numL, num_J, num_alpha), dtype=object)
Y = np.empty((numL, num_J, num_alpha), dtype=object)
signs = np.empty((steps,))
signs[::2] = +1
signs[1::2] = -1

plt.rcParams["axes.prop_cycle"] = plt.cycler("color", plt.cm.rainbow(np.linspace(0,1,numL)))

import matplotlib.cm as cm
from matplotlib.colors import to_hex

cmL = cm.get_cmap('turbo', numL)

for i in range(num_J):
    for j in range(num_alpha):
        m += 1
        plt.figure(m)
        for q in range(numL):
            L = Larray[q]
            print([J[i], alpha[j], L])

            X[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)-1] - 0.5) + 0.5
            Y[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)] - 0.5) + 0.5
            print(sigmaz[q,i,j][0,:])
            print(sigmaz[q,i,j][0,2*(L//4)], sigmaz[q,i,j][0,L-1])
            string = '$L = %d$' % L
            plt.plot(range(1,steps+1), X[q,i,j], label=string, color=to_hex(cmL(q)))
            plt.plot(range(1,steps+1), Y[q,i,j], label=string, linewidth=0.8, linestyle='--', color=to_hex(cmL(q)))

        plt.xlabel('$t$', fontsize=12)
        plt.ylabel('$(-1)^t(\langle\overline{S_k^z}(t)\\rangle-0.5)+0.5$', fontsize=12)
        plt.xticks(fontsize=10)
        plt.yticks(fontsize=10)
        plt.xscale('log')
        plt.ylim([-0.1,1.1])
        plt.legend(loc='lower center', bbox_to_anchor=(0.5, -0.45), fancybox=True, shadow=True, ncol=numL)

        txt = 'figures/sigmaz_avg_Dynamics_%s_Comparison_wrt_L_J%.5f_alpha%.2f_kick%.3f.pdf' % (init_state, J[i], alpha[j], kick)
        print(txt)
        plt.savefig(txt, dpi=600, bbox_inches='tight')
        print("\n")

#end = 20
#for i in range(num_J):
#    for j in range(num_alpha):
#        m += 1
#        plt.figure(m)
#        for q in range(numL):
#            L = Larray[q]
#            print([J[i], alpha[j], L])
#
#            #X[q, i, j] = signs * (sigmaz[q,i,j][:,2*(L//4)] - 0.5) + 0.5
#            #Y[q, i, j] = signs * (sigmaz[q,i,j][:,L-1] - 0.5) + 0.5
#            X[q, i, j] = sigmaz[q,i,j][:,2*(L//4)]
#            Y[q, i, j] = sigmaz[q,i,j][:,L-1]
#            print(sigmaz[q,i,j][0,:])
#            print(sigmaz[q,i,j][0,2*(L//4)], sigmaz[q,i,j][0,L-1])
#            string = '$L = %d$' % L
#            plt.plot(range(1,end+1), X[q,i,j][0:end], label=string, color=to_hex(cmL(q)))
#            #plt.plot(range(1,end+1), Y[q,i,j][0:end-1], label=string, linewidth=0.8, linestyle='--', color=to_hex(cmL(q)))
#
#
#        plt.xlabel('$t$', fontsize=12)
#        plt.ylabel('$(-1)^t(\overline{\sigma_k^z}(t)-0.5)+0.5$', fontsize=12)
#        plt.xticks(fontsize=10)
#        plt.yticks(fontsize=10)
#        plt.xscale('log')
#        plt.ylim([-0.1,1.1])
#        plt.legend(loc='lower center', bbox_to_anchor=(0.5, -0.45), fancybox=True, shadow=True, ncol=numL)
#
#        txt = 'figures/sigmaz_PARTIAL_avg_Dynamics_%s_Comparison_wrt_L_J%.5f_alpha%.2f_kick%.3f.pdf' % (init_state, J[i], alpha[j], kick)
#        print(txt)
#        plt.savefig(txt, dpi=600, bbox_inches='tight')
#        print("\n")


from tkinter import *

window = Tk()  # instantiate a window
window.geometry("420x420")
window.title("Bro Code first GUI program")

icon = PhotoImage(file='like.png')
window.iconphoto(True, icon)
window.config(background="#000000")

window.mainloop()  # place window on computer screen


import os
import shutil
from glob import glob

cleaning_root='../runs'

if __name__ == '__main__':
    exit(0)
    for x in list(glob(f'{cleaning_root}/*/checkpoint-*')):
        if 'mini_bert/' in x:
            continue
        print(f'removing {x}')
        shutil.rmtree(x)


#@+leo-ver=5-thin
#@+node:ekr.20221113062857.1: * @file ../unittests/commands/test_outlineCommands.py
"""
New unit tests for Leo's outline commands.

Older tests are in unittests/core/test_leoNodes.py
"""
import sys
from leo.core.leoTest2 import LeoUnitTest
from leo.core import leoGlobals as g
from leo.core.leoNodes import Position
assert g
assert sys

#@+others
#@+node:ekr.20221113062938.1: ** class TestOutlineCommands(LeoUnitTest)
class TestOutlineCommands(LeoUnitTest):
    """
    Unit tests for Leo's outline commands.
    """

    #@+others
    #@+node:ekr.20230724130924.1: *3* TestOutlineCommands.test_paste_as_template
    def test_paste_as_template(self):

        c = self.c
        p = c.p
        u = c.undoer

        #@+others  # Define test_tree function.
        #@+node:ekr.20230724130959.5: *4* function: test_tree (test_paste_as_template)
        def test_tree(pasted_flag: bool, tag: str) -> None:
            """Test the tree."""
            tag_s = f"kind: {test_kind} is_json? {int(is_json)} pasted? {int(pasted_flag)} {target_p.h}"
            try:
                # Test clone status and gnx. Set seen.
                seen = set()
                for p in c.all_positions():
                    seen.add(p.v)
                    if p.h == 'cc:child1':
                        assert p.isCloned(), f"{tag_s}: not cloned: {p.h}"
                    else:
                        assert not p.isCloned(), f"{tag_s}: is cloned: {p.h}"

                # Test bodies. A fairly weak test.
                for p in c.all_positions():
                    if p.h in ('cc', 'cc:child1', 'cc:child2'):
                        pass  # One copy will have a body, another won't.
                    else:
                        assert not p.b, f"{tag_s} unexpected body: {p.h}"

                # Test gnxs.
                for p in c.all_positions():
                    if p.h in ('cc', 'cc:child2'):
                        pass  # The pasted copies will have new gnxs.
                    else:
                        message = f"{tag_s}: p.gnx: {p.gnx} !=  {gnx_dict.get(p.h)}"
                        assert p.gnx == gnx_dict.get(p.h), message

                # Test that all and *only* the expected nodes exist.
                if test_kind == 'copy' or tag.startswith(('redo', 'paste-')):
                    for z in seen:
                        if z.h not in ('cc', 'cc:child2'):
                            assert z in vnodes, f"p.v not in vnodes: {z.gnx}, {z.h}"
                    for z in vnodes:
                        if z.h not in ('cc', 'cc:child2'):
                            assert z in seen, f"vnode not seen: {z.gnx}, {z.h}"
                else:
                    assert test_kind == 'cut' and tag.startswith('undo')
                    # All seen nodes should exist in vnodes.
                    for z in seen:
                        if z.h not in ('cc', 'cc:child2'):
                            assert z in vnodes, f"{z.h} not in vnodes"
                    # All vnodes should be seen except cc and cc:child2.
                    for z in vnodes:
                        if z.h in ('cc', 'cc:child2'):
                            assert z not in seen, f"{z.h} in seen after undo"
                        else:
                            assert z in seen, f"{z.h} not seen after undo"
            except Exception as e:
                message = f"clone_test failed! tag: {tag}: {e}"
                print(f"\n{message}\n")
                # self.dump_clone_info(c)
                self.dump_bodies(c)
                g.printObj(gnx_dict, tag='gnx_dict')
                # g.printObj(vnodes, tag='vnodes')
                # g.printObj([f"{z.gnx:30} {' '*z.level()}{z.h:10} {z.b!r}" for z in c.all_positions()], tag='bodies')
                self.fail(message)  # This throws another exception!
        #@-others

        # Every paste will invalidate positions, so search for headlines instead.
        valid_target_headlines = (
            'root', 'aa', 'aa:child1', 'bb', 'dd', 'dd:child1', 'dd:child1:child1', 'dd:child2', 'ee',
        )
        for target_headline in valid_target_headlines:
            for test_kind, is_json in (
                ('cut', True), ('cut', False), ('copy', True), ('copy', False),
            ):

                # print(f"TEST {test_kind} {target_headline}")

                # Create the tree and gnx_dict.
                self.clean_tree()
                cc = self.create_test_paste_outline()
                # Calculate vnodes and gnx_dict for test_node, before any changes.
                vnodes = list(set(list(c.all_nodes())))
                gnx_dict = {z.h: z.gnx for z in vnodes}
                self.assertEqual(0, c.checkOutline())

                # Change the body text of all the to-be-copied nodes.
                cc.b = 'cc body: changed'
                cc_child1 = cc.firstChild()
                cc_child2 = cc_child1.next()
                assert cc.h == 'cc', repr(cc)
                assert cc_child1.h == 'cc:child1', repr(cc_child1.h)
                assert cc_child2.h == 'cc:child2', repr(cc_child2.h)
                cc.b = 'cc body: changed'
                cc_child1.b = 'cc:child1 body: changed'
                cc_child2.b = 'cc:child2 body: changed'

                # Cut or copy cc.
                if test_kind == 'cut':
                    # Delete cc *without* undo.
                    c.selectPosition(cc)
                    self.copy_node(is_json)
                    back = cc.threadBack()
                    assert back
                    cc.doDelete()
                    c.selectPosition(back)
                else:
                    # *Copy*  node cc
                    c.selectPosition(cc)
                    self.copy_node(is_json)

                    # Restore the empty bodies of cc and cc:child1.
                    # Copy does not change these positions.
                    cc.b = cc_child1.b = cc_child2.b = ''

                self.assertEqual(0, c.checkOutline())

                # Pretest: select all positions in the tree.
                for p in c.all_positions():
                    c.selectPosition(p)

                # Find the target position by headline.
                target_p = g.findNodeAnywhere(c, target_headline)
                self.assertTrue(target_p, msg=target_headline)

                # Paste after the target.
                c.selectPosition(target_p)
                c.pasteAsTemplate()

                # Check the paste.
                self.assertEqual(0, c.checkOutline())
                test_tree(pasted_flag=True, tag='paste-as-template')

                # Check multiple undo/redo cycles.
                for i in range(3):
                    u.undo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=False, tag=f"undo {i}")
                    u.redo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=True, tag=f"redo {i}")
    #@+node:ekr.20230724064558.1: *3* TestOutlineCommands.test_paste_node
    def test_paste_node(self):

        c = self.c
        p = c.p
        u = c.undoer

        #@+others  # Define test_tree function.
        #@+node:ekr.20230724064558.5: *4* function: test_tree (test_paste_node)
        def test_tree(pasted_flag: bool, tag: str) -> None:
            """Test the tree"""
            seen = set()
            # All tests cut/copy cc.
            cloned_headline = None if test_kind == 'cut' else 'cc:child1'
            try:
                tag_s = f"{tag} kind: {test_kind} pasted? {int(pasted_flag)}"
                for p in c.all_positions():
                    seen.add(p.v)
                    if p.h == cloned_headline and p.gnx in gnx_dict.values():
                        assert p.isCloned(), f"{tag_s}: not cloned: {p.h}"
                        assert p.gnx in gnx_dict.get(p.h), f"{tag_s}: not in gnx_dict: {p.h}"
                    else:
                        assert not p.isCloned(), f"{tag_s}: is cloned: {p.h}"
            except Exception as e:
                message = f"clone_test failed! tag: {tag}: {e}"
                print(f"\n{message}\n")
                self.dump_clone_info(c)
                # g.printObj(gnx_dict, tag='gnx_dict')
                # g.printObj(vnodes, tag='vnodes')
                self.fail(message)
        #@-others

        self.clean_tree()
        cc = self.create_test_paste_outline()

        # All nodes except cc and its children itself are valid targets.
        valid_target_headlines = list(sorted(
            z.h for z in c.all_unique_positions() if z.h not in ('cc', 'cc:child1', 'cc:child2')
        ))
        # g.printObj(valid_target_headlines, tag='valid_target_headlines')
        for target_headline in valid_target_headlines:
            for test_kind, is_json in (
                ('cut', True), ('cut', False), ('copy', True), ('copy', False),
            ):

                # print('TEST', test_kind, target_headline)

                # Create the tree and gnx_dict.
                self.clean_tree()
                cc = self.create_test_paste_outline()
                # Calculate vnodes and gnx_dict for test_node, before any changes.
                vnodes = list(set(list(c.all_nodes())))
                gnx_dict = {z.h: z.gnx for z in vnodes}
                self.assertEqual(0, c.checkOutline())

                # Cut or copy cc.
                if test_kind == 'cut':
                    # Delete cc *without* undo.
                    c.selectPosition(cc)
                    self.copy_node(is_json)
                    back = cc.threadBack()
                    assert back
                    cc.doDelete()
                    c.selectPosition(back)
                else:
                    # *Copy*  node cc
                    c.selectPosition(cc)
                    self.copy_node(is_json)
                self.assertEqual(0, c.checkOutline())

                # Pretest: select all positions in the tree.
                for p in c.all_positions():
                    c.selectPosition(p)

                # Find the target position by headline.
                target_p = g.findNodeAnywhere(c, target_headline)
                self.assertTrue(target_p, msg=target_headline)

                # Paste after the target.
                c.selectPosition(target_p)
                c.pasteOutline()

                # Check the paste.
                self.assertEqual(0, c.checkOutline())
                test_tree(pasted_flag=True, tag='paste-node')

                # Check multiple undo/redo cycles.
                for i in range(3):
                    u.undo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=False, tag=f"undo {i}")
                    u.redo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=True, tag=f"redo {i}")
    #@+node:ekr.20230722104508.1: *3* TestOutlineCommands.test_paste_retaining_clones
    def test_paste_retaining_clones(self):

        c = self.c
        p = c.p
        u = c.undoer

        # These tests fail in Leo 6.7.4. To be corrected in Leo 6.7.5.
        # g.app.debug.extend(['test:strict'])
        # g.app.debug.extend(['test:verbose'])

        #@+others  # Define test_tree function.
        #@+node:ekr.20230723160812.1: *4* function: test_tree (test_paste_retaining_clones)
        def test_tree(pasted_flag: bool, tag: str) -> None:
            """Test the tree"""
            seen = set()
            if test_kind == 'cut':
                cloned_headlines = ('cc:child1',) if pasted_flag else ()
            else:
                cloned_headlines = ('cc:child1', 'cc') if pasted_flag else ('cc:child1',)
            try:
                tag_s = f"{tag} kind: {test_kind} pasted? {int(pasted_flag)}"
                for p in c.all_positions():
                    seen.add(p.v)
                    if p.h in cloned_headlines:
                        assert p.isCloned(), f"{tag_s}: not cloned: {p.h}"
                        assert p.b, f"{tag_s} {p.h}: unexpected empty body text: {p.b!r}"
                    else:
                        assert not p.isCloned(), f"{tag_s}: is cloned: {p.h}"
                    message = f"{tag}: p.gnx: {p.gnx} != expected {gnx_dict.get(p.h)}"
                    assert gnx_dict.get(p.h) == p.gnx, message

                # Test that all and *only* the expected nodes exist.
                if test_kind == 'copy' or tag.startswith(('redo', 'paste')):
                    for z in seen:
                        assert z in vnodes, f"p.v not in vnodes: {z.gnx}, {z.h}"
                    for z in vnodes:
                        assert z in seen, f"vnode not seen: {z.gnx}, {z.h}"
                else:
                    assert test_kind == 'cut' and tag.startswith('undo')
                    # All seen nodes should exist in vnodes.
                    for z in seen:
                        assert z in vnodes, f"{z.h} not in vnodes"
                    # All vnodes should be seen except cc and cc:child2.
                    for z in vnodes:
                        if z.h in ('cc', 'cc:child2'):
                            assert z not in seen, f"{z.h} in seen after undo"
                        else:
                            assert z in seen, f"{z.h} not seen after undo"
            except Exception as e:
                message = f"clone_test failed! tag: {tag}: {e}"
                print(f"\n{message}\n")
                self.dump_clone_info(c)
                # g.printObj(gnx_dict, tag='gnx_dict')
                # g.printObj(vnodes, tag='vnodes')
                # g.printObj([f"{z.gnx:30} {' '*z.level()}{z.h:10} {z.b!r}" for z in c.all_positions()], tag='bodies')
                self.fail(message)  # This throws another exception!
        #@-others

        # Every paste will invalidate positions, so search for headlines instead.
        valid_target_headlines = (
            'root', 'aa', 'aa:child1', 'bb', 'dd', 'dd:child1', 'dd:child1:child1', 'dd:child2', 'ee',
        )
        for target_headline in valid_target_headlines:

            # print(f"\nTarget headline: {target_headline}\n")

            for test_kind, is_json in (
                ('cut', True), ('cut', False), ('copy', True), ('copy', False),
            ):

                # print(f"TEST {test_kind} {target_headline}")

                # Create the tree and gnx_dict.
                self.clean_tree()
                cc = self.create_test_paste_outline()
                # Calculate vnodes and gnx_dict for test_node, before any changes.
                vnodes = list(set(list(c.all_nodes())))
                gnx_dict = {z.h: z.gnx for z in vnodes}
                self.assertEqual(0, c.checkOutline())

                # Change the body text of cc and cc:child1, the two cloned nodes.
                cc.b = 'cc body: changed'
                cc_child1 = cc.firstChild()
                assert cc_child1.h == 'cc:child1', repr(cc_child1.h)
                cc_child1.b = 'cc:child1 body: changed'

                # Cut or copy cc.
                if test_kind == 'cut':
                    # Delete cc *without* undo.
                    c.selectPosition(cc)
                    self.copy_node(is_json)
                    back = cc.threadBack()
                    assert back
                    cc.doDelete()
                    c.selectPosition(back)
                else:
                    # *Copy*  node cc
                    c.selectPosition(cc)
                    self.copy_node(is_json)

                self.assertEqual(0, c.checkOutline())

                # Pretest: select all positions in the tree.
                for p in c.all_positions():
                    c.selectPosition(p)

                # Find the target position by headline.
                target_p = g.findNodeAnywhere(c, target_headline)
                self.assertTrue(target_p, msg=target_headline)

                # Paste after the target.
                c.selectPosition(target_p)
                c.pasteOutlineRetainingClones()

                # Check the paste.
                self.assertEqual(0, c.checkOutline())
                test_tree(pasted_flag=True, tag='paste')

                # Check multiple undo/redo cycles.
                for i in range(3):
                    u.undo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=False, tag=f"undo {i}")
                    u.redo()
                    self.assertEqual(0, c.checkOutline())
                    test_tree(pasted_flag=True, tag=f"redo {i}")
    #@+node:ekr.20230729042305.1: *3* TestOutlineCommands.test_c_checkVnodeLinks
    def test_c_checkVnodeLinks(self):

        c = self.c

        # Create the initial tree.
        self.clean_tree()
        cc = self.create_test_paste_outline()
        self.assertEqual(cc.h, 'cc')

        # Create globals for restore_tree().
        vnodes_list = list(set(c.all_nodes()))
        children_dict = {}
        parents_dict = {}

        #@+others  # define helpers
        #@+node:ekr.20230730070124.1: *4* function: init_dicts
        def init_dicts() -> None:
            for z in vnodes_list:
                children_dict [z.gnx] = z.children[:]
            for z in vnodes_list:
                parents_dict [z.gnx] = z.parents[:]
        #@+node:ekr.20230730070250.1: *4* function: restore_tree
        def restore_tree():
            for v in vnodes_list:
                v.children = children_dict [v.gnx][:]
                v.parents = parents_dict [v.gnx][:]
        #@+node:ekr.20230729124541.1: *4* function: do_defect
        def do_defect(parent: Position, child: Position, defect: str) -> None:
            """
            Create the defect if possible. Return True if the defect was created.
            """
            if defect == 'parents:insert':
                child.v.parents.append(parent.v)
            elif defect == 'parents:delete':
                child.v.parents.remove(parent.v)
            elif defect == 'parents:delete-all':
                while parent.v in child.v.parents:
                    child.v.parents.remove(parent.v)
            elif defect == 'children:insert':
                parent.v.children.append(child.v)
            elif defect == 'children:delete':
                parent.v.children.remove(child.v)
            elif defect == 'children:delete-all':
                while child.v in parent.v.children:
                    parent.v.children.remove(child.v)
            else:
                assert False, defect
        #@+node:ekr.20230729124819.1: *4* function: enable_options
        def enable_options(parent: Position, options: list[tuple[str, str]]) -> None:
            """
            Enable options in g.app.debug for given list of option descriptors.

            Each descriptor has the form (selector, option).

            selector: 'all' or a headline.
            option: 's' for strict, 'v' for verbose.
            """
            g.app.debug = []
            for descriptor  in options:
                selector, option = descriptor
                if selector in ('all', parent.h):
                    if 's' in option and 'test:strict' not in g.app.debug:
                        g.app.debug.append('test:strict')
                    if 'v' in option and 'test:verbose' not in g.app.debug:
                        g.app.debug.append('test:verbose')

        #@+node:ekr.20230729124441.1: *4* function: test (test_c_checkVnodeLinks)
        def test(parent: Position, child: Position) -> int:
            """
            Run all tests on all positions with the given headline with all possible defects.

            Return the number of tests run.
            """
            # Create defects and run tests.
            enable_options(parent, options)
            n = 0
            for defect in defects:
                tag_s = f"defect: {defect} parent: {parent.h}, child: {child.h}"
                n += 1
                restore_tree()
                self.assertEqual(0, c.checkOutline(), msg=f"Before: {tag_s}")
                do_defect(parent, child, defect)
                self.assertEqual(0, c.checkOutline(), msg=f" After: {tag_s}")
            return n
        #@-others

        init_dicts()

        # Set options for enable_options().
        # Options: a tuple (selector, option):
        #          Selector: headline or 'all'
        #          Option: 's' for strict, 'v' for verbose or 'sv' for both.
        options: tuple[tuple[str, str]] = (
            # ('all', 'v'),
        )

        # The list of all possible defects. See do_defect.
        defects = [
            'parents:insert', 'parents:delete', 'parents:delete-all',
            'children:insert', 'children:delete', 'children:delete-all']

        # Test all defects on all positions.
        n_tests, n_positions = 0, 0
        for parent in c.all_positions():
            n_positions += 1
            for child in parent.children():
                n_positions += 1
                n_tests += test(parent, child)
        # g.trace('Done', n_tests, 'tests', n_positions, 'positions')
    #@+node:ekr.20230722083123.1: *3* TestOutlineCommands.test_restoreFromCopiedTree
    def test_restoreFromCopiedTree(self):

        c = self.c
        u = c.undoer

        #@+others  # Define helper functions.
        #@+node:ekr.20230724210028.1: *4* function: test_tree (test_restoreFromCopiedTree)
        def test_tree(tag: str) -> None:
            """Test the tree."""
            assert tag[0].isnumeric()
            try:
                for p in c.all_positions():
                    if p.h == 'cc:child1' and not tag.startswith('4'):
                        assert p.isCloned(), f"{p.h} is not cloned"
                    else:
                        assert not p.isCloned(), f"{p.h} is cloned"
                    assert p.gnx in gnx_dict.get(p.h), f"{p.h} not in gnx_dict"
            except Exception as e:
                message = f"Fail! tag: {tag}: {e}"
                print(f"\n{message}")
                self.dump_clone_info(c)
                # g.printObj(gnx_dict, tag='gnx_dict')
                # g.printObj(vnodes, tag='vnodes')
                self.fail(message)
        #@-others

        # Create the tree.
        self.clean_tree()
        cc = self.create_test_paste_outline()

        # No need to select cc. We use only vnodes.
        # c.selectPosition(cc)

        # Create the gnx_dict.
        vnodes = list(set(list(c.all_nodes())))
        gnx_dict = {z.h: z.gnx for z in vnodes}

        # s1: before inserting cc:child3.
        s1 = c.fileCommands.outline_to_clipboard_string(cc)
        assert s1

        # Insert cc:child3.
        cc_child3 = cc.insertAsLastChild()
        cc_child3.h = 'cc:child3'
        gnx_dict[cc_child3.h] = cc_child3.gnx

        self.assertEqual(0, c.checkOutline())
        test_tree(tag='1: before inserting cc:child3')

        s2 = c.fileCommands.outline_to_clipboard_string(cc)
        assert s2

        self.assertEqual(0, c.checkOutline())
        test_tree(tag='2: after inserting cc:child3')

         # Get back to the starting point.
        for (v, s, tag) in (
            (cc.v, s2, '2: undo'),
        ):
            u.restoreFromCopiedTree(v, s)
            self.assertEqual(0, c.checkOutline())
            test_tree(tag=tag)

        # Check multiple do/redo cycles.
        for i in range(3):
            for (v, s, tag) in (
                (cc.v, s1, f"1: redo{i}"),
                (cc.v, s2, f"2: undo{i}"),
            ):
                u.restoreFromCopiedTree(v, s)
                self.assertEqual(0, c.checkOutline())
                test_tree(tag=tag)
    #@+node:ekr.20221112051634.1: *3* TestOutlineCommands.test_sort_children
    def test_sort_children(self):
        c, u = self.c, self.c.undoer
        assert self.root_p.h == 'root'
        self.create_test_sort_outline()
        original_children = [z.h for z in self.root_p.v.children]
        sorted_children = sorted(original_children)
        c.sortChildren()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, sorted_children)
        u.undo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, original_children)
        u.redo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, sorted_children)
        u.undo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, original_children)
    #@+node:ekr.20221112051650.1: *3* TestOutlineCommands.test_sort_siblings
    def test_sort_siblings(self):
        c, u = self.c, self.c.undoer
        assert self.root_p.h == 'root'
        self.create_test_sort_outline()
        original_children = [z.h for z in self.root_p.v.children]
        sorted_children = sorted(original_children)
        c.selectPosition(self.root_p.firstChild())
        c.sortSiblings()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, sorted_children)
        u.undo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, original_children)
        u.redo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, sorted_children)
        u.undo()
        result_children = [z.h for z in self.root_p.v.children]
        self.assertEqual(result_children, original_children)
    #@+node:ekr.20230902053728.1: *3* TestOutlineCommands.test_move_outline_to_first_child
    def test_move_outline_to_first_child(self):
        # Setup.
        c, u = self.c, self.c.undoer
        root = self.root_p
        assert root.h == 'root'
        self.create_test_outline()
        original_children = root.v.children[:]
        last_child = root.lastChild()
        assert last_child
        last_gnx = last_child.gnx
        # Move.
        c.selectPosition(last_child)
        c.moveOutlineToFirstChild()
        # Tests.
        assert c.checkOutline() == 0
        first_child = root.firstChild()
        assert first_child
        first_gnx = first_child.gnx
        assert first_gnx == last_gnx, (first_gnx, last_gnx)
        u.undo()
        assert c.checkOutline() == 0
        assert root.v.children == original_children
        u.redo()
        assert c.checkOutline() == 0
        u.undo()
        assert c.checkOutline() == 0
        assert root.v.children == original_children

    #@+node:ekr.20230902053751.1: *3* TestOutlineCommands.test_move_outline_to_last_child
    def test_move_outline_to_last_child(self):
        # Setup.
        c, u = self.c, self.c.undoer
        root = self.root_p
        assert root.h == 'root'
        self.create_test_outline()
        original_children = root.v.children[:]
        first_child = root.firstChild()
        assert first_child
        first_gnx = first_child.gnx
        # Move.
        # c.selectPosition(first_child)
        c.selectPosition(first_child)
        c.moveOutlineToLastChild()
        # Tests.
        assert c.checkOutline() == 0
        last_child = root.lastChild()
        assert first_child
        last_gnx = last_child.gnx
        assert first_gnx == last_gnx, (first_gnx, last_gnx)
        u.undo()
        assert c.checkOutline() == 0
        assert root.v.children == original_children
        u.redo()
        assert c.checkOutline() == 0
        u.undo()
        assert c.checkOutline() == 0
        assert root.v.children == original_children

    #@-others
#@-others
#@-leo


from os.path import join

from django.test import TestCase
from django.conf import settings
from django.db import connection
from django.contrib.auth import get_user_model

from ..forms import CommunitiesFilterForm

User = get_user_model()

class TestCommunitiesFilterForm(TestCase):

    COMMUNITIES_NUM = 200

    def setUp(self):
        super().setUp()

        communities_ids = range(self.COMMUNITIES_NUM)

        with open(join(settings.BASE_DIR, '..', 'schema.sql'), encoding='utf-8') as fd:
            sql = fd.read()
        connection.cursor().execute(sql)

        sql = r'INSERT INTO "community" ("vkid","deactivated","type","name","description","status","site","members") VALUES'
        sql += ','.join(
            "({0:d},FALSE,1,'NAME_{0:d}','DESCRIPTION_{0:d}','STATUS_{0:d}','SITE_{0:d}',{1:d})".format(cid,
                                                                                cid * 1000 + 1)
            for cid in communities_ids
        )
        sql += ';'
        connection.cursor().execute(sql)

        sql = (r'INSERT INTO "profile" ("id","sex_vkid","age_range_id","country_vkid","app_id") '
               r'VALUES (1,1,1,1,1)')
        connection.cursor().execute(sql)

        sql = r'INSERT INTO "audience" ("profile_id","community_vkid","count") VALUES'
        sql += ','.join("(1,{0:d},{1:d})".format(cid, cid * 1000 + 1) for cid in communities_ids)
        sql += ';'
        connection.cursor().execute(sql)

    def test_correct_data(self):
        data = {
            'min_members': 0,
            'max_members': 1000000,

            'min_audience': 0,
            'max_audience': 1000000,

            'min_audience_perc': 0,
            'max_audience_perc': 100,

            'sex': [1],
            'age_ranges': [1],
            'countries': [1],
            'apps': [1],

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertTrue(form.is_valid())

    def test_constrains(self):
        data = {
            'min_members': -1,
            'max_members': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience': -1,
            'max_audience': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': -1,
            'max_audience_perc': 10,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': 0,
            'max_audience_perc': 101,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

    def test_ranges(self):
        data = {
            'min_members': 10,
            'max_members': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience': 10,
            'max_audience': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())

        data = {
            'min_audience_perc': 10,
            'max_audience_perc': 9,

            'ordering': 'audience_perc',
            'inverted': True
        }
        form = CommunitiesFilterForm(data)
        self.assertFalse(form.is_valid())