import os
import csv
import torch
import argparse
import random
import json
import numpy as np
from utils.load_model import load_victim
from utils.load_data import load_data
from pathlib import Path
from utils.predict import test, adv_test, fr_test, make_print_to_file
from model.linear import NonLinearClassifier

def arg_parse():
    parser = argparse.ArgumentParser(description='Test the performance of advencoder')
    parser.add_argument('--seed', default=100, type=int, help='which seed the code runs on')
    parser.add_argument('--gpu', default='0', type=str, help='which gpu the code runs on')
    parser.add_argument('--dataset', default='stl10', choices=['cifar10', 'stl10', 'gtsrb', 'imagenet'])
    parser.add_argument('--batch_size', type=int, default=256)
    parser.add_argument('--type', default='gan_per', choices=['gan_patch', 'gan_per'])
    parser.add_argument('--pre_dataset', default='imagenet', choices=['cifar10', 'imagenet'])
    parser.add_argument('--victim', default='mocov2plus', choices=['simclr', 'byol', 'barlow_twins', 'deepclusterv2', 'dino', 'mocov3', 'mocov2plus', 'nnclr', 'ressl', 'simsiam', 'supcon', 'swav', 'vibcreg', 'vicreg', 'wmse'])
    parser.add_argument('--sup_dataset', default='cifar10')
    parser.add_argument('--criterion', default='nce', choices=['cos', 'nt', 'nce'])
    parser.add_argument('--noise_percentage', type=float, default=0.03)
    parser.add_argument('--eps', type=int, default=10)
    args = parser.parse_args()
    return args

def classify(args, encoder, F, uap):
    data = args.dataset
    # log_save_path = path
    train_loader, test_loader = load_data(data, args.batch_size)

    results = {'clean_acc_t1': [], 'adv_acc_t1': [], 'decline_t1': [], 'clean_acc_t5': [], 'adv_acc_t5': [],
               'decline_t5': [], 'attack_success_rate': []}

    F.cuda()
    encoder.cuda()
    encoder.eval()
    clean_acc_t1, clean_acc_t5 = test(args, encoder, F, test_loader, data)
    adv_acc_t1, adv_acc_t5 = adv_test(args, encoder, F, test_loader, uap, data)
    attack_success_rate = fr_test(args, encoder, F, test_loader, uap, data)
    decline_t1 = ((clean_acc_t1 - adv_acc_t1) / clean_acc_t1) * 100
    decline_t5 = ((clean_acc_t5 - adv_acc_t5) / clean_acc_t5) * 100
    results['clean_acc_t1'].append(clean_acc_t1)
    results['clean_acc_t5'].append(clean_acc_t5)
    results['decline_t1'].append(decline_t1)
    results['adv_acc_t1'].append(adv_acc_t1)
    results['adv_acc_t5'].append(adv_acc_t5)
    results['decline_t5'].append(decline_t5)
    results['attack_success_rate'].append(attack_success_rate)

    print('Top1 test acc: %.4f, Top1 Adv_test acc: %.4f, Fooling rate: %.4f'
        % (clean_acc_t1, adv_acc_t1, attack_success_rate))

    return clean_acc_t1, adv_acc_t1, decline_t1, attack_success_rate, clean_acc_t5, adv_acc_t5, decline_t5

def main():

    args = arg_parse()
    # Set the seed and determine the GPU
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    random.seed(args.seed)
    os.environ['PYTHONHASHSEED'] = str(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    torch.cuda.manual_seed(args.seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True
    torch.set_printoptions(profile="full")
    torch.cuda.synchronize()

    args.eps = args.eps / 255

    # Logging
    log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'down_test', str(args.type), str(args.victim), str(args.sup_dataset), str(args.dataset))
    if not os.path.exists(log_save_path):
        os.makedirs(log_save_path)

    now_time = make_print_to_file(path=log_save_path)

    if not os.path.exists(log_save_path):
        os.makedirs(log_save_path)

    # Dump args
    with open(log_save_path + '/args.json', 'w') as fid:
        json.dump(args.__dict__, fid, indent=2)

    # load encoder
    model = load_victim(args)

    # downstream task
    if args.dataset == 'imagenet':
        num_classes = 100
    elif args.dataset == 'gtsrb':
        num_classes = 43
    else:
        num_classes = 10

    classifier = torch.nn.DataParallel(NonLinearClassifier(feat_dim=512, num_classes=num_classes))

    # load classifier
    classifier_path = os.path.join('victims', str(args.pre_dataset), str(args.victim), 'clean_model', str(args.dataset))
    if args.dataset == 'imagenet':
        encoder_path = [Path(classifier_path) / ckpt for ckpt in os.listdir(Path(classifier_path)) if ckpt.endswith("50.pth")][0]
    else:
        encoder_path = [Path(classifier_path) / ckpt for ckpt in os.listdir(Path(classifier_path)) if ckpt.endswith("20.pth")][0]

    checkpoint = torch.load(encoder_path)
    classifier.load_state_dict(checkpoint)

    print('Day: %s, Target encoder:%s, Attack type: %s, Downstream task:%s'% (now_time, args.victim, args.type, args.dataset))
    print("######################################  Test Attack! ######################################")

    if args.type == 'gan_per':
        uap_load_path = os.path.join('output', str(args.pre_dataset), 'uap_results', str(args.type), str(args.victim),
                                     str(args.sup_dataset), str(args.criterion), str(args.eps))
        uap_path = [Path(uap_load_path) / ckpt for ckpt in os.listdir(Path(uap_load_path)) if ckpt.endswith("20.pt")][0]

    elif args.type == 'gan_pat':
        uap_load_path = os.path.join('output', str(args.pre_dataset), 'uap_results', str(args.type), str(args.victim),
                                     str(args.sup_dataset), str(args.criterion), str(args.noise_percentage))
        uap_path = [Path(uap_load_path) / ckpt for ckpt in os.listdir(Path(uap_load_path)) if ckpt.endswith("20.pt")][0]

    UAP = torch.load(uap_path)

    clean_acc_t1, adv_acc_t1, decline_t1, attack_success_rate, clean_acc_t5, adv_acc_t5, decline_t5 = classify(args, model, classifier, UAP)

    print('Clean downstream accuracy: %.4f%%'% (clean_acc_t1))
    print('Adv downstream accuracy: %.4f%%'% (adv_acc_t1))
    print('Decline accuracy rate: %.4f%%' % (decline_t1))
    print('Downstream fooling rate: %.4f%%' % (attack_success_rate))

    final_log_save_path = os.path.join('output', str(args.pre_dataset), 'log', 'down_test', str(args.type))
    if not os.path.exists(final_log_save_path):
        os.makedirs(final_log_save_path)

    final_result = []

    if args.type == 'gan_per':
        final_result_ = {"now_time": now_time,
                         "attack_type": args.type,
                         "victim": str(args.victim),
                         "pre_dataset": str(args.pre_dataset),
                         "sup_dataset": str(args.sup_dataset),
                         "down_dataset": str(args.dataset),
                         "criterion": str(args.criterion),
                         "eps": str(round(args.eps, 4)),
                         "clean_acc_t1": round(clean_acc_t1, 4),
                         "clean_acc_t5": round(clean_acc_t5, 4),
                         "decline_t1": round(decline_t1, 4),
                         "adv_acc_t1": round(adv_acc_t1, 4),
                         "adv_acc_t5": round(adv_acc_t5, 4),
                         "decline_t5": round(decline_t5, 4),
                         "attack_success_rate": round(attack_success_rate, 4)}
        final_result.append(final_result_)
        header = ["now_time", "attack_type", "victim", "pre_dataset", "sup_dataset", "down_dataset", "criterion", "eps", "clean_acc_t1", "clean_acc_t5", "decline_t1",
                  "adv_acc_t1", "adv_acc_t5", "decline_t5", "attack_success_rate"]

    elif args.type == 'gan_patch':
        final_result_ = {"now_time": now_time,
                         "attack_type": args.type,
                         "victim": str(args.victim),
                         "pre_dataset": str(args.pre_dataset),
                         "sup_dataset": str(args.sup_dataset),
                         "down_dataset": str(args.dataset),
                         "criterion": str(args.criterion),
                         "noise_percentage": str(round(args.noise_percentage, 4)),
                         "clean_acc_t1": round(clean_acc_t1, 4),
                         "clean_acc_t5": round(clean_acc_t5, 4),
                         "decline_t1": round(decline_t1, 4),
                         "adv_acc_t1": round(adv_acc_t1, 4),
                         "adv_acc_t5": round(adv_acc_t5, 4),
                         "decline_t5": round(decline_t5, 4),
                         "attack_success_rate": round(attack_success_rate, 4)}
        final_result.append(final_result_)
        header = ["now_time", "attack_type", "victim", "pre_dataset", "sup_dataset", "down_dataset", "criterion", "noise_percentage", "clean_acc_t1","clean_acc_t5",
                  "decline_t1", "adv_acc_t1", "adv_acc_t5", "decline_t5", "attack_success_rate"]

    with open(final_log_save_path + '/all_final_results.csv', 'a', newline='') as f:
        writer = csv.DictWriter(f, fieldnames=header)
        writer.writeheader()
        writer.writerows(final_result)

if __name__ == "__main__":
    main()


from modules.base import Module, procedure

import docker

class InfrastructureModule(Module):
    name = 'infrastructure'

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.client = docker.from_env()

    def __post_init__(self, module_name):
        if module_name == self.name:
            for container in self.client.containers.list(
                filters=dict(
                    label='sales_channel_id',
                ),
            ):
                container.stop()
                container.remove()

    @procedure
    def spin_up_online_shop(self, sales_channel_id):
        images = self.client.images.list(
            filters=dict(label='name=commerce_engine_online_shop'),
        )

        if images:
            image = images[0]
        else:
            image, _ = self.client.images.build(
                path='/mnt',
                dockerfile='Dockerfile',
                labels=dict(name='commerce_engine_online_shop'),
            )

        containers = self.client.containers.list(
            filters=dict(
                label='sales_channel_id=' + str(sales_channel_id),
            )
        )

        if containers:
            for container in containers:
                container.stop()
                container.remove()

        infrastructure_host, infrastructure_port = self.address.split(':')
        infrastructure_port = int(infrastructure_port)

        port = 1 + infrastructure_port + len(self.client.containers.list())

        self.client.containers.run(
            image=image.attrs['Id'],
            command=['python3', 'online_shop.py'],
            environment={
                'DEBUG': 'true' if self.debug else 'false',
                'MASTER_MODULE_NAME': 'commerce_engine',
                'MODULE_NAME': 'online_shop',
                'COMMERCE_ENGINE_MODULE_ADDRESS': 'commerce_engine:5000',
                'WAREHOUSE_MODULE_ADDRESS': 'warehouse:5000',
                'FULFILLMENT_CENTER_MODULE_ADDRESS': 'fulfillment_center:5000',
                'SUPPLIER_MODULE_ADDRESS': 'supplier:5000',
                'SALES_CHANNEL_MODULE_ADDRESS': 'sales_channel:5000',
                'CART_MODULE_ADDRESS': 'cart:5000',
                'CUSTOMER_MODULE_ADDRESS': 'customer:5000',
                'PRODUCT_MODULE_ADDRESS': 'product:5000',
                'MANAGEMENT_MODULE_ADDRESS': 'management:5000',
                'INFRASTRUCTURE_MODULE_ADDRESS': 'infrastructure:' + str(infrastructure_port),
                'ONLINE_SHOP_MODULE_ADDRESS': '0.0.0.0:' + str(port),
                'SALES_CHANNEL_ID': str(sales_channel_id),
            },
            ports={
                str(port) + '/tcp': port,
            },
            network='commerce_engine_default',
            working_dir='/mnt',
            detach=True,
            labels=dict(
                sales_channel_id=str(sales_channel_id),
            ),
        )


from __future__ import print_function

import inspect
import json
import os
import os.path
import pickle
import re
import subprocess
import traceback as tb
from collections import OrderedDict
from pickle import UnpicklingError
from typing import Callable

import yaml
from flask import Flask, jsonify, redirect, request, render_template, flash, session
from flask import url_for
from flask_cors import CORS
from google.auth.transport.requests import Request
from google_auth_oauthlib.flow import Flow
from googleapiclient.discovery import build

class AuthenticationException(Exception):
    pass

app = Flask(__name__, template_folder='templates')
# app.wsgi_app = ReverseProxied(app.wsgi_app)
CORS(app)

app.config["creds"] = None
app.config["pickle"] = os.path.dirname(__file__) + "/configuration/token.pickle"
app.config["state"] = None
app.config["redirect"] = "/"

app.secret_key = os.getenv("SECRET")
viewIDs = []

# Items here will not be redirected to authenticate
oauth_whitelist = [
    r"^(?:\/api)?\/$",
    r"^(?:\/api)?\/oauth\/callback\/?$",
    r"^(?:\/api)?\/oauth\/authorize\/?$",
]

scopes = [
    "https://www.googleapis.com/auth/analytics.readonly"
]

discovery_urls = {
    ("analytics", "v4"): "https://analyticsreporting.googleapis.com/$discovery/rest"
}

progression = [
    "about", "primo", "google_cloud", "analytics", "confirm_and_complete"
]

class SetupError(Exception):
    pass

@app.before_first_request
def validate_environment():
    pass
    # if "API_URL" not in os.environ:
    #     os.environ["API_URL"] = "https://localhost:5000"

@app.route("/")
def index():
    return render_template("about.html", progression=get_progression("about"))

@app.route("/setup/about")
def about():
    return redirect(url_for("index"))

@app.route("/setup/primo", methods=["GET", "POST"])
def setup_primo():
    if request.method == "POST":
        fields = ["primo_url", "primo_key"]
        data = request.form
        state = session.setdefault("primo", {})

        """Validate data"""
        valid = True
        for field in fields:
            if not data.get(field, False):
                flash("{} cannot be empty".format(field.replace("_", " ").capitalize()))
                valid = False

        if valid:
            for field in fields:
                state[field] = data.get(field)
            if not state["primo_url"].startswith("https://"):
                state["primo_url"] = "https://" + state["primo_url"]
            if not state["primo_url"].endswith("/pnxs"):
                state["primo_url"] = state["primo_url"] + "/pnxs"
            return redirect(url_for('setup_google_cloud'))
    return render_template('primo.html', progression=get_progression("primo"))

@app.route("/setup/google_cloud", methods=["GET", "POST"])
def setup_google_cloud():
    print(session)
    if request.method == "POST":
        fields = ["client_id", "client_secret"]
        data = request.form
        state = session.setdefault("google_cloud", {})

        """Validate data"""
        valid = True
        for field in fields:
            if not data.get(field, False):
                flash("{} cannot be empty".format(field.replace("_", " ").capitalize()))
                valid = False

        if valid:
            state.update({
                "auth_uri": "https://accounts.google.com/o/oauth2/auth",
                "token_uri": "https://oauth2.googleapis.com/token",
                "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
            })
            for field in fields:
                state[field] = data.get(field)

            if not state["client_id"].endswith(".apps.googleusercontent.com"):
                state["client_id"] = state["client_id"] + ".apps.googleusercontent.com"

            session["google_cloud"] = state
            return redirect(url_for('setup_analytics'))
    return render_template('google_cloud.html', progression=get_progression("google_cloud"))

@app.route("/setup/analytics", methods=["GET", "POST"])
def setup_analytics():
    print(session)
    state = session.setdefault("analytics", {})
    if "token" in state:
        if request.method == "POST":
            data = request.form
            if "view_id" in data:
                state.update({"view_id": data.get("view_id")})
                session["analytics"] = state
                return redirect(url_for("confirm_and_complete"))
            flash("No idea how you managed to not pick any")
        if "available_views" not in session:
            auth = build("analytics", "v3", credentials=pickle.loads(state["token"]))
            session["available_views"] = get_views(auth)
        views = session["available_views"]
        return render_template('analytics.html', progression=get_progression("analytics"), views=list(enumerate(views)))
    else:
        print(request.method)
        print()
        if request.method == "POST":
            return redirect(url_for("oauth_authorize", return_to=url_for("setup_analytics")))

    return render_template('analytics.html', progression=get_progression("analytics"))

@app.route("/setup/confirm_and_complete", methods=["GET", "POST"])
def confirm_and_complete():
    if request.method == "POST":
        output_token_pickle()
        output_config_yaml()
        session["confirm_and_complete"] = True
        flash("Your configuration has been saved, run docker-compose restart to put changes in effect")
        return redirect(url_for("done"))

    sensitive_fields = ["token", "primo_key"]
    state = {
        key: list(
            (skey, str(value) if skey not in sensitive_fields else "*" * min(len(value), 50))
            for skey, value in session[key].items())
        for key in ["primo", "analytics"]
    }

    return render_template("complete.html", state=state, progression=get_progression("confirm_and_complete"))

@app.route("/setup/success")
def done():
    return render_template("successful.html", progression=get_progression("confirm_and_complete"))

def output_config_yaml():
    target_directory = os.getenv("CONFIG_DIR", "./configuration")
    config_file = os.path.join(target_directory, "config.yaml.docker")
    with open(config_file, "w") as config_f:
        yaml.safe_dump({
            "primo": {
                "host": session["primo"]["primo_url"],
                "api_key": session["primo"]["primo_key"],
            },
            "analytics": {
                "view_id": session["analytics"]["view_id"],
                "path_to_credentials": "/config/analytics.pickle"
            }
        }, config_f)

def output_token_pickle():
    target_directory = os.getenv("CONFIG_DIR", "./configuration")
    token_file = os.path.join(target_directory, "analytics.pickle")
    with open(token_file, "wb") as token_f:
        token_f.write(session["analytics"]["token"])

def get_progression(current_step: str):
    return OrderedDict(
        (progression[i], {
            "class": "active" if current_step == progression[i] else "",
            "url": "/setup/{}".format(progression[i]),
            "next": progression[i] if i < len(progression) else None,
            "completed": bool(session.get(progression[i]))
        })
        for i in range(len(progression))
        if i <= progression.index(current_step) or i > 0 and session.get(progression[i - 1], False) or session.get(
            progression[i])

    )

def update_config(new: dict):
    with open("./configuration/credentials.json", "r") as fin:
        data = json.load(fin)
    update_recursive(data, new)
    with open("./configuration/credentials.json", "w") as fout:
        json.dump(data, fout)

def update_recursive(dest: dict, values: dict):
    for key, value in values.items():
        if isinstance(value, dict):
            if key in dest and isinstance(dest[key], dict):
                update_recursive(dest[key], values[key])
                continue
        if not value:
            continue
        dest[key] = value

@app.route("/auth", methods=["POST"])
def oauth_check():
    if all(re.match(path, request.path) is None for path in oauth_whitelist):
        if not app.config["creds"] or not app.config["creds"].valid:
            # Retrieve data from POST request with form's name
            client_secret = request.form['client_secret']
            client_id = request.form['client_id']
            primo_url = request.form['primo_url']
            primo_key = request.form['primo_key']

            with open("./configuration/credentials.json", "r") as jsonFile:
                data = json.load(jsonFile)

            tmp = data["web"]
            data["web"]["client_secret"] = client_secret
            data["web"]["client_id"] = client_id
            with open("./configuration/credentials.json", "w") as jsonFile:
                json.dump(data, jsonFile)
            # New dict from the Form for necessary configuration
            new_yaml_data_dict = {
                'primo': {
                    'host': primo_url,
                    'api_key': primo_key,
                    'contexts': ["L", "PC"],
                    'common_fields': ['title', 'identifier', "doc_id", "language", "_type"],
                    'excluded_fields': ["_id"],
                    'name_mapping': {"lang3": "language", "pnx_id": "doc_id"},
                    'key_by': ""

                }
            }

            with open('./configuration/config.yaml.secret', 'w') as yamlfile:
                yaml.safe_dump(new_yaml_data_dict, yamlfile, default_flow_style=False, allow_unicode=True,
                               encoding=None)  # Also note the safe_dump
            app.config["redirect"] = "/viewslist"
            return redirect(os.getenv("API_URL") + url_for("oauth_authorize"))

def require_access(service_name, version):
    """
    Annotate a method with the access required
    :param service_name:
    :param version:
    :return:
    """
    if not os.path.isfile(app.config['pickle']):
        def receive(fn: Callable):
            def wrapper(*args, **kwargs):
                kwargs["auth"] = build(service_name, version, credentials=app.config["creds"])
                return fn(*args, **kwargs)

            wrapper.__name__ = fn.__name__
            wrapper.__signature__ = inspect.signature(fn)
            return wrapper
    else:
        def receive(fn: Callable):
            def wrapper(*args, **kwargs):
                try:
                    with open(app.config['pickle'], "rb") as tokenf:
                        token = pickle.load(tokenf)
                        if not token:
                            raise AuthenticationException(
                                f"The provided token file '{app.config['pickle']}' was empty")
                        auth = build(service_name, version, credentials=token, cache_discovery=False)
                        args = list(args)
                        args.append(auth)
                        return fn(*args, **kwargs)
                except UnpicklingError:
                    raise AuthenticationException(
                        f"The provided token file '{app.config['pickle']}' is corrupted and could not be loaded")

            wrapper.__name__ = fn.__name__
            wrapper.__signature__ = inspect.signature(fn)
            return wrapper
    return receive

def get_views(auth):
    view_list = []
    accounts = auth.management().accounts().list().execute()
    for account in accounts["items"]:
        account_id = account["id"]
        properties = auth.management().webproperties().list(accountId=account_id).execute()
        for webp in properties["items"]:
            web_id = webp["id"]
            views = auth.management().profiles().list(
                accountId=account_id,
                webPropertyId=web_id).execute()
            for view in views["items"]:
                view_list.append({"id": view['id'], "url": view.get("websiteUrl", '')})

    return view_list

@app.route("/viewslist")
@require_access("analytics", "v3")
def showListofViews(auth):
    return render_template('viewsList.html', viewIDs=get_views(auth))

@app.route("/finalise", methods=['GET', 'POST'])
def finalise():
    viewID = request.form['viewIDPicker']
    errors = []

    if not os.path.exists('./configuration/token.pickle'):
        errors.append("Cannot find token.pickle!!!")
    if viewID not in viewIDs:
        errors.append("View ID does not exist!!!")
    if not os.path.exists('./configuration/config.yaml.secret'):
        errors.append("Cannot find config.yaml.secret!!!")
    if errors:
        return render_template("/errors.html", errors=errors)

    subdir = f"{os.path.abspath('./configuration') + '/token.pickle'}"
    new_yaml_data_dict = {
        'view_id': viewID,
        'path_to_credentials': subdir
    }
    with open('./configuration/config.yaml.secret') as f:
        doc = yaml.load(f)

    doc['analytics'] = new_yaml_data_dict

    with open('./configuration/config.yaml.secret', 'w') as f:
        yaml.dump(doc, f)
    directory = "mv " + os.path.abspath('./configuration/config.yaml.secret') + " " + os.path.abspath(
        '../nebula-background-worker/')
    subprocess.call(directory, shell=True)
    return render_template("successful.html")

@app.route("/oauth/callback/")
def oauth_callback():
    state = request.args.get("state", None)
    if not state:
        return render_template("error.html", errors=["No state provided to callback"])

    host = os.getenv("API_URL", request.host_url[:-1])
    try:
        authorization = {"web": session["google_cloud"]}

        flow = Flow.from_client_config(authorization, scopes, state=state)
        flow.redirect_uri = "{}{}".format(host, url_for("oauth_callback"))

        auth_url = request.url

        if auth_url.startswith("http://"):
            auth_url = "https" + auth_url[4:]

        flow.fetch_token(authorization_response=auth_url)

        sess_state = session["analytics"]
        sess_state.update({"token": pickle.dumps(flow.credentials)})
        session["analytics"] = sess_state

        with open(app.config["pickle"], "wb") as fin:
            pickle.dump(app.config["creds"], fin)
    except:
        tb.print_exc()
        return render_template("error.html", error=["An exception occured while attempting to complete the oauth flow"])
    return redirect(session["redirect"])

@app.route("/oauth/authorize/", methods=['GET', 'POST'])
def oauth_authorize():
    if "return_to" in request.args:
        session["redirect"] = request.args["return_to"]
    state = session.setdefault("analytics", {})

    credentials = None
    if "token" in state:
        credentials = pickle.loads(state["token"])

    if credentials and credentials.expired and credentials.refresh_token:
        credentials.refresh(Request())

    if credentials and credentials.valid:
        return redirect(session.get("redirect", "/"))

    authorization = {"web": session["google_cloud"]}
    print(json.dumps(authorization, indent=4))

    flow = Flow.from_client_config(authorization, scopes)
    host = os.getenv("API_URL", request.host_url[:-1])
    # flow.redirect_uri = "https://localhost:5000/oauth/callback/"
    flow.redirect_uri = "{}{}".format(host, url_for("oauth_callback"))

    auth_url, app.config["state"] = flow.authorization_url(
        prompt='consent',
        access_type='offline',
        include_granted_scopes='true'
    )
    # For some reason redirect_uri is not attached by the lib
    return redirect(auth_url)

@app.route("/logout")
def logout():
    session.clear()
    flash("The information you entered has been cleared")
    return redirect(url_for("about"))

@app.route("/analytics/")
@require_access("analyticsreporting", "v4")
def access_reporting(auth):
    ids = require_analytics_id()
    data = auth.reports().batchGet(
        body={
            'reportRequests': [
                {
                    'viewId': ids,
                    'dateRanges': [{'startDate': '7daysAgo', 'endDate': 'today'}],
                    'dimensions': [{"name": "ga:pagePath"}, {"name": "ga:pageTitle"}],
                    'metrics': [{'expression': 'ga:pageviews'}, {'expression': 'ga:uniquePageviews'},
                                {'expression': 'ga:timeOnPage'}]
                }]
        }
    ).execute()
    return jsonify(data)

@app.route("/realtime/")
@require_access("analytics", "v3")
def access_realtime(auth):
    ids = require_analytics_id()
    data = auth.data().realtime().get(
        ids=f"ga:{ids}",
        metrics='rt:pageviews',
        dimensions='rt:pagePath,rt:minutesAgo,rt:country,rt:city,rt:pageTitle',
        filters="ga:pagePath=~/primo-explore/fulldisplay?/*",
        sort="rt:minutesAgo"
    ).execute()
    return jsonify(data)

@app.route("/analytics/views/")
@require_access("analytics", "v3")
def show_accounts(auth):
    accounts = auth.management().accounts().list().execute()
    results = []
    for account in accounts["items"]:
        account_id = account["id"]
        properties = auth.management().webproperties().list(accountId=account_id).execute()
        for webp in properties["items"]:
            web_id = webp["id"]
            views = auth.management().profiles().list(
                accountId=account_id,
                webPropertyId=web_id).execute()
            for view in views["items"]:
                results.append(view)
    return jsonify(
        {
            "user": accounts,
            "views": results
        }
    )

def require_analytics_id():
    if "ga" not in request.args:
        raise Exception("Google analytics required")
    ga_id = request.args.get("ga")
    if not re.match("^[0-9]+$", ga_id):
        raise Exception("Invalid GA ID")
    return ga_id

if __name__ == '__main__':
    app.run(host="0.0.0.0", threaded=False, debug=True, ssl_context='adhoc')


def identify(l):
	if len(l.split()) == 6:
		return 1
	else:
		return 2

def endOfSixEight(line):
	if len(line.split()) == 7:
		return True
	else:
		return False

amount = int(input())
text = []
total = 0
res = []
for i in range(amount):
	text.append(input())
i = 0
while i < amount:
	total += 1
	form = identify(text[i])
	res.append(form)
	if form == 1:
		i += 2
		while i < amount:
			if len(text[i].split()) == 6:
				i += 2
			else:
				break
	if form == 2:
		i += 4

print(total)
for result in res:
	print(result)


from drone import *
from point import *
def load_drones(input_path):
    drones = []
    file_content = open(input_path)
    content = file_content.read()
    lines = content.splitlines()[1:]
    for line in lines:
        ID, dep, b = line.split(', ')
        drones.append(Drone(ID, dep, b))
    return drones

def load_targets(input_path):
    target_points = []
    file_content = open(input_path)
    content = file_content.read()
    lines = content.splitlines()[1:]
    for line in lines:
        ID, time = line.split(', ')
        target_points.append(Point(ID, time))
    return target_points

def load_costs(input_path, points):
    file_content = open(input_path)
    content = file_content.read()
    lines = content.splitlines()[1:]
    for line in lines:
        src, dst, cost = line.split(', ')
        target_src = [t for t in points if t.ID == src][0]
        target_dst = [t for t in points if t.ID == dst][0]
        target_src.add_travelling_cost(target_dst, cost)
        target_dst.add_travelling_cost(target_src, cost)


# import open3d as o3d
import pathlib
import pickle
import time
from collections import defaultdict, OrderedDict
from functools import partial

import cv2
import numpy as np
import quaternion
from skimage import io as imgio
from rslo.utils.geometric import RT_to_tq, compute_rel_pose
# from rslo.core import box_np_ops
# from rslo.core import preprocess as prep
# from rslo.core.geometry import points_in_convex_polygon_3d_jit
from rslo.data import kitti_common as kitti
# from rslo.utils import simplevis
from rslo.utils.timer import simple_timer

import seaborn as sns
import matplotlib.pyplot as plt
from collections.abc import Iterable
# from rslo.data.kitti_dataset import generate_pointwise_local_transformation,generate_pointwise_local_transformation_3d,generate_pointwise_local_transformation_tch
import torch
import torch.nn.functional as F
from rslo.utils.pose_utils_np import *
import quaternion

def generate_cyc_vo(pose_seq):
    assert len(pose_seq) > 1
    seq_len = len(pose_seq)

    vos = []
    for i in range(0, seq_len):
        for j in range(i+1, seq_len):
            # vos.append(calc_vo(pose_seq[i], pose_seq[j]).squeeze(axis=0))
            vo = calc_vo(pose_seq[i], pose_seq[j]).squeeze(axis=0)
            vo[3:] *= np.sign(vo[3])
            vos.append(vo)
    vos = np.stack(vos, axis=0)  # (7*(seq_len(seq_len-1)/2),)
    return vos

def merge_second_batch(batch_list):
    # [batch][key][seq]->example[key][seq][batch]
    example_merged = defaultdict(list)
    for example in batch_list:  # batch dim
        for k, v in example.items():  # key dim
            # assert isinstance(v, list)
            if isinstance(v, list):

                seq_len = len(v)
                if k not in example_merged:
                    example_merged[k] = [[] for i in range(seq_len)]
                for i, vi in enumerate(v):  # seq dim
                    example_merged[k][i].append(vi)

            else:
                example_merged[k].append(v)

    ret = {}
    for key, elems in example_merged.items():
        if key in [
                'voxels', 'num_points'
        ]:
            if isinstance(elems[0], list):
                ret[key] = []
                for e in elems:
                    ret[key].append(np.concatenate(e, axis=0))
            else:
                ret[key] = np.concatenate(elems, axis=0)

        elif key == 'coordinates':
            coors = []
            for i, coor in enumerate(elems):  # different seq
                assert isinstance(
                    coor, list), f'{type(coor[0])} is not allowed!'
                coor_pad = []
                for j, _ in enumerate(coor):  # different batch
                    # coor_pad = np.pad(
                    #     coor[j], ((0, 0), (1, 0)), mode='constant', constant_values=i) # ! i is batch index
                    coor_pad.append(np.pad(
                        coor[j], ((0, 0), (1, 0)), mode='constant', constant_values=j))  # ! j is batch index
                # coors.append(coor_pad)
                coors.append(np.concatenate(coor_pad, axis=0))
            # ret[key] = np.concatenate(coors, axis=0)
            ret[key] = coors
        elif key == 'tq_maps':
            # ret[key] = torch.stack(elems, dim=0)
            ret[key] = []
            for e in elems:
                ret[key].append(torch.stack(e, dim=0))
        elif key in ['odometry', "icp_odometry"]:
            ret[key] = np.stack(elems, axis=0)
        elif key in ['lidar_seqs', "normal_gt_seqs"]:
            # ret[key] = elems
            min_len = elems[0][0].shape[0]
            for t in range(len(elems)):
                for b in range(len(elems[t]) ):
                    min_len = min(min_len, elems[t][b].shape[0] )
            for t in range(len(elems)):
                for b in range(len(elems[t])):
                    elems[t][b] = elems[t][b][ np.random.choice(len(elems[t][b]),min_len, replace=False )]
                elems[t] = np.stack(elems[t], axis=0)
            ret[key]=elems
        elif key in ['hier_points']:
            hier_min_lens = []
            for h in range(len(elems[0][0]) ):
                min_len = elems[0][0][h].shape[0]
                for t in range(len(elems)):
                    for b in range(len(elems[t]) ):
                        min_len = min(min_len, elems[t][b][h].shape[0] )
                hier_min_lens.append(min_len)
            new_elems=[]
            for t in range(len(elems)):
                for b in range(len(elems[t])):
                    for h in range(len(elems[t][b])):
                        elems[t][b][h] = elems[t][b][h][ np.random.choice(len(elems[t][b][h]), hier_min_lens[h], replace=False ) ]
                # elems[t] = np.stack(elems[t], axis=0)
                new_elems.append([])
                for h in range(len(elems[0][0] )):
                    new_elems[t].append(
                        np.stack([elems[t][b][h] for b in  range(len(elems[t])) ], axis=0)  )

            ret[key]=new_elems
        elif key == 'metrics':
            ret[key] = elems
        else:
            ret[key] = []
            for e in elems:
                ret[key].append(np.stack(e, axis=0))
    return ret

def merge_second_batch_multigpu(batch_list):
    # [batch][key][time]-> example_merged[key][time][batch]
    example_merged = defaultdict(list)
    for example in batch_list:  # batch
        for k, v in example.items():  # key
            # assert isinstance(v, list), f"{type(v)} is not allowed!"
            if isinstance(v, list):
                seq_len = len(v)
                if k not in example_merged:
                    example_merged[k] = [[] for i in range(seq_len)]
                for i, vi in enumerate(v):  # seq
                    example_merged[k][i].append(vi)
            else:
                example_merged[k].append(v)

            # example_merged[k].append(v)
    ret = {}
    for key, elems in example_merged.items():

        if key == 'coordinates':
            coors = []
            for i, coor in enumerate(elems):  # seq
                assert isinstance(
                    coor, list), f"{type(coor[0])} is not allowed!"
                coor_pad = []
                for j, _ in enumerate(coor):  # batch
                    coor_pad.append(np.pad(
                        coor[j], ((0, 0), (1, 0)), mode='constant', constant_values=j))
                coors.append(np.stack(coor_pad, axis=0))

            # ret[key] = np.stack(coors, axis=0)
            ret[key] = coors

        elif key in ['gt_names', 'gt_classes', 'gt_boxes']:
            continue
        elif key in ['tq_maps']:
            ret[key] = torch.stack(elems, dim=0)
        elif key in ['odometry']:
            ret[key] = np.stack(elems, axis=0)
        else:
            seqs = []
            for i, elem in enumerate(elems):  # seq
                seqs.append(np.stack(elem, axis=0))

            # ret[key] = np.stack(elems, axis=0)
            ret[key] = seqs
    return ret

def _dict_select(dict_, inds):
    for k, v in dict_.items():
        if isinstance(v, dict):
            _dict_select(v, inds)
        else:
            dict_[k] = v[inds]

def estimate_normal(points, radius=0.5, max_nn=30, camera_location=(0,0,0) ):
    '''
        points: Nx3
        down_sample_voxel_size: float scalar
    '''

    if isinstance(points, np.ndarray ):
        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(points)
    elif isinstance(points, o3d.geometry.PointCloud ):
        pcd = points
    else:
        raise NotImplementedError
    # pcd.estimate_normals(o3d.geometry.KDTreeSearchParamHybrid(radius=radius * 2, max_nn=30) )
    pcd.estimate_normals(o3d.geometry.KDTreeSearchParamHybrid(radius=radius, max_nn=30) )

    pcd.orient_normals_towards_camera_location(camera_location)
    return pcd
def down_sample_points(points, down_sample_voxel_size ):
    '''
        points: Nx3
        down_sample_voxel_size: float scalar
    '''

    if isinstance(points, np.ndarray):
        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(points)
    elif isinstance(points, o3d.geometry.PointCloud ):
        pcd = points
    else:
        raise NotImplementedError

    pcd = pcd.voxel_down_sample(down_sample_voxel_size)

    return pcd

def flip_odometry(old_odom):
    # old_odom = input_dict['odometry'][odom_idx]
    q_old = np.quaternion(*list(old_odom[3:]))
    r_old = quaternion.as_rotation_matrix(q_old)
    t_old = old_odom[:3]
    f = np.array([1, 0, 0,
                    0, -1, 0,
                    0, 0, 1]).reshape([3, 3])
    r_new = f@r_old@f.T
    t_new = f@t_old
    q_new = quaternion.from_rotation_matrix(r_new)
    q_new = quaternion.as_float_array(q_new)
    q_new *= np.sign(q_new[0])
    new_odom = np.concatenate(
        [t_new, q_new], axis=-1)
    return new_odom
def prep_pointcloud(input_dict,
                    root_path,
                    voxel_generator,
                    max_voxels=20000,
                    training=True,
                    shuffle_points=False,
                    num_point_features=4,
                    anchor_area_threshold=1,
                    random_crop=False,
                    multi_gpu=False,
                    use_dist=False,
                    min_points_in_gt=-1,
                    random_flip_x=False,
                    random_flip_y=False,
                    sample_importance=1.0,
                    rand_rotation_eps=0,
                    rand_translation_eps=0,
                    rand_aug_ratio=0,
                    gen_tq_map=False,
                    do_pre_transform=False,
                    out_dtype=np.float32,
                    seed=None,
                    cubic_tq_map=False,
                    downsample_voxel_sizes=None,
                    ):
    """convert point cloud to voxels, create targets if ground truths
    exists.

    input_dict format: dataset.get_sensor_data format

    """
    t = time.time()
    if seed is not None:
        np.random.seed(seed)
    # class_names = target_assigner.classes
    # points = input_dict["lidar"]["points"]
    """
      input_dict = {
           "seq_idx": [],
            "frame_idx": [],
            "lidar_seq": [],
            "cam_pose_seq": [],
        }
    """
    points_list = input_dict["lidar_seq"]
    # normal_gt_list=input_dict["normal_gt_seq"]

    if not isinstance(points_list, Iterable):
        points_list = [points_list]

    hier_points=input_dict.get("hier_points_seq", None)
    if hier_points is None:
        # print("Warning: key 'hier_points_seq' has not been found in the input dict")
        hier_points = [ [np.concatenate([p[:,:3], p[:,4:7]], axis=-1) ] for p in points_list]

    metrics = {}

    if training:
        """
        boxes_lidar = gt_dict["gt_boxes"]
        bev_map = simplevis.nuscene_vis(points, boxes_lidar)
        cv2.imshow('pre-noise', bev_map)
        """

        # if min_points_in_gt > 0:
        #     # points_count_rbbox takes 10ms with 10 sweeps nuscenes data
        #     point_counts = box_np_ops.points_count_rbbox(
        #         points, gt_dict["gt_boxes"])
        #     mask = point_counts >= min_points_in_gt
        #     _dict_select(gt_dict, mask)
        # gt_boxes_mask = np.array(
        #     [n in class_names for n in gt_dict["gt_names"]], dtype=np.bool_)

        pc_range = voxel_generator.point_cloud_range
        # group_ids = None
        # if "group_ids" in gt_dict:
        #     group_ids = gt_dict["group_ids"]

        # should remove unrelated objects after noise per object
        # for k, v in gt_dict.items():
        #     print(k, v.shape)

        bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]

    if 0:  # shuffle_points:
        # shuffle is a little slow.
        # np.random.shuffle(points)
        for points in points_list:
            np.random.shuffle(points)
    if random_flip_y and np.random.rand() > 0.5:
        #flip lidar points
        for i, points in enumerate(points_list):
            points_list[i][:, 1] = -points_list[i][:, 1]  # flip y
            # if len(points_list[i][0]) >= 8:
            if len(points_list[i][0]) >= 6:
                points_list[i][:, 5] = -points_list[i][:, 5]  # flip normal_y

            if len(points_list[i][0]) >= 9:
                points_list[i][:, 8] = -points_list[i][:, 8]  # flip normal_y
        #flip normal_gt (if exists)
        # for i,normal in enumerate(normal_gt_list):
        #     normal_gt_list[i][:,1]=-normal_gt_list[i][:,1] #flip normal_y

        #flip hier lidar points
        for i, hier_point in enumerate(hier_points):
            for h, _ in enumerate(hier_point):
                hier_points[i][h][:,1] = - hier_points[i][h][:,1] #flip y
                hier_points[i][h][:,4] = - hier_points[i][h][:,4] #flip normal_y

        seq_len = len(input_dict["frame_idx"])
        odom_idx = 0
        for i in range(0, seq_len):
            for j in range(i+1, seq_len):
                # [odom_idx:odom_idx+1]
                old_odom = input_dict['odometry'][odom_idx]
                # q_old = np.quaternion(*list(old_odom[3:]))
                # # print(q_old,'!!1')
                # r_old = quaternion.as_rotation_matrix(q_old)
                # t_old = old_odom[:3]

                # f = np.array([1, 0, 0,
                #               0, -1, 0,
                #               0, 0, 1]).reshape([3, 3])
                # r_new = f@r_old@f.T
                # t_new = f@t_old

                # q_new = quaternion.from_rotation_matrix(r_new)
                # q_new = quaternion.as_float_array(q_new)
                # q_new *= np.sign(q_new[0])
                # # print(t_new, q_new, '!!!')
                # input_dict['odometry'][odom_idx] = np.concatenate(
                #     [t_new, q_new], axis=-1)
                new_odom = flip_odometry(old_odom)
                input_dict['odometry'][odom_idx] = new_odom

                if input_dict.get('icp_odometry', None) is not None:
                    old_icp = input_dict['icp_odometry'][odom_idx]
                    input_dict['icp_odometry'][odom_idx] = flip_odometry(old_icp)

                odom_idx += 1

    if rand_aug_ratio > 0:
        # rot_vecs = [quaternion.as_rotation_vector(odom) for odom in odoms]
        seq_len = len(input_dict["frame_idx"])
        aug_ratio = np.random.uniform(-rand_aug_ratio, rand_aug_ratio, seq_len)

        abs_poses = [np.array([0, 0, 0, 1, 0, 0, 0]).reshape([-1, 7])]
        for i in range(seq_len-1):
            abs_poses.append(input_dict['odometry'][i:i+1])

        new_abs_poses = []
        for i in range(seq_len):
            if i==0: #do not augment the first one
                new_abs_poses.append(abs_poses[i])
                continue
            if i+1 < seq_len:
                # new_pose = abs_poses[i]+(abs_poses[i+1]-abs_poses[i])*aug_ratio[i]
                # new_pose = abs_poses[i] + \
                #     (abs_poses[i+1]-abs_poses[i])*aug_ratio[i]
                new_pose = abs_poses[i] + \
                    (abs_poses[i+1]-abs_poses[i])*aug_ratio[i]
                new_pose_q = quaternion.slerp(quaternion.from_float_array(
                    abs_poses[i][:, 3:]), quaternion.from_float_array(abs_poses[i+1][:, 3:]), 0, 1, aug_ratio[i])

                new_pose[:, 3:] = quaternion.as_float_array(new_pose_q)
                # print((new_pose-new_pose_bak)/(new_pose_bak+1e-12),'!!2')

            else:
                # new_pose = abs_poses[i] + \
                #     (abs_poses[i]-abs_poses[i-1])*aug_ratio[i]
                new_pose = abs_poses[i] + \
                    (abs_poses[i-1]-abs_poses[i])*(-aug_ratio[i])
                # print(new_pose,'1')
                new_pose_q = quaternion.slerp(quaternion.from_float_array(
                    abs_poses[i][:, 3:]), quaternion.from_float_array(abs_poses[i-1][:, 3:]), 0, 1, -aug_ratio[i])
                new_pose[:, 3:] = quaternion.as_float_array(new_pose_q)
                # print(new_pose,'2')

            new_pose[:, 3:] = normalize(new_pose[:, 3:], dim=1)
            new_pose[:, 3:] *= np.sign(new_pose[:, 3])
            # print(new_pose)
            new_abs_poses.append(new_pose)

        new_odoms = []

        input_dict['odometry'] = generate_cyc_vo(new_abs_poses)

        # apply the transformation
        for i, points in enumerate(points_list):
            transformation = calc_vo(new_abs_poses[i], abs_poses[i])

            points_list[i][:, :3] = rotate_vec_by_q(
                points[:, :3], transformation[:, 3:]) + transformation[:, :3]
            if len(points_list[i][0]) >= 7:
                # transform normals
                points_list[i][:, 4:7] = rotate_vec_by_q(
                    points[:, 4:7], transformation[:, 3:])  # + delta_ts[i:i+1]
            #TODO
            # if len(normal_gt_list)>0:
            if len(points_list[i][0]) >= 9:
                points_list[i][:, 7:]=rotate_vec_by_q(
                    points_list[i][:, 7:], transformation[:, 3:])
        #transform the hier lidar points
        for i, hier_point in enumerate(hier_points):
            transformation = calc_vo(new_abs_poses[i], abs_poses[i])
            for h, _ in enumerate(hier_point):
                hier_points[i][h][:,:3] =  rotate_vec_by_q(
                hier_points[i][h][:,:3], transformation[:, 3:]) + transformation[:, :3]
                # transform normals
                hier_points[i][h][:,3:6] = rotate_vec_by_q(
                    hier_points[i][h][:,3:6], transformation[:, 3:])

        # [0, -40, -3, 70.4, 40, 1]
    voxel_size = voxel_generator.voxel_size
    pc_range = voxel_generator.point_cloud_range
    grid_size = voxel_generator.grid_size

    # print(voxel_size, pc_range, grid_size,'!!')
    # [352, 400]
    t1 = time.time()

    voxels = []
    coordinates = []
    num_points = []
    num_voxels = []
    frame_inds = []
    seq_inds = []
    coor_to_voxelidx=[]
    if not multi_gpu or use_dist:
        for i, points in enumerate(points_list):
            # if do_pre_transform>0:
            #     points = points[points[:,-1]<do_pre_transform][:,:8] # remove the dynamic objects

                # print(points.shape, flush=True)
            # # if i >= 0 and do_pre_transform>0:
            # if i >= 1 and do_pre_transform>0:
            #     # only for testing purpose
            #     assert(len(input_dict['odometry']) == 1)
            #     t_ = input_dict['odometry'][0:1, :3]
            #     q_ = input_dict['odometry'][0:1, 3:]
            #     trans_xyz = rotate_vec_by_q(
            #         points[:, :3], t_) + t_
            #     # trans_xyz = points[:,:3]
            #     # trans_xyz[:,:2]=  150/(np.linalg.norm(points[:,:2], axis=1, keepdims=True)+0.1) **2 * trans_xyz[:,:2]
            #     points = np.concatenate([trans_xyz, points], axis=1)
            res = voxel_generator.generate(points, max_voxels)

            # if i >= 1 and do_pre_transform>0:
            #     # only for testing purpose
            #     res["voxels"] = res["voxels"][:, :, 3:]

            voxels.append(res["voxels"])
            # print(res["voxels"].shape, input_dict['velodyne_path'])
            coordinates.append(res["coordinates"])
            num_points.append(res["num_points_per_voxel"])
            num_voxels.append(np.array([voxels[-1].shape[0]], dtype=np.int64))
            # coor_to_voxelidx.append(voxel_generator._coor_to_voxelidx)
    else:
        for points in points_list:
            res = voxel_generator.generate_multi_gpu(
                points, max_voxels)
            voxels.append(res["voxels"])
            coordinates.append(res["coordinates"])
            num_points.append(res["num_points_per_voxel"])
            num_voxels.append(np.array([res["voxel_num"]], dtype=np.int64))

    ########################################## down sample lidar point ##########################################
    # hier_points = OrderedDict()
    # origin_pcds = [estimate_normal(points[:,:3]) for points in points_list  ]
    '''
    origin_pcds = []
    for points in points_list:
        pcd = o3d.geometry.PointCloud()
        pcd.points = o3d.utility.Vector3dVector(points[:,:3])
        if points.shape[-1]>4:
            pcd.normals = o3d.utility.Vector3dVector(points[:,4:7])
        else:
            pcd = estimate_normal(points[:,:3])
        origin_pcds.append(pcd)

    # origin_points = [np.concatenate([np.asarray(pcd.points), np.asarray(pcd.normals)], axis=-1) for pcd in origin_pcds ]
    # origin_points = [np.asarray(pcd.points) for pcd in origin_pcds ]
    # hier_points = [origin_points]
    hier_points = [] # sequence of hiearachical points
    for origin_pcd in origin_pcds:
        hier_points_list = []
        for i, ds in enumerate(downsample_voxel_sizes):
            # print(ds,flush=True)
            # for origin_pcd in origin_pcds:
            # for origin_pcd in hier_points[-1]:
            pcd = down_sample_points(origin_pcd, down_sample_voxel_size=ds)
            hier_points_list.append(np.concatenate([np.asarray(pcd.points), np.asarray(pcd.normals)], axis=-1) )
                # new_points_list.append(np.asarray(pcd.points) )
        hier_points.append(hier_points_list)
    '''
    # hier_points = [[p[:len(p)//2,:6]] for p in points_list]
    # hier_points_list =  input_dict["hier_points_seq"]
    # hier_points=input_dict.get("hier_points_seq", None)
    # # for i,_ in enumerate(hier_points):
    # #     hier_points[i] = [np.concatenate([points_list[i][:,:3], points_list[i][:,4:7]], axis=-1 ) ] + hier_points[i]
    # if hier_points is None:
    #     # print("Warning: key 'hier_points_seq' has not been found in the input dict")
    #     hier_points = [ [np.concatenate([p[:,:3], p[:,4:7]], axis=-1) ] for p in points_list]

    ########################################## \down sample lidar point #########################################

    if 0: #gen_tq_map:

        grid_size = np.array(voxel_generator.grid_size)//4
        # voxel_size = np.array(voxel_generator.voxel_size)*4  # x,y,z
        voxel_size = (np.array(pc_range[3:])-np.array(pc_range[0:3]))/grid_size
        pc_range = voxel_generator.point_cloud_range
        if cubic_tq_map:
            spatial_size = grid_size
            tq_map_generator = generate_pointwise_local_transformation_3d
        else:
            spatial_size = grid_size[:2]
            tq_map_generator = generate_pointwise_local_transformation
        # origin_loc = (0-pc_range[0])/(pc_range[3]-pc_range[0])*grid_size[0], (pc_range[4]-(
        #     pc_range[4]+pc_range[1])/2)/(pc_range[4]-pc_range[1])*grid_size[1], 0
        # # # fixed on 7/11/2019
        # origin_loc = (0-pc_range[0])/(pc_range[3]-pc_range[0])*(grid_size[0]
        #                                                         ), (pc_range[4]-0)/(pc_range[4]-pc_range[1])*(grid_size[1]), 0
        # the location of camera/lidar in voxel/bv coords
        origin_loc = (0-pc_range[0])/(pc_range[3]-pc_range[0])*(grid_size[0]
                                                                ), (pc_range[4]-0)/(pc_range[4]-pc_range[1])*(grid_size[1]), (0-pc_range[2])/(pc_range[5]-pc_range[2]) * grid_size[2]

        tq_maps = []
        for tq in input_dict['odometry']:
            tq_maps.append(tq_map_generator(
                tq,
                # spatial_size=grid_size[:2],
                spatial_size=spatial_size,
                # origin_loc=np.array([0, grid_size[1]/2, 0]),
                origin_loc=origin_loc,  # np.array(origin_loc),
                voxel_size=voxel_size,
                inv_trans_factor=-1 if not do_pre_transform else 150
            ))
        # tq_maps = np.stack(tq_maps, axis=0)
        tq_maps = torch.stack(tq_maps, dim=0)
        tq_maps = [F.interpolate(
            tq_maps, scale_factor=0.5**(i+1), mode='nearest')for i in range(1)]
        # tq_maps = [tq_maps]
    else:
        tq_maps = [torch.Tensor([0])]

    metrics["voxel_gene_time"] = time.time() - t1
    example = {
        'lidar_seqs': points_list,
        'hier_points': hier_points,
        'voxels': voxels,
        'num_points': num_points,
        'coordinates': coordinates,
        "num_voxels": num_voxels,
        "metrics": metrics,
        "frame_inds": input_dict["frame_idx"],
        "seq_inds": input_dict["seq_idx"],
        "pose_seq": input_dict["pose_seq"],
        "odometry": input_dict["odometry"],
        "skip": input_dict["skip"],
        "tq_maps": tq_maps,
        "icp_odometry": input_dict.get("icp_odometry", np.zeros(7) ),
        # "normal_gt_seq":normal_gt_list,
    }

    metrics["prep_time"] = time.time() - t

    # for RT in input_dict["cam_pose_seq"]:
    #     example["cam_pose_seq"].append(RT_to_qt(RT))

    # example["odometry"] = compute_rel_pose(
    #     example["cam_pose_seq"][-2], example["cam_pose_seq"][-1])

    return example


import os
from os.path import join as fullpath
import argparse

import numpy as np
from tensorflow.keras.callbacks import ModelCheckpoint

from model import (
    create_model,
    load_model_with_weights
)
from utils import (
    load_preprocessed_data,
    evaluate,
)

parser = argparse.ArgumentParser(description='')
parser.add_argument('--model', type=str, default='model.h5')
parser.add_argument('--data_folder', type=str, default='../data/processed/')
parser.add_argument('--train_file', type=str, default='training.mat')
parser.add_argument('--test_file', type=str, default='test.mat')
parser.add_argument('--num_key', type=int, default=10)
parser.add_argument('--batch_size', type=int, default=220)
parser.add_argument('--epochs', type=int, default=40)
parser.add_argument('--training', type=bool, default=True)
args = parser.parse_args()

os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'

def train(model, data, labels, input_length, label_length, test_data, test_labels, test_input_length, test_label_length, callbacks):

    inputs = {
        'input': data,
        'labels': labels,
        'input_length': input_length,
        'label_length': label_length,
    }
    outputs = {'ctc': np.zeros([data.shape[0]])}

    validate_inputs = {
        'input': test_data,
        'labels': test_labels,
        'input_length': test_input_length,
        'label_length': test_label_length,
    }
    validate_outputs = {'ctc': np.zeros([test_data.shape[0]])}

    # training
    model.fit(
        inputs, outputs,
        batch_size=args.batch_size,
        epochs=args.epochs,
        validation_data=(validate_inputs, validate_outputs),
        callbacks=callbacks
    )
    return model

def main():

    data_folder = args.data_folder

    training_filename = args.train_file
    test_filename = args.test_file
    model_name = fullpath('saved_model', args.model)

    should_training = args.training

    test_data, test_labels, test_input_length, test_label_length, num_key = load_preprocessed_data(fullpath(data_folder, test_filename))

    model = create_model(test_data[0].shape[1], num_key)

    # Training
    if should_training:
        data, labels, input_length, label_length, num_key = load_preprocessed_data(fullpath(data_folder, training_filename))
        checkpoint = ModelCheckpoint(model_name, monitor='val_loss', verbose=0, save_weights_only=True, save_best_only=False, mode='min', period=1)
        try:
            train(model, data, labels, input_length, label_length,
                test_data, test_labels, test_input_length, test_label_length,
                [checkpoint])
        except KeyboardInterrupt:
            # do nothing here
            pass

    # Test
    model = load_model_with_weights(model, model_name)
    evaluate(model, test_data, test_labels, test_input_length, test_label_length, num_key)

if __name__ == "__main__":
    main()


menu = {}
ordini = []

flag_ = True
while flag_:
    scelta = input('''Premi 1 per aggiungere un piatto al menu, \n 2 per visualizzare il menu completo, \n 3 per cancellare un piatto dal menu,
                   4 prendere gli ordini, \n 5 fare il conto \n 6 per uscire''')
    if scelta == '6':                                                                #esco da tutto
        flag_ = False
    elif  scelta == '1':
        nome = input('inserisci il nome del piatto')
        prezzo = float(input('insrisci il prezzo del piatto'))
        if nome not in menu.keys():                                                #se non c'è lo inserisco
            menu[nome]= prezzo
        else:
            print('Piatto già esistente, voui aggiornare il prezzo?')
            scelta_agg_prezzo = input('1 per aggiornare, 0 per uscire')            # se già c'è chiedo se vuole aggiornare il prezzo
            if scelta_agg_prezzo == '1':
                menu[nome]= prezzo
    elif  scelta == '2':                                                          #stampo il menu completo (come dizionario)
        print(menu)
    elif  scelta == '3':
        nome_piatto = input('inserisci il nome del piatto da eliminare')
        if nome_piatto not in menu.keys():
            print('Il piatto inserito non esiste')
        else:
            del menu[nome_piatto]                                                  #elimino il piatto dal menu
    elif  scelta == '4':
        ord_flag = True
        while ord_flag:                                                         # per ordinare piu piatti senza tornare al menu principale
            piatto = input('quale piatto vuoi ordinare? ')
            if piatto not in menu.keys():
                print('Non abbiamo questo piatto nel nostro menu')
            else:
                quantita = int(input('quanti ? '))
                ordini.append(piatto)                                             # aggiungo alla lista ordini il piatto ordinato e la quantità
                ordini.append(quantita)
            continua_ordine=input('0 per uscire 1 per ordinare un altro piatto')
            if continua_ordine==0:
                ord_flag=False                                                   # quando ho finito l'ordine esco
    elif  scelta == '5':
        conto = 0
        if len(ordini)==0:
            print('non hai ancora ordinato')
        else:
            for i in range(0,len(ordini),2):                                    # nella lista ordini ho piatto, quantità, piatto, quantità.....
                conto += ordini[i+1]*(menu[ordini[i]])                          #dal menu vedo il prezzo
        print('ecco il conto: ', conto)
    else:
        print('ERRORE!')


from collections import defaultdict

class Graph:
    def __init__(self):
        self.graph = defaultdict(list)

    def add_edge(self,u,v):
        self.graph[u].append(v)

    def print_DFS(self,v,visited):
        visited[v] = True
        print(v)

        for i in self.graph[v]:
            if visited[i] == False:
                self.print_DFS(i,visited)

    def DFS(self,v):
        visited = [False] * (len(self.graph))

        self.print_DFS(v,visited)

def main():
    g = Graph()
    g.add_edge(0,1)
    g.add_edge(0,2)
    g.add_edge(1,2)
    g.add_edge(2,0)
    g.add_edge(2,3)
    g.add_edge(3,3)

    g.DFS(2)

main()

Amazon_interview/Graph/Depth First Search or DFS for a Graph.py


#!/usr/bin/python

DIR_LANG = "src/assets/text/"
DIR_DIST = "dist/"

import json

from os import walk
from shutil import copyfile
from pack_gui import pack_gui
from pack_sprites import pack_sprites
from pack_parts import pack_parts

def copy_langfiles():
    print("Copying language files:")

    for (dir_path, dir_names, file_names) in walk(DIR_LANG):
        for file in file_names:
            print("- '" + file + "'")

            json_src_file = open(dir_path + file, encoding="utf-8-sig")

            json_dest_file = open(DIR_DIST + file, "w", encoding="utf-8-sig")
            json_dest_file.write(json.dumps(
                json.load(json_src_file),
                indent=None,
                separators=(",", ":"),
                ensure_ascii=False))
            json_dest_file.close()

            json_src_file.close()

pack_sprites()
pack_gui()
pack_parts()
copy_langfiles()


import time
import pytest
import os
from unittest.mock import patch
from unittest.mock import mock_open
from ai_manager.tools.common_tools import CommonTools
VERSION_INFO_CORRECT = """GaussDB-Kernel-V500R001C20
92.301
cbafe281"""
VERSION_INFO_WRONG = """GaussDB-Kernel-V500R001C20
92.301"""
JSON_INFO_CORRECT = """{"A":"B"}"""
JSON_INFO_WRONG = """{"A":'B'}"""
PATH_CHECK_LIST = ["|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"",
                   "{", "}", "(", ")", "[", "]", "~", "*", "?", " ", "!", "\n"]

class TestCommonTools:

    @patch("builtins.open", new_callable=mock_open, read_data=VERSION_INFO_CORRECT)
    def test_correct_get_version_info_from_file(self, mock_read_file):
        version = CommonTools.get_version_info_from_file('./test')
        assert version == '92301'

    @patch("builtins.open", new_callable=mock_open, read_data=VERSION_INFO_WRONG)
    def test_wrong_get_version_info_from_file(self, mock_read_file):
        with pytest.raises(Exception, match=r"is wrong."):
            CommonTools.get_version_info_from_file('./test')

    @patch('subprocess.getstatusoutput')
    def test_extract_file_to_dir(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"extract file"):
            CommonTools.extract_file_to_dir('path1', 'path2')

    @patch('subprocess.getstatusoutput')
    def test_copy_file_to_dest_path(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"copy files"):
            CommonTools.copy_file_to_dest_path('path1', 'path2')

    def test_check_dir_access(self):
        # current path check full authority
        CommonTools.check_dir_access('./', 'full')
        # check exist
        with pytest.raises(Exception, match=r"can not access"):
            CommonTools.check_dir_access('./notexitpath', 'exist')
        # check rw
        with pytest.raises(Exception, match=r"can not access"):
            CommonTools.check_dir_access('./notexitpath', 'rw')
        # check wrong params
        with pytest.raises(Exception, match=r"is wrong"):
            CommonTools.check_dir_access('./notexitpath', 'wrong')

    @patch('subprocess.getstatusoutput')
    def test_mkdir_with_mode(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"mkdir"):
            CommonTools.mkdir_with_mode('path1', '0700')

    @patch('ai_manager.tools.common_tools.CommonTools.remote_execute_cmd')
    def test_remote_mkdir_with_mode(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"remote mkdir"):
            CommonTools.remote_mkdir_with_mode('path', 'mode', 'ip', 'username', 'password')

    @patch('subprocess.getstatusoutput')
    def test_clean_dir(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"clean dir"):
            CommonTools.clean_dir('path')

    @patch('ai_manager.tools.common_tools.CommonTools.remote_execute_cmd')
    def test_remote_clean_dir(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"remote mkdir"):
            CommonTools.remote_clean_dir('ip', 'username', 'password', 'cmd')

    @patch('subprocess.getstatusoutput')
    def test_remove_files(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"remove file"):
            CommonTools.remove_files('path')

    @patch("builtins.open", new_callable=mock_open, read_data=JSON_INFO_CORRECT)
    @patch("os.path.isfile")
    def test_correct_json_file_to_dict(self, is_file, mock_read_file):
        is_file.return_value = True
        ret = CommonTools.json_file_to_dict('/test')
        assert ret == {'A': 'B'}

    @patch('subprocess.getstatusoutput')
    def test_add_cron(self, result):
        result.return_value = (1, 'aa')
        CommonTools.add_cron('install_path', 'cmd', 'frequency')

    @patch('subprocess.getstatusoutput')
    def test_del_cron(self, result):
        result.return_value = (1, 'aa')
        CommonTools.del_cron('install_path', 'cmd', 'frequency')

    def test_delete_early_record(self):
        with pytest.raises(Exception, match=r"is not exist."):
            CommonTools.delete_early_record('path', 1024)

    @patch("builtins.open", new_callable=mock_open, read_data=VERSION_INFO_WRONG)
    def test_read_last_line_from_file(self, mock_read_file):
        ret = CommonTools.read_last_line_from_file('./')
        assert ret == '92.301'

    @patch('subprocess.getstatusoutput')
    def test_grep_process_and_kill(self, result):
        result.return_value = (1, 'aa')
        CommonTools.grep_process_and_kill('process')

    @patch('subprocess.getstatusoutput')
    def test_check_is_root(self, result):
        result.return_value = True
        ret = CommonTools.check_is_root()
        assert ret in [True, False]

    @patch('subprocess.getstatusoutput')
    def test_check_process(self, result):
        result.return_value = (1, 'aa')
        with pytest.raises(Exception, match=r"check process"):
            CommonTools.check_process('path')
        result.return_value = (0, 'aa')
        with pytest.raises(Exception, match=r"check process num"):
            CommonTools.check_process('path')
        result.return_value = (0, '5')
        ret = CommonTools.check_process('path')
        assert ret == 5

    def test_check_path_valid(self):
        for item in PATH_CHECK_LIST:
            wrong_path = os.path.realpath('./') + item
            with pytest.raises(Exception, match=r"There are illegal character"):
                CommonTools.check_path_valid(wrong_path)

    def test_get_funcs(self):
        from ai_manager.tools import params_checkers
        from ai_manager.tools.params_checkers import check_ip
        ret = CommonTools.get_funcs(params_checkers)
        assert check_ip in ret.values()

    def test_parallel_execute(self):
        para_list = [0.5] * 10
        start = time.time()
        CommonTools.parallel_execute(lambda x: time.sleep(x), para_list)
        end = time.time()
        assert end - start < 0.5 * 10

    @patch('ai_manager.tools.common_tools.CommonTools.get_status_output_error')
    def test_remote_copy_files(self, result):
        result.return_value = (1, 'aa')
        CommonTools.remote_copy_files('remote_ip', 'user', 'password', 'path_from', 'path_to')

    @patch('ai_manager.tools.common_tools.CommonTools.get_status_output_error')
    def test_remote_execute_cmd(self, result):
        result.return_value = (1, 'aa')
        CommonTools.remote_execute_cmd('remote_ip', 'user', 'password', 'cmd')

    @patch('ai_manager.tools.common_tools.CommonTools.get_status_output_error')
    def test_get_local_ips(self, result):
        result.return_value = (0, "eth0 : df df 10.10.72.57 test "
                                  "string 10.10.72.57 test string 10.10.7.5")
        ret = CommonTools.get_local_ips()
        assert ret.sort() == ['10.10.72.57', '10.10.7.5'].sort()

    @patch("ai_manager.tools.common_tools.CommonTools.get_local_ips")
    def test_modify_agent_config_file(self, ips):
        ips.return_value = ['10.10.72.57', '10.10.7.5']
        content = {"security": {"tls": "False"}}
        with pytest.raises(Exception, match=r"not found in the config file or section"):
            CommonTools.modify_agent_config_file('config_path', content, allow_add=False)

    @patch("ai_manager.tools.common_tools.CommonTools.get_local_ips")
    def test_modify_config_file(self, ips):
        ips.return_value = ['10.10.72.57', '10.10.7.5']
        content = {"security": {"tls": "False"}}
        with pytest.raises(Exception, match=r"not found in the config file or section"):
            CommonTools.modify_config_file('config_path', content, allow_add=False)

    @patch('subprocess.getstatusoutput')
    def test_get_current_usr(self, result):
        result.return_value = (1, 'a')
        with pytest.raises(Exception, match=r"get user"):
            CommonTools.get_current_usr()

    @patch('os.mknod')
    def test_create_file_if_not_exist(self, result):
        ret = CommonTools.create_file_if_not_exist('path')
        assert ret is True

    @patch('subprocess.getstatusoutput')
    def test_read_info_from_config_file(self, result):
        result.return_value = (1, 'a')
        with pytest.raises(Exception, match=r"not found in the config file or section"):
            CommonTools.read_info_from_config_file('file_path', 'section', 'option')

    @patch('subprocess.getstatusoutput')
    def test_get_local_ip(self, result):
        result.return_value = (0, 'eth0 : inet aa 10.30.30.30 bb eth1: cc dd 10.10.10.10')
        ret = CommonTools.get_local_ip(ignore=True)
        assert ret == '127.0.0.1'
        result.return_value = {0, """eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.30.30.30  netmask 255.255.254.0"""}
        ret = CommonTools.get_local_ip(ignore=True)
        assert ret == '10.30.30.30'

    @patch("ai_manager.tools.common_tools.CommonTools.chmod_files_with_execute_permission")
    @patch("platform.machine")
    @patch('ai_manager.tools.common_tools.CommonTools.get_status_output_error')
    def test_encrypt_with_path(self, result, plat, execute):
        with pytest.raises(Exception, match=r"Unsupported operating system"):
            CommonTools.encrypt_with_path('password', 'path')
        plat.return_value = 'x86_64'
        result.return_value = (1, 'a')
        with pytest.raises(Exception, match=r"Failed to encrypt random string"):
            CommonTools.encrypt_with_path('password', 'path')

    @patch('ai_manager.tools.common_tools.CommonTools.get_status_output_error')
    def test_chmod_files_with_execute_permission(self, result):
        result.return_value = (1, 'a')
        with pytest.raises(Exception, match=r"change file authority"):
            CommonTools.chmod_files_with_execute_permission(['file_list'])


# Editor: Khalid
"""This program plays a game of Rock, Paper, Scissors between two Players,
and reports both Player's scores each round."""
import random

moves = ['rock', 'paper', 'scissors']

"""The Player class is the parent class for all of the Players
in this game"""

# Main Class "Player"
class Player:
    def move(self):
        return 'rock'

    def learn(self, my_move, their_move):
        self.my_move = my_move
        self.their_move = their_move

    # Chick Who wins the game
    def beats(self, one, two):
        return ((one == 'rock' and two == 'scissors') or
                (one == 'scissors' and two == 'paper') or
                (one == 'paper' and two == 'rock'))

# subclasses starting with RandomPlayer
class RandomPlayer(Player):
    def move(self):
        return random.choice(moves)

# HumanPlayer subclass
class HumanPlayer(Player):
    def move(self):
        request = ""

        # loop till the valid move
        while request not in moves:
            request = input("Rock, paper, scissors? >").lower()

        return request

# ReflectPlayer subclass
class ReflectPlayer(Player):
    # reflect = "rock"
    reflect = random.choice(moves)

    # selecting the last move
    def move(self):
        return self.reflect

    # saving the last move
    def learn(self, my_move, their_move):
        self.reflect = their_move

class CyclePlayer(Player):
    # cycle = 'rock'
    cycle = random.choice(moves)

    def move(self):
        return self.cycle

    # doing cycle you can do it with an array, but its only 3 elements!
    def learn(self, my_move, their_move):
        if my_move == 'rock':
            self.cycle = 'paper'
        elif my_move == 'paper':
            self.cycle = 'scissors'
        elif my_move == 'scissors':
            self.cycle = 'rock'

class Game:
    # Initial values and scores
    def __init__(self, p1, p2):
        self.p1 = p1
        self.p2 = p2
        self.p1_score = 0
        self.p2_score = 0

    # Stat new round!
    def play_round(self):
        move1 = self.p1.move()
        move2 = self.p2.move()
        # print(f"Player 1: {move1}  Player 2: {move2}")
        self.p1.learn(move1, move2)
        self.p2.learn(move2, move1)

        # Loop as long as it's not None, no need for more conditionals check
        # HumanPlayer move method
        if move1 is not None:
            print(f"You played {move1}.\nOpponent played {move2}.")

            # Determine the scores using beats method
            if self.p1.beats(move1, move2):  # or is True
                print("** PLAYER ONE WINS **")
                # print(f"This Round : Player 1 WIN")
                self.p1_score += 1
                print(f"Score: Player One {self.p1_score}, "
                      f"Player Two {self.p2_score}")

            elif self.p2.beats(move2, move1):  # or is True
                print("** PLAYER TWO WINS **")
                # print(f"This Round : Player 2 WIN")
                self.p2_score += 1
                print(f"Score: Player One {self.p1_score}, "
                      f"Player Two {self.p2_score}")

            else:
                print("** TIE **")
                print(f"Score: Player One {self.p1_score}, "
                      f"Player Two {self.p2_score}")

    # Game start point!
    def play_game(self):
        # print("Game start!")
        print("Rock Paper Scissors, Go!")
        for round in range(8):
            print(f"\nRound {round} --")
            self.play_round()
        # print("Game over!")

        # Last required thing: At the end of the game, have it print out which
        # player won, and what the final scores are.
        if self.p1_score > self.p2_score:
            print("\nPlayer 1 is the winner ^_^")
        elif self.p1_score < self.p2_score:
            print("Player 2 is the winner ^_^")
        else:
            print("Its Tie >_<")
        print(f"Final score: Player One {self.p1_score}, "
              f"Player Two {self.p2_score}")

# Main
if __name__ == '__main__':
    game = Game(HumanPlayer(), ReflectPlayer())
    game.play_game()

# pycodestyle RPS.py 100% done!
# End! clone on May 10, 2020


# Lesson index: https://pandas.pydata.org/pandas-docs/stable/tutorials.html under Lessons for new pandas users
# Pandas exercises https://nbviewer.jupyter.org/urls/bitbucket.org/hrojas/learn-pandas/raw/master/lessons/07%20-%20Lesson.ipynb

# Configuration
## Import Libraries
from pprint import pprint
import pandas as pd
import sys

## Check versions
print('Python version ' + sys.version)
print('Pandas version ' + pd.__version__)

# Prepare Data

## Create dataframe with dates as index
States = ['NY', 'NY', 'NY', 'NY', 'FL', 'FL', 'GA', 'GA', 'FL', 'FL']
data = [1.0, 2, 3, 4, 5, 6, 7, 8, 9, 10]
idx = pd.date_range('1/1/2012', periods=10, freq='MS')
df1 = pd.DataFrame(data, index=idx, columns=['Revenue'])
df1['State'] = States
df1

## Create a second dataframe
data2 = [10.0, 10.0, 9, 9, 8, 8, 7, 7, 6, 6]
idx2 = pd.date_range('1/1/2013', periods=10, freq='MS')
df2 = pd.DataFrame(data2, index=idx2, columns=['Revenue'])
df2['State'] = States

## Combine the data frames
df = pd.concat([df1,df2])
df
len(df)

# Calculating Outliers
## NOTE: AVG and StDev only work for Normal distribution
## Method 1

### make a copy of original df
newdf = df.copy()

### layer in the distance from average of revenue into new column
newdf['x-Mean'] = abs(newdf['Revenue'] - newdf['Revenue'].mean())

### derive the threshold for statistical outlier: 1.96*standard deviation of the revenue
newdf['1.96*std'] = 1.96*newdf['Revenue'].std()

### determine if mean of each month's revenue is more than 1.96 * standard deviations
newdf['Outlier'] = abs(newdf['Revenue'] - newdf['Revenue'].mean()) > 1.96*newdf['Revenue'].std()

###
newdf

## Method 2
### Group by item, in this case the state

### make a copy of original df
newdf = df.copy()

### group by the state column
State = newdf.groupby('State')

### use of lambda function to determine if calculated distance from mean is greater than 1.96*the standard deviation of the state's revenue
newdf['Outlier'] = State.transform( lambda x: abs(x-x.mean()) > 1.96*x.std() )
### NOTE: the bottom two functions are unncessary, as the clever use of a lambda does the entire necessary function in a single pass

### determine the mean
newdf['x-Mean'] = State.transform( lambda x: abs(x-x.mean()) )

### determine the standard deviation
newdf['1.96*std'] = State.transform( lambda x: 1.96*x.std() )

newdf

## Method 2.5
### Group by multiple items

### make a copy of original df
newdf = df.copy()

### group by state and month
StateMonth = newdf.groupby(['State', lambda x: x.month])

### calculate if the month is an outlier using same lambda as Method 2 above
newdf['Outlier'] = StateMonth.transform( lambda x: abs(x-x.mean()) > 1.96*x.std() )

### calculation methods for supporitng data, but unecessary
newdf['x-Mean'] = StateMonth.transform( lambda x: abs(x-x.mean()) )
newdf['1.96*std'] = StateMonth.transform( lambda x: 1.96*x.std() )
newdf

## Method 3
## Group by item

### make a copy of original df
newdf = df.copy()

### group by the a particular item
State = newdf.groupby('State')

### define a function taking the input of calling .apply object (.apply applies a specific custom function - weird but clever)
### NOTE: Examples of .apply(s)
def s(group):
    group['x-Mean'] = abs(group['Revenue'] - group['Revenue'].mean())
    group['1.96*std'] = 1.96*group['Revenue'].std()
    group['Outlier'] = abs(group['Revenue'] - group['Revenue'].mean()) > 1.96*group['Revenue'].std()
    return group

Newdf2 = State.apply(s)
Newdf2

## Method 3.5
## Group by multiple items

### make a copy of original df
newdf = df.copy()

### apply group by function
StateMonth = newdf.groupby(['State', lambda x: x.month])

### another custom function to be applied using .apply call
def s(group):
    group['x-Mean'] = abs(group['Revenue'] - group['Revenue'].mean())
    group['1.96*std'] = 1.96*group['Revenue'].std()
    group['Outlier'] = abs(group['Revenue'] - group['Revenue'].mean()) > 1.96*group['Revenue'].std()
    return group

### apply custom function
Newdf2 = StateMonth.apply(s)
Newdf2

## Method 4: Find outliers Assuming a non-gaussian distribution:
### make a copy of original df
newdf = df.copy()

### apply the group by
State = newdf.groupby('State')

### calculate upper/lower bounds of revenue using quantiles
newdf['Lower'] = State['Revenue'].transform( lambda x: x.quantile(q=.25) - (1.5*(x.quantile(q=.75)-x.quantile(q=.25))) )
newdf['Upper'] = State['Revenue'].transform( lambda x: x.quantile(q=.75) + (1.5*(x.quantile(q=.75)-x.quantile(q=.25))) )
newdf['Outlier'] = (newdf['Revenue'] < newdf['Lower']) | (newdf['Revenue'] > newdf['Upper'])
newdf


import re
import warnings

from django.conf import settings
from django.contrib.sites.models import Site
from django.utils.cache import patch_vary_headers

from subdomains.exceptions import IncorrectSiteException

class SubdomainMiddleware(object):
    def process_request(self, request):
        """
        Adds a `subdomain` attribute to the request object, which corresponds
        to the portion of the URL before the current Site object's `domain`
        attribute.
        """
        site = Site.objects.get_current()
        domain = site.domain

        # To allow for case-insensitive comparison, force the site.domain and
        # the HTTP Host to lowercase.
        domain, host = domain.lower(), request.get_host().lower()

        REMOVE_WWW_FROM_DOMAIN = getattr(settings, 'REMOVE_WWW_FROM_DOMAIN',
            False)
        if REMOVE_WWW_FROM_DOMAIN and domain.startswith("www."):
            domain = domain.replace("www.", "", 1)

        pattern = r'^(?:(?P<subdomain>.*?)\.)?%s(?::.*)?$' % re.escape(domain)
        matches = re.match(pattern, host)

        request.subdomain = None

        if matches:
            request.subdomain = matches.group('subdomain')
        else:
            error = 'The current host, %s, does not match the Site instance ' \
                'specified by SITE_ID.' % request.get_host()

            if getattr(settings, 'USE_SUBDOMAIN_EXCEPTION', False):
                raise IncorrectSiteException(error)
            else:
                warnings.warn('%s The URLconf for this host will fall back to '
                    'the ROOT_URLCONF.' % error, UserWarning)

        # Continue processing the request as normal.
        return None

class SubdomainURLRoutingMiddleware(SubdomainMiddleware):
    def process_request(self, request):
        """
        Sets the current request's `urlconf` attribute to the URL conf
        associated with the subdomain, if listed in `SUBDOMAIN_URLCONFS`.
        """
        super(SubdomainURLRoutingMiddleware, self).process_request(request)

        subdomain = getattr(request, 'subdomain', False)

        if subdomain is not False:
            try:
                request.urlconf = settings.SUBDOMAIN_URLCONFS[subdomain]
            except KeyError:
                # There was no match in the SUBDOMAIN_URLCONFS setting, so let
                # the ROOT_URLCONF handle the URL routing for this request.
                pass

        # Continue processing the request as normal.
        return None

    def process_response(self, request, response):
        """
        Forces the HTTP Vary header onto requests to avoid having responses
        cached from incorrect urlconfs.

        If you'd like to disable this for some reason, set `FORCE_VARY_ON_HOST`
        in your Django settings file to `False`.
        """
        if getattr(settings, 'FORCE_VARY_ON_HOST', True):
            patch_vary_headers(response, ('Host',))

        return response