from datetime import datetime

from bottle import get, post, request, abort
from peewee import JOIN

from bomber.auth import check_api_user
from bomber.constant_mapping import (
    CallActionType,
    AutoListStatus,
    SpecialBomber,
    Cycle
)
from bomber.plugins import ip_whitelist_plugin
from bomber.db import db
from bomber.models import (
    ApplicationStatus,
    BombingHistory,
    AutoCallList,
    Application,
    CallActions,
    Bomber,
)
from bomber.serializers import (
    bombing_history_serializer,
    call_history_serializer,
)
from bomber.validator import collection_validator, cs_ptp_validator
from bomber.controllers.asserts import set_ptp_for_special_bomber
from bomber.utils import get_cycle_by_overdue_days

@get('/api/v1/applications/<app_id:int>/collection_history')
def get_collection_history(bomber, application):
    result = (CallActions.select(CallActions, Bomber)
              .join(Bomber, JOIN.INNER, on=(CallActions.bomber_id == Bomber.id)
                    .alias('operator'))
              .where(CallActions.application == application.id)
              .order_by(-CallActions.created_at))
    return call_history_serializer.dump(result, many=True).data

@get('/api/v1/applications/<app_id:int>/user_collection_history')
def get_user_collection_history(bomber, application):
    bombing_history = BombingHistory.filter(
        BombingHistory.application != application.id,
        BombingHistory.user_id == application.user_id,
    ).order_by(-BombingHistory.created_at)
    return bombing_history_serializer.dump(bombing_history, many=True).data

@post('/api/v1/applications/<app_id:int>/collection_history')
def add_collection_history(bomber, application):
    form = collection_validator(request.json)
    with db.atomic():
        promised_date = (form['promised_date']
                         if 'promised_date' in form else None)
        remark = form['remark'] if 'remark' in form else None
        promised_amount = (form['promised_amount']
                           if 'promised_amount' in form
                           else None)
        if promised_date:
            real_cycle = get_cycle_by_overdue_days(application.overdue_days)
            if real_cycle > application.cycle:
                abort(400, 'Can not extend PTP')
        bombing_history = BombingHistory.create(
            application=application.id,
            user_id=application.user_id,
            ektp=application.id_ektp,
            cycle=application.cycle,
            bomber=bomber.id,
            promised_amount=promised_amount,
            promised_date=promised_date,
            follow_up_date=form['follow_up_date'],
            result=form['result'],
            remark=remark,
        )
        CallActions.create(
            cycle=application.cycle,
            bomber_id=bomber.id,
            call_record_id=bombing_history.id,
            application=application.id,
            note=remark,
            promised_amount=promised_amount,
            promised_date=promised_date,
            follow_up_date=form['follow_up_date']
        )
        if 'promised_amount' in form:
            application.promised_amount = form['promised_amount']
        if 'promised_date' in form:
            if bomber.id == SpecialBomber.OLD_APP_BOMBER.value:
                set_ptp_for_special_bomber(application.id, promised_date)
            if application.cycle == Cycle.C1A.value:
                # 1a不允许员工续p
                if (application.promised_date and
                    application.promised_date.date() >= datetime.now().date()):
                    abort(400, "Can not extend PTP")
                # 1a p过期件给新下p的人
                if (application.promised_date and
                    application.promised_date.date() < datetime.now().date()):
                    application.latest_bomber = bomber.id
            # 下p时没有latest_bomber，件分给下p的人
            if not application.latest_bomber:
                application.latest_bomber = bomber.id
            application.promised_date = form['promised_date']
            application.latest_call = bomber.id
            application.ptp_bomber = bomber.id
            application.status = ApplicationStatus.PROCESSING.value
            if form['promised_date'] >= datetime.today().date():
                update_auto_call_list = (
                    AutoCallList
                    .update(status=AutoListStatus.REMOVED.value,
                            description='set ptp')
                    .where(AutoCallList.application == application.id)
                )
                update_auto_call_list.execute()

        application.follow_up_date = form['follow_up_date']

        # 更新自动呼出队列
        update_auto_call_list = (
            AutoCallList
            .update(follow_up_date=form['follow_up_date'],
                    description='set followed up date')
            .where(AutoCallList.application == application.id)
        )
        update_auto_call_list.execute()

        application.latest_bombing_time = datetime.now()
        application.save()
    return bombing_history_serializer.dump(bombing_history).data

@post('/api/v1/applications/<app_id:int>/cs-ptp', skip=[ip_whitelist_plugin])
def add_collection_history(app_id):
    check_api_user()
    form = cs_ptp_validator(request.json)

    application = (Application
                   .filter(Application.external_id == app_id,
                           Application.status != ApplicationStatus.REPAID.value)
                   .first())

    if not application:
        # 如果这个件 在 cs ptp 的时候还没有进入催收 则 直接添加记录
        bombing_history = BombingHistory.create(
            application=app_id,
            bomber=72,
            promised_date=form['promised_date'],
            follow_up_date=form['promised_date'],
        )
        return bombing_history_serializer.dump(bombing_history).data

    with db.atomic():
        bombing_history = BombingHistory.create(
            application=application.id,
            user_id=application.user_id,
            ektp=application.id_ektp,
            cycle=application.cycle,
            bomber=72,
            promised_date=form['promised_date'],
            follow_up_date=form['promised_date'],
        )
        CallActions.create(
            cycle=application.cycle,
            bomber_id=72,
            call_record_id=bombing_history.id,
            application=application.id,
            promised_date=form['promised_date'],
            follow_up_date=form['promised_date'],
        )
        application.promised_date = form['promised_date']

        if form['promised_date'] >= datetime.today().date():
            update_auto_call_list = (
                AutoCallList
                .update(status=AutoListStatus.REMOVED.value,
                        description='set ptp')
                .where(AutoCallList.application == application.id)
            )
            update_auto_call_list.execute()

        application.follow_up_date = form['promised_date']

        # 更新自动呼出队列
        update_auto_call_list = (
            AutoCallList
            .update(follow_up_date=form['promised_date'],
                    description='set followed up date')
            .where(AutoCallList.application == application.id)
        )
        update_auto_call_list.execute()
        if not application.latest_bomber:
            application.latest_bomber = 72
        if application.status == ApplicationStatus.UNCLAIMED.value:
            application.status = ApplicationStatus.PROCESSING.value
        application.save()
    return bombing_history_serializer.dump(bombing_history).data


# pylint: disable=C0321,C0103,C0301,E1305,E1121,C0302,C0330,C0111,W0613,W0611,R1705
# -*- coding: utf-8 -*-
"""
https://github.com/arita37/pytorch_tabular

python torch_tabular.py test

The core model which orchestrates everything from initializing the datamodule, the model, trainer, etc.
Args:
    config (Optional[Union[DictConfig, str]], optional): Single OmegaConf DictConfig object or
        the path to the yaml file holding all the config parameters. Defaults to None.
    data_config (Optional[Union[DataConfig, str]], optional): DataConfig object or path to the yaml file. Defaults to None.
    model_config (Optional[Union[ModelConfig, str]], optional): A subclass of ModelConfig or path to the yaml file.
        Determines which model to run from the type of config. Defaults to None.
    optimizer_config (Optional[Union[OptimizerConfig, str]], optional): OptimizerConfig object or path to the yaml file.
        Defaults to None.
    trainer_config (Optional[Union[TrainerConfig, str]], optional): TrainerConfig object or path to the yaml file.
        Defaults to None.
    experiment_config (Optional[Union[ExperimentConfig, str]], optional): ExperimentConfig object or path to the yaml file.
        If Provided configures the experiment tracking. Defaults to None.
    model_callable (Optional[Callable], optional): If provided, will override the model callable that will be loaded from the config.
        Typically used when providing Custom Models

"""
import os, sys,  numpy as np,  pandas as pd

try :
    from pytorch_tabular import TabularModel
    from pytorch_tabular.models import CategoryEmbeddingModelConfig
    from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig
except :
    os.system("pip install pytorch_tabular[all]")

# torch.manual_seed(0)
# np.random.seed(0)
# torch.set_deterministic(True)
# from torch.utils import data
from sklearn.model_selection import train_test_split
from pathlib import Path
import wget

####################################################################################################
VERBOSE = False

def log(*s):
    print(*s, flush=True)

####################################################################################################
global model, session

def init(*kw, **kwargs):
    global model, session
    model = Model(*kw, **kwargs)
    session = None

class Model(object):
    def __init__(self, model_pars=None, data_pars=None, compute_pars=None):
        self.model_pars, self.compute_pars, self.data_pars = model_pars, compute_pars, data_pars

        if model_pars is None:
            self.model = None

        else:
            ###############################################################
            dm          = data_pars['cols_model_group_custom']
            data_config = DataConfig(
              target           = dm['coly'], #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
              continuous_cols  = dm['colnum'],
              categorical_cols = dm['colcat'],
            )

            model_config     = CategoryEmbeddingModelConfig( **model_pars['model_pars'],   )
            trainer_config   = TrainerConfig( **compute_pars.get('compute_pars', {} ) )
            optimizer_config = OptimizerConfig()

            self.config_pars = { 'data_config' : data_config,
                        'model_config'     : model_config,
                        'optimizer_config' : optimizer_config,
                        'trainer_config'   : trainer_config,
            }

            self.model = TabularModel(**self.config_pars)
            self.guide = None
            self.pred_summary = None  ### All MC summary

            if VERBOSE: log(self.guide, self.model)

def fit(data_pars=None, compute_pars=None, out_pars=None, **kw):
    """
    """
    global model, session
    session = None  # Session type for compute

    # if data_pars is not None :
    Xtrain, ytrain, Xtest, ytest = get_dataset(data_pars, task_type="train")

    if VERBOSE: log(Xtrain, model.model)

    #Xtrain = torch.tensor(Xtrain.values, dtype=torch.float)
    #Xtest  = torch.tensor(Xtest.values, dtype=torch.float)
    #ytrain = torch.tensor(ytrain.values, dtype=torch.float)
    #ytest  = torch.tensor(ytest.values, dtype=torch.float)

    train = pd.concat((Xtrain,ytrain))
    val   = pd.concat((Xtest,ytest))

    ###############################################################
    compute_pars2 = compute_pars.get('compute_pars', {})

    model.model.fit(train=train, validation=val)

    #############################################################

def predict(Xpred=None, data_pars={}, compute_pars=None, out_pars={}, **kw):
    global model, session

    compute_pars2 = model.compute_pars if compute_pars is None else compute_pars
    num_samples   = compute_pars2.get('num_samples', 300)

    ###### Data load
    if Xpred is None:
        Xpred = get_dataset(data_pars, task_type="predict")
    cols_Xpred = list(Xpred.columns)

    max_size = compute_pars2.get('max_size', len(Xpred))
    Xpred    = Xpred.iloc[:max_size, :]

    # Xpred_   = torch.tensor(Xpred.values, dtype=torch.float)

    ###### Post processing normalization
    post_process_fun = model.model_pars.get('post_process_fun', None)
    if post_process_fun is None:
        def post_process_fun(y):
            return y

    #####################################################################
    ypred = model.model.predict(Xpred)

    ypred = ypred #### Nornalized

    ypred_proba = None  ### No proba
    if compute_pars.get("probability", False):
         ypred_proba = model.model.predict_proba(Xpred)
    return ypred, ypred_proba

def reset():
    global model, session
    model, session = None, None

def save(path=None, info=None):
    """
       Custom saving
    """

    ""
    global model, session
    import cloudpickle as pickle
    os.makedirs(path, exist_ok=True)

    filename = "model.pkl"
    model.model.save_model(f"{path}/filename")
    pickle.dump(model, open(f"{path}/{filename}", mode='wb'))  # , protocol=pickle.HIGHEST_PROTOCOL )

    model.model.save_model(f"{path}/torch_checkpoint/")

    filename = "info.pkl"
    pickle.dump(info, open(f"{path}/{filename}", mode='wb'))  # ,protocol=pickle.HIGHEST_PROTOCOL )

def load_model(path=""):
    global model, session
    import cloudpickle as pickle
    #model0 = pickle.load(open(f"{path}/model.pkl", mode='rb'))

    model = Model()  # Empty model
    model.model_pars   = model0.model_pars
    model.compute_pars = model0.compute_pars
    model.data_pars    = model0.data_pars

    model.model        = model0.model.load_from_checkpoint(f"{path}/")
    session = None
    return model, session

def load_info(path=""):
    import cloudpickle as pickle, glob
    dd = {}
    for fp in glob.glob(f"{path}/*.pkl"):
        if not "model.pkl" in fp:
            obj = pickle.load(open(fp, mode='rb'))
            key = fp.split("/")[-1]
            dd[key] = obj
    return dd

def preprocess(prepro_pars):
    if prepro_pars['type'] == 'test':
        from sklearn.datasets import make_classification
        from sklearn.model_selection import train_test_split

        X, y = make_classification(n_features=10, n_redundant=0, n_informative=2,
                                   random_state=1, n_clusters_per_class=1)

        # log(X,y)
        Xtrain, Xtest, ytrain, ytest = train_test_split(X, y)
        return Xtrain, ytrain, Xtest, ytest

    if prepro_pars['type'] == 'train':
        from sklearn.model_selection import train_test_split
        df = pd.read_csv(prepro_pars['path'])
        dfX = df[prepro_pars['colX']]
        dfy = df[prepro_pars['coly']]
        Xtrain, Xtest, ytrain, ytest = train_test_split(dfX.values, dfy.values)
        return Xtrain, ytrain, Xtest, ytest

    else:
        df = pd.read_csv(prepro_pars['path'])
        dfX = df[prepro_pars['colX']]

        Xtest, ytest = dfX, None
        return None, None, Xtest, ytest

####################################################################################################
############ Do not change #########################################################################
def get_dataset(data_pars=None, task_type="train", **kw):
    """
      "ram"  :
      "file" :
    """
    # log(data_pars)
    data_type = data_pars.get('type', 'ram')
    if data_type == "ram":
        if task_type == "predict":
            d = data_pars[task_type]
            return d["X"]

        if task_type == "eval":
            d = data_pars[task_type]
            return d["X"], d["y"]

        if task_type == "train":
            d = data_pars[task_type]
            return d["Xtrain"], d["ytrain"], d["Xtest"], d["ytest"]

    elif data_type == "file":
        raise Exception(f' {data_type} data_type Not implemented ')

    raise Exception(f' Requires  Xtrain", "Xtest", "ytrain", "ytest" ')

def get_params(param_pars={}, **kw):
    import json
    # from jsoncomment import JsonComment ; json = JsonComment()
    pp = param_pars
    choice = pp['choice']
    config_mode = pp['config_mode']
    data_path = pp['data_path']

    if choice == "json":
        cf = json.load(open(data_path, mode='r'))
        cf = cf[config_mode]
        return cf['model_pars'], cf['data_pars'], cf['compute_pars'], cf['out_pars']

    else:
        raise Exception(f"Not support choice {choice} yet")

def test2(nrow=10000):
    """
       python source/models/torch_tabular.py test

    """
    global model, session

    #X = np.random.rand(10000,20)
    #y = np.random.binomial(n=1, p=0.5, size=[10000])

    BASE_DIR = Path.home().joinpath('data/input/covtype/')
    datafile = BASE_DIR.joinpath('covtype.data.gz')
    datafile.parent.mkdir(parents=True, exist_ok=True)
    url = "https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz"
    if not datafile.exists():
        wget.download(url, datafile.as_posix())

    target_name = ["Covertype"]

    colcat = [ "Wilderness_Area1", "Wilderness_Area2", "Wilderness_Area3", "Wilderness_Area4", "Soil_Type1", "Soil_Type2", "Soil_Type3", "Soil_Type4", "Soil_Type5", "Soil_Type6", "Soil_Type7", "Soil_Type8", "Soil_Type9", "Soil_Type10", "Soil_Type11", "Soil_Type12", "Soil_Type13", "Soil_Type14", "Soil_Type15", "Soil_Type16", "Soil_Type17", "Soil_Type18", "Soil_Type19", "Soil_Type20", "Soil_Type21", "Soil_Type22", "Soil_Type23", "Soil_Type24", "Soil_Type25", "Soil_Type26", "Soil_Type27", "Soil_Type28", "Soil_Type29", "Soil_Type30", "Soil_Type31", "Soil_Type32", "Soil_Type33", "Soil_Type34", "Soil_Type35", "Soil_Type36", "Soil_Type37", "Soil_Type38", "Soil_Type39", "Soil_Type40"
                      ]

    colnum = [ "Elevation", "Aspect", "Slope", "Horizontal_Distance_To_Hydrology", "Vertical_Distance_To_Hydrology", "Horizontal_Distance_To_Roadways", "Hillshade_9am", "Hillshade_Noon", "Hillshade_3pm", "Horizontal_Distance_To_Fire_Points"
    ]

    feature_columns = (  colnum + colcat + target_name)

    df = pd.read_csv(datafile, header=None, names=feature_columns, nrows= nrows)

    df.head()
    train, test = train_test_split(df, random_state=42)
    train, val  = train_test_split(train, random_state=42)
    num_classes = len(set(train[target_name].values.ravel()))

    data_config = DataConfig(
        target=target_name,
        continuous_cols=colnum,
        categorical_cols=colcat,
        continuous_feature_transform=None,#"quantile_normal",
        normalize_continuous_features=False
    )
    model_config = CategoryEmbeddingModelConfig(task="classification",
                                                metrics=["f1","accuracy"],
                                                metrics_params=[{"num_classes":num_classes},{}])

    trainer_config = TrainerConfig(gpus=None, fast_dev_run=True)
    experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example",
                                         run_name="node_forest_cov",
                                         exp_watch="gradients",
                                         log_target="wandb",
                                         log_logits=True)
    optimizer_config = OptimizerConfig()

    tabular_model = TabularModel(
        data_config=data_config,
        model_config=model_config,
        optimizer_config=optimizer_config,
        trainer_config=trainer_config,
        # experiment_config=experiment_config,
    )

    tabular_model.fit(  train=train, validation=val)
    result = tabular_model.evaluate(val)
    log(result)

    test.drop(columns=target_name, inplace=True)
    pred_df = tabular_model.predict(val.iloc[:100,:])

    log(pred_df)
    # pred_df.to_csv("output/temp2.csv")
    # tabular_model.save_model("test_save")
    # new_model = TabularModel.load_from_checkpoint("test_save")
    # result = new_model.evaluate(test)

def test(nrows=1000):

    log("start")
    global model, session

    #X = np.random.rand(10000,20)
    #y = np.random.binomial(n=1, p=0.5, size=[10000])
    root = os.getcwd()  +"/ztmp/"

    BASE_DIR = Path.home().joinpath( root + 'data/input/covtype/')
    datafile = BASE_DIR.joinpath('covtype.data.gz')
    datafile.parent.mkdir(parents=True, exist_ok=True)
    url = "https://archive.ics.uci.edu/ml/machine-learning-databases/covtype/covtype.data.gz"
    if not datafile.exists():
        wget.download(url, datafile.as_posix())

    target_name = ["Covertype"]

    colcat = [ "Wilderness_Area1", "Wilderness_Area2", "Wilderness_Area3", "Wilderness_Area4", "Soil_Type1", "Soil_Type2", "Soil_Type3", "Soil_Type4", "Soil_Type5", "Soil_Type6", "Soil_Type7", "Soil_Type8", "Soil_Type9", "Soil_Type10", "Soil_Type11", "Soil_Type12", "Soil_Type13", "Soil_Type14", "Soil_Type15", "Soil_Type16", "Soil_Type17", "Soil_Type18", "Soil_Type19", "Soil_Type20", "Soil_Type21", "Soil_Type22", "Soil_Type23", "Soil_Type24", "Soil_Type25", "Soil_Type26", "Soil_Type27", "Soil_Type28", "Soil_Type29", "Soil_Type30", "Soil_Type31", "Soil_Type32", "Soil_Type33", "Soil_Type34", "Soil_Type35", "Soil_Type36", "Soil_Type37", "Soil_Type38", "Soil_Type39", "Soil_Type40"
                      ]

    colnum = [ "Elevation", "Aspect", "Slope", "Horizontal_Distance_To_Hydrology", "Vertical_Distance_To_Hydrology", "Horizontal_Distance_To_Roadways", "Hillshade_9am", "Hillshade_Noon", "Hillshade_3pm", "Horizontal_Distance_To_Fire_Points"
    ]

    feature_columns = (  colnum + colcat + target_name)

    df = pd.read_csv(datafile, header=None, names=feature_columns, nrows=1000)

    ####### Using API
    X = df
    y = df[target_name].astype('uint8')
    log('y', np.sum(y[y==1]) )

    X_train_full, X_test, y_train_full, y_test = train_test_split(X, y, test_size=0.05, random_state=2021, stratify=y)
    X_train, X_valid, y_train, y_valid         = train_test_split(X_train_full, y_train_full, random_state=2021, stratify=y_train_full)
    num_classes = len(set(y_train_full[target_name].values.ravel()))
    log(X_train)

    cols_input_type_1 = []
    n_sample = 100
    def post_process_fun(y):
        return int(y)

    def pre_process_fun(y):
        return int(y)

    m = {'model_pars': {
        ### LightGBM API model   #######################################
         'model_class':  'torch_tabular.py::model'
        ,'model_pars' : { 'task': "classification",
                          'metrics' : ["f1","accuracy"],
                          'metrics_params' : [{"num_classes":num_classes},{}]
                        }

        , 'post_process_fun' : post_process_fun   ### After prediction  ##########################################
        , 'pre_process_pars' : {'y_norm_fun' :  pre_process_fun ,  ### Before training  ##########################

        ### Pipeline for data processing ##############################
        'pipe_list': [  #### coly target prorcessing
        {'uri': 'source/prepro.py::pd_coly',                 'pars': {}, 'cols_family': 'coly',       'cols_out': 'coly',           'type': 'coly'         },

        {'uri': 'source/prepro.py::pd_colnum_bin',           'pars': {}, 'cols_family': 'colnum',     'cols_out': 'colnum_bin',     'type': ''             },
        {'uri': 'source/prepro.py::pd_colnum_binto_onehot',  'pars': {}, 'cols_family': 'colnum_bin', 'cols_out': 'colnum_onehot',  'type': ''             },

        #### catcol INTO integer,   colcat into OneHot
        {'uri': 'source/prepro.py::pd_colcat_bin',           'pars': {}, 'cols_family': 'colcat',     'cols_out': 'colcat_bin',     'type': ''             },
        {'uri': 'source/prepro.py::pd_colcat_to_onehot',     'pars': {}, 'cols_family': 'colcat_bin', 'cols_out': 'colcat_onehot',  'type': ''             },

        ],
               }
        },

      'compute_pars': { 'metric_list': ['accuracy_score','average_precision_score']
                        ,'mlflow_pars' : {}   ### Not empty --> use mlflow
                      },

      'data_pars': { 'n_sample' : n_sample,

          'download_pars' : None,

          'cols_input_type' : cols_input_type_1,
          ### family of columns for MODEL  #########################################################
          'cols_model_group': [ 'colnum_bin',
                                'colcat_bin',
                              ]

          ,'cols_model_group_custom' :  { 'colnum' : colnum,
                                         'colcat' : colcat,
                                         'coly' : target_name
                              }

          ,'train': {'Xtrain': X_train,
                           'ytrain': y_train,
                           'Xtest': X_valid,
                           'ytest': y_valid},
                 'eval': {'X': X_valid,
                          'y': y_valid},
                 'predict': {'X': X_valid}

          ### Filter data rows   ##################################################################
         ,'filter_pars': { 'ymax' : 2 ,'ymin' : -1 }

         }
      }

    log('Setup model..')
    model = Model(model_pars=m['model_pars'], data_pars=m['data_pars'], compute_pars= m['compute_pars'] )

    log('\n\nTraining the model..')
    fit(data_pars=m['data_pars'], compute_pars= m['compute_pars'], out_pars=None)
    log('Training completed!\n\n')

    log('Predict data..')
    ypred, ypred_proba = predict(Xpred=None, data_pars=m['data_pars'], compute_pars=m['compute_pars'])
    log(f'Top 5 y_pred: {np.squeeze(ypred)[:5]}')

    log('Evaluating the model..')
    # log(eval(data_pars= m['data_pars'], compute_pars= m['compute_pars']))

    log('Saving model..')
    save(path= root + 'data/output/torch_tabular/model/')

    log('Load model..')
    model, session = load_model(path="model_dir/")

    log('Model architecture:')
    log(model)

def test3():
    pass

if __name__ == "__main__":
    import fire
    fire.Fire()


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, url

import views as wiki_views

class Patterns(object):
	def __init__(self):
		self.app_name = 'wiki'
		self.name = 'wiki'

	@property
	def urls(self):
		urlpatterns = patterns('wiki.views',
			url(r'^$', wiki_views.show_page, name = 'home'),
			url(r'(?P<slug>[-\w]+)/create/$', wiki_views.PageCreateView.as_view(), {'create': True}, name = 'create'),
			url(r'(?P<slug>[-\w]+)/edit/$', wiki_views.PageUpdateView.as_view(), {'create': False}, name = 'edit'),
			url(r'(?P<slug>[-\w]+)/history/(?P<history>\d+)/$', wiki_views.show_page, name = 'page-history'),
			url(r'(?P<slug>[-\w]+)/history/(?P<history>\d+)/(?P<page>\d+)/$', wiki_views.show_page, name = 'page-history-page'),
			url(r'(?P<slug>[-\w]+)/(?:(?P<page>\d+)/)?$', wiki_views.show_page, name = 'page'),
		)
		return (urlpatterns, self.app_name, self.name)

urlpatterns = Patterns().urls


# -*- coding: utf-8 -*-
import psycopg2
from create_filmdb.items import awardsItem,castingItem,filmographyItem
from os import environ

con = psycopg2.connect("host={} port={} dbname={} user={} password={}".format(environ['POSTGRES_HOST'],environ['POSTGRES_PORT'],environ['POSTGRES_DBNAME'],environ['POSTGRES_USER'],environ['POSTGRES_PASS']))
cur = con.cursor()

class CreateFilmdbPipeline(object):

    def getActorId(self,name):
        cur.execute("SELECT * FROM actors WHERE name = '{}'".format(name))
        if cur.rowcount == 0:
            cur.execute("INSERT INTO actors (name) VALUES ('{}')".format(name))
            con.commit()
            cur.execute("SELECT * FROM actors WHERE name = '{}'".format(name))
            return cur.fetchone()[0]
        else:
            return cur.fetchone()[0]

    def getFilmId(self,title):
        cur.execute("SELECT * FROM films WHERE title = '{}'".format(title))
        if cur.rowcount == 0:
            cur.execute("INSERT INTO films (title) VALUES ('{}')".format(title))
            con.commit()
            cur.execute("SELECT * FROM films WHERE title = '{}'".format(title))
            return cur.fetchone()[0]
        else:
            return cur.fetchone()[0]

    def process_item(self, item, spider):
        if isinstance(item,awardsItem):
            if "'" in item['awards']['title']:
                return item
            year  = item['awards']['year']
            award = item['awards']['award']
            title = item['awards']['title']
            name  = item['awards']['name']
            win   = item['awards']['win']
            film_id  = self.getFilmId(title)
            if name is not None:
                actor_id = self.getActorId(name)
            else:
                actor_id = 0
            cur.execute("insert into awards (year,award,film_id,actor_id,win) values ('%s','%s','%s','%s','%s')" % (year,award,film_id,actor_id,win))
            con.commit()
            return item

        elif isinstance(item,castingItem):
            title = item['casting']['title']
            name  = item['casting']['name']
            role  = item['casting']['role']
            film_id  = self.getFilmId(title)
            actor_id = self.getActorId(name)
            cur.execute("insert into casting (film_id,actor_id,role) values ('%s','%s','%s')" % (film_id,actor_id,role))
            con.commit()
            return item

        elif isinstance(item,filmographyItem):
            return item
        con.close()
        cur.close()


import json
import os

from lEmoji.generator.web_extractor import WebExtractor
from lEmoji.data.tree import tree

class Generator:
    def __init__(self, proxies=None):
        self.extractor = WebExtractor(proxies=proxies)
        self.tree = self.extractor.build()
        self.working_dir = os.path.join(os.getcwd(), 'lEmoji')
        self.data_file = os.path.join(os.getcwd(), 'lEmoji', 'data', 'tree.py')

    @staticmethod
    def find_node(node_list, name):
        for node in node_list:
            if node['name'] == name:
                return node

    def generate(self, his_nodes=None, cur_nodes=None, version=None):
        if not version:
            his_nodes = tree
            cur_nodes = self.tree
        else:
            version = version.replace('.', '_')

        emoji_list = []

        for node in cur_nodes:
            his_node = self.find_node(his_nodes, node['name'])
            if node['type'] == 'folder':
                if not his_node:
                    self.generate([], node['children'], version=node['name'])
                elif node['time'] != his_node['time']:
                    self.generate(his_node['children'], node['children'], version=node['name'])
            elif node['name'].startswith('emoji'):
                emoji_list.extend(self.extractor.emoji_fetcher(node['link']))

        if version:
            path = os.path.join(self.working_dir, 'emoji_v%s.py' % version)
            emoji_list = list(set(emoji_list))
            with open(path, 'wb+') as f:
                s = 'EMOJI_LIST = ' + json.dumps(emoji_list, ensure_ascii=False)
                f.write(s.encode())
        else:
            with open(self.data_file, 'wb+') as f:
                s = 'tree = ' + json.dumps(cur_nodes, ensure_ascii=False) + '\n'
                f.write(s.encode())
            with open(os.path.join(self.working_dir, '__init__.py'), 'wb+') as f:
                for node in cur_nodes:
                    name = node['name'].replace('.', '_')
                    s = 'from .emoji_v%s import EMOJI_LIST as EMOJI_LIST_%s\n' % (
                        name, name.upper())
                    f.write(s.encode())

                s = '\n\nEMOJI_LIST = EMOJI_LIST_%s\n' % \
                    cur_nodes[-1]['name'].replace('.', '_').upper()
                f.write(s.encode())


import codecs
#This script parses the engdish gf-wordnet and outputs a csv-file containing the function and its status
with codecs.open('WordNetEng.gf', 'r', encoding='ISO-8859-2') as inputfile:
    with codecs.open('WordNetChecked.csv', 'w', encoding='ISO-8859-2') as outputfile:
        #This file must be run first of the python parsing files
        outputfile.writelines("function;status;langcode\n")

        for line in inputfile:
            if("lin" not in line):
                continue
            line = line[4:]
            function = line.split(" ")[0]
            if("--" not in line):
                #In English, if there is no comment then the word is considered checked
                outputfile.writelines("\"" + function + "\";\"checked\";\"en-US\"\n")
                continue
            if("--" in line):
                #If there is a comment in English, I am not sure of what to do with the word, so lets consider it guessed...
                outputfile.writelines("\"" + function + "\";\"guessed\";\"en-US\"\n")


import re
from fangjia.wuyuhang.spider.get_next_pagelist import get_next_pagelist
from fangjia.wuyuhang.spider.get_page import get_page
from fangjia.wuyuhang.scheduler.global_var import global_var
from fangjia.wuyuhang.spider.stack import stack

def get_city_all_url(whole_url):
    long_url_head=re.search(r'http:\D+\e-',whole_url).group(0)
    global_var.long_url_head=long_url_head
    shirt_url_tail=re.search(r'#\D+',whole_url).group(0)
    global_var.shirt_url_tail=shirt_url_tail
    g_var=global_var
    i=1
    while(i):
        whole_url=long_url_head+str(i)+shirt_url_tail
        get_page(whole_url)
        if(get_next_pagelist()==''):
            g_var.city_url_s.push(whole_url)
            break
        else:
            g_var.city_url_s.push(whole_url)
            i=i+1


""" Assignment for module0 in ITA19 COMPAS course
"""

__author__ = "Anton T Johansson"

import random
from typing import List
import pdb

def mul_matrix(matrix: List[float], multiplier: float) -> List[float]:
    return [i * multiplier for i in matrix]

def vector_on_plane(point: List[float],
                         normal: List[float]) -> List[float]:
    '''
    '''
    # plane equation
    # Ax + By + Cz = D

    # std equation plane (abc is normal, xyz1 is given point, xyz is vector)
    # a(x-x1) + b(y-y1) + c(z-z1) = 0

    # isolate one vector component
    # x = ( -b(y-y1) -c(z-z1) / a +x1

    a, b, c = normal
    x1, y1, z1 = point
    y, z = random.random(), random.random()

    x = (-b * (y - y1) - c * (z - z1)) / a + x1

    # https://www.youtube.com/watch?v=p_JItfXpGlQ

    return [x, y, z]

if __name__ == "__main__":

    print("Module0 bonus, by " + __author__ + "\n")
    print(
        "Returns vector that rests within plane defined by point on plane and plane's normal...\n"
    )

    input_point = input("Enter a point, three values separated by comma: ")
    input_normal = input(
        "Enter the normal vector, three values separated by comma: ")

    point = list(map(float, input_point.split(",")))
    normal = list(map(float, input_normal.split(",")))

    perp_vector = vector_on_plane(point, normal)

    pdb.set_trace()

    print("The vector ({}, {}, {}) rests within given plane".format(
        *perp_vector))


#!/usr/bin/python
#-*-coding:utf-8-*-
#4.0增加了random函数,优化UI
#5.0新增输入检测机制,开始采用isinstance，后来采用for in循环,后来采用def定义函数，不然代码过于冗长
#5.1新增次数提示功能
import random
answer = random.randint(1,9)
print('\n****游戏程序版本五****')
print('Build By Jason\n')
dic=('0','1','2','3','4','5','6','7','8','9')

#定义一个函数检测输入有效性
def check(temp):
	if temp  in dic:
		return True
	else:
		return False
#再定义一个函数，用于完整的检查
def test (x) :
	c = 3
	while not check(x) :
		c-=1
		if c==0:
			print('输入次数已超过三次，程序已停止')
			exit()
		else:
			print('请检查输入对象！还有',c,'次机会')
			temp=input('请输入一个个位数：')
#开始执行实体代码
temp=input('请输入一个个位数：')
test(temp)
guess = int(temp)
i=3
while  i>0 :
	if guess == answer :
		print('答对了，很强势！')
		break
	else:
		if guess >= answer :
			print('   整大了\n')
		else:
			print('   整小了\n')
	i-=1
	if i != 0 :
		print('你还有',i,'次机会')
		temp = input('请输入一个个位数: ')
		test
		guess = int(temp)
	else:
		break
print('game over')


import codecs
import tensorflow as tf
import numpy as np
import json
import re
from collections import Counter

#V3版本已经能够猜对5个左右的字符，猜测，可能是数据量不足，或者是参数空间不足够拟合DES的全部变换
#V4版本采用全新的数据集，共有13.4M的英文小说做成数据去拟合DES的全部映射参数
#V5版本是更换了数据集的制作方式，每4个英文字母加密一次，因为DES的分组密码特性，64位比特相当于两个英文字母
#V7版本的代码内容和V5一样，只是采用了师兄的建议，首先分析5000对明密文攻击的可行性，
#V8版本加了一个统计数据,统计错误出现的位置，发现了强相关性,可以用来降低破解的时间
#TODO 加入summary
#TODO 加入之前论文10的反向分析
#TODO 加入tfargs 控制可变参数
#TODO 计算错误共现概率

#比较好的实验结果如下：3层x100,5x60 参数量为300
#餐数量200的效果也不错

#加入个统计方法
error_distribution = Counter()

def fetch_error_distribution(data):#用于查看错误的位置
    #然后又加入了一些改动，把这些分布写到一个文件中去。
    fr = codecs.open('error_distribution','w',encoding = 'utf-8')
    mylist = []
    for index,item in enumerate(data):
        if not item:
            mylist.append(index)
            error_distribution[index] += 1
    fr.write(json.dumps(mylist)+'\n')
    return error_distribution,mylist

def clean(data):
    # 用于将队列的返回结果进行重新构建
    data = np.array(json.loads(data))
    return np.reshape(data, [-1, 1])

def func(x):
    return tf.cast(tf.greater(x,0.5),tf.float32)

train_data = []
target_data = []
#我在V8版本采用了最大的数据集，这样做的目的就是可以不需要验证集
with codecs.open(r'C:\Users\ruin2\DES密码分析\train_data/bytes_data_1203','r',encoding = 'utf-8') as fr:
    mydata = fr.readlines()
    for line in mydata:
        line = line.strip('\n')
        line = line.split('###')
        #左边是明文，右边是密文
        #明文是label，密文是data
        target_data.append(line[0])
        train_data.append(line[1])
fr.close()
del mydata

# 开始搭建网络
def fc(inputs, units, layer_name,activation = None):  # 只在最后一层添加relu作为激活
    output = tf.layers.dense(inputs=inputs,
                             units=units,
                             activation=activation,
                             name=layer_name)
    return output

def get_weight(shape, name):
    initial = tf.truncated_normal(shape, mean=0, stddev=0.1)  # 服从均匀正太分布
    return tf.Variable(initial, name=name)

def get_bias(shape, name):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial, name=name)

def inference_loop(inputs):  # 将【72，】的密文作为模型的输入
    # num_neural = 10#每一层的神经元数量  #ada 的时候，30以内会取得不错的效果,但奇怪的是，使用SGD就会出现NAN。
    # ori = inputs
    num_neural = 30
    for i in range(2):
        name = str(i)
        # if i % 3 == 0:#skip layer#在V5版本中，加上skip层的效果明显不好
        #     inputs = fc(inputs, num_neural, name) + ori
        #     continue
        inputs = fc(inputs, num_neural ,layer_name = name)#输出的大小是
    final_w_l = get_weight([64, 64], 'final_w_l')#左乘矩阵

    final_w_r = get_weight([num_neural,1],'final_w_r')#右乘矩阵
    logits = tf.matmul(final_w_l,inputs)
    logits = tf.matmul(logits,final_w_r)
    return logits

def evaluation(logits, labels):
    with tf.variable_scope('accuracy') as scope:
        # logits = tf.cast(logits,tf.int32)#这个表达不是很好，所以我们在V5版本换成tf.map_fn,这样的话，就可以用0.5作为分割界限了
        logits = tf.map_fn(lambda x : func(x),logits)
        logits = tf.cast(logits,tf.int32)
        labels = tf.cast(labels,tf.int32)
        correct = tf.equal(logits,labels)
        correct_list = correct
        correct = tf.cast(correct, tf.float32)
        accuracy = tf.reduce_sum(correct)
        # tf.summary.scalar('accuracy',accuracy)
    return accuracy,correct_list

def losses(data, labels):
    with tf.name_scope('losses') as scope:
        loss = tf.losses.absolute_difference(labels=labels,predictions=data)
        # tf.summary.scalar('loss',loss)
    return loss

def trainning(loss):
    with tf.name_scope('optimizer') as scope:
        optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
        train_op = optimizer.minimize(loss)
    return train_op

xs = tf.placeholder(tf.float32, shape=[64, 1])
ys = tf.placeholder(tf.float32, shape=[64, 1])

train_logits = inference_loop(xs)
train_loss = losses(train_logits, ys)
summary_loss = tf.summary.scalar('loss',train_loss)
train_step = trainning(train_loss)
train_acc = evaluation(train_logits,ys)
accuracy = tf.cast(train_acc[0],tf.int32)
summary_acc = tf.summary.scalar('accuracy',accuracy)
summary_op = tf.summary.merge([summary_loss,summary_acc])
# sum_ops = tf.summary.merge_all()

queue = tf.FIFOQueue(capacity=2000, dtypes=[tf.string, tf.string], )
enqueue_op = queue.enqueue_many([train_data, target_data])
x, y = queue.dequeue()
qr = tf.train.QueueRunner(queue, [enqueue_op])

# with tf.Graph().as_default():
#     session_conf = tf.ConfigProto(
#         allow_soft_placement=True,
#         log_device_placement=False)
#     sess = tf.Session(config=session_conf)
#     with sess.as_default():
with tf.Session() as sess:
    sess.run([tf.local_variables_initializer(), tf.global_variables_initializer()])
    saver = tf.train.Saver()
    coord = tf.train.Coordinator()
    # threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    enqueue_threads = qr.create_threads(sess, coord=coord, start=True)
    train_summary_writer = tf.summary.FileWriter('logs/',sess.graph)

    for i in range(2000):
        datas, labels = sess.run([x, y])
        step, logits, loss, acc , summaries= sess.run([train_step, train_logits, train_loss, train_acc, summary_op],
                                           feed_dict={xs: clean(datas),
                                                      ys: clean(labels)})
        train_summary_writer.add_summary(summaries, i)
        fetch_error_distribution(acc[1])

        if i % 1000 == 0:
            print('step:{}，loss:{:.2f}, acc:{} '.format(i, loss, acc[0]))
            res_dict,res_list = fetch_error_distribution(acc[1])
            print('{}'.format(res_dict))
            print('\n'+'---------------------------------')
            print('{}'.format(res_list))

            saver.save(sess,save_path = './model/des', global_step=i)
    coord.request_stop()
    # coord.join(threads)
    sess.close()

    #tensorboard --logdir C:\Users\ruin2\PycharmProjects\untitled\logs
    #启动tensorboard的时候，参数不要用带''的这种。当成linux操作
    #http: // localhost: 6006


###### for testing #########
from first_fit import *
from initial import *
############################
import copy

# Function to deallocate processes from memory
def deallocate( name , memory_arr ):
    memory = copy.deepcopy(memory_arr)
    l = len(name)
    n = len(memory)
    for i in range (n):
        if len(memory) <= i:
            break
        if len(memory[i][2])> l:
            if ( memory[i][2][:l] == name ) and (memory[i][2][l] =='(') :
                if i == 0 :
                    if memory[i+1][2] == 'free':
                        memory[i+1][0] = memory[i][0]
                        memory.pop(i)
                    else :
                        memory[i][2]= 'free'

                elif i < ( len(memory) -1) :
                    if (memory[i-1][2] == 'free') and ( memory[i+1][2] == 'free'):
                        memory[i-1][1] = memory[i+1][1]
                        memory.pop(i)
                        memory.pop(i)

                    elif memory[i-1][2] == 'free':
                        memory[i-1][1] = memory[i][1]
                        memory.pop(i)
                    elif memory[i+1][2] == 'free':
                        memory[i + 1][0] = memory[i][0]
                        memory.pop(i)
                    else :
                        memory[i][2]='free'
                else :
                    if memory[i - 1][2] == 'free':
                        memory[i - 1][1] = memory[i][1]
                        memory.pop(i)
                    else :
                        memory[i][2]='free'

    for i in range (len(memory)):
        if len(memory) <= i:
            break
        if len(memory[i][2]) > l:
            if (memory[i][2][:l] == name) and (memory[i][2][l] == '('):
                memory = deallocate(name, memory)
                break

    return memory
if __name__ == '__main__':
    #block entry [ [start,end ,free] ]
    h = [[400, 125], [100, 250], [1000, 500],[550,450] ]
    z = intialAlloc(1500, h)
    print(z)
    processSize = [['code',212],
				   ['IDE', 426],
				   ['data',417] ,
				   ['stack',112],
				   ['zz',12]]
    result, mymem = firstFit(z, processSize, 'p1')
    print(result)
    print(mymem)
    deal= deallocate( 'p1' , mymem )
    print (deal)
    mem = deallocate('old1', deal)
    print(mem)
    mem = deallocate('old0', mem)
    print(mem)
    mem = deallocate('old2', mem)
    print(mem)


import numpy as np
import tensorflow as tf
import Nn
from utils.sth import sth
from .policy import Policy

class DDDQN(Policy):
    '''
    Dueling Double DQN
    '''

    def __init__(self,
                 s_dim,
                 visual_sources,
                 visual_resolution,
                 a_dim_or_list,
                 action_type,
                 gamma=0.99,
                 max_episode=50000,
                 batch_size=128,
                 buffer_size=10000,
                 base_dir=None,

                 lr=5.0e-4,
                 epsilon=0.2,
                 assign_interval=2,
                 logger2file=False,
                 out_graph=False):
        assert action_type == 'discrete', 'dueling double dqn only support discrete action space'
        super().__init__(
            s_dim=s_dim,
            visual_sources=visual_sources,
            visual_resolution=visual_resolution,
            a_dim_or_list=a_dim_or_list,
            action_type=action_type,
            gamma=gamma,
            max_episode=max_episode,
            base_dir=base_dir,
            policy_mode='OFF',
            batch_size=batch_size,
            buffer_size=buffer_size)
        self.epsilon = epsilon
        self.assign_interval = assign_interval
        with self.graph.as_default():

            self.lr = tf.train.polynomial_decay(lr, self.episode, self.max_episode, 1e-10, power=1.0)
            self.v, self.a = Nn.critic_dueling('dueling_net', self.pl_s, self.pl_visual_s, self.a_counts)
            self.action = tf.argmax(self.a, axis=1, name='action_int')
            tf.identity(self.action, 'action')
            self.average_a = tf.reduce_mean(self.a, axis=1)[:, np.newaxis]

            self.v_next, self.a_next = Nn.critic_dueling('dueling_net', self.pl_s_, self.pl_visual_s_, self.a_counts)
            self.next_max_action = tf.argmax(self.a_next, axis=1, name='next_action_int')
            self.next_max_action_one_hot = tf.one_hot(tf.squeeze(self.next_max_action), self.a_counts, 1., 0., dtype=tf.float32)
            self.v_target_next, self.a_target_next = Nn.critic_dueling('dueling_target_net', self.pl_s_, self.pl_visual_s_, self.a_counts)
            self.average_a_target_next = tf.reduce_mean(self.a_target_next, axis=1)[:, np.newaxis]

            self.q_eval = tf.reduce_sum(tf.multiply(self.v + self.a - self.average_a, self.pl_a), axis=1)[:, np.newaxis]
            self.q_target_next_max = tf.reduce_sum(
                tf.multiply(self.v_target_next + self.a_target_next - self.average_a_target_next, self.next_max_action_one_hot),
                axis=1)[:, np.newaxis]
            self.q_target = tf.stop_gradient(self.pl_r + self.gamma * (1 - self.pl_done) * self.q_target_next_max)

            self.q_loss = tf.reduce_mean(tf.squared_difference(self.q_eval, self.q_target))

            self.q_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='dueling_net')
            self.q_target_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='dueling_target_net')

            self.train_q = tf.train.AdamOptimizer(self.lr).minimize(self.q_loss, var_list=self.q_vars, global_step=self.global_step)
            self.assign_target = self.update_target_net_weights(self.q_target_vars, self.q_vars)

            tf.summary.scalar('LOSS/loss', tf.reduce_mean(self.q_loss))
            tf.summary.scalar('LEARNING_RATE/lr', tf.reduce_mean(self.lr))
            self.summaries = tf.summary.merge_all()
            self.generate_recorder(
                logger2file=logger2file,
                graph=self.graph if out_graph else None
            )
            self.recorder.logger.info('''
　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　　　ｘｘｘｘｘｘ　　　　　　ｘｘｘｘ　　　ｘｘｘｘ
　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘ　ｘｘｘｘ　　　　　　　ｘｘｘ　　　　ｘ
　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　ｘｘｘ　　　ｘｘｘｘ　　　　　　ｘｘｘｘ　　　ｘ
　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　ｘｘｘ　　　　ｘｘｘ　　　　　　ｘｘｘｘｘ　　ｘ
　　　　ｘｘ　　　　　ｘｘ　　　　　　ｘｘ　　　　　ｘｘ　　　　　　ｘｘ　　　　　ｘｘ　　　　　ｘｘ　　　　　ｘｘｘ　　　　　　ｘ　ｘｘｘｘ　ｘ
　　　　ｘｘ　　　　　ｘｘ　　　　　　ｘｘ　　　　　ｘｘ　　　　　　ｘｘ　　　　　ｘｘ　　　　　ｘｘｘ　　　　ｘｘｘ　　　　　　ｘ　　ｘｘｘｘｘ
　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　　ｘｘ　　　　ｘｘｘ　　　　　ｘｘｘ　　　　ｘｘｘ　　　　　　ｘ　　　ｘｘｘｘ
　　　　ｘｘ　　　ｘｘｘｘ　　　　　　ｘｘ　　　ｘｘｘｘ　　　　　　ｘｘ　　　ｘｘｘｘ　　　　　ｘｘｘ　　　ｘｘｘ　　　　　　　ｘ　　　　ｘｘｘ
　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　　ｘｘｘｘｘｘｘｘ　　　　　　ｘｘｘ　　　　ｘｘ
　　　ｘｘｘｘｘｘｘ　　　　　　　　ｘｘｘｘｘｘｘ　　　　　　　　ｘｘｘｘｘｘｘ　　　　　　　　　　ｘｘｘｘｘ
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　ｘｘｘｘ
　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　ｘｘｘ
            ''')

    def choose_action(self, s, visual_s):
        if np.random.uniform() < self.epsilon:
            a = np.random.randint(0, self.a_counts, len(s))
        else:
            a = self.sess.run(self.action, feed_dict={
                self.pl_visual_s: visual_s,
                self.pl_s: s
            })
        return sth.int2action_index(a, self.a_dim_or_list)

    def choose_inference_action(self, s, visual_s):
        return sth.int2action_index(
            self.sess.run(self.action, feed_dict={
                self.pl_visual_s: visual_s,
                self.pl_s: s
            }),
            self.a_dim_or_list
        )

    def store_data(self, s, visual_s, a, r, s_, visual_s_, done):
        self.off_store(s, visual_s, a, r[:, np.newaxis], s_, visual_s_, done[:, np.newaxis])

    def learn(self, **kwargs):
        episode = kwargs['episode']
        for i in range(kwargs['step']):
            s, visual_s, a, r, s_, visual_s_, done = self.data.sample()
            _a = sth.action_index2one_hot(a, self.a_dim_or_list)
            summaries, _ = self.sess.run([self.summaries, self.train_q], feed_dict={
                self.pl_visual_s: visual_s,
                self.pl_s: s,
                self.pl_a: _a,
                self.pl_r: r,
                self.pl_visual_s_: visual_s_,
                self.pl_s_: s_,
                self.pl_done: done,
                self.episode: episode
            })
            if self.sess.run(self.global_step) % self.assign_interval == 0:
                self.sess.run(self.assign_target)
            self.recorder.writer.add_summary(summaries, self.sess.run(self.global_step))


from opentrons import labware, instruments, robot
from otcustomizers import FileInput, StringSelection

reservoir_name = 'biotix-resevoir'
if reservoir_name not in labware.list():
    labware.create(
        reservoir_name,
        grid=(1, 1),
        spacing=(0, 0),
        diameter=72,
        depth=23
        )

tiprack_slots = ['1', '4', '7', '10']
resevoir = labware.load(reservoir_name, '2')
source = resevoir.wells('A1')
trash = robot.fixed_trash[0]

example_csv = """
90,168,187,13,70,189,196,93
56,197,147,139,74,61,44,157
106,198,45,6,46,113,111,33
28,143,185,17,199,155,78,93
185,96,60,105,143,151,18,102
139,48,111,68,179,126,59,172
111,25,84,12,63,31,34,8
24,128,106,88,124,65,133,26
61,71,109,84,85,62,89,168
58,101,121,5,122,88,27,59
43,16,156,175,190,41,78,8
66,60,164,129,106,7,198,195

"""

def transpose_matrix(m):
    return [[r[i] for r in reversed(m)] for i in range(len(m[0]))]

def flatten_matrix(m):
    return [cell for row in m for cell in row]

def well_csv_to_list(csv_string):
    """
    Takes a csv string and flattens it to a list, re-ordering to match
    Opentrons well order convention (A1, B1, C1, ..., A2, B2, B2, ...)
    """
    data = [
        line.split(',')
        for line in reversed(csv_string.split('\n')) if line.strip()
        if line
    ]
    if len(data[0]) > len(data):
        # row length > column length ==> "landscape", so transpose
        return flatten_matrix(transpose_matrix(data))
    # "portrait"
    return flatten_matrix(data)

def run_custom_protocol(
        volumes_csv: FileInput=example_csv,
        pipette_mount: StringSelection(
            'right', 'left')='right',
        pipette_model: StringSelection(
            'p300-Single', 'p50-Single', 'p10-Single')='p300-Single',
        plate_type: StringSelection('96-flat', '384-plate')='96-flat',
        tip_reuse: StringSelection(
            'new tip each time', 'reuse tip')='new tip each time'):
    tipracks = [
        labware.load('opentrons-tiprack-300ul', slot)
        for slot in tiprack_slots]

    if pipette_model == 'p300-Single':
        pipette = instruments.P300_Single(
            mount=pipette_mount,
            tip_racks=tipracks)
    elif pipette_model == 'p50-Single':
        pipette = instruments.P50_Single(
            mount=pipette_mount,
            tip_racks=tipracks)
    elif pipette_model == 'p10-Single':
        tipracks = [
            labware.load('tiprack-10ul', slot)
            for slot in tiprack_slots]
        pipette = instruments.P10_Single(
            mount=pipette_mount,
            tip_racks=tipracks)

    plate = labware.load(plate_type, '3')

    volumes = [float(cell) for cell in well_csv_to_list(volumes_csv)]

    for vol in volumes:
        if 0 < vol < pipette.min_volume:
            robot.comment(
                'WARNING: volume {} is below pipette\'s minimum volume.'
                .format(vol))

    tip_strategy = 'always' if tip_reuse == 'new tip each time' else 'once'
    pipette.transfer(volumes, source, plate, new_tip=tip_strategy)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#@Author: Paweł Grochowski
#

from machinelearning.ConfusionMatrix import ConfusionMatrix

from machinelearning.classifier.ClassifierBase import ClassifierBase

class EnsemblerBaseParams():
    DEFAULT_CLASSIFIER_PARAMS = {}
    DEFAULT_CLASSIFIER_AMOUNT = 3
    DEFAULT_SAMPLE_SIZE_RATIO = 1.0
    DEFAULT_SAMPLE_DUPLICATES = True

class EnsemblerBase():

    def __init__(
        self,
        trainDataset,
        classifierType,
        classifierParams=EnsemblerBaseParams.DEFAULT_CLASSIFIER_PARAMS,
        classifierAmount=EnsemblerBaseParams.DEFAULT_CLASSIFIER_AMOUNT,
        sampleSizeRatio=EnsemblerBaseParams.DEFAULT_SAMPLE_SIZE_RATIO,
        sampleWithDuplicates=EnsemblerBaseParams.DEFAULT_SAMPLE_DUPLICATES,
        **kwargs
    ):
        if not issubclass(classifierType, ClassifierBase):
            raise Exception("Parameter 'classifierType' must derive of 'ClassifierBase'!")
        print("----------------------------------------------------------")
        print("Chosen classifier type: %s" %(str(classifierType)))
        print("Chosen classifier amount: %s" %(classifierAmount))
        print("Chosen sample size ratio: %s" %(sampleSizeRatio))
        for name, value in kwargs.items():
            setattr(self, name, value)
    ##

    @classmethod
    def fit(
        cls,
        trainDataset,
        classifierType,
        classifierParams=EnsemblerBaseParams.DEFAULT_CLASSIFIER_PARAMS,
        classifierAmount=EnsemblerBaseParams.DEFAULT_CLASSIFIER_AMOUNT,
        sampleSizeRatio=EnsemblerBaseParams.DEFAULT_SAMPLE_SIZE_RATIO,
        sampleWithDuplicates=EnsemblerBaseParams.DEFAULT_SAMPLE_DUPLICATES,
        **kwargs
    ):
        return cls(
            trainDataset,
            classifierType,
            classifierParams,
            classifierAmount,
            sampleSizeRatio,
            sampleWithDuplicates,
            **kwargs
        )
    ##

    def predict(self, testDataset):
        cm = ConfusionMatrix()
        for record in testDataset:
            clsLblTrue = record[-1]
            clsLblPred = self._makePrediction(record)
            cm.addPrediction(clsLblTrue, clsLblPred)
        return cm
    ##

    def _makePrediction(self, record):
        raise NotImplemented("This method must be overridden in subclasses!")
##


#
# @lc app=leetcode id=1073 lang=python3
#
# [1073] Adding Two Negabinary Numbers
#
# https://leetcode.com/problems/adding-two-negabinary-numbers/description/
#
# algorithms
# Medium (31.55%)
# Likes:    44
# Dislikes: 25
# Total Accepted:    3K
# Total Submissions: 9.4K
# Testcase Example:  '[1,1,1,1,1]\n[1,0,1]'
#
# Given two numbers arr1 and arr2 in base -2, return the result of adding them
# together.
#
# Each number is given in array format:  as an array of 0s and 1s, from most
# significant bit to least significant bit.  For example, arr = [1,1,0,1]
# represents the number (-2)^3 + (-2)^2 + (-2)^0 = -3.  A number arr in array
# format is also guaranteed to have no leading zeros: either arr == [0] or
# arr[0] == 1.
#
# Return the result of adding arr1 and arr2 in the same format: as an array of
# 0s and 1s with no leading zeros.
#
#
#
# Example 1:
#
#
# Input: arr1 = [1,1,1,1,1], arr2 = [1,0,1]
# Output: [1,0,0,0,0]
# Explanation: arr1 represents 11, arr2 represents 5, the output represents
# 16.
#
#
#
#
# Note:
#
#
# 1 <= arr1.length <= 1000
# 1 <= arr2.length <= 1000
# arr1 and arr2 have no leading zeros
# arr1[i] is 0 or 1
# arr2[i] is 0 or 1
#
#

class Solution:
    # NAIVE:
    # first convert to normal num, and add them
    # then convert back to base -2
    # def addNegabinary(self, arr1, arr2):
    #     def toneg2(n):
    #         if n == 0:
    #             return [0]
    #         # if n is odd, then n = n//(-2)+1
    #         # NOTE: -5//(-2) = 2, 5//(-2) = -3, -5//(2) = -3 in python
    #         # NOTE: -5//(-2) = 2, 5//(-2) = -2, -5//(2) = -2 in java/c++
    #         res = []
    #         while n != 1:
    #             res.append(abs(n % 2))
    #             if res[-1] == 0:
    #                 n = n // (-2)
    #             else:
    #                 n = n // (-2) + 1
    #         res.append(1)
    #         return res[::-1]

    #     def fromneg2(arr):
    #         res = 0
    #         n = 0
    #         for a in arr[::-1]:
    #             res += a*((-2) ** n)
    #             n += 1
    #         return res

    #     return toneg2(fromneg2(arr1)+fromneg2(arr2))

    def addBinary(self, arr1, arr2):
        res = []
        carry = 0
        while arr1 or arr2 or carry:
            carry += (arr1 or [0]).pop()+(arr2 or [0]).pop()
            res.append(carry & 1)
            carry >>= 1
        # need to remove leading 0
        while len(res) > 1 and res[-1] == 0:
            res.pop()
        return res[::-1]

    def addNegabinary(self, arr1, arr2):
        res = []
        carry = 0
        while arr1 or arr2 or carry:
            carry += (arr1 or [0]).pop()+(arr2 or [0]).pop()
            res.append(carry & 1)
            carry = -(carry >> 1)
        # need to remove leading 0
        while len(res) > 1 and res[-1] == 0:
            res.pop()
        return res[::-1]

if __name__ == "__main__":
    print(Solution().addNegabinary(arr1=[0], arr2=[1, 0, 1]))
    print(Solution().addNegabinary(arr1=[1, 1, 1, 1, 1], arr2=[1, 0, 1]))


import torch
import torchvision.models as models
import torch.nn as nn
from torch.nn import functional as F
from networks.loss import TransferNetLoss
from networks.metric import TransferNetMetrics
from networks.utils import layer_init
from networks.base import BaseNet

class TransferNet(BaseNet):
    def __init__(self, cfg, logger=None, random_state=1234):
        super(TransferNet, self).__init__(cfg, logger, random_state)

        self.class_scores = nn.Linear(self.out_dim, cfg.TRAIN.NUM_CLASSES)
        self._init_modules()

    def _init_modules(self):
        if self.cfg.MODEL.CONV_BODY.ARCH == 'res50':
            first_block = self.cfg.MODEL.CONV_BODY.RESNET50_FIRST_BLOCK_INDEX

            for i, layer in enumerate(self.feature_extractor):
                if i < first_block + self.cfg.MODEL.CONV_BODY.FREEZE_NUM_BLOCKS:
                    for param in layer.parameters():
                        param.requires_grad = False
                else:
                    for param in layer.parameters():
                        param.requires_grad = True
        else:
            raise NotImplementedError

        layer_init(self.class_scores)

    def forward(self, inputs, labels):
        feats = self.feature_extractor(inputs)
        feats = feats.squeeze(2).squeeze(2)
        class_scores = self.class_scores(feats)

        return class_scores

class FinetuneNet(BaseNet):
    def __init__(self, cfg, logger=None, random_state=1234):
        super(FinetuneNet, self).__init__(cfg, logger, random_state)

        if self.cfg.MODEL.NORM_PROTOTYPES:
            self.class_scores_w = torch.nn.Parameter(torch.FloatTensor(self.out_dim, cfg.TRAIN.NUM_CLASSES))
            self.class_scores = self._class_scores
            torch.nn.init.xavier_normal_(self.class_scores_w)
        else:
            self.class_scores = nn.Linear(self.out_dim, cfg.TRAIN.NUM_CLASSES)
            layer_init(self.class_scores)

        self._init_modules()

    def _init_modules(self):
        if self.cfg.MODEL.CONV_BODY.ARCH == 'res50':
            first_block = self.cfg.MODEL.CONV_BODY.RESNET50_FIRST_BLOCK_INDEX

            for i, layer in enumerate(self.feature_extractor):
                if i < first_block + self.cfg.MODEL.CONV_BODY.FREEZE_NUM_BLOCKS:
                    for param in layer.parameters():
                        param.requires_grad = False
                else:
                    for param in layer.parameters():
                        param.requires_grad = True
        else:
            raise NotImplementedError

    def _w_dot_x(self, x, w):
        if self.cfg.MODEL.NORM_FEATURES:
            normed_x = F.normalize(x, p=2, dim=1)
        if self.cfg.MODEL.NORM_PROTOTYPES:
            normed_w = F.normalize(w, p=2, dim=0)

        return self.cfg.MODEL.RADIUS_PROTOTYPES * torch.matmul(normed_x, normed_w)

    def _class_scores(self, x):
        return self._w_dot_x(x, self.class_scores_w)

    def forward(self, inputs, labels):
        feats = self.feature_extractor(inputs)
        feats = feats.squeeze(2).squeeze(2)
        class_scores = self.class_scores(feats)

        return class_scores


# nrbf.py - .NET Remoting Binary Format reading library for Python 3.6+
# Copyright (C) 2017 Christopher Gurnee
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

__all__ = ['read_stream', 'serialization', 'JSONEncoder']

import enum, itertools, json, re, sys
from array       import array as Array, typecodes
from collections import OrderedDict, namedtuple
from copy        import deepcopy
from datetime    import datetime, timedelta, timezone
from decimal     import Decimal
from keyword     import iskeyword
from struct      import Struct, calcsize, pack
from namedlist   import namedlist

# Decorator which adds an enum value (an int or a length-one bytes) and its associated
# reader function to a registration dict (either PrimitiveType_readers or RecordType_readers)
def _register_reader(type_dict, enum_value):
    if isinstance(enum_value, int):
        enum_value = bytes([enum_value])  # convert from int to length-one bytes
    else:
        assert isinstance(enum_value, bytes) and len(enum_value) == 1
    def decorator(read_func):
        type_dict[enum_value] = read_func
        return read_func
    return decorator

class serialization:
    def __init__(self, streamfile, can_overwrite_member = False):
        '''
        :param streamfile: a file-like object in .NET Remoting Binary Format
        :param can_overwrite_member: iff True, overwrite_member() may be called
        '''
        assert streamfile.readable()
        if can_overwrite_member:
            assert streamfile.writable()
            assert streamfile.seekable()
        self._file                  = streamfile
        self._Class_by_id           = {}    # see _read_ClassInfo()
        self._objects_by_id         = {}    # all referenceable objects indexed by ObjectId
        self._root_id               = None  # the id of the single root object
        self._member_references     = []    # all seen simple references, see _read_MemberReference()
        self._collection_references = []    # all seen collection references, e.g. .NET dicts and lists
        self._add_overwrite_info     = can_overwrite_member
        self._overwrite_info_by_pyid = {}   # if above is True, overwrite_info objs indexed by python id()

    # Below are a set of "readers" and other support types, one per defined structure
    # in revision 11.0 of the ".NET Remoting: Binary Format Data Structure" specification
    # in approximately the same order of each structure's definition. Each one reads in
    # the structure's elements, and depending on the reader type may return some or all
    # of the read values. This first section implements the Primitive Types, each one
    # returning the closest Python analogous type to the Primitive Type read.
    ######## Reference: https://msdn.microsoft.com/en-us/library/cc236844.aspx ########

    # A dict of all PrimitiveType readers indexed by a length-one PrimitiveTypeEnumeration bytes object
    _PrimitiveType_readers = {}

    @_register_reader(_PrimitiveType_readers, 3)
    def _read_Char(self):
        utf8_bytes = bytearray()
        while True:
            utf8_bytes += self._file.read(1)
            try:
                return utf8_bytes.decode('utf-8')
            except UnicodeDecodeError:
                if len(utf8_bytes) > 4:
                    raise

    @_register_reader(_PrimitiveType_readers, 12)
    def _read_TimeSpan(self):
        return timedelta(microseconds= self._read_Int64() / 10)  # units of 100 nanoseconds

    @_register_reader(_PrimitiveType_readers, 13)
    def _read_DateTime(self):
        ticks = self._read_UInt64()
        kind  = ticks >> 62     # the 2 most significant bits store the "kind"
        ticks &= (1 << 62) - 1  # all but the above
        if ticks >= 1 << 61:    # if negative, reinterpret
            ticks -= 1 << 62    # as 62-bit two's complement
        time = datetime(1, 1, 1)
        try:
            time += timedelta(microseconds= ticks / 10)
        except OverflowError:
            pass
        if kind == 1:
            time = time.replace(tzinfo=timezone.utc)
        elif kind == 2:
            try:
                time = time.astimezone()  # kind 2 is the local time zone
            except OSError:
                pass
        return time

    @_register_reader(_PrimitiveType_readers, 18)
    def _read_LengthPrefixedString(self):
        length = 0
        for bit_range in range(0, 5*7, 7):  # each bit range is 7 bits long, with at most 5 of them
            byte    = self._read_Byte()
            length += (byte & ((1 << 7) - 1)) << bit_range  # highest bit masked out, then shifted
            if byte & (1 << 7) == 0:        # the "is there more?" (highest/most significant) bit
                break
        else:
            raise OverflowError('LengthPrefixedString overflow')
        return self._file.read(length).decode('utf-8')

    @_register_reader(_PrimitiveType_readers, 5)
    def _read_Decimal(self):
        return Decimal(self._read_LengthPrefixedString())

    @_register_reader(_PrimitiveType_readers, 17)
    def _read_Null(self):
        return None

    # Creates the rest of the PrimitiveType readers (those based on struct.unpack);
    # this only works when it's called after the class has been fully defined.
    _array_format_by_primitive_type  = {}  # array-format-specs  indexed by PrimitiveTypeEnumeration length-one bytes
    _struct_format_by_primitive_type = {}  # struct-format-specs indexed by PrimitiveTypeEnumeration length-one bytes
    @classmethod
    def _create_PrimitiveType_readers(cls):
        for enum_value, name, format in (
                ( 1, '_read_Boolean',  '?'),
                ( 2, '_read_Byte',     'B'),
                ( 6, '_read_Double',  '<d'),
                ( 7, '_read_Int16',   '<h'),
                ( 8, '_read_Int32',   '<l'),
                ( 9, '_read_Int64',   '<q'),
                (10, '_read_SByte',    'b'),
                (11, '_read_Single',  '<f'),
                (14, '_read_UInt16',  '<H'),
                (15, '_read_UInt32',  '<L'),
                (16, '_read_UInt64',  '<Q')):
            enum_value = bytes([enum_value])  # convert from int to length-one bytes
            if format[-1] in typecodes:  # if the format can also be used to create an array type
                cls._array_format_by_primitive_type[enum_value] = format[-1]  # (the [-1] skips the '<' flag)
            cls._struct_format_by_primitive_type[enum_value] = format
            struct = Struct(format)
            length = calcsize(format)
            reader = lambda self, s=struct, l=length: s.unpack(self._file.read(l))[0]
            setattr(cls, name, _register_reader(cls._PrimitiveType_readers, enum_value)(reader))

    # The ClassTypeInfo structure is read and ignored
    def _read_ClassTypeInfo(self):
        self._read_LengthPrefixedString()  # TypeName
        self._read_Int32()                 # LibraryId

    @enum.unique
    class _MessageFlags(enum.Flag):
        NoArgs                 = 0x00000001
        ArgsInline             = 0x00000002
        ArgsIsArray            = 0x00000004
        ArgsInArray            = 0x00000008
        NoContext              = 0x00000010
        ContextInline          = 0x00000020
        ContextInArray         = 0x00000040
        MethodSignatureInArray = 0x00000080
        PropertiesInArray      = 0x00000100
        NoReturnValue          = 0x00000200
        ReturnValueVoid        = 0x00000400
        ReturnValueInline      = 0x00000800
        ReturnValueInArray     = 0x00001000
        ExceptionInArray       = 0x00002000
        GenericMethod          = 0x00008000

    def _read_ValueWithCode(self):
        return self._PrimitiveType_readers[self._file.read(1)](self)

    _read_StringValueWithCode = _read_ValueWithCode

    def _read_ArrayOfValueWithCode(self):
        return [self._read_ValueWithCode() for i in range(self._read_Int32())]

    # A dict of all RecordType readers indexed by a length-one RecordTypeEnumeration bytes object
    _RecordType_readers = {}

    @_register_reader(_RecordType_readers, 21)
    def _read_BinaryMethodCall(self):
        raise NotImplementedError('BinaryMethodCall')
        # flags = self._MessageFlags(self._read_Int32())  # MessageEnum
        # self._read_StringValueWithCode()                # MethodName
        # self._read_StringValueWithCode()                # TypeName
        # if flags & self._MessageFlags.ContextInline:
        #     self._read_StringValueWithCode()            # CallContext
        # if flags & self._MessageFlags.ArgsInline:
        #     self._read_ArrayOfValueWithCode()           # Args

    @_register_reader(_RecordType_readers, 22)
    def _read_BinaryMethodReturn(self):
        raise NotImplementedError('BinaryMethodReturn')
        # flags = self._MessageFlags(self._read_Int32())  # MessageEnum
        # if flags & self._MessageFlags.ReturnValueInline:
        #     self._read_ValueWithCode()                  # ReturnValue
        # if flags & self._MessageFlags.ContextInline:
        #     self._read_StringValueWithCode()            # CallContext
        # if flags & self._MessageFlags.ArgsInline:
        #     self._read_ArrayOfValueWithCode()           # Args

    ######## Classes ########

    # Reads a ClassInfo structure, and if the Class's ObjectId hasn't yet been seen, adds to
    # self._Class_by_id a new Python class with the same members as the ClassInfo specifies.
    def _read_ClassInfo(self):
        object_id    = self._read_Int32()
        class_name   = self._read_LengthPrefixedString()
        member_count = self._read_Int32()
        member_names = [self._read_LengthPrefixedString() for i in range(member_count)]
        unique_members = set()
        for member_num, member_name in enumerate(member_names):
            member_name = make_unique(sanitize_identifier(member_name), unique_members)
            unique_members.add(member_name)
            member_names[member_num] = member_name
        Class = namedlist(sanitize_identifier(class_name), member_names, default=None)
        Class._id = object_id
        Class._primitive_types = {}  # filled in below by _read_MemberTypeInfo()
        Class._is_system_class = False
        self._Class_by_id[object_id] = Class
        return Class

    # Readers for the AdditionalInfos member of MemberTypeInfo indexed by
    # BinaryTypeEnumeration ints; created after this class is fully defined
    _AdditionalInfo_readers = ()

    def _read_MemberTypeInfo(self, Class):
        binary_types = [self._read_Byte() for m in Class._fields]  # BinaryTypeEnums
        for member_num, binary_type in enumerate(binary_types):    # AdditionalInfos
            additional_info = self._AdditionalInfo_readers[binary_type](self)
            if binary_type == 0:  # (0 == BinaryTypeEnumeration.Primitive)
                Class._primitive_types[member_num] = additional_info  # save any PrimitiveType for later

    @_register_reader(_RecordType_readers, 5)
    def _read_ClassWithMembersAndTypes(self):
        Class = self._read_ClassInfo()
        self._read_MemberTypeInfo(Class)
        self._read_Int32()  # LibraryId is ignored
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 3)
    def _read_ClassWithMembers(self):
        Class = self._read_ClassInfo()
        self._read_Int32()  # LibraryId is ignored
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 4)
    def _read_SystemClassWithMembersAndTypes(self):
        Class = self._read_ClassInfo()
        Class._is_system_class = True
        self._read_MemberTypeInfo(Class)
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 2)
    def _read_SystemClassWithMembers(self):
        Class = self._read_ClassInfo()
        Class._is_system_class = True
        return self._read_members_into(Class(), Class._id, Class._primitive_types.get)

    @_register_reader(_RecordType_readers, 1)
    def _read_ClassWithId(self):
        object_id = self._read_Int32()
        Class = self._Class_by_id[self._read_Int32()]  # MetadataId
        return self._read_members_into(Class(), object_id, Class._primitive_types.get)

    # If primitive_type is not None, read the specified primitive_type with one of the
    # PrimitiveType_readers. Otherwise read the next RecordType in the streamfile with
    # one of the RecordType_readers. If overwrite_info is not None, add overwrite info
    # (a tuple containing: file position, struct type) to overwrite_info[overwrite_index]
    # if the value read is an overwritable primitive. Finally, returns the value read.
    def _read_Record_or_Primitive(self, primitive_type, overwrite_info = None, overwrite_index = None):
        if primitive_type:
            if overwrite_info is not None:
                format = self._struct_format_by_primitive_type.get(primitive_type)
                if format:
                    overwrite_info[overwrite_index] = self._file.tell(), format
            return self._PrimitiveType_readers[primitive_type](self)
        else:
            record_type = self._file.read(1)
            # If overwrite_info is not None and record_type == MemberPrimitiveTyped, parse it ourselves--
            # read in the PrimitiveTypeEnum and call ourselves to finish parsing and add the overwrite_info
            if overwrite_info is not None and record_type == b'\x08':
                return self._read_Record_or_Primitive(self._file.read(1), overwrite_info, overwrite_index)
            return self._RecordType_readers[record_type](self)

    _Reference = namedlist('_Reference',  # see _read_MemberReference()
        'id parent index_in_parent resolved collection_resolver orig_obj', default=None)

    # Reads members or array elements into the 'obj' pre-allocated list or class instance
    def _read_members_into(self, obj, object_id, members_primitive_type):
        assert callable(members_primitive_type)  # when called with the member_num, returns any respective primitive type
        if self._add_overwrite_info:
            # create the object which will store the overwrite info for all the members in obj
            overwrite_info = [None] * len(obj) if isinstance(obj, list) else obj.__class__()
        else:
            overwrite_info = None
        member_num = 0
        while member_num < len(obj):
            primitive_type = members_primitive_type(member_num)
            val = self._read_Record_or_Primitive(primitive_type, overwrite_info, member_num)
            if isinstance(val, self._BinaryLibrary):  # a BinaryLibrary can precede the actual member, it's ignored
                val = self._read_Record_or_Primitive(primitive_type, overwrite_info, member_num)
            if isinstance(val, self._ObjectNullMultiple):  # represents one or more empty members
                member_num += val.count
                continue
            if isinstance(val, self._Reference):      # see _read_MemberReference()
                val.parent          = obj
                val.index_in_parent = member_num
            obj[member_num] = val
            member_num += 1
        if overwrite_info is not None:
            self._overwrite_info_by_pyid[id(obj)] = overwrite_info
        # If this object is a .NET collection (e.g. a Generic dict or list) which can be
        # replaced by a native Python type, insert a collection _Reference instead of the raw
        # object which will be resolved later in read_stream() using a "collection resolver"
        if getattr(obj.__class__, '_is_system_class', False):
            for collection_name, resolver in self._collection_resolvers:
                if obj.__class__.__name__.startswith(f'System_Collections_Generic_{collection_name}_'):
                    obj = self._Reference(object_id, collection_resolver=resolver, orig_obj=obj)
                    self._collection_references.append(obj)
                    break
        self._objects_by_id[object_id] = obj
        return obj

    ######## Arrays ########

    @enum.unique
    class _BinaryArrayTypeEnumeration(enum.Enum):
        Single            = 0
        Jagged            = 1
        Rectangular       = 2
        SingleOffset      = 3
        JaggedOffset      = 4
        RectangularOffset = 5
        def is_offset(self):
            return self in (self.SingleOffset, self.JaggedOffset, self.RectangularOffset)
        def is_rectangular(self):
            return self in (self.Rectangular, self.RectangularOffset)

    def _read_ArrayInfo(self):
        return (self._read_Int32(),  # ObjectId
                self._read_Int32())  # Length

    # This might not be able to use _read_members_into() since BinaryArrays are much more complex than
    # other types, thus it has no choice but to reimplement much of what _read_members_into() does
    @_register_reader(_RecordType_readers, 7)
    def _read_BinaryArray(self):
        object_id  = self._read_Int32()
        array_type = self._BinaryArrayTypeEnumeration(self._read_Byte())
        rank       = self._read_Int32()
        lengths    = [self._read_Int32() for i in range(rank)]
        if array_type.is_offset():
            [self._read_Int32() for i in range(rank)]  # LowerBounds are not implemented; they're ignored
        binary_type     = self._read_Byte()
        additional_info = self._AdditionalInfo_readers[binary_type](self)
        primitive_type  = additional_info if binary_type == 0 else None  # (0 == BinaryTypeEnumeration.Primitive)
        # If the BinaryArray is multidimensional, the complex code branch is required:
        if array_type.is_rectangular():
            array_format = self._array_format_by_primitive_type.get(primitive_type) if primitive_type else None
            if array_format:  # if not None, a Python Array can be used for the *last* dimension...
                array_length = lengths.pop()  # ...which is removed from the lengths list here
            array = multidimensional_array(lengths)  # preallocate a list of lists (it's not a Python Array)
            skip  = 0
            for indexes in itertools.product(*[range(l) for l in lengths]):  # iterates through all of the indexes
                if array_format:
                    val = self._read_Array_native_elements(array_format, array_length)  # read in the last index in one call
                else:
                    if skip > 0:
                        skip -= 1
                        continue
                    val = self._read_Record_or_Primitive(primitive_type)
                    if isinstance(val, self._BinaryLibrary):   # a BinaryLibrary can precede the actual array element, it's ignored
                        val = self._read_Record_or_Primitive(primitive_type)
                    if isinstance(val, self._ObjectNullMultiple):  # represents one or more empty elements
                        skip = val.count - 1  # counts this iteration which we're skipping right now
                        continue
                    if isinstance(val, self._Reference):       # see _read_MemberReference()
                        list_indexes = ''.join(f'[{i}]' for i in indexes[:-1])
                        val.parent          = eval(f'array{list_indexes}')  # the parent list, i.e. all but the last index
                        val.index_in_parent = indexes[-1]                   # the last index
                list_indexes = ''.join(f'[{i}]' for i in indexes)
                exec(f'array{list_indexes} = val')
            self._objects_by_id[object_id] = array
            return array
        else:  # else it's not multidimensional, the standard code branch can be used:
            assert len(lengths) == 1
            return self._read_Array_elements(lengths[0], object_id, primitive_type)

    @_register_reader(_RecordType_readers, 16)
    def _read_ArraySingleObject(self):
        object_id, length = self._read_ArrayInfo()
        return self._read_Array_elements(length, object_id)

    @_register_reader(_RecordType_readers, 15)
    def _read_ArraySinglePrimitive(self):
        object_id, length = self._read_ArrayInfo()
        primitive_type    = self._file.read(1)
        return self._read_Array_elements(length, object_id, primitive_type)

    _read_ArraySingleString = _register_reader(_RecordType_readers, 17)(_read_ArraySingleObject)

    def _read_Array_elements(self, length, object_id, primitive_type = None):
        array_format = self._array_format_by_primitive_type.get(primitive_type) if primitive_type else None
        if array_format:  # if not None, a Python Array can be used
            array = self._read_Array_native_elements(array_format, length)
            self._objects_by_id[object_id] = array
            return array
        return self._read_members_into([None] * length, object_id, lambda i: primitive_type)

    assert sys.byteorder in ('little', 'big')
    def _read_Array_native_elements(self, format, length):
        if self._add_overwrite_info:
            pos = self._file.tell()
        array = Array(format)
        array.fromfile(self._file, length)  # read them in one call
        if sys.byteorder == 'big':
            array.byteswap()
        if self._add_overwrite_info:
            format = '<' + format  # always little-endian
            self._overwrite_info_by_pyid[id(array)] = [
                (pos + offset, format) for offset in range(0, length * array.itemsize, array.itemsize) ]
        return array

    _read_MemberPrimitiveTyped = _register_reader(_RecordType_readers, 8)(_read_ValueWithCode)

    # Wherever an ObjectId is encountered above, it's added to the self._objects_by_id map
    # along with the newly-created object. A MemberReference contains an IdRef which refers
    # to an object in this map, however because MemberReferences can appear before the object
    # to which they refer, they can't be resolved until the very end. Instead, a _Reference
    # Python object is temporarily created which stores enough information (its parent and
    # its index inside the parent) to eventually replace it with the actual object referenced
    # once all of the objects have been read from the stream in read_stream().
    @_register_reader(_RecordType_readers, 9)
    def _read_MemberReference(self):
        member_ref = self._Reference(self._read_Int32())  # IdRef
        self._member_references.append(member_ref)  # (parent and index_in_parent should be set by the caller)
        return member_ref

    _read_ObjectNull = _register_reader(_RecordType_readers, 10)(_read_Null)

    _ObjectNullMultiple = namedtuple('_ObjectNullMultiple', 'count')

    @_register_reader(_RecordType_readers, 14)
    def _read_ObjectNullMultiple(self):
        return self._ObjectNullMultiple(self._read_Int32())

    @_register_reader(_RecordType_readers, 13)
    def _read_ObjectNullMultiple256(self):
        return self._ObjectNullMultiple(self._read_Byte())

    @_register_reader(_RecordType_readers, 6)
    def _read_BinaryObjectString(self):
        object_id = self._read_Int32()
        string    = self._read_LengthPrefixedString()
        self._objects_by_id[object_id] = string
        return string

    @_register_reader(_RecordType_readers, 0)
    def _read_SerializationHeaderRecord(self):
        self._root_id = self._read_Int32()
        self._read_Int32()  # HeaderId is ignored
        major_version = self._read_Int32()
        minor_version = self._read_Int32()
        if major_version != 1:
            raise NotImplementedError(f'SerializationHeaderRecord.MajorVersion == {major_version}')
        if minor_version != 0:
            raise NotImplementedError(f'SerializationHeaderRecord.MinorVersion == {minor_version}')
        if self._root_id == 0:
            raise NotImplementedError('SerializationHeaderRecord.RootId == 0')

    class _BinaryLibrary: pass
    @_register_reader(_RecordType_readers, 12)
    def _read_BinaryLibrary(self):
        self._read_Int32()                 # MinorVersion and
        self._read_LengthPrefixedString()  # LibraryName are ignored
        return self._BinaryLibrary()

    class _MessageEnd: pass
    @_register_reader(_RecordType_readers, 11)
    def _read_MessageEnd(self):
        return self._MessageEnd()

    def read_header(self):
        '''Reads just the SerializationHeaderRecord from the streamfile.
        It's not necessary to call this, however it may be called before read_stream() if desired.
        Otherwise, read_stream() will raise an exception if the SerializationHeaderRecord isn't found.

        :return: True if the streamfile is in a supported .NET Remoting Binary Format
        '''
        assert self._root_id is None, 'read_header() has not already been called'
        try:
            self._read_Record_or_Primitive(primitive_type=False)
        except Exception:
            return False
        return self._root_id is not None

    def read_stream(self):
        '''Read the streamfile in .NET Remoting Binary Format and extract its root object

        :return: the root object contained in the stream
        '''
        if self._root_id is None and not self.read_header():
            raise RuntimeError('SerializationHeaderRecord not found (probably not an NRBF file)')
        obj = None
        while not isinstance(obj, self._MessageEnd):
            obj = self._read_Record_or_Primitive(primitive_type=False)
        self._Class_by_id.clear()

        # Resolve all the collection references
        for reference in self._collection_references:
            replacement = reference.collection_resolver(self, reference)  # calls one of the non-simple resolvers below
            # The final steps common to all collection resolvers are completed below
            if reference.parent:
                reference.parent[reference.index_in_parent] = replacement
            self._objects_by_id[reference.id] = replacement
        self._collection_references.clear()

        # Resolve all the (remaining) simple member references
        for reference in self._member_references:
            self._resolve_simple_reference(reference)
        self._member_references.clear()

        obj = self._objects_by_id[self._root_id]
        self._objects_by_id.clear()
        self._root_id = None
        return obj

    # Convert a _Reference representing a .NET dictionary collection into a Python dict
    def _resolve_dict_reference(self, dict_ref):
        orig_obj = dict_ref.orig_obj
        # If KeyValuePairs is itself a _Reference, it must be resolved first
        if isinstance(orig_obj.KeyValuePairs, self._Reference):
            self._resolve_simple_reference(orig_obj.KeyValuePairs)
        replacement    = {}
        overwrite_info = {} if self._add_overwrite_info else None
        for item in orig_obj.KeyValuePairs:
            try:
                # If any key is a _Reference, it must be resolved first
                # (value _References will be resolved later)
                if isinstance(item.key, self._Reference):
                    self._resolve_simple_reference(item.key)
                assert item.key not in replacement
                replacement[item.key] = item.value
                if overwrite_info is not None:
                    overwrite_info[item.key] = self._overwrite_info_by_pyid[id(item)].value
            except (AssertionError, TypeError):  # not all .NET dictionaries can be converted to Python dicts;
                replacement = orig_obj           # if the conversion fails, just proceed w/the original object
                break
        else:
            # If any dict value is a _Reference, fix its parent and index_in_parent
            for key, value in replacement.items():
                if isinstance(value, self._Reference):
                    value.parent          = replacement
                    value.index_in_parent = key
            if overwrite_info is not None:
                self._overwrite_info_by_pyid[id(replacement)] = overwrite_info
        return replacement

    # Convert a _Reference representing a .NET list collection into a Python list
    def _resolve_list_reference(self, list_ref):
        orig_obj = list_ref.orig_obj
        # If items is itself a _Reference, it must be resolved first
        if isinstance(orig_obj.items, self._Reference):
            self._resolve_simple_reference(orig_obj.items)
        replacement = orig_obj.items
        # If any list element is a _Reference, fix its parent
        for element in replacement:
            if isinstance(element, self._Reference):
                element.parent = replacement  # (the index_in_parent remains the same)
        return replacement

    _collection_resolvers = (
        ('Dictionary', _resolve_dict_reference),
        ('List',       _resolve_list_reference)
    )

    # Convert a _Reference representing a MemberReference into its referenced object
    def _resolve_simple_reference(self, member_ref):
        assert not member_ref.collection_resolver
        if member_ref.resolved:
            return
        replacement = self._objects_by_id[member_ref.id]
        member_ref.parent[member_ref.index_in_parent] = replacement
        member_ref.resolved = True

    def overwrite_member(self, obj, member, value):
        '''Overwrite an object's member with a new value in the streamfile (does not change obj).

        :param obj: an object in the hierarchy returned by read_stream()
        :param member: a writable member (attribute) name, index (for lists/arrays), or key (for dicts) in obj
        :param value: the new value to be written to the streamfile
        '''
        assert self._add_overwrite_info, 'serialization object must have been constructed with can_overwrite_member == True'
        overwrite_info = self._overwrite_info_by_pyid[id(obj)]
        if isinstance(member, int) or isinstance(obj, dict):
            pos, format = overwrite_info[member]
        else:
            pos, format = getattr(overwrite_info, member)
        value   = pack(format, value)
        old_pos = self._file.tell()
        try:
            self._file.seek(pos)
            self._file.write(value)
        finally:
            self._file.seek(old_pos)

    def is_member_writable(self, obj, member):
        '''Returns True if the object's member can be overwritten.
        Can (but isn't guaranteed to) raise an exception if the member doesn't exist.

        :param obj: an object in the hierarchy returned by read_stream()
        :param member: a member (attribute) name, index (for lists/arrays), or key (for dicts) in obj
        '''
        assert self._add_overwrite_info, 'serialization object must have been constructed with can_overwrite_member == True'
        overwrite_info = self._overwrite_info_by_pyid.get(id(obj))
        if overwrite_info is None:
            return False
        if isinstance(member, int) or isinstance(obj, dict):
            return overwrite_info[member]          is not None
        else:
            return getattr(overwrite_info, member) is not None

# Finish setting up the serialization class
serialization._create_PrimitiveType_readers()
#
serialization._AdditionalInfo_readers = (
    lambda self: self._file.read(1),           # 0 Primitive
    serialization._read_Null,                  # 1 String
    serialization._read_Null,                  # 2 Object
    serialization._read_LengthPrefixedString,  # 3 SystemClass
    serialization._read_ClassTypeInfo,         # 4 Class
    serialization._read_Null,                  # 5 ObjectArray
    serialization._read_Null,                  # 6 StringArray
    lambda self: self._file.read(1),           # 7 PrimitiveArray
)

# Now that we're done adding PrimitiveType and RecordType readers, ensure they're
# all present (except PrimitiveType 4 and RecordTypes 18-20 which aren't defined)
assert all(bytes([i]) in serialization._PrimitiveType_readers if i != 4            else True for i in range(1, 19))
assert all(bytes([i]) in serialization._RecordType_readers    if not 18 <= i <= 20 else True for i in range(23))

def read_stream(streamfile):
    '''Read a file in .NET Remoting Binary Format and extract its root object

    :param streamfile: a file-like object
    :return: the root object contained in the stream
    '''
    return serialization(streamfile).read_stream()

# A JSONEncoder which can convert an object returned by read_stream() into json
# (can't handle circular references; primarily intended for debugging purposes)
class JSONEncoder(json.JSONEncoder):
    def default(self, o):
        if hasattr(o, '_asdict'):
            d = OrderedDict(_class_name=o.__class__.__name__)  # prepend the class name
            d.update(o._asdict())
            return d
        if isinstance(o, Array):
            return o.tolist()
        if isinstance(o, (datetime, timedelta)):
            return str(o)
        if isinstance(o, Decimal):
            return repr(o)
        return super().default(o)

# Returns a version of the identifier suitable to pass to namedlist
_identifier_re = re.compile('[^a-z0-9_]', flags=re.IGNORECASE)
def sanitize_identifier(identifier):
    identifier = _identifier_re.sub('_', identifier).lstrip('0123456789_')
    if not identifier:
        return 'invalid_identifier'
    if iskeyword(identifier):
        identifier += '_'
    assert identifier.isidentifier()
    return identifier

# Returns a version of the name which isn't present in the unique_set
def make_unique(name, unique_set):
    if name not in unique_set:
        return name
    for append in itertools.count(2):
        replacement = name + str(append)
        if replacement not in unique_set:
            return replacement

# Pre-allocates a "multidimensional array", i.e. a list of lists of Nones
def multidimensional_array(lengths):
    if not lengths:
        return None
    arrays = [None] * lengths[-1]
    for length in reversed(lengths[:-1]):
        arrays = [deepcopy(arrays) for i in range(length)]
    return arrays

if __name__ == '__main__':
    if len(sys.argv) != 2:
        sys.exit(f'Usage: {sys.argv[0]} streamfile')
    json_encoder = JSONEncoder(indent=4)
    with open(sys.argv[1], 'rb') as streamfile:
        while True:
            print(json_encoder.encode(read_stream(streamfile)))
            if streamfile.peek(1) == b'':
                break