from typing import Optional

import torch
import numpy as np
import argparse
import os
from os.path import dirname, join
from pathlib import Path
import yaml

from dingo.core.models import PosteriorModel
from dingo.core.utils.plotting import plot_corner_multi
from dingo.gw.data.event_dataset import EventDataset
from dingo.gw.inference.gw_samplers import GWSampler, GWSamplerGNPE
from dingo.gw.data.data_preparation import get_event_data_and_domain, \
    parse_settings_for_raw_data
from dingo.gw.inference.visualization import load_ref_samples

def parse_args():
    parser = argparse.ArgumentParser(
        description="Infer the posterior of a GW event using a trained dingo model.",
    )
    parser.add_argument(
        "--out_directory",
        type=str,
        default=None,
        help="Directory for output of analysis (samples, plots).",
    )
    parser.add_argument(
        "--model",
        type=str,
        help="Path to trained dingo model.",
    )
    parser.add_argument(
        "--model_init",
        type=str,
        default=None,
        help="Path to trained dingo model for initialization of gnpe. Only required if "
        "gnpe is used.",
    )
    parser.add_argument(
        # TODO: this should be renamed --events or similar
        "--gps_time_event",
        type=str,
        required=True,
        nargs="+",
        help="List of GPS times of the events to be analyzed. Used to download the GW "
        "event data, or search for it in the dataset file. Can also be a string with "
        "the path to a file containing the event data.",
    )
    parser.add_argument(
        "--event_dataset",
        type=str,
        default=None,
        help="Path to dataset for GW event data. If set, GW event data is cached in "
        "this dataset, such that it only needs to be downloaded once.",
    )
    parser.add_argument(
        "--event_dataset_init",
        type=str,
        default=None,
        help="Path to dataset for GW event data, for gnpe initialization model "
        "model_init. If set, GW event data is cached in this dataset, such that it "
        "only needs to be downloaded once.",
    )
    parser.add_argument(
        "--num_samples",
        type=int,
        default=50_000,
        help="Number of posterior samples per event.",
    )
    parser.add_argument(
        "--num_gnpe_iterations",
        type=int,
        default=30,
        help="Number of gnpe iterations.",
    )
    parser.add_argument(
        "--batch_size",
        type=int,
        default=None,
        help="Batch size for sampling.",
    )
    parser.add_argument(
        "--reference_file",
        type=str,
        default=None,
        help="File with path to reference samples (e.g., bilby, LALInference). If set, "
        "the corresponding samples are loaded and compared in a cornerplot.",
    )
    # settings for raw GW data
    parser.add_argument(
        "--time_psd",
        type=float,
        default=1024,
        help="Time in seconds used for PSD estimation.",
    )
    parser.add_argument(
        "--time_buffer",
        type=float,
        default=2.0,
        help="Buffer time in seconds. The analyzed strain segment extends up to "
        "gps_time_event + time_buffer.",
    )
    parser.add_argument(
        "--suffix",
        type=str,
        default="",
        help="Optional suffix for sample name.",
    )
    parser.add_argument(
        "--get_log_prob",
        action="store_true",
        help="If set, log_probs of samples are saved. For GNPE models, this invokes "
        "training of an unconditional density estimator to recover the density.",
    )
    parser.add_argument(
        "--save_low_latency",
        action="store_true",
        help="Only relevant for GNPE models and if args.get_log_prob is set. In that "
        "case, and if save_low_latency is set, intermediate results without log_prob "
        "are saved.",
    )
    parser.add_argument(
        "--density_settings",
        type=str,
        default=None,
        help="Path to yaml file with settings for density estimation. Only used if "
        "log_prob is requested for a gnpe model.",
    )
    parser.add_argument(
        "--exit_command",
        type=str,
        default=None,
        nargs="+",
        help="If set, run os.system(args.exit_command) before exiting.",
    )

    args = parser.parse_args()

    return args

def get_event_data(event, args, model, ref=None):
    # if event is a float: interpret it as gps time, download the data
    # else: interpret it as path to file with event data
    try:
        time_event = float(event)
    except ValueError:
        time_event = None

    if time_event is not None:
        # get raw event data, and prepare it for the network domain
        event_data, _ = get_event_data_and_domain(
            model.metadata,
            time_event,
            args.time_psd,
            args.time_buffer,
            args.event_dataset,
        )

        event_metadata = parse_settings_for_raw_data(model.metadata, args.time_psd,
                                                     args.time_buffer)

        # Put the metadata in the same format as provided by dingo_pipe data_generation.
        # (This is a bit ad hoc, should be improved.)
        event_metadata["time_event"] = time_event
        event_metadata["psd_duration"] = event_metadata.pop("time_psd")
        window = event_metadata.pop("window")
        event_metadata["T"] = window["T"]
        del event_metadata["time_segment"]
        event_metadata["window_type"] = window["type"]
        event_metadata["roll_off"] = window["roll_off"]
        event_metadata["f_min"] = model.metadata["dataset_settings"]["domain"]["f_min"]
        event_metadata["f_max"] = model.metadata["dataset_settings"]["domain"]["f_max"]

        if ref is None or time_event not in ref:
            label = f"gps-{time_event}{args.suffix}"
        else:
            name_event = ref[time_event]["event_name"]
            label = name_event + args.suffix

    else:
        # load file with event data
        event_dataset = EventDataset(file_name=event)
        event_data = event_dataset.data
        event_metadata = event_dataset.settings
        label = Path(event).stem + args.suffix

    return event_data, event_metadata, label

def prepare_log_prob(
    sampler,
    num_samples: int,
    nde_settings: dict,
    batch_size: Optional[int] = None,
    threshold_std: Optional[float] = np.inf,
    remove_init_outliers: Optional[float] = 0.0,
    low_latency_label: str = None,
    outdir: str = None,
):
    """
    Prepare gnpe sampling with log_prob. This is required, since in its vanilla
    form gnpe does not provide the density for its samples.

    Specifically, we train an unconditional neural density estimator (nde) for the
    gnpe proxies. This requires running the gnpe sampler till convergence, and
    extracting the gnpe proxies after the final gnpe iteration. The nde is trained
    to match the distribution over gnpe proxies, which provides a way of rapidly
    sampling (converged!) gnpe proxies *and* evaluating the log_prob.

    After this preparation step, self.run_sampler can leverage
    self.gnpe_proxy_sampler (which is based on the aforementioned trained nde) to
    sample gnpe proxies, such that one gnpe iteration is sufficient. The
    log_prob of
    the samples in the *joint* space (inference parameters + gnpe proxies) is then
    simply given by the sum of the corresponding log_probs (from self.model and
    self.gnpe_proxy_sampler.model).

    Parameters
    ----------
    num_samples: int
        number of samples for training of nde
    batch_size: int = None
        batch size for sampler
    threshold_std: float = np.inf
        gnpe proxies deviating by more then threshold_std standard deviations from
        the proxy mean (along any axis) are discarded.
    low_latency_label: str = None
        File label for low latency samples (= samples used for training nde).
        If None, these samples are not saved.
    outdir: str = None
        Directory in which low latency samples are saved. Needs to be set if
        low_latency_label is not None.
    """
    sampler.remove_init_outliers = remove_init_outliers
    sampler.run_sampler(num_samples, batch_size)
    if low_latency_label is not None:
        sampler.to_hdf5(label=low_latency_label, outdir=outdir)
    result = sampler.to_result()
    nde_settings["training"]["device"] = str(sampler.model.device)
    unconditional_model = result.train_unconditional_flow(
        sampler.gnpe_proxy_parameters,
        nde_settings,
        threshold_std=threshold_std,
    )

    # Prepare sampler with unconditional model as initialization. This should only use
    # one iteration and also not remove any outliers.
    sampler.init_sampler = GWSampler(model=unconditional_model)
    sampler.num_iterations = 1
    sampler.remove_init_outliers = 0.0  # Turn off for final sampler.

def analyze_event():
    args = parse_args()

    if torch.cuda.is_available():
        device = "cuda"
    else:
        device = "cpu"

    model = PosteriorModel(args.model, device=device, load_training_info=False)
    epoch = model.epoch
    wf_model = model.metadata["dataset_settings"]["waveform_generator"]["approximant"]

    # create analysis directory
    if args.out_directory is None:
        args.out_directory = join(dirname(args.model), f"analysis_epoch-{epoch}")
    os.makedirs(args.out_directory, exist_ok=True)

    #
    if args.reference_file is not None:
        with open(args.reference_file, "r") as f:
            ref = yaml.safe_load(f)[wf_model]
    else:
        ref = None

    if args.model_init is not None:
        gnpe = True
        init_model = PosteriorModel(
            args.model_init, device=device, load_training_info=False
        )
        init_sampler = GWSampler(model=init_model)
        sampler = GWSamplerGNPE(
            model=model,
            init_sampler=init_sampler,
            num_iterations=args.num_gnpe_iterations,
        )
    else:
        gnpe = False
        sampler = GWSampler(model=model)

    # sample posterior for events
    for time_event in args.gps_time_event:
        print(f"Analyzing event at {time_event}.")

        event_data, event_metadata, label = get_event_data(time_event, args, model, ref)

        sampler.context = event_data
        sampler.event_metadata = event_metadata

        if gnpe and args.get_log_prob:
            # GNPE generally does not provide straightforward access to the log_prob.
            # If requested, need to train an initialization model for the GNPE proxies.
            with open(args.density_settings) as f:
                density_settings = yaml.safe_load(f)
            if args.save_low_latency:
                low_latency_label = label + "_low-latency"
            else:
                low_latency_label = None
            prepare_log_prob(
                sampler,
                batch_size=args.batch_size,
                low_latency_label=low_latency_label,
                outdir=args.out_directory,
                **density_settings,
            )
            if low_latency_label is not None and sampler.iteration_tracker.store_data:
                np.save(
                    join(
                        args.out_directory,
                        f"gnpe_trajectories_{low_latency_label}.npy",
                    ),
                    sampler.iteration_tracker.data,
                )

        sampler.run_sampler(
            args.num_samples,
            batch_size=args.batch_size,
        )

        # if no reference samples are available, simply save the dingo samples
        if ref is None or time_event not in ref:
            sampler.to_hdf5(label=label, outdir=args.out_directory)

        # if reference samples are available, save dingo samples and additionally
        # compare to the reference method in a corner plot
        else:
            ref_samples_file = ref[time_event]["reference_samples"]["file"]
            ref_method = ref[time_event]["reference_samples"]["method"]
            sampler.to_hdf5(label=label, outdir=args.out_directory)
            ref_samples = load_ref_samples(ref_samples_file)
            plot_corner_multi(
                [ref_samples, sampler.samples],
                labels=[ref_method, "Dingo"],
                filename=join(args.out_directory, f"cornerplot_{label}.pdf")
            )
    if args.exit_command:
        os.system(" ".join(args.exit_command))

if __name__ == "__main__":
    analyze_event()


from timeit import default_timer
from typing import List, Dict, Any, Callable

__all__ = ["time_this"]

# A module-level store of all the evaluation times of things you ran with the
# @time_this decorator
runtime_table: List[Dict[str, Any]] = list()

def time_this(f):
    def timed_function(*args, **kwargs):
        ts = default_timer()
        print(f'Running {f.__name__} ...')

        result = f(*args, **kwargs)
        te = default_timer()
        t = 1000 * (te - ts)
        print(f'Completed {f.__name__} in {round(t, 3)} milliseconds')
        print()
        return result

    return timed_function


from setuptools import find_packages, setup

with open("README.md", "r") as fh:
    LONG_DESC = fh.read()
    setup(
        name="hydra-range-sweeper",
        version="0.1.0",
        author="Jan-Matthis Lueckmann, Omry Yadan",
        author_email="mail@jan-matthis.de, omry@fb.com",
        description="Ray Launcher for Hydra apps",
        long_description=LONG_DESC,
        long_description_content_type="text/markdown",
        url="https://github.com/facebookresearch/hydra/",
        packages=find_packages(exclude=["tests", "example"]),
        classifiers=[
            "License :: OSI Approved :: MIT License",
            "Programming Language :: Python :: 3.6",
            "Programming Language :: Python :: 3.7",
            "Programming Language :: Python :: 3.8",
            "Operating System :: OS Independent",
        ],
        install_requires=["hydra-core>=1.0.0rc1"],
        include_package_data=True,
    )


"""
A program a raktáron levő zöldségekkel foglalkozik.
Bizonyos feladatokat kell ellátnia a feldolgozott szövegállományból (adatok.txt).
A feladat megoldásokhoz használja a programozási tételeket!
a) Írjon programot zoldsegek.py néven!
b) Hozzon létre egy osztályt, majd végezze el a beolvasást és eltárolja az adatokat
   egy megfelelő adatszerkezetben. A kódolást állítsa utf-8-ra.
c) Határozza meg és írja ki a képernyőre, a termékek számát a raktáron.
d) Számítsa ki, melyik termékből van a legtöbb. A kiíratásnál szerepeljen a termék neve és mennyisége.
e) Írassa ki, hogy mennyi az átlag termék ár.
f) Számolja ki a raktáron található termékek összértékét. A kapott értéket millióban jelenítse meg.
g) Az átlag feletti termékeket írassa ki a képernyőre. A kiírtatásban szerepeljen az ára és a termék neve.
"""

class ZoldsegRaktar:
    def __init__(self, bemenet):

        # Az adatok beolvasasa:
        data = open(bemenet, "r", encoding="utf-8").read().replace("\n", ";").split(";")
        adatok = list(data)
        self.adatok = adatok
        # print(adatok)

        # A termekek kulonvalogatasa:
        termekek = [adatok[i] for i in range(1, len(adatok), 3)]
        self.termekek = termekek
        # print(termekek)

        # A termekek mennyisegenek kulonvalogatasa
        dl = [float(adatok[i]) for i in range(0, len(adatok), 3)]
        self.dl = dl
        # print(dl)

        # Az ar kulonvalogatasa, atlag kiszamitasa
        ar = [float(adatok[i]) for i in range(2, len(adatok), 3)]
        self.ar = ar
        # print(ar)
        atlag_ossz = sum(ar)
        self.atlag = atlag_ossz / len(termekek)
        # print(self.atlag)

    def termek_szam(self):
        print("\nA termekek szama:", len(self.termekek))

    def legtobb_termek(self):
        legtobb = [0, 0]  # Az elso az index a masodik a legnagyobb mennyiseg
        for i, mennyiseg in enumerate(self.dl):
            if mennyiseg > legtobb[1]:
                legtobb = [i, mennyiseg]
        print("\nA legtobb a(z)", self.termekek[legtobb[0]] + "-bol van es", legtobb[1], "darab van belole")

    def atlagar(self):
        print("\nAz atlagar:", round(self.atlag), "Ft")

    def keszlet_ertek(self):
        keszlet_ertek = 0
        for dl, ar in zip(self.dl, self.ar):
            keszlet_ertek += ar * dl
        print("\nA keszlet erteke:", round(keszlet_ertek / 1000000, 2), "millio forint")

    def atlag_feletti(self):
        print("\nAz atlag aron feluli termekek:")
        for i, ar in enumerate(self.ar):
            if self.atlag < ar:
                print(ar, "Ft   -   ", self.termekek[i])

zoldseg = ZoldsegRaktar("adatok.txt")
zoldseg.termek_szam()
zoldseg.legtobb_termek()
zoldseg.atlagar()
zoldseg.keszlet_ertek()
zoldseg.atlag_feletti()


from __future__ import annotations

import logging
from typing import Optional

from langchain_core.callbacks import CallbackManagerForToolRun

from langchain_community.tools.edenai.edenai_base_tool import EdenaiTool

logger = logging.getLogger(__name__)

class EdenAiParsingInvoiceTool(EdenaiTool):
    """Tool that queries the Eden AI Invoice parsing API.

    for api reference check edenai documentation:
    https://docs.edenai.co/reference/ocr_invoice_parser_create.

    To use, you should have
    the environment variable ``EDENAI_API_KEY`` set with your API token.
    You can find your token here: https://app.edenai.run/admin/account/settings

    """

    name = "edenai_invoice_parsing"

    description = (
        "A wrapper around edenai Services invoice parsing. "
        """Useful for when you have to extract information from
        an image it enables to take invoices
        in a variety of formats and returns the data in contains
        (items, prices, addresses, vendor name, etc.)
        in a structured format to automate the invoice processing """
        "Input should be the string url of the document to parse."
    )

    language: Optional[str] = None
    """
    language of the image passed to the model.
    """

    feature = "ocr"
    subfeature = "invoice_parser"

    def _parse_response(self, response: list) -> str:
        formatted_list: list = []

        if len(response) == 1:
            self._parse_json_multilevel(
                response[0]["extracted_data"][0], formatted_list
            )
        else:
            for entry in response:
                if entry.get("provider") == "eden-ai":
                    self._parse_json_multilevel(
                        entry["extracted_data"][0], formatted_list
                    )

        return "\n".join(formatted_list)

    def _run(
        self,
        query: str,
        run_manager: Optional[CallbackManagerForToolRun] = None,
    ) -> str:
        """Use the tool."""
        query_params = {
            "file_url": query,
            "language": self.language,
            "attributes_as_list": False,
        }

        return self._call_eden_ai(query_params)


# https://diagrams.mingrammer.com/docs/nodes/aws
# https://www.graphviz.org/doc/info/attrs.html
from diagrams import Cluster, Diagram, Edge
from diagrams.aws.compute import Lambda
from diagrams.aws.storage import S3
from diagrams.aws.network import APIGateway
from diagrams.aws.database import DynamodbTable
from diagrams.aws.security import IdentityAndAccessManagementIam
from diagrams.aws.devtools import Codebuild
from diagrams.aws.devtools import Codecommit
from diagrams.aws.devtools import Codedeploy
from diagrams.aws.devtools import Codepipeline
from diagrams.aws.management import Cloudformation
from diagrams.aws.devtools import CommandLineInterface
from diagrams.aws.general import User
from diagrams.aws.general import General
from diagrams.onprem.client import Client

graph_attr = {
    "fontsize": "45"
}

cluster_attr = {
    "bgcolor": "lightgrey"
}

with Diagram("Canary Deployments", show=False, direction="LR", filename="canary-deployments", graph_attr=graph_attr):

    with Cluster("Launch API URL", direction="LR"):
        user = User("User")
        console = Client("Browser")
        user >> console

    with Cluster("Cloud9", direction="LR"):
        builder = User("Builder")
        cli = CommandLineInterface("AWS CLI")
        builder >> cli

    with Cluster("CloudFormation", direction="LR"):
        cloudformation = Cloudformation("Stack")
        cli >> cloudformation

    with Cluster("CodePipeline"):
        codepipeline = Codepipeline("Pipeline")
        cfn  = Cloudformation("Deploy")
        codepipeline >> cfn

    with Cluster("Serverless Application Model (SAM) Stack"):
        sam = Cloudformation("SAM Template")
        apigateway = APIGateway("API Gateway")
        mylambda = Lambda("Lambda")
        ddb  = DynamodbTable("DynamoDB")
        codedeploy  = Codedeploy("CodeDeploy")
        mylambda >> codedeploy
        sam >> apigateway
        sam >> mylambda
        sam >> ddb
        mylambda >> ddb

        cloudformation >> codepipeline
        cfn >> sam
        console >> apigateway


import numpy as np
import scipy.io
from load_data import loading_data
from load_data import split_data

# environment and parameters
checkpoint_path = './checkpoint'

SEMANTIC_EMBED = 512
MAX_ITER = 100
batch_size = 64

images, tags, labels = loading_data()
dimTxt = tags.shape[1]
dimLab = labels.shape[1]

DATABASE_SIZE = 18015
TRAINING_SIZE = 18015
QUERY_SIZE = 2000

X, Y, L = split_data(images, tags, labels, QUERY_SIZE, TRAINING_SIZE, DATABASE_SIZE)
train_L = L['train']
train_x = X['train']
train_y = Y['train']

query_L = L['query']
query_x = X['query']
query_y = Y['query']

retrieval_L = L['retrieval']
retrieval_x = X['retrieval']
retrieval_y = Y['retrieval']

num_train = train_x.shape[0]
numClass = train_L.shape[1]

Sim = (np.dot(train_L, train_L.transpose()) > 0).astype(int)*0.999

Epoch = 30
k_lab_net = 10
k_img_net = 15
k_txt_net = 15

bit = 16
# hyper here

# Learning rate
lr_lab = [np.power(0.1, x) for x in np.arange(2.0, MAX_ITER, 0.5)]
lr_img = [np.power(0.1, x) for x in np.arange(4.5, MAX_ITER, 0.5)]
lr_txt = [np.power(0.1, x) for x in np.arange(3.5, MAX_ITER, 0.5)]
lr_dis = [np.power(0.1, x) for x in np.arange(3.0, MAX_ITER, 0.5)]


"""initial migration

Revision ID: 282408d6d0f2
Revises:
Create Date: 2017-11-17 12:01:28.240094

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '282408d6d0f2'
down_revision = None
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('login',
    sa.Column('id', sa.INTEGER(), nullable=False),
    sa.Column('user_id', sa.INTEGER(), nullable=True),
    sa.Column('passwd', sa.VARCHAR(length=20), nullable=True),
    sa.Column('identity', sa.VARCHAR(length=10), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_table('suggestion')
    op.drop_table('goods_common')
    op.drop_table('card_info')
    op.drop_table('django_migrations')
    op.alter_column('user', 'username',
               existing_type=mysql.VARCHAR(length=30),
               nullable=True)
    op.create_unique_constraint(None, 'user', ['email'])
    op.create_unique_constraint(None, 'user', ['username'])
    op.drop_column('user', 'department_number')
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('user', sa.Column('department_number', mysql.INTEGER(display_width=10), autoincrement=False, nullable=True))
    op.drop_constraint(None, 'user', type_='unique')
    op.drop_constraint(None, 'user', type_='unique')
    op.alter_column('user', 'username',
               existing_type=mysql.VARCHAR(length=30),
               nullable=False)
    op.create_table('django_migrations',
    sa.Column('id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('app', mysql.VARCHAR(length=255), nullable=False),
    sa.Column('name', mysql.VARCHAR(length=255), nullable=False),
    sa.Column('applied', mysql.DATETIME(fsp=6), nullable=False),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('card_info',
    sa.Column('id', mysql.INTEGER(display_width=50), nullable=False),
    sa.Column('student_id', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.Column('kinds', mysql.VARCHAR(length=30), nullable=True),
    sa.Column('location', mysql.VARCHAR(length=20), nullable=True),
    sa.Column('pay_type', mysql.VARCHAR(length=20), nullable=True),
    sa.Column('pay_time', mysql.DATETIME(), nullable=True),
    sa.Column('money', mysql.VARCHAR(length=10), nullable=True),
    sa.Column('left_money', mysql.VARCHAR(length=10), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('goods_common',
    sa.Column('id', mysql.INTEGER(display_width=11), nullable=False),
    sa.Column('title', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('vardate', mysql.DATETIME(), nullable=True),
    sa.Column('linkman', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('contact', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('pulldate', mysql.DATETIME(), nullable=True),
    sa.Column('validdate', mysql.DATETIME(), nullable=True),
    sa.Column('haulway', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('ps', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('websource', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('websourcelink', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('longterm', mysql.VARCHAR(length=255), nullable=True),
    sa.Column('linkaddress', mysql.VARCHAR(length=255), nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.create_table('suggestion',
    sa.Column('id', mysql.INTEGER(display_width=50), nullable=False),
    sa.Column('student_id', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.Column('type', mysql.VARCHAR(length=10), nullable=True),
    sa.Column('mean', mysql.DOUBLE(precision=20, scale=0, asdecimal=True), nullable=True),
    sa.Column('amount', mysql.INTEGER(display_width=20), autoincrement=False, nullable=True),
    sa.PrimaryKeyConstraint('id'),
    mysql_default_charset='utf8',
    mysql_engine='InnoDB'
    )
    op.drop_table('login')
    # ### end Alembic commands ###


import math

from Cryptodome.Random import random

from schemes import (
    precompute_both_scheme,
    precompute_gm_scheme,
    precompute_gnr_scheme,
    scheme1,
    scheme3,
)
from schemes.config import POWER

def testScheme1(m1, m2):
    ps = scheme1.PaillierScheme()

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testScheme3(m1, m2):
    ps = scheme3.PaillierScheme()

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeGm(m1, m2):
    ps = precompute_gm_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_gm-2022-01-06_22:16:03.993283.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeGnr(m1, m2):
    ps = precompute_gnr_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_both-2022-01-07_14.47.27.047353.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

def testPrecomputeBoth(m1, m2):
    ps = precompute_both_scheme.PaillierScheme.constructFromJsonFile(
        "precompute_both-2022-01-07_14.47.27.047353.json"
    )

    ct1 = ps.encrypt(m1)
    ct2 = ps.encrypt(m2)

    pt1 = ps.decrypt(ct1)
    pt2 = ps.decrypt(ct2)

    pt3 = ps.decrypt(ps.add_two_ciphertexts(ct1, ct2))

    assert pt1 == m1
    assert pt2 == m2
    assert pt1 + pt2 == pt3

if __name__ == "__main__":
    m1 = random.getrandbits(int(math.log2(POWER)) * 2)
    m2 = random.getrandbits(int(math.log2(POWER)) * 2)

    print("Testing scheme1")
    testScheme1(m1, m2)

    print("Testing scheme3")
    testScheme3(m1, m2)

    print("Testing precompute_gm")
    testPrecomputeGm(m1, m2)

    print("Testing precompute_gnr")
    testPrecomputeGnr(m1, m2)

    print("Testing precompute_both")
    testPrecomputeBoth(m1, m2)

    print("Finished successfully")


import os
import numpy as np
import cv2
import yaml
from shutil import copyfile
import multiprocessing
import matplotlib.pyplot as plt

def create_config(config_path):
    if not os.path.exists(config_path):
        copyfile('config.yml.example', config_path)

def create_dir(dir):
    if not os.path.exists(dir):
        os.makedirs(dir)

def resave_config(config, config_path):
    with open(config_path, 'w') as f_obj:
        yaml.dump(config, f_obj)
    with open(config_path, 'r') as f_obj:
        config = yaml.load(f_obj, Loader=yaml.FullLoader)
    return config

def point_linear_dis(xy2, xy1, point_xy):
    linear = np.array([xy2[0]-xy1[0], xy2[1]-xy1[1]])
    point_linePoint = np.array([xy2[0]-point_xy[0], xy2[1]-point_xy[1]])
    array_temp = (float(point_linePoint.dot(linear)) /
                  linear.dot(linear))
    array_temp = linear.dot(array_temp)
    distance = np.sqrt(
        (point_linePoint - array_temp).dot(point_linePoint - array_temp))
    return distance

def set_flist_config(config_path, train_val_test_path, flag='data'):
    with open(config_path, 'r') as f_obj:
        config = yaml.load(f_obj, Loader=yaml.FullLoader)
    if flag == 'mask':
        config['TRAIN_MASK_FLIST'] = train_val_test_path[0]
        config['VAL_MASK_FLIST'] = train_val_test_path[1]
        config['TEST_MASK_FLIST'] = train_val_test_path[2]
    if flag == 'data':
        config['TRAIN_FLIST'] = train_val_test_path[0]
        config['VAL_FLIST'] = train_val_test_path[1]
        config['TEST_FLIST'] = train_val_test_path[2]
    if flag == 'edge':
        config['TRAIN_EDGE_FLIST'] = train_val_test_path[0]
        config['VAL_EDGE_FLIST'] = train_val_test_path[1]
        config['TEST_EDGE_FLIST'] = train_val_test_path[2]
    with open(config_path, 'w') as f_obj:
        yaml.dump(config, f_obj)

def data_split(split_num, LOAD_PATH, SAVE_PATH, key_name, id_img, flist_path, RGB=False):
    # Generate split_num^2 small images
    print("******* data split ->START ******\n")
    edge_flist_list = np.genfromtxt(LOAD_PATH, dtype=np.str, encoding='utf-8')
    if len(edge_flist_list.shape) == 0:
        edge_flist_list_ = np.array([])
        edge_flist_list = np.append(edge_flist_list_, edge_flist_list)
    id_img_last = id_img
    image_flist = []
    for img_path in edge_flist_list:
        if RGB:
            img = cv2.imread(img_path, 3)
        else:
            img = cv2.imread(img_path, 0)
    #     cv2.imshow('edge', img)
    #     k = cv2.waitKey(0)
        imh, imw = img.shape[0:2]
        interval = [imh//split_num, imw//split_num]
        start_lists = [[0, 0], [interval[0]//2, 0],
                       [0, interval[1]//2], [interval[0]//2, interval[1]//2]]
        for start_list in start_lists:
            row = list(range(start_list[0], imh+1, interval[0]))
            col = list(range(start_list[1], imw+1, interval[1]))
            for i_row in range(0, len(row)-1):
                for i_col in range(0, len(col)-1):
                    if RGB:
                        cv2.imwrite(SAVE_PATH+'/'+str(id_img)+'_'+key_name+'.png',
                                    img[row[i_row]:row[i_row+1], col[i_col]:col[i_col+1], :])
                    else:
                        cv2.imwrite(SAVE_PATH+'/'+str(id_img)+'_'+key_name+'.png',
                                    img[row[i_row]:row[i_row+1], col[i_col]:col[i_col+1]])
                    image_flist.append(
                        SAVE_PATH+'/'+str(id_img)+'_'+key_name+'.png')
                    id_img += 1
        print('save '+img_path + "'s "+' sub image:' +
              str(id_img_last)+'-'+str(id_img))
        id_img_last = id_img
    np.savetxt(flist_path, image_flist, fmt='%s')
    print("******* data split ->END ******\n")
    return id_img

class Multiprocessing():
    def __init__(self, nThreads):
        self.nThreads = nThreads
        self.pool = multiprocessing.Pool(processes=self.nThreads)

    def process(self, fun, flist):
        self.pool.map(fun, flist)

    def close(self):
        self.pool.close()
        self.pool.join()

def imshow(img, title=''):
    fig = plt.gcf()
    fig.canvas.set_window_title(title)
    plt.axis('off')
    if len(img.size) == 3:
        plt.imshow(img, interpolation='none')
        plt.show()
    else:
        plt.imshow(img, cmap='Greys_r')
        plt.show()

def imshow_img_point(img, points_all, title=''):
    fig = plt.gcf()
    fig.canvas.set_window_title(title)
    plt.axis('on')
    color = ['b', 'r', 'y', 'g']
    i = 0
    for points_i in points_all:
        for points in points_i:
            plt.plot(points[0], points[1], '.'+color[i])
            if i >= 2:
                points[0] = np.append(points[0], points[0][0])
                points[1] = np.append(points[1], points[1][0])
                plt.plot(points[0], points[1], '-'+color[i])
        i += 1

    if len(img.size) == 3:
        plt.imshow(img, interpolation='none')
        plt.show()
    else:
        plt.imshow(img, cmap='Greys_r')
        plt.show()


# The isBadVersion API is already defined for you.
# @param version, an integer
# @return a bool
def isBadVersion(version):
    if version < 4:
        return False
    return True

class Solution:
    def firstBadVersion(self, n):
        """
        :type n: int
        :rtype: int
        """
        l, r = 1, n + 1
        while l < r:
            m = l + (r - l) // 2
            if isBadVersion(m):
                r = m
            else:
                l = m + 1
        return l

def test():
    s = Solution()
    assert s.firstBadVersion(100) == 4
    assert s.firstBadVersion(4) == 4

if __name__ == '__main__':
    test()


from vyper.compiler import (
    compile_codes,
    mk_full_signature,
)

def test_only_init_function():
    code = """
x: int128

@public
def __init__():
    self.x = 1
    """
    code_init_empty = """
x: int128

@public
def __init__():
    pass
    """

    empty_sig = [{
        'outputs': [],
        'inputs': [],
        'constant': False,
        'payable': False,
        'type': 'constructor'
    }]

    assert mk_full_signature(code) == empty_sig
    assert mk_full_signature(code_init_empty) == empty_sig

def test_default_abi():
    default_code = """
@payable
@public
def __default__():
    pass
    """

    assert mk_full_signature(default_code) == [{
        'constant': False,
        'payable': True,
        'type': 'fallback'
    }]

def test_method_identifiers():
    code = """
x: public(int128)

@public
def foo(x: uint256) -> bytes[100]:
    return b"hello"
    """

    out = compile_codes(
        codes={'t.vy': code},
        output_formats=['method_identifiers'],
        output_type='list'
    )[0]

    assert out['method_identifiers'] == {
        'foo(uint256)': '0x2fbebd38',
        'x()': '0xc55699c'
    }


#!/usr/bin/env python3
import sys

def Hours(Time):
    if Time < 0:
        raise ValueError("Parameter Error")
    else:
        Hour = int(Time // 60)
        Minut = int(Time % 60)
        print("{} H, {} M".format(Hour, Minut))

if __name__ == '__main__':
    try:
        Hours(int(sys.argv[1]))
    except:
        print("Parameter Error")


import calendar
import time
import datetime
from datetime import datetime, timedelta
from datetime import datetime
import os
import os.path as op
import zipfile
import random

# calendar: 获取一年的日历字符串
# 参数
# w = 每个日期之间的间隔字符数
# l = 每周所占用的行数
# c = 每个月之间的间隔字符数
cal = calendar.calendar(2017, l=0, c=3)

# isleap：判断某一年是否闰年
isleap = calendar.isleap(2007)

# leapdays：获取指定年份之间的闰年的个数
leapdays = calendar.leapdays(1998, 2018)

# mouth() 获取某个月的日历字符串
# 格式：calendar.mouth(年，月)
# 返回值：月日历的字符串
m3 = calendar.month(2018, 2)

# monthrange() 获取一个月的周几开始及总天数
# 格式：calendar.monthrange（年，月）
# 返回值：元组（周几开始，总天数）
# 注意：周默认 0-6，表示周一到周天
w, t = calendar.monthrange(2017, 3)

# monthcalendar() 返回一个月每天的矩阵列表
# 格式：calendar.monthcalendar(年，月)
# 返回值：二级列表
# 注意：矩阵中没有天数用0表示
m = calendar.monthcalendar(2018, 3)

# prcal：直接打印日历
# calendar.prcal(2018)
# help(calendar.prcal)

# prmonth() 直接打印整个月的日历
# 格式：calendar.permonth(年，月)
# 返回值：无
# calendar.prmonth(2018, 3)

# weekday() 获取周几
# 格式：calendar.weekday(年，月，日)
# 返回值：周几对应的数字
weekday = calendar.weekday(2018, 2, 2)

# 时间模块
# timezone：当期时区和UTC时间差的描述，在没有夏令时的情况下的间隔，东八区的是-28800
# altzone：获取当前时区与UTC时间差的秒数，在有夏令时的情况下
# daylight 测试当前是否是夏令时时间状态，
# print(time.timezone)

# 得到时间戳
t = time.time()

# localtime，得到当前时间的时间结构
# 可以通过点号操作符得到相应的属性元素的内容
t = time.localtime()

# asctime() 返回元组的正常字符串化之后的时间格式
# 格式：time.asctime（时间元组）
# 返回值：字符串 Tue Jun 6 11:11:00 2017
t = time.asctime()

# ctime：获取字符串化的当前时间
t = time.ctime()

# mktime() 使用时间元组获取对应的时间戳
# 格式：time.mktime(时间元组)
# 返回值：浮点数时间戳
lt = time.localtime()
ts = time.mktime(lt)

# clock：获取cpu时间，3.0 - 3.3版本直接使用，3.6调用有问题

# sleep：是程序进入睡眠，n秒后继续
time.sleep(1)

# 把时间表示称，2018年3月26日 21:05
t = time.localtime()
ft = time.strftime("%Y %m %d %H:%M", t)

# datetime常见属性
# datetime.date：一个理想的日期，提供year，mouth，day属性
# print(datetime.date(2018, 3, 26))

# datetime.time：提供一个理想的时间，提供hour，minute，sec，microsec等内容
# datetime.datetime：提供日期跟时间的组合
# datetime.timedelta：提供一个时间差，时间长度

# datetime.datetime

# 常用方法
# today：
# now
# utcnow
# fromtimestamp：从时间戳中返回本地时间
# dt = datetime(2018, 9, 26)
# print(dt.today())
# print(dt.now())
# print(dt.fromtimestamp(time.time()))

t1 = datetime.now()
# print(t1)
# print(t1.strftime("%Y-%m-%d %H:%M:%S"))
# td表示一小时的时间长度
td = timedelta(hours=1)

# 当前时间加上时间间隔后，得到一个小时后的时间
# print((t1 + td).strftime("%Y-%m-%d %H:%M:%S"))

# timeit-时间测量工具
# 测量测序运行时间间隔实验
def p():
    time.sleep(3.6)

t1 = time.time()
# p()
# print(time.time() - t1)
# 生成列表两种方法的比较
# 如果单纯比较生成一个列表的时间，可能很难实现
c = '''
sum = []
for i in range(1000):
    sum.append(i)
'''
import timeit

# 利用timeit调用代码，执行100000，查看运行时间
# t1 = timeit.timeit(stmt="[i for i in range(1000)]", number=100000)
# 测量代码c执行100000次运行结果
# t2 = timeit.timeit(stmt=c, number=100000)
# print(t1)
# print(t2)
# timeit 可以执行一个函数，来测量一个函数的执行时间
# def doIt():
#     num = 3
#     for i in range(num):
#         print("Repeat for {0}".format(i))
#
#
# t = timeit.timeit(stmt=doIt, number=10)
# print(t)
s = """
def doIt(num):
    for i in range(num):
        print("Repeat for {0}".format(i))
"""
# 执行doIt(num)
# setup负责吧环境变量准备好
# 实际相当于给timi创再来个一个小环境
# 在创建的小环境中，代码执行的顺序大致是
#
"""
def doIt(num):
    ....
num = 3
doIt(num)
"""
# t = timeit.timeit("doIt(num)", setup=s + "num=3", number=10)

# getcwd() 获取当期的工作目录
# 格式： os.getcwd()
# 返回值：当前工作目录的字符串
# 当前工作目录就是程序在进行文件相关操作，默认查找文件的目录
mydir = os.getcwd()

# chdir() 改变当前的工作目录
# change directory
# 格式： os.chdir(路径)
# 返回值：无
# os.chdir('G:\Python\marketdown')
# mydir = os.getcwd()

# listdiir() 获取一个目录中所有子目录和文件的名称列表
# 格式： os.listdir（路径）
# 返回值：所有子目录和文件名称的列表
ld = os.listdir()

# makedir() 递归创建文件夹
# 格式：os.makedirs(递归路径)
# 返回值：无
# 递归路径：多个文件夹层层包换的路径就是递归路径，例如 a/b/c
# rst = os.makedirs('test')

# system() 运行系统shell命令
# 格式： os.system(系统命令)
# 返回值：打开一个shell或者终端界面
# 一般推荐使用subprocess代替
# rst = os.system("dir")

# getenv() 获取指定的系统环境变量值
# 格式：os.getenv('环境变量名')
# 返回值：指定环境变量名对应的值
rst = os.getenv('PATH')

# exit() 退出当前程序
# 格式：exit()
# 返回值：无

# abspath() 将路径转换成绝对路径
# abselute 绝对
# 格式：os.path.abspath('路径')
# 返回值：路径的绝对路径形式
# 在linux中
# . 点号，代表当前目录
# .. 双点，代表父目录
absp = op.abspath(".")

# basename() 获取路径中的文件名部分
# 格式：os.path.basename('路径')
# 返回值：文件名字符串

# join() 将多个路径拼合成一个路径
# 格式：os.path.join(路径1，路径2)
# 返回值：组合之后的新路径字符串

# split() 将路径切割成文件夹部分和当期文件部分
# 格式：os.path.split(路径)
# 返回值：路径和文件名组成的元祖

# isdir() 检测是否是目录
# 格式： os.path.isdir(路径)
# 返回值：布尔值

# exists() 检测文件或者目录是否存在
# 格式：os.path.exists(路径)
# 返回值：布尔值

# copy() 复制文件
# 格式： shutil.copy(来源路径，目标路径)
# 返回值：返回目标路径
# 拷贝的同时，可以给文件重命名

# copy2() 复制文件，保留元数据（文件信息）
# 格式：shutil.copy2(来源路径，目标路径)
# 返回值：返回目标路径
# 注意：copy和copy2的唯一区别在于copy2复制文件时尽量保留元数据

# copyfile() 将一个文件中的内容复制到另一个文件当中
# 格式：shutil.copyfile（'源路径','目标路径'）
# 返回值：无

# move() 移动文件/文件夹
# 格式： shutil.move('源路径','目标路径')
# 返回值：目标路径

# make_archive() 归档操作
# 格式：shutil.make_archive('归档之后的目录和文件名','后缀','需要归档的文件夹')
# 返回值：归档之后的地址

# unpack_archive() 解包操作
# 格式：shutil.unpack_archive('归档文件地址','解包之后的地址')
# 返回值：解包之后的地址

# zipfile.ZipFile(file[.mode[,compression[,allowZip64]]])
# 创建一个ZipFile对象，表示一个zip文件。参数file表示文件的路径或者类文件对象

# zipfile.getinfo(name)
# 获取zip文档内指定文件的信息。返回一个zipfile.ZipInfo对象，它包括文件的详细信息。

# zipfile.namelist()
# 获取zip文档内所有文件的名称列表。

# ZipFile.extractall([path.[,members[,pwd]]])
# 解压zip文档中的所有文件当当前目录。参数members的默认值为zip文档内的所有文件名称...

# random() 获取0-1之间的随机小数
# 格式：random.random()
# 返回值：随机0-1之间的小数

# choice() 随机返回序列中的某个值
# 格式：random.choice(序列)
# 返回值：序列中的某个值
l = [str(i) + "haha" for i in range(10)]
rst = random.choice(l)

# shuffle() 随机打乱列表
# 格式：random.shuffle(列表)
# 返回值：None，打乱源列表

# randint() 随机产生一个证书
# 格式：random.randint(a,b)
# 返回值：返回一个a到b之间的随机整数，包含a和b


'''
分析字幕文件中出现的所有英文单词，并累计出现的次数
统计单词中所有字母的频次

steps:
    1. 识别文件类型，txt、rar、zip
        zip: 50 4B 03
        rar: 52 61 72 21 1A 07 00
        else:
    2. 打开/解压文件
        7z x 14.zip -y -oE:\tmp
    3. 提取单词
    4. 统计

https://blog.csdn.net/weixin_42902669/article/details/102806154
'''
import os
import shutil
import re
import subprocess
import json
import stat

class data():
    def __init__(self):
        self.words = dict()
        self.letters = dict()

        for i in range(65, 91):
            self.letters[chr(i)] = 0
        for i in range(97, 123):
            self.letters[chr(i)] = 0

    def add_word(self, word):
        # print('add_word: ', word)
        if word in self.words:
            self.words[word] += 1
        else:
            self.words[word] = 1

        for a in word:
            self.letters[a] += 1

    def save(self, filename):
        with open(f'{results}\{filename}_letters.json', mode='wt') as f:
            f.write(json.dumps(sorted(self.letters.items(),
                                      key=lambda i: i[1], reverse=True)))

        with open(f'{results}\{filename}_words.json', mode='wt') as f:
            f.write(json.dumps(sorted(self.words.items(),
                                      key=lambda i: i[1], reverse=True)))

    def save_total(dl: list):
        words = dict()
        letters = dict()

        for i in range(65, 91):
            letters[chr(i)] = 0
        for i in range(97, 123):
            letters[chr(i)] = 0

        for d in dl:
            for k, v in d.letters.items():
                letters[k] += v

            for k, v in d.words.items():
                if k in words:
                    words[k] += v
                else:
                    words[k] = v

        with open(f'{results}\\total_letters.json', mode='wt') as f:
            f.write(json.dumps(sorted(letters.items(),
                                      key=lambda i: i[1], reverse=True)))

        with open(f'{results}\\total_words.json', mode='wt') as f:
            f.write(json.dumps(sorted(words.items(),
                                      key=lambda i: i[1], reverse=True)))

ds = 'requests\\download'
# ds = 'test'
tmp = os.path.join(os.getcwd(), '.tmp')
results = 'results'

def get_file_type(file):
    '''
    判断文件类型：txt, zip,  rar
    '''
    if os.path.getsize(file) < 8:
        return 'txt'

    with open(file, 'rb') as f:
        bs = f.read(8)
        if bs[0] == 0x50 and bs[1] == 0x4B and bs[2] == 0x03:
            return 'zip'
        elif bs[0] == 0x52 and bs[1] == 0x61 and bs[2] == 0x72:
            return 'rar'
        else:
            return 'txt'

def readonly_handler(func, path, execinfo):
    '''
    PermissionError: [WinError 5] 拒绝访问
    '''
    os.chmod(path, stat.S_IWRITE)
    func(path)

def mk_tmp():
    if os.path.exists(tmp):
        shutil.rmtree(tmp, onerror=readonly_handler)
        os.mkdir(tmp)
    else:
        os.mkdir(tmp)

def unfile(file):
    mk_tmp()
    # os.popen(f"7z x {file} -y -o{tmp}")
    ex=subprocess.Popen(f"7z x {file} -y -o{tmp}",
                          stdout = subprocess.PIPE, shell = True)
    out, err=ex.communicate()
    status=ex.wait()
    # print("cmd out: ", out.decode('gbk'))

def yield_tmp():
    for path, dir_list, file_list in os.walk('.tmp'):
        for file in file_list:
            f=os.path.join(path, file)
            yield f

def get_files(file):
    t=get_file_type(file)

    if t == 'txt':
        yield file
        return

    if t == 'rar' or t == 'zip':
        unfile(file)

    for f in yield_tmp():
        yield f

def da(file):
    d=data()
    r=re.compile(r'\b([A-Za-z]+)\b')
    rl = re.compile(r"}([a-zA-Z\d\s,\.?!']+)\n") # 取对白部分

    with open(file, mode = 'rt', encoding = 'utf-8', errors = 'ignore') as f:
        fl = f.readline().replace('\x00', '')
        if '[Script Info]' in fl:
            suf = 'ass'
        else:
            suf = 'srt'

        for line in f:
            line=line.replace('\x00', '')

            if suf=='ass':
                line = line.replace('{\\r}', '')
                m = rl.search(line)
                if m:
                    line = m.group(1)
                else:
                    continue

            # print(line)
            for m in r.finditer(line):
                w=m.group(1)
                d.add_word(w)

    return d

if __name__ == '__main__':
    if(os.path.exists(results)):
        shutil.rmtree(results)
    os.mkdir(results)

    dl=list()
    for path, dir_list, file_list in os.walk(ds):
        for file in file_list:
            f=os.path.join(path, file)
            for fp in get_files(f):
                print(fp)
                d=da(fp)
                d.save(os.path.basename(fp))
                dl.append(d)
    data.save_total(dl)


import pytest
import datetime
from snekdate.snekdate import SnekDate

############offset
def test_offset_type_returned():
    assert isinstance(SnekDate().offset(days=1), SnekDate)

@pytest.mark.parametrize(
    'end_date,offset_date,days,hours,minutes,seconds',
    [
        (
                datetime.datetime(day=20, year=2020, month=7),
                datetime.datetime(day=19, year=2020, month=7),
                1,
                0,
                0,
                0,
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=20),
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=15),
                0,
                0,
                0,
                5,
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=5, second=20),
                datetime.datetime(day=20, year=2020, month=7, hour=4, minute=3, second=50),
                0,
                0,
                0,
                90,
        ),
    ]
)
def test_offset_value_returned(end_date, offset_date, days, hours, minutes, seconds):
    assert SnekDate(end_date).offset(
        days=days,
        hours=hours,
        minutes=minutes,
        seconds=seconds,
    ).start == offset_date

############round_down_date
def test_round_down_type_returned():
    assert isinstance(SnekDate().round_down(increment='minute').start, datetime.datetime)

def test_round_down_exception():
    with pytest.raises(Exception):
        SnekDate.round_down_date(SnekDate().round_down(increment='minute'), increment='min')

@pytest.mark.parametrize(
    'date_raw,date_rounded,increment',
    [
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=0),
                'minute',
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=0, second=0),
                'hour',
        ),
        (
                datetime.datetime(day=20, year=2020, month=7, hour=3, minute=5, second=15),
                datetime.datetime(day=20, year=2020, month=7, hour=0, minute=0, second=0),
                'day',
        ),
    ]
)
def test_round_down_value_returned(date_raw, date_rounded, increment):
    assert SnekDate(date_raw).round_down(increment=increment).start == date_rounded


import numpy as np
import pandas as pd
import random

"""
KNN

1.  数据集标准化       X_normalization = (df - Df_min)/(Df_max- Df_min)
        对dataframe里面每个数都要做这步，so参数为df, 返回也是df形式
2. 求距离， 用test的一条数据(array形式) 减去 train_df (把标签剔除)    formular: [ (x1-y2)**2 + (x2 -y2)**2------] 再开根号 也就是*0.5
3. 距离按小到大排列， 去前K个
4. value_counts 一下前K个的标签， 取value_counts.index[0], 就是概率最大的那个
5. 把概率最大那个，作为预测类别

6. 预测结果.append to list, then, add to test_df
7. 看原标签列和预测列是否一样， 一样的为True， 否则False ，也弄一列correction_check
8. calculate_accuracy: correction_check.mean()
"""

# prepare data
"""
Dataset 1: Binary classification with breast_cancle Dataset

#path = "D:/Data/Dataset/breast_cancle.csv"
#df = pd.read_csv(path,)
#df["label"] = df.diagnosis
#df = df.drop(["id", "diagnosis"], axis = 1)
"""

#Dataset 2: Multiclass classification with iris Dataset
path = "D:/Data/Dataset/iris.csv"
df = pd.read_csv(path)
df = df.drop("Id", axis =1)
df = df.rename(columns={"species": "Label"})

print(df.head())
print(df.info())

def train_test_split(df, test_size):
        if isinstance(test_size, float):
            test_size = round(test_size * len(df)) # Round function to take only Int

        data_index = df.index.tolist()
        test_index = random.sample(population = data_index, k = test_size)

        test_df = df.loc[test_index]
        train_df = df.drop(test_index)
        #print("train\n{}\n test\n{}".format(train_df,  test_df))

        return train_df, test_df

#normalization
def normalization(df):
        label = df.iloc[:, -1]
        df = df.iloc[:, :-1]
        min_df = df.min()
        max_df = df.max()
        normed_df = (df - min_df) / (max_df - min_df)
        normed_df["label"] = label
        return normed_df

def calculate_distance(train, one_test):  # 一次只算一条test_data
        train_label = train.iloc[:, -1]
        train = train.iloc[:, :-1]

        one_test = one_test.values                                                  #  运算时是用array 减去 dataframe
        distance = ((one_test - train) ** 2).sum(axis = 1) * 0.5            #axis=1 =columns, 才是按行求和 #是列上相加
        #print("distance{}".format(distance))

        distance_df = pd.DataFrame(data = distance, columns = ["distance"])
        distance_df["label"] = train_label
        #print("distance_df:\n", distance_df)

        return distance_df

def classify(distance_df, k):
       sorted_distance = distance_df.sort_values(by ="distance")[: k]
       label_value = sorted_distance.iloc[:, -1]. value_counts()  # 标签列统计
       majority_label = label_value.index[0]
       return majority_label

def Knn_algorithm(df, k):
    normed_df = normalization(df)                                               #標準化
    train, test = train_test_split(normed_df , test_size= 0.1)        #train, test, split

    prediction_list = []
    for i in range(len(test)):                                                            # 对这个测试集:
          distance = calculate_distance(train, test.iloc[i, :-1])                      # 计算距离: array - df
          prediction = classify(distance, k)                                                    #  做预测
          prediction_list.append(prediction)

    test["prediction"] = prediction_list                                   # 结果确认
    correction_check = test["prediction"] == test["label"]
    #correction_check = test.iloc[:, -2] == test.iloc[:, -1]
    test["correction_check"] = correction_check
    accuracy = correction_check.mean()

    #global  None
    if accuracy==1.0:
        error = None
    else:
        label_True = test.iloc[:, -1].value_counts().index[0]      # true
        True_index = test[test["correction_check"] == label_True].index    # 把预测正确的index给出来，
        error = test.drop(index = True_index, axis = 0)                                #再把那些行给drop掉， 返回分类错的dataframe

    return accuracy, error, test

accuracy_list = []
column_names = df.columns
error_df = pd.DataFrame(columns = column_names, index = [0])
for i in range(10):
        accuracy, error, test = Knn_algorithm(df, k = 5)
        accuracy_list.append(accuracy)

        error_df = pd.concat([error_df, error], axis = 0)
        print("\nfor loop {}, test is:\n{}, \nerror:\n{}".format(i, test, error))
        print("accuracy:{}".format(accuracy))

error_df = error_df.drop(error_df.index[0], axis = 0)
error_counts = (error_df.index).value_counts()
average_accuracy = np.array(accuracy_list).mean()

print("\naccuracy_list:\n{}".format(accuracy_list, ))
print("\naverage_accuracy:{}".format(average_accuracy))
print("\nerror_df:\n{}\n".format(error_df))
print("\nerror_counts:\n{}\n".format(error_counts))


"""
solutions/directed.py
=====================

Solution for /content/directed.md.

"""

def directed():
    """Read the graph from stdin, and return the nodes with highest outdegree
    and indegree, respectively.

    This function reads every edge from stdin, computes the degrees of each
    node and then sorts them by degree. This wouldn't be efficient for
    large graphs. In that case, it would be better to keep count of the
    degrees as the edges are read, and never store the edges in memory.

    """
    num_nodes, num_edges = [int(i) for i in input().split(" ")]

    # nodes are just labeled by numbers
    nodes = range(num_nodes)

    # edges are a list of two nodes
    edges = []
    for _ in range(num_edges):
        edges.append([int(n) for n in input().split(" ")])

    # compute the degrees of each node by counting edges that include them
    # every edge is of the form 'source target'
    outdegree = {node: len([e for e in edges if e[0] == node])
                 for node in nodes}

    indegree = {node: len([e for e in edges if e[1] == node])
                for node in nodes}

    # sort nodes by degree and get out desired node
    highest_out = sorted(nodes, key=outdegree.get)[-1]
    highest_in = sorted(nodes, key=indegree.get)[-1]

    return highest_out, highest_in

if __name__ == '__main__':
    print("{0} {1}".format(*directed()))


#!/bin/python

import sys, os, fnmatch

if len(sys.argv) < 2:
	print('Missing root path.')
	sys.exit(1)

rootPath = sys.argv[1]
print('Rebuilding resource files at {}'.format(rootPath))

for root, dirnames, filenames in os.walk(rootPath):
	for filename in fnmatch.filter(filenames, '*.qrc'):
		baseName = filename[:-4]

		qrcFileName = baseName + '.qrc'
		pyFileName = baseName + '.py'
		relPath = root[len(rootPath)+1:]
		print('... {} => {}'.format(os.path.join(relPath, qrcFileName), os.path.join(relPath, pyFileName)))

		qrcPath = os.path.join(root, qrcFileName)
		pyPath = os.path.join(root, pyFileName)
		command = 'pyrcc5 -o {} {}'.format(pyPath, qrcPath)
		os.system(command)


import sys, glob, os, shutil
import numpy as np
import pandas as pd
import pickle
from argparse import ArgumentParser
from distutils.util import strtobool
from Bio.PDB import PDBParser
from Bio.PDB.DSSP import DSSP

'''
the purpose of this script is to take in PDB models straight from diffusion and run some
analysis on them, including calculating:
    - RMSD to motif
    - ROG
    - secondary structure
    if AF2 structures have been predicted:
        - AF2 plddt
        - AF2 pae
        - AF2 RMSD
'''

def get_args(argv=None):
    p = ArgumentParser(description=__doc__)
    p.add_argument("--folder", type=str, help="path to folder you want to run analysis on (with pdb and trb files)")
    p.add_argument("--temp_fn", type=str, help="fn of template pdb with motif")
    p.add_argument("--af2_metrics", strtobool,default=True, help="whether to look at af2 metrics")
    p.add_argument("--out_fn", type=str, help="fn to write output csv to")
    args = p.parse_args()
    if argv is not None:
        args = p.parse_args(argv) # for use when testing
    else:
        args = p.parse_args()
    return args

'''
DEFINE GOLBAL VARIABLES
'''
num2aa=[
    'ALA','ARG','ASN','ASP','CYS',
    'GLN','GLU','GLY','HIS','ILE',
    'LEU','LYS','MET','PHE','PRO',
    'SER','THR','TRP','TYR','VAL',
    ]

aa2long=[
    (" N  "," CA "," C  "," O  "," CB ",  None,  None,  None,  None,  None,  None,  None,  None,  None), # ala
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD "," NE "," CZ "," NH1"," NH2",  None,  None,  None), # arg
    (" N  "," CA "," C  "," O  "," CB "," CG "," OD1"," ND2",  None,  None,  None,  None,  None,  None), # asn
    (" N  "," CA "," C  "," O  "," CB "," CG "," OD1"," OD2",  None,  None,  None,  None,  None,  None), # asp
    (" N  "," CA "," C  "," O  "," CB "," SG ",  None,  None,  None,  None,  None,  None,  None,  None), # cys
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD "," OE1"," NE2",  None,  None,  None,  None,  None), # gln
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD "," OE1"," OE2",  None,  None,  None,  None,  None), # glu
    (" N  "," CA "," C  "," O  ",  None,  None,  None,  None,  None,  None,  None,  None,  None,  None), # gly
    (" N  "," CA "," C  "," O  "," CB "," CG "," ND1"," CD2"," CE1"," NE2",  None,  None,  None,  None), # his
    (" N  "," CA "," C  "," O  "," CB "," CG1"," CG2"," CD1",  None,  None,  None,  None,  None,  None), # ile
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD1"," CD2",  None,  None,  None,  None,  None,  None), # leu
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD "," CE "," NZ ",  None,  None,  None,  None,  None), # lys
    (" N  "," CA "," C  "," O  "," CB "," CG "," SD "," CE ",  None,  None,  None,  None,  None,  None), # met
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD1"," CD2"," CE1"," CE2"," CZ ",  None,  None,  None), # phe
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD ",  None,  None,  None,  None,  None,  None,  None), # pro
    (" N  "," CA "," C  "," O  "," CB "," OG ",  None,  None,  None,  None,  None,  None,  None,  None), # ser
    (" N  "," CA "," C  "," O  "," CB "," OG1"," CG2",  None,  None,  None,  None,  None,  None,  None), # thr
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD1"," CD2"," CE2"," CE3"," NE1"," CZ2"," CZ3"," CH2"), # trp
    (" N  "," CA "," C  "," O  "," CB "," CG "," CD1"," CD2"," CE1"," CE2"," CZ "," OH ",  None,  None), # tyr
    (" N  "," CA "," C  "," O  "," CB "," CG1"," CG2",  None,  None,  None,  None,  None,  None,  None), # val
]

aa2num= {x:i for i,x in enumerate(num2aa)}

'''
Functions
'''

def parse_pdb_lines(lines, parse_hetatom=False, ignore_het_h=True):
    '''
    Reading in PDB file
    '''
    # indices of residues observed in the structure
    res = [(l[22:26],l[17:20]) for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"]
    seq = [aa2num[r[1]] if r[1] in aa2num.keys() else 20 for r in res]
    pdb_idx = [( l[21:22].strip(), int(l[22:26].strip()) ) for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"]  # chain letter, res num
    plddt_val = [(float(l[61:66].strip()) ) for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"]  # chain letter, res num

    # 4 BB + up to 10 SC atoms
    xyz = np.full((len(res), 14, 3), np.nan, dtype=np.float32)
    for l in lines:
        if l[:4] != "ATOM":
            continue
        chain, resNo, atom, aa = l[21:22], int(l[22:26]), ' '+l[12:16].strip().ljust(3), l[17:20]
        idx = pdb_idx.index((chain,resNo))
        for i_atm, tgtatm in enumerate(aa2long[aa2num[aa]]):
            if tgtatm is not None and tgtatm.strip() == atom.strip(): # ignore whitespace
                xyz[idx,i_atm,:] = [float(l[30:38]), float(l[38:46]), float(l[46:54])]
                break

    # save atom mask
    mask = np.logical_not(np.isnan(xyz[...,0]))
    xyz[np.isnan(xyz[...,0])] = 0.0

    # remove duplicated (chain, resi)
    new_idx = []
    i_unique = []
    for i,idx in enumerate(pdb_idx):
        if idx not in new_idx:
            new_idx.append(idx)
            i_unique.append(i)

    pdb_idx = new_idx
    xyz = xyz[i_unique]
    mask = mask[i_unique]
    seq = np.array(seq)[i_unique]

    out = {'xyz':xyz, # cartesian coordinates, [Lx14]
            'mask':mask, # mask showing which atoms are present in the PDB file, [Lx14]
            'idx':np.array([i[1] for i in pdb_idx]), # residue numbers in the PDB file, [L]
            'seq':np.array(seq), # amino acid sequence, [L]
            'pdb_idx': pdb_idx,  # list of (chain letter, residue number) in the pdb file, [L]
            'plddt_val':plddt_val
           }

    # heteroatoms (ligands, etc)
    if parse_hetatom:
        xyz_het, info_het = [], []
        for l in lines:
            if l[:6]=='HETATM' and not (ignore_het_h and l[77]=='H'):
                info_het.append(dict(
                    idx=int(l[7:11]),
                    atom_id=l[12:16],
                    atom_type=l[77],
                    name=l[16:20]
                ))
                xyz_het.append([float(l[30:38]), float(l[38:46]), float(l[46:54])])

        out['xyz_het'] = np.array(xyz_het)
        out['info_het'] = info_het

    return out
def parse_pdb(filename, **kwargs):
    '''extract xyz coords for all heavy atoms'''
    lines = open(filename,'r').readlines()
    return parse_pdb_lines(lines, **kwargs)

def calc_rmsd(xyz1, xyz2, eps=1e-6):
    '''
    calculate RMSD between two sets of xyz coordinates
    '''
    # center to CA centroid
    xyz1 = xyz1 - xyz1.mean(0)
    xyz2 = xyz2 - xyz2.mean(0)

    # Computation of the covariance matrix
    C = xyz2.T @ xyz1

    # Compute optimal rotation matrix using SVD
    V, S, W = np.linalg.svd(C)

    # get sign to ensure right-handedness
    d = np.ones([3,3])
    d[:,-1] = np.sign(np.linalg.det(V)*np.linalg.det(W))

    # Rotation matrix U
    U = (d*V) @ W

    # Rotate xyz2
    xyz2_ = xyz2 @ U

    L = xyz2_.shape[0]
    rmsd = np.sqrt(np.sum((xyz2_-xyz1)*(xyz2_-xyz1), axis=(0,1)) / L + eps)

    return rmsd
# ROG calculator
def Rg(filename):
    '''
    Calculates the Radius of Gyration (Rg) of a protein given its .pdb
    structure file. Returns the Rg integer value in Angstrom.
    '''
    coord = list()
    mass = list()
    Structure = open(filename, 'r')
    for line in Structure:
        try:
            line = line.split()
            x = float(line[6])
            y = float(line[7])
            z = float(line[8])
            coord.append([x, y, z])
            if 'C' in line[2]:
                mass.append(12.0107)
            elif line[2] == 'O':
                mass.append(15.9994)
            elif line[2] == 'N':
                mass.append(14.0067)
            elif line[2] == 'S':
                mass.append(32.065)
        except:
            pass
    xm = [(m*i, m*j, m*k) for (i, j, k), m in zip(coord, mass)]
    tmass = sum(mass)
    rr = sum(mi*i + mj*j + mk*k for (i, j, k), (mi, mj, mk) in zip(coord, xm))
    mm = sum((sum(i) / tmass)**2 for i in zip(*xm))
    rg = math.sqrt(rr / tmass-mm)
    return(round(rg, 3))

def main():
    '''
    where everything is loaded in and calculated
    '''
    # load in arguments
    args = get_args()
    folder = args.folder
    temp_fn =args.temp_fn
    out_fn = args.out_fn

    # load in fns
    pdb_fns = glob.glob(f'{folder}/*pdb') # diffusion PDBS
    temp_pdb = parse_pdb(temp_fn) # template PDB of motif

    # build RMSD dict
    out_dict={} # dictionary of metrics
    for fn in pdb_fns:
        prefix =fn.split('/')[-1].split('.pdb')[0] # get prefix for naming
        itter_num = int(prefix.split('_')[-1]) # diffusion itteration number
        out_dict[itter_num]={}
        out_dict[itter_num]['file_name'] = prefix
        #load in pdb
        model_pdb = parse_pdb(fn)
        out_dict[itter_num]['ROG'] = Rg(fn) # ROG of diffused structure
        # load in trb file --> has some info on this
        trb_fn = f'{fn[0:-4]}.trb'
        trb_pkl = pd.read_pickle(trb_fn)
        #final plddt
        out_dict[itter_num]['diff_plddt'] = np.mean(trb_pkl['plddt'][-1]) # plddt from diffusion (not super meaninful)
        # get motif residue info
        temp_resi = trb_pkl['con_ref_pdb_idx'] # motif residues idx on template structure
        model_resi = trb_pkl['con_hal_pdb_idx']  # motif residue idx on diffusion structure
        # find idx of motif residues in template
        temp_motif_resi_idx=[]
        for idx, resi in enumerate(temp_pdb['pdb_idx']):
            if resi in temp_resi:
                temp_motif_resi_idx.append(idx)
        # find idx of motif residues in diffused model
        model_motif_resi_idx=[]
        for idx, resi in enumerate(model_pdb['pdb_idx']):
            if resi in model_resi:
                model_motif_resi_idx.append(idx)

        mot_len=len(temp_motif_resi_idx)

        # get MPNN sequence
        mpnn_seq = f'{folder}/seqs/{prefix}.fa'
        with open(mpnn_seq) as seq_fn:
            seq=seq_fn.readlines()[1]
        out_dict[itter_num]['seq']=seq

        # find RMSD
        xyz_temp = temp_pdb['xyz'][temp_motif_resi_idx]
        xyz_model = model_pdb['xyz'][model_motif_resi_idx]

        # bb only (C,N,CA,O)
        bb_rmsd = calc_rmsd(xyz_temp[:,:4].reshape(len(xyz_temp)*4,3),
                            xyz_model[:,:4].reshape(len(xyz_temp)*4,3))
        out_dict[itter_num]['model_temp_bb_rmsd'] = bb_rmsd

        # get secondary structure using DSSP of diffusion model
        p = PDBParser()
        structure = p.get_structure("idk", fn)
        model = structure[0]
        dssp = DSSP(model, fn)
        dssp_vals = np.array([dssp[e][2] for e in list(dssp.keys())])[model_motif_resi_idx]
        out_dict[itter_num]['DSSP_model'] = dssp_vals
        out_dict[itter_num]['len'] = len(out_dict[itter_num]['DSSP_model'])

        # if we also want to look at AF2 predictions:
        if args.af2_metrics:
            af2_fn = f'{folder}/af2/{prefix}.pdb'
            af2_pdb = parse_pdb(af2_fn)
            xyz_af2 = af2_pdb['xyz']
            if len(xyz_af2) == len(xyz_model):
                out_dict[itter_num]['DSSP_AF2'] = get_dssp(af2_fn)
                out_dict[itter_num]['ROG_AF2'] = Rg(af2_fn)

                        # RMSD for AF2 stuff
                out_dict[itter_num]['model_af2_all_bb_rmsd'] = calc_rmsd(xyz_af2[:,:4].reshape(len(xyz_af2)*4,3), xyz_model[:,:4].reshape(len(xyz_af2)*4,3))
                out_dict[itter_num]['model_af2_mot_bb_rmsd'] = calc_rmsd(xyz_af2[model_motif_resi_idx][:,:4].reshape(mot_len*4,3),xyz_model[model_motif_resi_idx][:,:4].reshape(mot_len*4,3))
                out_dict[itter_num]['af2_temp_mot_bb_rmsd'] = calc_rmsd(xyz_temp[temp_motif_resi_idx][:,:4].reshape(mot_len*4,3), xyz_af2[model_motif_resi_idx][:,:4].reshape(mot_len*4,3))
                out_dict[itter_num]['temp_af2_mot_cb_rmsd'] = calc_rmsd(xyz_temp[temp_motif_resi_idx][:,:5].reshape(mot_len*5,3), xyz_af2[model_motif_resi_idx][:,:5].reshape(mot_len*5,3))

    # turn dictionary into a dataframe to be saved
    keys = list(out_dict.keys())
    keys.sort()
    sorted_dict = {i: out_dict[i] for i in keys}
    df_rmsd=pd.DataFrame()
    df_rmsd['itter'] = sorted_dict.keys()
    for k in list(out_dict[list(out_dict.keys())[0]].keys()):
        df_rmsd[k] = [d[k] for d in sorted_dict.values()]
    df_rmsd.to_csv(out_fn)
    df_rmsd.to_csv(out_fn)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

import scrapy
from bs4 import BeautifulSoup
import re
import traceback
from pokemons.items import PokemonsItem

class pokemonsSpider(scrapy.Spider):
    '''
    定义一个继承自 scrapy.Spider 的类，用于爬虫
    '''
    # 用于在 terminal 中执行 scrapy 爬虫命令的标示
    name = 'pokemons'

    # 爬取的链接
    start_urls = [
        'https://wiki.52poke.com/wiki/宝可梦列表（按全国图鉴编号）/简单版'
    ]

    # scrapy 输出设置
    custom_settings = {
        'LOG_LEVEL': 'ERROR' # 只显示错误信息
    }

    # 解析网页
    def parse(self, response):
        '''
        解析初始网页
        '''
        # 爬虫获取到的 url 为相对链接
        # 因此定义一个 domain name
        domain = 'https://wiki.52poke.com'
        # 此处的 response 是 start_urls 中返回的数据
        soup = BeautifulSoup(response.body, 'html.parser')
        tags = soup.find_all(class_='mw-redirect',
                             href=re.compile(r'/wiki/\w.*'))
        for tag in tags:
            # 每个 Pokemon 详情界面的 url
            url = (domain+tag.get('href'))
            yield scrapy.Request(url, callback=self.parse_details)

    def parse_details(self, response):
        '''
        解析详情网页
        '''
        # 此处的 response 是 Pokemon 详情界面返回的数据
        soup = BeautifulSoup(response.body, 'html.parser')
        # 返回一个 ResultSet（list）
        base_info_tables = soup.find_all('table',
                                        class_=re.compile('roundy a-r at-c'))
        # 获取 ResultSet 中的第一个元素
        base_info_table = base_info_tables[0]
        # 获取所有 tr 节点
        base_info_trs = base_info_table.find_all('tr')

        # 获取 Pokemon 的种族值信息
        stats_info_table = soup.find_all('table',
                            class_=re.compile('roundy alignt-center'))[0]
        stats_info_trs = stats_info_table.find_all('tr')

        # 获取 Pokemon 的属性相克信息
        against_info_table = soup.find_all(class_="bgwhite bw-1 bd-变化")[0].parent
        against_info_tds = against_info_table.find_all('td')

        try:
            # 获取 Pokemon 中文名字
            chinese_name = self.extract_chinese_name(base_info_trs)
            if chinese_name is None:
                raise Exception('Pokemon Chinese name not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 英文名字
            english_name = self.extract_english_name(base_info_trs)
            if english_name is None:
                raise Exception('Pokemon English name not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 序号
            pokdex = self.extract_pokdex(base_info_trs, soup)
            if pokdex is None:
                raise Exception('Pokemon index not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 属性
            type_1, type_2 = self.extract_types(base_info_trs)
            if type_1 is None:
                raise Exception('Pokemon Type 1 not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 分类
            category = self.extract_category(base_info_trs)
            if category is None:
                raise Exception('Pokemon category not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 特性
            ability_1, ability_2 = self.extract_abilities(base_info_trs)
            if ability_1 is None:
                raise Exception('Pokemon Ability 1 not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 100级时的经验值
            exp_100 = self.extract_exp_100(base_info_trs)
            if exp_100 is None:
                raise Exception('Pokemon Experience not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 身高
            height_m = self.extract_height(base_info_trs)
            if height_m is None:
                raise Exception('Pokemon Height not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 100级时的经验值
            weight_kg = self.extract_weight(base_info_trs)
            if weight_kg is None:
                raise Exception('Pokemon Weight not found for {}'\
                    .format(response.url))

            # 获取 普通的精灵球在 Pokemon 满体力下的捕获率
            catch_rate = self.extract_catch_rate(base_info_trs)
            if catch_rate is None:
                raise Exception('Pokemon Catch Rate not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 是 male 的比例
            male_ratio = self.extract_male_ratio(base_info_trs)
            if male_ratio is None:
                raise Exception('Pokemon Male Ratio not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 孵化步数
            hatch_time = self.extract_hatch_time(base_info_trs)
            if hatch_time is None:
                raise Exception('Pokemon Hatch Time not found for {}'\
                    .format(response.url))

            # 获取 Pokemon HP
            hp = self.extract_hp(stats_info_trs)
            if hp is None:
                raise Exception('Pokemon HP not found for {}'\
                    .format(response.url))

            # 获取 Pokemon attack
            atk = self.extract_atk(stats_info_trs)
            if atk is None:
                raise Exception('Pokemon Attack not found for {}'\
                    .format(response.url))

            # 获取 Pokemon HP
            defense = self.extract_def(stats_info_trs)
            if defense is None:
                raise Exception('Pokemon Defense not found for {}'\
                    .format(response.url))

            # 获取 Pokemon special attack
            spatk = self.extract_spatk(stats_info_trs)
            if spatk is None:
                raise Exception('Pokemon Special Attack not found for {}'\
                    .format(response.url))

            # 获取 Pokemon special defense
            spdef = self.extract_spdef(stats_info_trs)
            if spdef is None:
                raise Exception('Pokemon Special Defense not found for {}'\
                    .format(response.url))

            # 获取 Pokemon Speed
            speed = self.extract_speed(stats_info_trs)
            if speed is None:
                raise Exception('Pokemon Speed not found for {}'\
                    .format(response.url))

            # 获取 Pokemon total
            total = self.extract_total(stats_info_trs)
            if total is None:
                raise Exception('Pokemon Total not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 世代
            generation = self.extract_generation(soup)
            if generation is None:
                raise Exception('Pokemon Generation not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 图片链接
            image_url = self.extract_image_url(base_info_trs)
            image_urls = []
            image_urls.append(image_url)
            if image_url is None:
                raise Exception('Pokemon Image URL not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 一般 属性的抗性
            against_normal = self.extract_against_norml(against_info_tds, soup)
            if against_normal is None:
                raise Exception('Pokemon Against Normal not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 格斗 属性的抗性
            against_fight = self.extract_against_fight(against_info_tds)
            if against_fight is None:
                raise Exception('Pokemon Against Fight not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 飞行 属性的抗性
            against_flying = self.extract_against_flying(against_info_tds)
            if against_flying is None:
                raise Exception('Pokemon Against Flying not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 毒 属性的抗性
            against_poison = self.extract_against_poison(against_info_tds)
            if against_poison is None:
                raise Exception('Pokemon Against Poison not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 地面 属性的抗性
            against_ground = self.extract_against_ground(against_info_tds)
            if against_ground is None:
                raise Exception('Pokemon Against Ground not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 岩石 属性的抗性
            against_rock = self.extract_against_rock(against_info_tds)
            if against_rock is None:
                raise Exception('Pokemon Against Rock not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 虫 属性的抗性
            against_bug = self.extract_against_bug(against_info_tds)
            if against_bug is None:
                raise Exception('Pokemon Against Bug not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 幽灵 属性的抗性
            against_ghost = self.extract_against_ghost(against_info_tds)
            if against_ghost is None:
                raise Exception('Pokemon Against Ghost not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 钢 属性的抗性
            against_steel = self.extract_against_steel(against_info_tds)
            if against_steel is None:
                raise Exception('Pokemon Against Steel not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 火 属性的抗性
            against_fire = self.extract_against_fire(against_info_tds)
            if against_fire is None:
                raise Exception('Pokemon Against Fire not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 水 属性的抗性
            against_water = self.extract_against_water(against_info_tds)
            if against_water is None:
                raise Exception('Pokemon Against Water not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 草 属性的抗性
            against_grass = self.extract_against_grass(against_info_tds)
            if against_grass is None:
                raise Exception('Pokemon Against Grass not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 电 属性的抗性
            against_electric = self.extract_against_electric(against_info_tds)
            if against_electric is None:
                raise Exception('Pokemon Against Electric not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 超能力 属性的抗性
            against_psychic = self.extract_against_psychic(against_info_tds)
            if against_psychic is None:
                raise Exception('Pokemon Against Psychic not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 冰 属性的抗性
            against_ice = self.extract_against_ice(against_info_tds)
            if against_ice is None:
                raise Exception('Pokemon Against Ice not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 龙 属性的抗性
            against_dragon = self.extract_against_dragon(against_info_tds)
            if against_dragon is None:
                raise Exception('Pokemon Against Dragon not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 恶 属性的抗性
            against_dark = self.extract_against_dark(against_info_tds)
            if against_dark is None:
                raise Exception('Pokemon Against Dark not found for {}'\
                    .format(response.url))

            # 获取 Pokemon 对 妖精 属性的抗性
            against_fairy = self.extract_against_fairy(against_info_tds)
            if against_fairy is None:
                raise Exception('Pokemon Against Fairy not found for {}'\
                    .format(response.url))

            # 保存成 item 以存储至 mongodb
            pokemon_item = PokemonsItem(_id=pokdex, chinese_name=chinese_name,
                                        english_name=english_name,
                                        type_1=type_1, type_2=type_2,
                                        category=category,
                                        ability_1=ability_1, ability_2=ability_2,
                                        exp_100=exp_100, height_m=height_m,
                                        weight_kg=weight_kg,
                                        catch_rate=catch_rate,
                                        male_ratio=male_ratio,
                                        hatch_time=hatch_time, hp=hp, atk=atk,
                                        defense=defense,  spatk=spatk,
                                        spdef=spdef, speed=speed,
                                        total=total, generation=generation,
                                        image_urls=image_urls,
                                        against_normal=against_normal,
                                        against_fight=against_fight,
                                        against_flying=against_flying,
                                        against_poison=against_poison,
                                        against_ground=against_ground,
                                        against_rock=against_rock,
                                        against_bug=against_bug,
                                        against_ghost=against_ghost,
                                        against_steel=against_steel,
                                        against_fire=against_fire,
                                        against_water=against_water,
                                        against_grass=against_grass,
                                        against_electric=against_electric,
                                        against_psychic=against_psychic,
                                        against_ice=against_ice,
                                        against_dragon=against_dragon,
                                        against_dark=against_dark,
                                        against_fairy=against_fairy)

            yield pokemon_item

        except Exception as err:
            self.logger.error(str(err))
            self.logger.error(traceback.format_exc())

    @staticmethod
    def extract_chinese_name(trs):
        '''
        提取 Pokemon 中文名字
        '''
        return trs[0].find_all('b')[0].text

    @staticmethod
    def extract_english_name(trs):
        '''
        提取 Pokemon 英文名字
        '''
        return trs[0].find_all('b')[-1].text

    @staticmethod
    def extract_pokdex(trs, soup):
        '''
        提取 Pokemon 序号
        '''
        return re.findall(r'\d{3}', trs[0].text)[0]

    @staticmethod
    def extract_types(trs):
        '''
        提取 Pokemon 属性
        '''
        types = types = trs[6].text.strip().split('\xa0\xa0')
        if len(types) == 2: # 如果该 Pokemon 有两种属性
            # 返回两种属性
            return types[0], types[1]
        else:
            # 返回一种属性
            return types[0], None

    @staticmethod
    def extract_category(trs):
        '''
        提取 Pokemon 类别
        '''
        if len(trs) == 62: # 基格尔德
            return trs[9].text.strip()
        return trs[8].text.strip()

    @staticmethod
    def extract_abilities(trs):
        '''
        提取 Pokemon 特性
        '''
        abilities = trs[11].text.strip().replace('\xa0', ' ').split('\n\n')
        if len(abilities) == 2: # 如果该 Pokemon 有两种特性
            # 返回两种特性
            return abilities[0], abilities[1]
        else:
            # 返回一种特性
            return abilities[0], None

    @staticmethod
    def extract_exp_100(trs):
        '''
        提取 Pokemon 100级时的经验值
        '''
        if len(trs) == 60:
            return trs[13].text.strip()
        elif len(trs) == 62: # 基格尔德
            return trs[15].text.strip()
        return trs[14].text.strip()

    @staticmethod
    def extract_height(trs):
        '''
        提取 Pokemon 身高（m）
        '''
        if len(trs) == 60:
            return re.findall(r'\d.\d', trs[31].text.strip())[0]
        if len(trs) == 62: # 基格尔德
            return 0
        return re.findall(r'\d.\d', trs[32].text.strip())[0]

    @staticmethod
    def extract_weight(trs):
        '''
        提取 Pokemon 体重（kg）
        '''
        if len(trs) == 60:
            return re.findall(r'\d.\d', trs[33].text.strip())[0]
        if len(trs) == 62: # 基格尔德
            return 0
        return re.findall(r'\d.\d', trs[34].text.strip())[0]

    @staticmethod
    def extract_catch_rate(trs):
        '''
        提取 普通的精灵球在 Pokemon 满体力下的捕获率
        '''
        if len(trs) == 60:
            return trs[43].find('span').text
        return trs[44].find('span').text

    @staticmethod
    def extract_male_ratio(trs):
        '''
        提取 Pokemon 是 male 的比例
        '''
        if len(trs) == 60:
            if trs[48].find('span'):
                return re.findall(r'[\d.\d]*%', trs[48].find('span').text)[0]
            else:
                return trs[48].text.strip()
        elif len(trs) == 61:
            if trs[49].find('span'):
                return re.findall(r'[\d.\d]*%', trs[49].find('span').text)[0]
            else:
                return trs[49].text.strip()
        else: # 基格尔德
            return trs[50].text.strip()

    @staticmethod
    def extract_hatch_time(trs):
        '''
        提取 Pokemon 的孵化步数
        '''
        if len(trs) == 60:
            return re.findall(r'[\d]*步', trs[51].find_all('td')[-1].text)[0]
        return re.findall(r'[\d]*步', trs[52].find_all('td')[-1].text)[0]

    @staticmethod
    def extract_hp(trs):
        '''
        提取 Pokemon hp
        '''
        return trs[3].find_all('th', class_='bgl-HP')[-1].text.strip()

    @staticmethod
    def extract_atk(trs):
        '''
        提取 Pokemon attack
        '''
        return trs[5].find_all('th', class_='bgl-攻击')[-1].text.strip()

    @staticmethod
    def extract_def(trs):
        '''
        提取 Pokemon defense
        '''
        return trs[7].find_all('th', class_='bgl-防御')[-1].text.strip()

    @staticmethod
    def extract_spatk(trs):
        '''
        提取 Pokemon special attack
        '''
        return trs[9].find_all('th', class_='bgl-特攻')[-1].text.strip()

    @staticmethod
    def extract_spdef(trs):
        '''
        提取 Pokemon special defense
        '''
        return trs[11].find_all('th', class_='bgl-特防')[-1].text.strip()

    @staticmethod
    def extract_speed(trs):
        '''
        提取 Pokemon speed
        '''
        return trs[13].find_all('th', class_='bgl-速度')[-1].text.strip()

    @staticmethod
    def extract_total(trs):
        '''
        提取 Pokemon 种族值总值
        '''
        return re.findall(r'\d{3}', trs[15].text)[0].strip()

    @staticmethod
    def extract_generation(soup):
        '''
        提取 Pokemon 世代
        '''
        catlinks = soup.find(class_='mw-normal-catlinks')
        return re.search(r'第[\u4e00-\u9fa5]世代寶可夢', catlinks.text)[0]

    @staticmethod
    def extract_image_url(trs):
        '''
        提取 Pokemon 图片链接
        '''
        domain = 'https://'
        try:
            relative_url = re.findall(r'media.*png', trs[3]\
                .find('img').get('data-srcset').split(',')[-1])[0]
        except:
            relative_url = trs[3].find('img').get('data-url')

        img_url = domain + relative_url
        # self.img_urls.append(img_url)
        return img_url

    @staticmethod
    def extract_against_norml(tds, soup):
        '''
        提取 Pokemon 对 一般 属性的抗性
        '''
        if tds[3]:
            return tds[3].text.strip()

    @staticmethod
    def extract_against_fight(tds):
        '''
        提取 Pokemon 对 格斗 属性的抗性
        '''
        if tds[4]:
            return tds[4].text.strip()

    @staticmethod
    def extract_against_flying(tds):
        '''
        提取 Pokemon 对 飞行 属性的抗性
        '''
        if tds[5]:
            return tds[5].text.strip()

    @staticmethod
    def extract_against_poison(tds):
        '''
        提取 Pokemon 对 毒 属性的抗性
        '''
        if tds[6]:
            return tds[6].text.strip()

    @staticmethod
    def extract_against_ground(tds):
        '''
        提取 Pokemon 对 地面 属性的抗性
        '''
        if tds[7]:
            return tds[7].text.strip()

    @staticmethod
    def extract_against_rock(tds):
        '''
        提取 Pokemon 对 岩石 属性的抗性
        '''
        if tds[8]:
            return tds[8].text.strip()

    @staticmethod
    def extract_against_bug(tds):
        '''
        提取 Pokemon 对 虫 属性的抗性
        '''
        if tds[9]:
            return tds[9].text.strip()

    @staticmethod
    def extract_against_ghost(tds):
        '''
        提取 Pokemon 对 幽灵 属性的抗性
        '''
        if tds[10]:
            return tds[10].text.strip()

    @staticmethod
    def extract_against_steel(tds):
        '''
        提取 Pokemon 对 钢 属性的抗性
        '''
        if tds[11]:
            return tds[11].text.strip()

    @staticmethod
    def extract_against_fire(tds):
        '''
        提取 Pokemon 对 火 属性的抗性
        '''
        if tds[12]:
            return tds[12].text.strip()

    @staticmethod
    def extract_against_water(tds):
        '''
        提取 Pokemon 对 水 属性的抗性
        '''
        if tds[13]:
            return tds[13].text.strip()

    @staticmethod
    def extract_against_grass(tds):
        '''
        提取 Pokemon 对 草 属性的抗性
        '''
        if tds[14]:
            return tds[14].text.strip()

    @staticmethod
    def extract_against_electric(tds):
        '''
        提取 Pokemon 对 电 属性的抗性
        '''
        if tds[15]:
            return tds[15].text.strip()

    @staticmethod
    def extract_against_psychic(tds):
        '''
        提取 Pokemon 对 超能力 属性的抗性
        '''
        if tds[16]:
            return tds[16].text.strip()

    @staticmethod
    def extract_against_ice(tds):
        '''
        提取 Pokemon 对 冰 属性的抗性
        '''
        if tds[17]:
            return tds[17].text.strip()

    @staticmethod
    def extract_against_dragon(tds):
        '''
        提取 Pokemon 对 龙 属性的抗性
        '''
        if tds[18]:
            return tds[18].text.strip()

    @staticmethod
    def extract_against_dark(tds):
        '''
        提取 Pokemon 对 恶 属性的抗性
        '''
        if tds[19]:
            return tds[19].text.strip()

    @staticmethod
    def extract_against_fairy(tds):
        '''
        提取 Pokemon 对 妖精 属性的抗性
        '''
        if tds[20]:
            return tds[20].text.strip()


import numpy as np
import os
import sys

import skopi.geometry as pg
import skopi.util as pu
from skopi.util import xp, asnumpy

from .base import DetectorBase

class UserDefinedDetector(DetectorBase):
    """
    Class for user defined detector. The user has to provide the necessary information
    with a dictionary with proper entries to use this class.
    """

    def __init__(self, geom, beam=None):
        """
        Initialize the detector
        :param geom:  The dictionary containing all the necessary information to initialize
                        the detector.
        :param beam: The beam object
        """
        super(UserDefinedDetector, self).__init__()
        self.initialize(geom=geom, beam=beam)

    def initialize(self, geom, beam):
        """
        Initialize the detector with user defined parameters
        :param geom: The dictionary containing all the necessary information to initialized the
                    detector.
        :param beam: The beam object
        :return: None
        """

        """
        Doc:
            To use this class, the user has to provide the necessary information to initialize
             the detector.
            All the necessary entries are listed in the example notebook.
        """

        # 'detector distance': detector distance in (m)

        ##########################################################################################
        # Extract necessary information
        ##########################################################################################

        # Define the hierarchy system. For simplicity, we only use two-layer structure.
        for key in {'panel number', 'panel pixel num x', 'panel pixel num y'}:
            if key not in geom:
                raise KeyError("Missing required '{}' key.".format(key))
        self.panel_num = int(geom['panel number'])
        self.panel_pixel_num_x = int(geom['panel pixel num x'])
        self.panel_pixel_num_y = int(geom['panel pixel num y'])
        self._shape = (self.panel_num, self.panel_pixel_num_x,
                       self.panel_pixel_num_y)

        # Define all properties the detector should have
        self._distance = None
        if 'pixel center z' in geom:
            if 'detector distance' in geom:
                raise ValueError("Please provide one of "
                                 "'pixel center z' or 'detector distance'.")
            self.center_z = xp.asarray(geom['pixel center z'], dtype=xp.float64)
            self._distance = float(self.center_z.mean())
        else:
            if 'detector distance' not in geom:
                KeyError("Missing required 'detector distance' key.")
            self._distance = float(geom['detector distance'])
            self.center_z = self._distance * xp.ones(self._shape,
                                                    dtype=xp.float64)

        # Below: [panel number, pixel num x, pixel num y]  in (m)
        # Change dtype and make numpy/cupy array
        self.pixel_width = xp.asarray(geom['pixel width'], dtype=xp.float64)
        self.pixel_height = xp.asarray(geom['pixel height'], dtype=xp.float64)
        self.center_x = xp.asarray(geom['pixel center x'], dtype=xp.float64)
        self.center_y = xp.asarray(geom['pixel center y'], dtype=xp.float64)
        self.orientation = np.array([0, 0, 1])

        # construct the the pixel position array
        self.pixel_position = xp.zeros(self._shape + (3,))
        self.pixel_position[..., 0] = self.center_x
        self.pixel_position[..., 1] = self.center_y
        self.pixel_position[..., 2] = self.center_z

        # Pixel map
        if 'pixel map' in geom:
            # [panel number, pixel num x, pixel num y]
            self.pixel_index_map = xp.asarray(geom['pixel map'], dtype=xp.int64)
            # Detector pixel number info
            self.detector_pixel_num_x = asnumpy(
                xp.max(self.pixel_index_map[..., 0]) + 1)
            self.detector_pixel_num_y = asnumpy(
                xp.max(self.pixel_index_map[..., 1]) + 1)

            # Panel pixel number info
            # number of pixels in each panel in x/y direction
            self.panel_pixel_num_x = self.pixel_index_map.shape[1]
            self.panel_pixel_num_y = self.pixel_index_map.shape[2]

        # total number of pixels (px*py)
        self.pixel_num_total = np.prod(self._shape)

        ###########################################################################################
        # Do necessary calculation to finishes the initialization
        ###########################################################################################
        # self.geometry currently only work for the pre-defined detectors
        self.geometry = geom

        # Calculate the pixel area
        self.pixel_area = xp.multiply(self.pixel_height, self.pixel_width)

        # Get reciprocal space configurations and corrections.
        self.initialize_pixels_with_beam(beam=beam)

        ##########################################################################################
        # Do necessary calculation to finishes the initialization
        ##########################################################################################
        # Detector effects
        if 'pedestal' in geom:
            self._pedestal = xp.asarray(geom['pedestal'], dtype=xp.float64)
        else:
            self._pedestal = xp.zeros(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))

        if 'pixel rms' in geom:
            self._pixel_rms = xp.asarray(geom['pixel rms'], dtype=xp.float64)
        else:
            self._pixel_rms = xp.zeros(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))

        if 'pixel bkgd' in geom:
            self._pixel_bkgd = xp.asarray(geom['pixel bkgd'], dtype=xp.float64)
        else:
            self._pixel_bkgd = xp.zeros(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))

        if 'pixel status' in geom:
            self._pixel_status = xp.asarray(geom['pixel status'], dtype=xp.float64)
        else:
            self._pixel_status = xp.zeros(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))

        if 'pixel mask' in geom:
            self._pixel_mask = xp.asarray(geom['pixel mask'], dtype=xp.float64)
        else:
            self._pixel_mask = xp.zeros(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))

        if 'pixel gain' in geom:
            self._pixel_gain = xp.asarray(geom['pixel gain'], dtype=xp.float64)
        else:
            self._pixel_gain = xp.ones(
                (self.panel_num, self.panel_pixel_num_x, self.panel_pixel_num_y))