"""VirusTotal v3 API."""
import asyncio
from enum import Enum
from typing import Dict, List, Optional, Set

import pandas as pd
from IPython.display import HTML, display

from ..common.exceptions import MsticpyImportExtraError
from ..common.provider_settings import get_provider_settings

try:
    import vt
    from vt_graph_api import VTGraph
    from vt_graph_api import errors as vt_graph_errs
except ImportError as imp_err:
    raise MsticpyImportExtraError(
        "Cannot use this feature without vt-py and vt-graph-api packages installed.",
        title="Error importing VirusTotal modules.",
        extra="vt3",
    ) from imp_err

class MsticpyVTNoDataError(Exception):
    """No data returned from VT API."""

class MsticpyVTGraphSaveGraphError(Exception):
    """Could not save VT Graph."""

class VTEntityType(Enum):
    """VTEntityType: Enum class for VirusTotal entity types."""

    FILE = "file"
    DOMAIN = "domain"
    IP_ADDRESS = "ip_address"
    URL = "url"

class ColumnNames(Enum):
    """Column name enum for DataFrame output."""

    ID = "id"
    TYPE = "type"
    DETECTIONS = "detections"
    SCANS = "scans"
    SOURCE = "source"
    TARGET = "target"
    RELATIONSHIP_TYPE = "relationship_type"
    SOURCE_TYPE = "source_type"
    TARGET_TYPE = "target_type"

class VTObjectProperties(Enum):
    """Enum for VT Object properties."""

    ATTRIBUTES = "attributes"
    RELATIONSHIPS = "relationship"
    LAST_ANALYSIS_STATS = "last_analysis_stats"
    MALICIOUS = "malicious"

def _make_sync(future):
    """Wait for an async call, making it sync."""
    try:
        event_loop = asyncio.get_event_loop()
    except RuntimeError:
        # Generate an event loop if there isn't any.
        event_loop = asyncio.new_event_loop()
        asyncio.set_event_loop(event_loop)
    return event_loop.run_until_complete(future)

class VTLookupV3:
    """VTLookupV3: VirusTotal lookup of IoC reports."""

    _SUPPORTED_VT_TYPES: Set[VTEntityType] = {
        VTEntityType.FILE,
        VTEntityType.URL,
        VTEntityType.IP_ADDRESS,
        VTEntityType.DOMAIN,
    }

    _MAPPING_TYPES_ENDPOINT: Dict[VTEntityType, str] = {
        VTEntityType.FILE: "files",
        VTEntityType.URL: "urls",
        VTEntityType.IP_ADDRESS: "ip_addresses",
        VTEntityType.DOMAIN: "domains",
    }

    _BASIC_PROPERTIES_PER_TYPE: Dict[VTEntityType, Set[str]] = {
        VTEntityType.FILE: {
            "type_description",
            "size",
            "first_submission_date",
            "last_submission_date",
            "times_submitted",
            "meaningful_name",
        },
        VTEntityType.URL: {
            "first_submission_date",
            "last_submission_date",
            "times_submitted",
        },
        VTEntityType.IP_ADDRESS: {"date", "country", "asn", "as_owner"},
        VTEntityType.DOMAIN: {"id", "creation_date", "last_update_date", "country"},
    }

    @property
    def supported_vt_types(self) -> List[str]:
        """
        Return list of VirusTotal supported IoC type names.

        Returns
        -------
        List[str]:
            List of VirusTotal supported IoC type names.

        """
        return [str(i_type) for i_type in self._SUPPORTED_VT_TYPES]

    @classmethod
    def _get_endpoint_name(cls, vt_type: str) -> str:
        if VTEntityType(vt_type) not in cls._SUPPORTED_VT_TYPES:
            raise KeyError(f"Property type {vt_type} not supported")

        return cls._MAPPING_TYPES_ENDPOINT[VTEntityType(vt_type)]

    @classmethod
    def _parse_vt_object(cls, vt_object: vt.object.Object) -> pd.DataFrame:
        obj_dict = vt_object.to_dict()
        if VTObjectProperties.ATTRIBUTES.value in obj_dict:
            attributes = obj_dict[VTObjectProperties.ATTRIBUTES.value]
            vt_type = VTEntityType(vt_object.type)
            if vt_type not in cls._SUPPORTED_VT_TYPES:
                raise KeyError(f"Property type {vt_type} not supported")
            obj = {
                key: attributes[key]
                for key in cls._BASIC_PROPERTIES_PER_TYPE[vt_type]
                if key in attributes
            }
            vt_df = pd.json_normalize(data=[obj])
            last_analysis_stats = attributes[
                VTObjectProperties.LAST_ANALYSIS_STATS.value
            ]
            vt_df[ColumnNames.DETECTIONS.value] = last_analysis_stats[
                VTObjectProperties.MALICIOUS.value
            ]
            vt_df[ColumnNames.SCANS.value] = sum(last_analysis_stats.values())
            # Format dates for pandas
            for date_col in ("first_submission_date", "last_submission_date"):
                if date_col in vt_df.columns:
                    vt_df[date_col.replace("_date", "")] = pd.to_datetime(
                        vt_df[date_col], unit="s", utc=True
                    )
        else:
            vt_df = pd.DataFrame()

        # Inject ID and Type columns
        vt_df[ColumnNames.ID.value] = [vt_object.id]
        vt_df[ColumnNames.TYPE.value] = [vt_object.type]
        return vt_df.set_index([ColumnNames.ID.value])

    def __init__(self, vt_key: Optional[str] = None):
        """
        Create a new instance of VTLookupV3 class.

        Parameters
        ----------
        vt_key: str, optional
            VirusTotal API key, if not supplied, this is read from
            user configuration.

        """
        if not vt_key:
            prov_settings = get_provider_settings("TIProviders")
            vt_settings = prov_settings.get("VirusTotal")
            if vt_settings:
                self._vt_key = vt_settings.args.get("AuthKey")
        else:
            self._vt_key = vt_key
        self._vt_client = vt.Client(apikey=vt_key)

    async def _lookup_ioc_async(self, observable: str, vt_type: str) -> pd.DataFrame:
        """
        Look up and single IoC observable.

        Parameters
        ----------
        observable: str
            The observable value
        vt_type: str
            The VT entity type

        Returns
        -------
            Attributes Pandas DataFrame with the properties of the entity

        Raises
        ------
        KeyError
            Unknown vt_type

        """
        if VTEntityType(vt_type) not in self._SUPPORTED_VT_TYPES:
            raise KeyError(
                f"Property type {vt_type} not supported",
                "Valid types are",
                ", ".join(x.value for x in VTEntityType.__members__.values()),
            )

        endpoint_name = self._get_endpoint_name(vt_type)
        try:
            response = self._vt_client.get_object(f"/{endpoint_name}/{observable}")
            return self._parse_vt_object(response)
        except vt.APIError as err:
            raise MsticpyVTNoDataError(
                "An error occurred requesting data from VirusTotal"
            ) from err

    def lookup_ioc(self, observable: str, vt_type: str) -> pd.DataFrame:
        """
        Look up and single IoC observable.

        Parameters
        ----------
        observable: str
            The observable value
        vt_type: str
            The VT entity type

        Returns
        -------
            Attributes Pandas DataFrame with the properties of the entity

        Raises
        ------
        KeyError
            Unknown vt_type

        """
        try:
            return _make_sync(self._lookup_ioc_async(observable, vt_type))
        finally:
            self._vt_client.close()

    async def _lookup_iocs_async(
        self,
        observables_df: pd.DataFrame,
        observable_column: str = ColumnNames.TARGET.value,
        observable_type_column: str = ColumnNames.TARGET_TYPE.value,
    ):
        """
        Look up and multiple IoC observables.

        Parameters
        ----------
        observables_df: pd.DataFrame
            A Pandas DataFrame, where each row is an observable
        observable_column:
            ID column of each observable
        observable_type_column:
            Type column of each observable

        Returns
        -------
            Future Attributes Pandas DataFrame with the properties of the entities

        Raises
        ------
        KeyError
            Column not found in observables_df

        """
        _observables_df = observables_df.reset_index()

        for column in [observable_column, observable_type_column]:
            if column not in _observables_df.columns:
                raise KeyError(f"Column {column} not found in observables_df")

        observables_list = _observables_df[observable_column]
        types_list = _observables_df[observable_type_column]
        dfs_futures = []
        for observable, observable_type in zip(observables_list, types_list):
            try:
                ioc_df_future = self._lookup_ioc_async(observable, observable_type)
                dfs_futures.append(ioc_df_future)
            except KeyError:
                print(
                    "ERROR\t It was not possible to obtain results for",
                    f"{observable_type} {observable}",
                )
        dfs = await asyncio.gather(*dfs_futures)
        return pd.concat(dfs) if dfs else pd.DataFrame()

    def lookup_iocs(
        self,
        observables_df: pd.DataFrame,
        observable_column: str = ColumnNames.TARGET.value,
        observable_type_column: str = ColumnNames.TARGET_TYPE.value,
    ):
        """
        Look up and multiple IoC observables.

        Parameters
        ----------
        observables_df: pd.DataFrame
            A Pandas DataFrame, where each row is an observable
        observable_column:
            ID column of each observable
        observable_type_column:
            Type column of each observable

        Returns
        -------
            Attributes Pandas DataFrame with the properties of the entities

        """
        try:
            return _make_sync(
                self._lookup_iocs_async(
                    observables_df, observable_column, observable_type_column
                )
            )
        finally:
            self._vt_client.close()

    async def _lookup_ioc_relationships_async(
        self, observable: str, vt_type: str, relationship: str, limit: int = None
    ):
        """
        Look up and single IoC observable relationships.

        Parameters
        ----------
        observable: str
            The observable value
        vt_type: str
            The VT entity type
        relationship: str
            Desired relationship
        limit: int
            Relations limit

        Returns
        -------
            Future Relationship Pandas DataFrame with the relationships of the entity

        Raises
        ------
        KeyError
            Unknown vt_type

        """
        if VTEntityType(vt_type) not in self._SUPPORTED_VT_TYPES:
            raise KeyError(f"Property type {vt_type} not supported")

        endpoint_name = self._get_endpoint_name(vt_type)
        response: vt.object.Object

        if limit is None:
            try:
                response = self._vt_client.get_object(
                    f"/{endpoint_name}/{observable}?relationship_counters=true"
                )
                relationships = response.relationships
                limit = (
                    relationships[relationship]["meta"]["count"]
                    if relationship in relationships
                    else 0
                )
            except KeyError:
                print(
                    f"ERROR: Could not obtain relationship limit for {vt_type} {observable}"
                )
                return pd.DataFrame()

        if limit == 0 or limit is None:
            return pd.DataFrame()

        try:
            # print(f"Obtaining {limit} relationships for {vt_type} {observable}")
            response = self._vt_client.iterator(
                f"/{endpoint_name}/{observable}/relationships/{relationship}",
                batch_size=40,
                limit=limit,
            )
            vt_objects = [self._parse_vt_object(r) for r in response]
            result_df = pd.concat(vt_objects) if vt_objects else pd.DataFrame()

            if vt_objects:
                # Inject source and target columns
                result_df[ColumnNames.SOURCE.value] = observable
                result_df[ColumnNames.SOURCE_TYPE.value] = VTEntityType(vt_type).value
                result_df[ColumnNames.RELATIONSHIP_TYPE.value] = relationship
                result_df.reset_index(inplace=True)
                result_df.rename(
                    columns={
                        ColumnNames.ID.value: ColumnNames.TARGET.value,
                        ColumnNames.TYPE.value: ColumnNames.TARGET_TYPE.value,
                    },
                    inplace=True,
                )
                result_df.set_index(
                    [ColumnNames.SOURCE.value, ColumnNames.TARGET.value], inplace=True
                )
        except vt.APIError as err:
            raise MsticpyVTNoDataError(
                "An error occurred requesting data from VirusTotal"
            ) from err

        return result_df

    def lookup_ioc_relationships(
        self, observable: str, vt_type: str, relationship: str, limit: int = None
    ) -> pd.DataFrame:
        """
        Look up and single IoC observable relationships.

        Parameters
        ----------
        observable: str
            The observable value
        vt_type: str
            The VT entity type
        relationship: str
            Desired relationship
        limit: int
            Relations limit

        Returns
        -------
            Relationship Pandas DataFrame with the relationships of the entity

        """
        try:
            return _make_sync(
                self._lookup_ioc_relationships_async(
                    observable, vt_type, relationship, limit
                )
            )
        finally:
            self._vt_client.close()

    async def _lookup_iocs_relationships_async(
        self,
        observables_df: pd.DataFrame,
        relationship: str,
        observable_column: str = ColumnNames.TARGET.value,
        observable_type_column: str = ColumnNames.TARGET_TYPE.value,
        limit: int = None,
    ) -> pd.DataFrame:
        """
        Look up and single IoC observable relationships.

        Parameters
        ----------
        observables_df: pd.DataFrame
            A Pandas DataFrame, where each row is an observable
        relationship: str
            Desired relationship
        observable_column:
            ID column of each observable
        observable_type_column:
            Type column of each observable.
        limit: int
            Relations limit

        Returns
        -------
            Future Relationship Pandas DataFrame with the relationships of each observable.

        Raises
        ------
        KeyError
            Column not found in observables_df

        """
        _observables_df = observables_df.reset_index()

        for column in [observable_column, observable_type_column]:
            if column not in _observables_df.columns:
                raise KeyError(f"Column {column} not found in observables df")

        observables_list = _observables_df[observable_column]
        types_list = _observables_df[observable_type_column]
        dfs_futures = []

        for observable, observable_type in zip(observables_list, types_list):
            try:
                result_df_future = self._lookup_ioc_relationships_async(
                    observable, observable_type, relationship, limit
                )
                dfs_futures.append(result_df_future)
            except KeyError:
                print(
                    "ERROR:\t It was not possible to get the data for",
                    f"{observable_type} {observable}",
                )
        dfs = await asyncio.gather(*dfs_futures)
        return pd.concat(dfs) if len(dfs) > 0 else pd.DataFrame()

    def lookup_iocs_relationships(
        self,
        observables_df: pd.DataFrame,
        relationship: str,
        observable_column: str = ColumnNames.TARGET.value,
        observable_type_column: str = ColumnNames.TARGET_TYPE.value,
        limit: int = None,
    ) -> pd.DataFrame:
        """
        Look up and single IoC observable relationships.

        Parameters
        ----------
        observables_df: pd.DataFrame
            A Pandas DataFrame, where each row is an observable
        relationship: str
            Desired relationship
        observable_column:
            ID column of each observable
        observable_type_column:
            Type column of each observable.
        limit: int
            Relations limit

        Returns
        -------
            Relationship Pandas DataFrame with the relationships of each observable.

        """
        try:
            return _make_sync(
                self._lookup_iocs_relationships_async(
                    observables_df,
                    relationship,
                    observable_column,
                    observable_type_column,
                    limit,
                )
            )

        finally:
            self._vt_client.close()

    def create_vt_graph(
        self, relationship_dfs: List[pd.DataFrame], name: str, private: bool
    ) -> str:
        """
        Create a VirusTotal Graph with a set of Relationship DataFrames.

        Parameters
        ----------
        relationship_dfs:
            List of Relationship DataFrames
        name:
            New graph name
        private
            Indicates if the Graph is private or not.

        Returns
        -------
            Graph ID

        Raises
        ------
            ValueError when private is not indicated.
            ValueError when there are no relationship DataFrames
            MsticpyVTGraphSaveGraphError when Graph can not be saved

        """
        if not relationship_dfs:
            raise ValueError("There are no relationship DataFrames")

        if not isinstance(private, bool):
            raise ValueError("Please indicate if Graph is private or not")

        concatenated_df = pd.concat(relationship_dfs).reset_index()

        # Create nodes DF, with source and target
        sources_df = (
            concatenated_df.groupby(ColumnNames.SOURCE.value)[
                ColumnNames.SOURCE_TYPE.value
            ]
            .first()
            .reset_index()
            .rename(
                columns={
                    ColumnNames.SOURCE.value: ColumnNames.ID.value,
                    ColumnNames.SOURCE_TYPE.value: ColumnNames.TYPE.value,
                }
            )
        )

        target_df = (
            concatenated_df.groupby(ColumnNames.TARGET.value)[
                ColumnNames.TARGET_TYPE.value
            ]
            .first()
            .reset_index()
            .rename(
                columns={
                    ColumnNames.TARGET.value: ColumnNames.ID.value,
                    ColumnNames.TARGET_TYPE.value: ColumnNames.TYPE.value,
                }
            )
        )

        nodes_df = pd.concat([sources_df, target_df])

        graph = VTGraph(self._vt_key, name=name, private=private)

        nodes = [
            {
                "node_id": row[ColumnNames.ID.value],
                "node_type": row[ColumnNames.TYPE.value],
            }
            for _, row in nodes_df.iterrows()
        ]

        graph.add_nodes(nodes)

        for _, row in concatenated_df.iterrows():
            graph.add_link(
                source_node=row[ColumnNames.SOURCE.value],
                target_node=row[ColumnNames.TARGET.value],
                connection_type=row[ColumnNames.RELATIONSHIP_TYPE.value],
            )
        try:
            graph.save_graph()
        except vt_graph_errs.SaveGraphError as graph_err:
            raise MsticpyVTGraphSaveGraphError(
                "Could not save Graph. %s" % ""
                if not private
                else "Please check you have Private Graph premium feature enabled in"
                "your subscription. It is possible to create public Graphs"
                "with 'private=False' input argument"
            ) from graph_err

        return graph.graph_id

    @staticmethod
    def render_vt_graph(graph_id: str, width: int = 800, height: int = 600):
        """
        Display a VTGraph in a Jupyter Notebook.

        Parameters
        ----------
        graph_id:
            Graph ID
        width
            Graph width.
        height
            Graph height

        """
        display(
            HTML(
                f"""
              <iframe
                src="https://www.virustotal.com/graph/embed/{graph_id}"
                width="{width}"
                height="{height}">
              </iframe>

            """
            )
        )

    def get_object(self, vt_id: str, vt_type: str) -> pd.DataFrame:
        """
        Return the full VT object as a DataFrame.

        Parameters
        ----------
        vt_id : str
            The ID of the object
        vt_type : str
            The type of object to query.

        Returns
        -------
        pd.DataFrame
            Single column DataFrame with attribute names as
            index and values as data column.

        Raises
        ------
        KeyError
            Unrecognized VT Type
        MsticpyVTNoDataError
            Error requesting data from VT.

        """
        if VTEntityType(vt_type) not in self._SUPPORTED_VT_TYPES:
            raise KeyError(
                f"Property type {vt_type} not supported",
                "Valid types are",
                ", ".join(x.value for x in VTEntityType.__members__.values()),
            )

        endpoint_name = self._get_endpoint_name(vt_type)
        try:
            response: vt.object.Object = self._vt_client.get_object(
                f"/{endpoint_name}/{vt_id}"
            )
            return _ts_to_pydate(
                pd.DataFrame(data=response.to_dict()).drop(columns=["id", "type"])
            )
        except vt.APIError as err:
            raise MsticpyVTNoDataError(
                "An error occurred requesting data from VirusTotal"
            ) from err
        finally:
            self._vt_client.close()

def _ts_to_pydate(data):
    """Replace Unix timestamps in VT data with Py/pandas Timestamp."""
    for date_col in (col for col in data.columns if col.endswith("_date")):
        data[date_col] = pd.to_datetime(data[date_col], unit="s", utc=True)
    return data


import numpy as np
from sklearn.neighbors import KNeighborsClassifier
from sklearn.cross_validation import train_test_split, cross_val_score
from sklearn import svm
from sklearn.decomposition import PCA

def load_data(filename):
    return np.genfromtxt(filename, delimiter=',')

def perform_pca(Xs, n_components):
    # FIXME: TOTALLY WRONG! use fit() to train and then
    # transform
    print("Performing PCA with %s components" % (str(n_components),))
    pca = PCA(n_components=n_components)
    return [pca.fit_transform(X) for X in Xs]

def find_best_svm(X, y):
    print("Searching SVMs")
    scores = []
    for kernel in ['linear', 'rbf', 'sigmoid']:
        print(kernel)
        # For some reason, poly needed really long or
        # did even get stuck, so removed it
        clf = svm.SVC(kernel=kernel, C=1.0)
        score = cross_val_score(clf, X, y, cv=5)
        print("%s: %0.2f" % (kernel, score.mean()))

        scores.append((score.mean(), kernel))

    return max(scores)

def semi_supervised_set(clf, X, y, X_test, thresh=0.9):
    """
    Create a semi-supervised training set
    using estimations from classifier clf which will be
    trained on X and y before, but only take
    guesses where the probability is at least thresh
    """
    clf.fit(X, y)
    y_test = clf.predict(X_test)

    new_X = np.zeros((X.shape[0] + X_test.shape[0], X.shape[1]))
    new_y = np.zeros(X.shape[0] + X_test.shape[0])
    new_X[0:X.shape[0], :] = X
    new_y[0:X.shape[0]] = y

    added_count = 0
    for i, probs in enumerate(clf.predict_proba(X_test)):
        if probs[y_test[i]] > thresh:
            new_X[X.shape[0] + added_count, :] = X_test[i, :]
            new_y[X.shape[0] + added_count] = y_test[i]

            added_count += 1

    new_X.resize((X.shape[0] + added_count, X.shape[1]))
    new_y.resize(X.shape[0] + added_count)

    return new_X, new_y

def use_classifier(clf, X, y, X_test, filename):
    """
    Perform classification onto test data and save the
    result into filename
    """
    clf.fit(X, y)

    res = clf.predict(X_test)
    output = np.zeros((res.shape[0], 2), dtype='int')
    output[:, 0] = np.arange(1, res.shape[0] + 1)
    output[:, 1] = res
    np.savetxt(filename, output, fmt='%d', delimiter=',',
               header='Id,Solution', comments='')

X = load_data('train.csv')
y = load_data('trainLabels.csv')
X_test = load_data('test.csv')

scores_svm = []
for components in range(1, 2):  # removed mle, due to bad result
    #X_pca, X_test_pca = perform_pca((X, X_test), components)

    #score, kernel = find_best_svm(X_pca, y)
    score, kernel = find_best_svm(X, y)
    scores_svm.append((round(score, 2), components, kernel))

score, components, kernel = max(scores_svm, key=lambda x: (x[0], -x[1]))
print("Using %s kernel with %s components due to score %0.2f" %
      (kernel, str(components), score))
#X_pca, X_test_pca = perform_pca((X, X_test), components)
clf = svm.SVC(kernel=kernel, C=1.0, probability=True)
for i in range(2):
    X, y = semi_supervised_set(clf, X, y, X_test, 0.9)
clf = svm.SVC(kernel=kernel, C=1.0, probability=True)
#use_classifier(clf, X_pca, y, X_test_pca, 'submission_svm.csv')
use_classifier(clf, X, y, X_test, 'submission_svm_semi.csv')


# Version: 1.0
# Modified: 04.02.2018
# Author: Sergey Talash
import matplotlib

from sys import platform as sys_pf
from sys import path as sys_path

if sys_pf == 'darwin':
  import matplotlib

  matplotlib.use("TkAgg")
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
from matplotlib.figure import Figure

import gc
import datetime
import os
from os.path import split as p_split
import json
from tkinter import *
from tkinter import ttk
import numpy as np
import matplotlib.pyplot as plt

settings_home = p_split(p_split(os.getcwd())[0])[0]
sys_path.insert(0, settings_home)
# import procedures
# from procedures import Settings
from lib.core import *

import collections

def nm2pix(nm, abc, add=0):
  nm = float(nm)
  pix = 0
  if 270 < nm < 350:
    ans_nm = pix2nm(abc, pix, 1, add)
    while ans_nm < nm and pix < 1500:
      pix += 1
      ans_nm = pix2nm(abc, pix, 1, add)
      # print(pix,ans_nm)
  elif 350 <= nm < 430 and 1500 < pix < 3692:
    pix = 1500
    ans_nm = pix2nm(abc, pix, 1, add)
    while ans_nm < nm:
      pix += 1
      ans_nm = pix2nm(abc, pix, 1, add)
      # print(pix, ans_nm)
  else:
    print(nm, 'nm2pix: error')
  return pix

def pix2nm(abc, pix, digs, add):
  """
  Обработка одного пикселя
  abc - массив коэффициентов полинома
  pix - номер пиксела
  dig - количество знаков после запятой
  add - сдвиг для зенитных измерений
  """
  try:
    return round(eval(abc[0]) * pix ** 2 + eval(abc[1]) * pix + eval(abc[2]) + add, digs)
  except:
    return 0

class Calc:
  def __init__(self, PARS):
    self.dates = {}
    PARS = PARS
    self.dates_no_data = []
    self.files = {}
    self.pixels = {}
    self.type = 'Morning'
    self.start_mu = True
    self.home = 'calibration_files'
    self.dict_polynoms = {}
    self.dict_polynom_tmp = {}
    self.Ls = {'sko': [], 'mean': [], 'mu': []}
    self.data2file = []
    confZ = PARS['calibration']['nm(pix)']['Z']
    self.p_zero1 = nm2pix(285, confZ, 0)
    self.p_zero2 = nm2pix(290, confZ, 0)
    self.sensitivityZ = read_sensitivity("Z")

  def get_dates(self):
    with open(os.path.join(self.home, 'Calibration_dates.txt')) as f:
      data = f.readlines()
    for i in data:
      d = i.split()
      self.dates[d[0]] = d[1]  # 'YYYY.MM.DD': 'Ozone'
    return self.dates

  def get_filenames(self):
    for date in self.dates.keys():
      d = date.split('.')  # YYYY.MM.DD
      path = os.path.join(settings_home,
                          'Ufos_{}'.format(PARS['device']['id']),
                          'Mesurements',
                          str(d[0]),
                          '{}-{}'.format(d[0], d[1]),
                          '{}-{}-{}'.format(d[0], d[1], d[2]))
      if os.path.exists(path):
        self.files[date] = [os.path.join(path, name) for name in os.listdir(path) if name.count('txt') > 0]

  def calculate(self, abc, x, num):
    deg = len(abc) - 1
    out = 0
    for i in abc:
      out += eval(str(i)) * float(x) ** deg
      deg -= 1
    return round(out, num)

  def pix2nm(self, pix, chan):
    return self.calculate(PARS['calibration']['nm(pix)'][chan], pix, 3)

  def nm2pix(self, nm, chan, pix=0):
    nm = float(nm)
    if not 270 < nm < 440:
      print(nm, 'nm2pix: error')
    if 350 <= nm < 440:
      pix = 1500
    ans_nm = 0
    while ans_nm < nm:
      pix += 1
      ans_nm = self.pix2nm(pix, chan)
    return pix

  def get_pixels(self, chan):
    for pair in PARS['calibration']['points'].keys():
      self.pixels[pair] = []

      if pair != 'Fraunhofer_pair':
        self.Ls[pair] = []
      for nm in PARS['calibration']['points'][pair]:
        self.pixels[pair].append(self.nm2pix(nm, chan))

  def open_file(self, path):
    with open(path) as f:
      data = json.load(f)
    return data

  def split_mas(self, x, y, num_mas):
    """Разделение массива на интервалы
        x = mu
        y - значения
        num_mas - массив чисел, ограничивающих интервалы
        num_mas = [3,5]"""
    new_x = [[]]
    new_y = [[]]
    for i in num_mas:
      new_x.append([])
      new_y.append([])
    for xi, yi in zip(x, y):
      if xi <= num_mas[0] * 1.1:
        new_x[0].append(xi)
        new_y[0].append(yi)
      if num_mas[0] * 0.9 <= xi <= num_mas[1] * 1.1:
        new_x[1].append(xi)
        new_y[1].append(yi)
      if num_mas[1] * 0.9 <= xi:
        new_x[2].append(xi)
        new_y[2].append(yi)
    return new_x, new_y

  def get_Ls(self, date, chan):
    self.Ls = {'sko': [], 'mean': [], 'mu': []}
    for pair in self.pixels.keys():
      if pair != 'Fraunhofer_pair':
        self.Ls.setdefault(pair, [])
    if date in self.files:
      for file in sorted(self.files[date]):  # ['YYYY.MM.DD']
        if file.count(chan + 'D') > 0:
          try:
            file_data = self.open_file(file)
            mv_list = [file_data['spectr'][i] for i in range(self.p_zero1, self.p_zero2 + 1)]
            mv = sum(mv_list) / len(mv_list)

            spectr = [i - mv for i in file_data['spectr']]
            len_sp, len_sens = len(spectr), len(self.sensitivityZ)
            if len_sp != len_sens:
              print("Spectr and sensitivity length not equal for {}"
                    "(Spectr: {}, SensitivityZ: {})".format(date, len_sp, len_sens))
            len_min = min(len_sp, len_sens)

            spectr = [i * k for i, k in zip(spectr[:len_min], self.sensitivityZ[:len_min])]

            sko = file_data['calculated']['sko']
            mean = file_data['calculated']['mean']
            mu = file_data['calculated']['mu']
            for pair_name, pixels in self.pixels.items():
              if pair_name != 'Fraunhofer_pair':
                p = {str(i): int(pixels[i]) for i in [0, 1]}
                I = {str(i): spectr[
                             p[str(i)] - int(PARS['calibration']['pix_interval']):
                             p[str(i)] + int(PARS['calibration']['pix_interval'])] for i in [0, 1]}
                self.Ls[pair_name].append(round(sum(I['0']) / sum(I['1']), 6))
            self.Ls['sko'].append(sko)
            self.Ls['mean'].append(mean)
            self.Ls['mu'].append(mu)
          except Exception as err:
            print(err, file)
            raise err

  def polynom_add(self, date, dates_count, *event):
    if date[dates_count] not in calc.dates_no_data:
      self.dict_polynoms = self.update(self.dict_polynoms, self.dict_polynom_tmp)

  def update(self, d, u):
    for k, v in u.items():
      if isinstance(v, collections.Mapping):
        d[k] = self.update(d.get(k, {}), v)
      else:
        d[k] = v
    return d

  def table_create(self, name, date, mu, condition):
    text = ''
    for pair in ['o3_pair_', 'cloud_pair_']:
      L = 0
      if self.dict_polynoms[date][pair + name][condition][1]:
        L = self.calculate(self.dict_polynoms[date][pair + name][condition][0], mu, 6)
      text += ';{}'.format(L)
    return text

class GUI():
  def __init__(self, root, dates):
    self.plt = {}
    self.fig = {}
    self.file_meta = 'Sens_dev{}_{}_'.format(PARS["device"]["id"],
                                        datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d-%H%M'))
    self.file_format = '.csv'
    self.text_polynoms = '{}{}_polynoms_{}{}'
    self.text_tables = '{}{}_table_{}{}'
    self.mus = np.arange(PARS["calibration"]["polynom"]["mu_start"],
                         PARS["calibration"]["polynom"]["mu_end"],
                         PARS["calibration"]["polynom"]["mu_step"])
    self.channel = 'Z'  # Take ZD files (or set 'S' to taket ZS files)
    self.graph_names = ['1', '2']  # For pairs ['o3_pair_','cloud_pair_']
    self.polynom_vars = {i: [] for i in self.graph_names}
    self.polynom_chkbtns = {i: [] for i in self.graph_names}
    self.canvas = {}
    self.dates_count = 0
    self.dates_len = len(dates)
    self.dates = dates
    self.date = list(dates.keys())
    self.date.sort()
    self.root = root
    self.chk_btns = []
    self.root.title('УФОС Калибровка {}'.format(calc.type))
    self.root.geometry('+100+0')
    self.root.resizable(False, False)
    self.gui_elements = []
    self.left_frame = ttk.Frame(self.root)
    self.right_frame = ttk.Frame(self.root)
    self.gui_elements.append(ttk.Label(self.left_frame, text='Номер прибора:'))
    self.gui_elements.append(ttk.Label(self.left_frame, text=PARS['device']['id']))
    self.gui_elements.append(ttk.Label(self.left_frame, text='Номер станции:'))
    self.gui_elements.append(ttk.Label(self.left_frame, text=PARS['station']['id']))
    self.gui_elements.append(ttk.Label(self.left_frame))
    self.gui_elements.append(ttk.Label(self.left_frame))
    self.gui_elements.append(ttk.Label(self.left_frame, text='Дата'))
    self.gui_elements.append(ttk.Label(self.left_frame, text='Озон'))
    for i in self.date:
      self.gui_elements.append(ttk.Label(self.left_frame, text=i))
      self.gui_elements.append(ttk.Label(self.left_frame, text=dates[i]))

    self.btn_save_table = ttk.Button(self.right_frame, text='Сохранить табличные данные', command=self.save_to_table)
    self.btn_save_koeff = ttk.Button(self.right_frame, text='Сохранить коэффициенты', command=self.save_to_file)
    self.btn_save_poly = ttk.Button(self.right_frame, text='Запомнить эти полиномы',
                                    command=lambda: calc.polynom_add(self.date, self.dates_count))
    self.btn_next = ttk.Button(self.right_frame, text='Далее >', command=self.next_graphs)
    self.btn_refresh = ttk.Button(self.right_frame, text='Обновить', command=self.refresh_graphs)
    self.btn_prev = ttk.Button(self.right_frame, text='< Назад', command=self.prev_graphs)

  def draw_elements(self):
    self.left_frame.grid(row=0, column=0, sticky='wesn')
    self.right_frame.grid(row=0, column=1, sticky='wesn')
    r = 0
    c = 0
    for i in self.gui_elements:
      i.grid(row=r, column=c, sticky='w')
      if c == 0:
        c = 1
      else:
        c = 0
        r += 1

  def graph(self, plt_name, size, pos, calc):
    try:
      self.plt[plt_name].close()
      del self.plt[plt_name]
    except:
      pass
    for i in range(6):
      intvar = IntVar()
      self.polynom_vars[plt_name].append(intvar)
      chkbtn = ttk.Checkbutton(self.right_frame, text=str(i), variable=intvar)
      self.polynom_chkbtns[plt_name].append(chkbtn)
      chkbtn.grid(row=pos[0] + 1, column=i, sticky='w')
    self.plt[plt_name] = plt
    self.fig[plt_name] = Figure(figsize=size, tight_layout=True)
    self.canvas[plt_name] = FigureCanvasTkAgg(self.fig[plt_name], master=self.right_frame)
    self.canvas[plt_name].get_tk_widget().grid(row=pos[0], column=pos[1], sticky='wesn', columnspan=6)
    self.canvas[plt_name].draw()

  def buttons(self, r):
    self.btn_save_table.grid(row=r, column=5, sticky='w')
    self.btn_save_koeff.grid(row=r, column=4, sticky='w')
    self.btn_save_poly.grid(row=r, column=3, sticky='w')
    self.btn_next.grid(row=r, column=2, sticky='w')
    self.btn_refresh.grid(row=r, column=1, sticky='w')
    self.btn_prev.grid(row=r, column=0, sticky='w')
    self.btn_prev.configure(state=DISABLED)

  def subplot(self, plt_name, x, y, title, pos, legend, plot_pos):
    ax = self.fig[plt_name].add_subplot(plot_pos)
    ax.set_ylim(0, max(y) * 1.1)
    ax.set_xlim(min(x) * 0.9, max(x) * 1.1)
    ax.set_title(title)
    ax.grid(True)  # координатную сетку рисовать
    xx, yy = [x, x, x], [y, y, y]
    lines = []
    calc.dict_polynom_tmp[self.date[self.dates_count]][title] = {}
    calc.dict_polynom_tmp[self.date[self.dates_count]][title + '_mesure'] = {}
    for xi, yi, color, lab in zip(xx, yy, ['-r', '-g', '-b'], ['mu<3', '3<mu<5', '5<mu']):
      try:
        calc.dict_polynom_tmp[self.date[self.dates_count]][title + '_mesure']['mu'] = xi
        calc.dict_polynom_tmp[self.date[self.dates_count]][title + '_mesure']['L'] = yi
        sko = round(float(np.std(yi)), 4)
        lines.append(ax.plot(xi, yi, color, label=title + ' ' + lab + ' ' + str(sko))[0])
        z = np.polyfit(xi, yi, PARS["calibration"]["polynom"]["degree"])
        calc.dict_polynom_tmp[self.date[self.dates_count]][title][lab] = [z.tolist(), sko]
        y = [calc.calculate(z.tolist(), i, 6) for i in xi]
      except Exception as err:
        calc.dict_polynom_tmp[self.date[self.dates_count]][title][lab] = [[], None]

    ax.legend(loc='best', fancybox=True).get_frame().set_alpha(0.2)

  def next_graphs(self, *event):
    self.dates_count += 1
    self.refresh_graphs()
    if 0 < self.dates_count <= self.dates_len:
      self.btn_prev.configure(state=NORMAL)
    if self.dates_count >= self.dates_len - 1:
      self.btn_next.configure(state=DISABLED)

  def prev_graphs(self, *event):
    self.dates_count -= 1
    self.refresh_graphs()
    if 0 <= self.dates_count < self.dates_len:
      self.btn_next.configure(state=NORMAL)
    if self.dates_count <= 0:
      self.btn_prev.configure(state=DISABLED)

  def refresh_graphs(self, *event):
    curr_date = self.date[self.dates_count]
    for i in self.gui_elements:
      if i.cget("text") == curr_date:
        i.configure(font='Arial 10 bold')
      else:
        i.configure(font='Arial 10')
    calc.get_Ls(curr_date, self.channel)
    calc.dict_polynom_tmp = {curr_date: {}}
    self.chk_btns = []
    out = False
    for name, row in zip(self.graph_names,
                         [i * 2 for i in range(len(self.graph_names))]):  # graph_names = ['1','2']
      self.graph(name, (12, 3), (row, 0), calc)
      for pair, column, left, plot_pos in zip(['o3_pair_', 'cloud_pair_'], [0, 1], [0.050, 0.545], [121, 122]):
        if calc.Ls[pair + name]:
          self.subplot(name, calc.Ls['mu'], calc.Ls[pair + name], pair + name, (0, column),
                       [left, 0.1, 0.12, 0.17], plot_pos)
          self.canvas[name].draw()
        else:
          if not out:
            out = True
            calc.dates_no_data.append(curr_date)
            print("No data: {}".format(curr_date))
            break

  def save_to_file(self, *event):
    if not calc.dict_polynoms:
      print("No data to save")
    else:
      for name in self.graph_names:
        with open(os.path.join(
          calc.home, self.text_polynoms.format(self.file_meta, calc.type, name,
                                               self.file_format)), 'w') as f:
          # Headers Date;Ozone;
          f.write('Date;Ozone')
          for pair in ['o3_pair_', 'cloud_pair_']:
            for mu in ['mu<3', '3<mu<5', '5<mu']:
              for coeff in 'abcdef'[:PARS["calibration"]["polynom"]["degree"] + 1]:
                f.write(';{}{}_{}'.format(pair, mu, coeff))
              f.write(';SKO;Status')
          f.write('\n')

          for date in calc.dict_polynoms.keys():
            f.write('{};{}'.format(date, self.dates[date]))
            for pair in ['o3_pair_', 'cloud_pair_']:
              for mu in ['mu<3', '3<mu<5', '5<mu']:
                if calc.dict_polynoms[date][pair + name][mu][1]:
                  f.write(';{};{};{}'.format(
                    str(calc.dict_polynoms[date][pair + name][mu][0]).replace('[', '').replace(']',
                                                                                '').replace(
                      ', ', ';'), calc.dict_polynoms[date][pair + name][mu][1], 1))
                else:
                  f.write(';{};{};{}'.format(
                    str([0] * (PARS["calibration"]["polynom"]["degree"] + 1)).replace('[', '').replace(
                      ']', '').replace(', ', ';'), 'NaN', 0))
            f.write('\n')
        print("Saved: " + self.text_polynoms.format(self.file_meta, calc.type, name, self.file_format))

  def save_to_table(self, *event):
    if not calc.dict_polynoms:
      print("No data to save")
    else:
      for name in self.graph_names:
        with open(os.path.join(
          calc.home, self.text_tables.format(self.file_meta, calc.type,
                                             name, self.file_format)), 'w') as f:
          # Headers Date;Ozone;
          f.write('Date;Ozone;Mu')
          for pair in ['o3_pair_', 'cloud_pair_']:
            f.write(';{}'.format(pair + name))
          f.write(';;Mu')
          for pair in ['o3_pair_', 'cloud_pair_']:
            f.write(';{}'.format(pair + name + '_mesure'))
          f.write('\n')
          for date in sorted(calc.dict_polynoms.keys()):
            i = 0
            self.mus = calc.dict_polynoms[date][pair + name + '_mesure']['mu']
            while i < len(self.mus):
              mu = self.mus[i]
              text = '{};{};{}'.format(date, self.dates[date], mu)
              if mu < PARS["calibration"]["polynom"]["mu_intervals"][0]:  # mu<3
                text += calc.table_create(name, date, mu, 'mu<3')
              elif PARS["calibration"]["polynom"]["mu_intervals"][0] <= mu < \
                PARS["calibration"]["polynom"]["mu_intervals"][1]:  # 3<=mu<5
                text += calc.table_create(name, date, mu, '3<mu<5')
              elif mu >= PARS["calibration"]["polynom"]["mu_intervals"][1]:  # mu>=5
                text += calc.table_create(name, date, mu, '5<mu')
              try:
                if i < len(calc.dict_polynoms[date][pair + name + '_mesure']['mu']):
                  text += ';;{}'.format(calc.dict_polynoms[date][pair + name + '_mesure']['mu'][i])
                  for pair in ['o3_pair_', 'cloud_pair_']:
                    text += ';{}'.format(calc.dict_polynoms[date][pair + name + '_mesure']['L'][i])
              except Exception as err:
                pass
              f.write('{}\n'.format(text))
              i += 1
        print('Saved: ' + self.text_tables.format(self.file_meta, calc.type, name, self.file_format))

if __name__ == "__main__":
  PARS = Settings.get_device(settings_home, Settings.get_common(settings_home).get('device').get('id'))
  calc = Calc(PARS)
  dates = calc.get_dates()
  calc.get_filenames()
  calc.get_pixels('Z')

  root = Tk()
  main = GUI(root, dates)
  main.draw_elements()
  main.refresh_graphs()
  main.buttons(4)

  root.mainloop()


'''
Created on 07-Feb-2017

@author: rjoshi
'''
def height(node):
    if node is None:
        return -1
    else:
        return node.height

def size(node):
    if node is None:
        return 0
    else:
        return node.size

def update_size(node):
    node.size = size(node.left) + size(node.right) + 1

def update_height(node):
    node.height = max(height(node.left), height(node.right)) + 1

class BST(object):
    def __init__(self):
        self.root = None
        self.size = 0
        self.right = None
        self.left = None
        self.parent = None
        self.height = -1
        self.key = 0

    def insert(self, key):
        count = 0
        node = BST()
        node.key = key
        node.size = 1
        if self.root == None:
            self.root = node
        else:
            croot = self.root
            while True:
                if croot.key > key:
                    count += size(croot.right) + 1
                    if croot.left == None:
                        croot.left = node
                        node.parent = croot
                        break
                    else:

                        croot = croot.left
                else:
                    if croot.right == None:
                        croot.right = node
                        node.parent = croot
                        break
                    else:
                        croot = croot.right
        print(count)
        return node

    def printT(self):
        if self.left != None:
            self.left.printT()
        print(self.key, end=" ")
        print( self.height, self.size)
        if self.right != None:
            self.right.printT()

    def leftRotate(self, x):
        y = x.right
        y.parent = x.parent
        if y.parent is None:
            self.root = y
        else:
            if x.parent.right is x:
                x.parent.right = y
            if x.parent.left is x:
                x.parent.left = y
        x.right = y.left
        if x.right is not None:
            x.right.parent = x
        x.parent = y
        y.left = x
        update_height(x)
        update_height(y)
        update_size(x)
        update_size(y)

    def rightRotate(self, x):
        y = x.left
        y.parent = x.parent
        if y.parent is None:
            self.root = y
        else:
            if x.parent.right is x:
                x.parent.right = y
            if x.parent.left is x:
                x.parent.right = y
        x.left = y.right
        if x.left is not None:
            x.left.parent = x
        y.right = x
        x.parent = y
        update_height(x)
        update_height(y)
        update_size(x)
        update_size(y)

    def rebalance(self, node):
        while node is not None:
            update_height(node)
            update_size(node)
            if height(node.left) >= 2 + height(node.right):
                if height(node.left.left) >= height(node.left.right):
                    self.rightRotate(node)
                else:
                    self.leftRotate(node.left)
                    self.rightRotate(node)
            elif height(node.right) >= 2 + height(node.left):
                if height(node.right.right) >= height(node.right.left):
                    self.leftRotate(node)
                else:
                    self.rightRotate(node.right)
                    self.leftRotate(node)
            node = node.parent

    def insertAVL(self, t):
        node = self.insert(t)
        self.rebalance(node)
#print("10101".count("1"))
tree = BST()
tree.insertAVL(1)
tree.insertAVL(2)
tree.insertAVL(5)
tree.insertAVL(4)
# tree.insertAVL(4)
# tree.insertAVL(5)
# tree.insertAVL(6)
# tree.insertAVL(7)
# tree.insertAVL(8)
# tree.insertAVL(9)

# tree.insert(5)
# tree.insert(23)
# tree.insert(12)
# tree.insert(21)
# tree.insert(31)
# tree.insert(42)
# tree.insert(41)
# tree.insert(123)
#tree.leftRotate(tree.root.right.right.right.right.right)
#tree.root.printT()


import csv

def bestCSV(fileName, indexList, *cleanFlag):

    nums = 1
    account = 1
    result = {}
    valueList = []

    # Open and read the file
    with open(fileName) as files:
        line = csv.reader(files)
        for value in indexList:
            for row in line:
                valueList.append(row[value])

    # Clean or reject invalid data
    for flag in range(0, len(cleanFlag) - 1):
        for num in range(0, len(valueList)):
            if valueList[num] == '' or valueList[num] == cleanFlag[flag]:
                valueList.pop(num)
    maxNum = len(valueList)

    # Data statistics
    while True:

        # When the list is not empty
        if maxNum != 0:

            # When a round of statistics does not reach the end of the list
            if nums != maxNum:
                if valueList[0] == valueList[nums]:
                    account += 1
                    valueList.pop(nums)
                    maxNum -= 1
                    result[valueList[0]] = account
                else:
                    nums += 1

            # When a round of statistics reaches the end of the list
            else:

                # The last round of statistics may have only one list element left,
                # so the element and its statistical results must be added to the dictionary before the element is popped.
                if maxNum == 1 and nums == 1:
                    result[valueList[0]] = account
                    valueList.pop(0)
                    break

                # When not the last round of statistics for a list element or the last round of statistics, as usual
                else:
                    valueList.pop(0)
                    maxNum -= 1
                    nums = 1
                    account = 1

        # When the list is empty
        else:
            break
    return result


from __future__ import absolute_import
import argparse
import logging
import os
import torch
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from tqdm import tqdm
import random
import numpy as np
import pandas as pd
from sklearn.metrics import f1_score, accuracy_score
from transformers.modeling_bert import *
from transformers.modeling_xlnet import *
from transformers.tokenization_bert import BertTokenizer
from transformers.tokenization_xlnet import XLNetTokenizer
from transformers import AdamW, WarmupLinearSchedule
from models import SplitLinearModel, LinearBertModel, RnnBertModel,SPRnnBertModel,CnnBertModel,RCnnBertModel, MyRnnBertModel, MLPBertModel, PoolingBertModel
from torch import nn
import torch
import time
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, SubsetRandomSampler, WeightedRandomSampler, TensorDataset
from itertools import cycle
from data_utils import Corpus, split_transform, base_transform
import json

# from ipdb import launch_ipdb_on_exception,set_trace

import torch
class PGD():
    """
    using pgd to do adversarial training, but the result is not as good as fgm
    """
    def __init__(self, model):
        self.model = model
        self.emb_backup = {}
        self.grad_backup = {}

    def attack(self, epsilon=1e-6, alpha=0.3, emb_name='embeddings', is_first_attack=False):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name and 'LayerNorm' not in name:
                if is_first_attack:
                    self.emb_backup[name] = param.data.clone()
                norm = torch.norm(param.grad)
                if norm != 0 and not torch.isnan(norm):
                    r_at = alpha * param.grad / norm
                    param.data.add_(r_at)
                    param.data = self.project(name, param.data, epsilon)

    def restore(self, emb_name='embeddings'):
        # emb_name这个参数要换成你模型中embedding的参数名
        for name, param in self.model.named_parameters():
            if param.requires_grad and emb_name in name and 'LayerNorm' not in name:
                assert name in self.emb_backup, '%s is not in backup'.format(name)
                param.data = self.emb_backup[name]
        self.emb_backup = {}

    def project(self, param_name, param_data, epsilon):
        r = param_data - self.emb_backup[param_name]
        if torch.norm(r) > epsilon:
            r = epsilon * r / torch.norm(r)
        return param_data + r

    def backup_grad(self):
        for name, param in self.model.named_parameters():
            if param.requires_grad:
                self.grad_backup[name] = param.grad

    def restore_grad(self):
        for name, param in self.model.named_parameters():
            if param.requires_grad:
                param.grad = self.grad_backup[name]

MODEL_MAP = {
'bert_SPLi':(BertConfig,SplitLinearModel,BertTokenizer,split_transform),
'bert_SPRNN':(BertConfig,SPRnnBertModel,BertTokenizer,split_transform),
'bert_RCNN':(BertConfig,RCnnBertModel,BertTokenizer,split_transform),
'bert_Li':(BertConfig,LinearBertModel,BertTokenizer,base_transform),
'bert_RNN':(BertConfig,RnnBertModel,BertTokenizer,base_transform),
'bert_MyRNN':(BertConfig,MyRnnBertModel,BertTokenizer,base_transform),
'bert_MLP':(BertConfig,MLPBertModel,BertTokenizer,base_transform),
'bert_Pool':(BertConfig,PoolingBertModel,BertTokenizer,base_transform),
'bert_CNN':(BertConfig,CnnBertModel,BertTokenizer,base_transform),
'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer,base_transform),
}

def get_logger(args=None):
    logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                        datefmt='%m/%d/%Y %H:%M:%S',
                        level=logging.INFO)
    return logging.getLogger(__name__)
logger=get_logger()

def set_seed(args):
    random.seed(args.seed) ##设置种子
    np.random.seed(args.seed) ##
    torch.manual_seed(args.seed) ##setting random seed for torch
    if args.n_gpus > 0:
        torch.cuda.manual_seed_all(args.seed)

def load_model(Model, args, config):
    if os.path.isfile(args.resume_model_path) and args.to_resume_model:
        # BertForSequenceClassification.from_pretrained("./model_roberta_full/pytorch_model.bin",args,config=config)
        model = Model(config=config)
        logger.info("resuming model from {} ...".format(args.resume_model_path))
        model.load_state_dict(torch.load(args.resume_model_path))
    else:
        # model = Model.from_pretrained(args.pretrained_model_path, args,config=config)
        model = Model.from_pretrained(args.pretrained_model_path, config=config)
    return model

def add_args_to_config(args, config):
    config.linear_hidden_size = args.linear_hidden_size
    config.lstm_hidden_size = args.lstm_hidden_size
    config.lstm_layers = args.lstm_layers
    config.lstm_dropout = args.lstm_dropout
    config.kmax = args.kmax
    config.kernel_sizes =args.kernel_sizes
    config.out_channels = args.out_channels
    config.weight = args.weight
    config.output_hidden_states=args.output_hidden_states
    return config

def stacking_models(args, stacker):
    model_list = args.ensemble_models
    origin_out_dir = args.out_dir
    train_outs = []
    label_outs = []
    test_outs = []
    for idx, model in enumerate(model_list):
        k = args.folds
        args.out_dir = os.path.join(origin_out_dir, "model_{}_{}".format(model,idx))
        args.model_name = model
        args.resume_model_path = os.path.join(args.out_dir,"pytorch_model.bin")
        args.to_resume_model=False
        if not os.path.isdir(args.out_dir):
            os.mkdir(args.out_dir)
        for i in range(k):
            args.data_dir = "./data/data_{}".format(i)
            dev_set_list = []
            dev_label_list = []
            test_set_list = []
            dev_set, dev_label, test_set = train(args) ##
            dev_set_list.append(dev_set)
            dev_label_list.append(dev_label)
            test_set_list.append(test_set.reshape(test_set.shape[0],test_set.shape[1],1))
            args.to_resume_model = True
        train_features = np.concatenate(dev_set_list, axis=0)
        train_labels = np.concatenate(dev_label_list, axis=0)
        test_features = np.concatenate(test_set_list, axis=2)
        test_features = test_features.mean(axis=2) ##calculate the mean score of five fold
        train_outs.append(train_features)
        label_outs.append(train_labels)
        test_outs.append(test_features)
    train_set = np.concatenate(train_outs,axis=1)
    assert all(label_outs[0]==label_outs[1])
    train_labels = label_outs[0]
    test_set = np.concatenate(test_outs,axis=1)
    ###output the middle features for better use
    train_df = pd.DataFrame(train_set)
    label_df = pd.DataFrame(train_labels)
    test_df = pd.DataFrame(test_set)
    train_df.to_csv(os.path.join(origin_out_dir,'train.csv'),index=False)
    label_df.to_csv(os.path.join(origin_out_dir,'label.csv'),index=False)
    test_df.to_csv(os.path.join(origin_out_dir,'test.csv'),index=False)
    ##train test split
    train_x, test_x,train_y,test_y = train_test_split(train_set, train_labels, test_size=0.33, shuffle=True,random_state=676)
    stacker = stacker.fit(train_x,train_y)
    pre_label = stacker.predict(test_set)
    filename = time.ctime().replace(' ', '-')
    label_filename = "label-" + filename
    label_filename = label_filename.replace(':', '-') + ".csv"
    df = pd.read_csv("./data/data_0/test.csv")
    df['label_pre']=pre_label
    df[['id', 'label_pre']].to_csv(os.path.join(origin_out_dir, label_filename), index=False)
    ###evaluate model###
    test_pre = stacker.predict(test_x)
    f1 = f1_score(test_y,test_pre, average='macro')
    logger.info("the f1 score over test set is: %10f"%f1)

def train(args):
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    best_f1 = 0
    logger.info("the current config is :\n {}".format(str(vars(args))))
    set_seed(args)
    if args.model_name in MODEL_MAP:
        Config, Model, Tokenizer, Transform = MODEL_MAP[args.model_name]
        config = BertConfig.from_pretrained(args.pretrained_model_path, num_labels=args.num_labels)
        config = add_args_to_config(args,config) ##add customized args
        tokenizer = BertTokenizer.from_pretrained(args.pretrained_model_path, do_lower_case=args.do_lower_case)
        model = load_model(LinearBertModel, args, config)
        model = model.to(device)
        if args.n_gpus >1:
            model = nn.DataParallel(model)
        ###adv training
        pgd = PGD(model)
        ###adv training
        transform = base_transform(tokenizer,args)
        train_data = Corpus(args, "train.csv", transform)
        dev_data = Corpus(args, 'dev.csv', transform)
        dev_sampler = SequentialSampler(dev_data)
        dev_loader = DataLoader(dev_data, batch_size=args.eval_batch_size, sampler=dev_sampler)

        # Run prediction for full data
        eval_sampler = SequentialSampler(dev_data)
        dev_loader = DataLoader(dev_data, sampler=eval_sampler, batch_size=args.eval_batch_size)
        train_sampler = RandomSampler(train_data)
        test_sampler = SubsetRandomSampler(np.random.randint(low=0,high=(len(train_data)),size=len(dev_data)))
        train_loader = DataLoader(train_data,batch_size=args.batch_size, sampler= train_sampler,drop_last=True)

        test_loader = DataLoader(train_data,batch_size=args.eval_batch_size, sampler= test_sampler)
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_data))
        logger.info("  Batch size = %d", args.batch_size)
        logger.info("  Num steps = %d", args.epochs)
        logger.info("  Early Stopping dev_loss = %f",args.dev_loss)
        bar = tqdm(total=len(train_loader)*args.epochs)
        no_decay = ['bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
             'weight_decay': args.weight_decay},
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=0 , t_total=len(bar))
        steps = 0
        total_train_loss = 0
        set_seed(args)
        model.zero_grad()
        for _ in range(args.epochs):
            for step, data_batch in enumerate(train_loader):
                bar.update(1)
                model.train()
                for k,v in data_batch.items():
                    data_batch[k] = v.to(device)
                loss = model(batch=data_batch,feed_labels = True)
                if args.n_gpus>1:
                    loss = loss.mean()
                loss.backward()
                ###adv training
                pgd.backup_grad()
                for t in range(2):
                    pgd.attack(is_first_attack=(t == 0))
                    if t != 2 - 1:
                        model.zero_grad()
                    else:
                        pgd.restore_grad()
                    loss_adv = model(batch=data_batch, feed_labels=True)
                    if args.n_gpus > 1:
                        loss_adv = loss_adv.mean()
                    loss_adv.backward()  # 反向传播，并在正常的grad基础上，累加对抗训练的梯度
                pgd.restore()  # 恢复embedding参数
                ###adv training
                torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                optimizer.step()
                scheduler.step()
                model.zero_grad()##delete the grad in the graph
                ##setting bar
                steps += 1
                total_train_loss += loss.item()
                bar.set_description("training loss {}".format(loss.item()))
                if (steps)%args.eval_steps==0:
                    logits, loss, dev_labels= do_inference(model, dev_loader, device)
                    test_logits, test_loss,test_labels = do_inference(model, test_loader, device)
                    inference_labels = logits.argmax(axis=1)
                    test_inference_labels = test_logits.argmax(axis=1)
                    f1 = f1_score(y_true=dev_labels, y_pred=inference_labels)
                    test_f1 = f1_score(y_true=test_labels, y_pred=test_inference_labels)
                    acc = accuracy_score(dev_labels, inference_labels)
                    logger.info("=========eval report =========")
                    logger.info("step : %s ", str(steps))
                    logger.info("average_train loss: %s" %(str(total_train_loss/steps)))
                    logger.info("subset train loss: %s" %(str(test_loss)))
                    logger.info("subset train f1 score: %s", str(test_f1))
                    logger.info("eval loss: %s",str(loss))
                    logger.info("eval f1 score: %s",str(f1))
                    logger.info("eval acc: %s", str(acc))
                    output_eval_file = os.path.join(args.out_dir, "eval_records.txt")
                    with open(output_eval_file, "a") as writer:
                        if steps==args.eval_steps:
                            writer.write("\n%s\n"%(args.memo))
                        writer.write("=========eval report =========\n")
                        writer.write("step : %s \n" %(str(steps)))
                        writer.write("average_train loss: %s\n" % (str(total_train_loss/steps)))
                        writer.write("subset train loss: %s\n" % (str(test_loss)))
                        writer.write("subset f1 score: %s\n" % (str(test_f1)))
                        writer.write("eval loss: %s\n" %(str(loss)))
                        writer.write("eval f1 score: %s\n"%( str(f1)))
                        writer.write('\n')
                    if f1 > best_f1:
                        logger.info("we get a best dev f1 %s saving model....",str(f1))
                        output_path = os.path.join(args.out_dir,"pytorch_model.bin")
                        if hasattr(model, 'module'):
                            logger.info("model has module")
                        model_to_save = model.module if hasattr(model, 'module') else model
                        torch.save(model_to_save.state_dict(), output_path)
                        logger.info("model saved")
                        best_f1 = f1
        save_config(args)
        logger.info("args saved")
        ##load the final model
        args.to_resume_model=True
        model = load_model(Model, args, config)
        model = model.to(device)
        if args.n_gpus > 1:
            model = nn.DataParallel(model)
        dev_logits, loss, dev_labels = do_inference(model, dev_loader, device) ##do the inference for dev set
        pub_data = Corpus(args, 'test.csv', transform)
        pub_sampler = SequentialSampler(pub_data)
        pub_loader = DataLoader(pub_data, batch_size=args.eval_batch_size, sampler=pub_sampler)
        # logits, loss, dev_labels = do_inference(model, dev_loader, device)
        test_logits, _, _ = do_inference(model, pub_loader, device)
        return dev_logits, dev_labels, test_logits
    else:
        logger.info("the model %s is not registered", args.model_name)
        return

def kfold_train(args):
    k = args.folds
    origin_data_dir = args.data_dir
    origin_out_dir = args.out_dir
    if len(args.weight_list)==k: ##using weight list as weight
        weight_list = args.weight_list
    for i in range(k):
        args.weight = weight_list[i]
        args.data_dir = origin_data_dir.format(i)
        args.out_dir = os.path.join(origin_out_dir,"model_{}".format(i))
        args.resume_model_path = os.path.join(args.out_dir,"pytorch_model.bin")
        if not os.path.isdir(args.out_dir):
            os.mkdir(args.out_dir)
        logger.info("training fold %d"%i)
        args.to_resume_model = False
        try:
            train(args)
        except:
            train(args)
        ###get the evaluation##
        args.to_resume_model=True ##set to resume the model
        predict(args,is_eval=True) ##output the dev result
        ###do the test set prediction###
        predict(args,is_eval=False)
        torch.cuda.empty_cache()

def save_config(args):
    filename = time.ctime().replace(' ', '-')
    filename = "config_"+filename.replace(':', '-')+".json"
    filename = os.path.join(args.out_dir,filename)
    with open(filename,'w') as file:
        json.dump(vars(args), file)

def predict(args,is_eval=False):
    args.to_resume_model = True
    if is_eval:
        input_file_name = "dev.csv"
    else:
        input_file_name = "test.csv"
    device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    if args.model_name in MODEL_MAP:
        Config, Model, Tokenizer,Tansform = MODEL_MAP[args.model_name]
        config = Config.from_pretrained(args.pretrained_model_path, num_labels=args.num_labels)
        config = add_args_to_config(args,config)
        tokenizer = Tokenizer.from_pretrained(args.pretrained_model_path, do_lower_case=args.do_lower_case)
        transform = Tansform(tokenizer,args)
        model = load_model(Model, args, config)
        model = model.to(device)
        if args.n_gpus >1:
            model = nn.DataParallel(model)
        pub_data = Corpus(args,input_file_name,transform)
        pub_sampler = SequentialSampler(pub_data)
        pub_loader = DataLoader(pub_data, batch_size=args.eval_batch_size,sampler= pub_sampler)
        logits, _ , _ = do_inference(model, pub_loader, device)
        df = pd.read_csv(os.path.join(args.data_dir, input_file_name))
        inference_label = logits.argmax(axis =1)
        df['label_pre'] = inference_label
        if is_eval:
            df['label_0'] = logits[:, 0]
            df['label_1'] = logits[:, 1]
            df[['id', 'label', 'label_pre','label_0', 'label_1']].to_csv(os.path.join(args.out_dir, "dev_sub.csv"),index=False)
        else:
            df['label_0'] = logits[:, 0]
            df['label_1'] = logits[:, 1]
            filename = time.ctime().replace(' ', '-')
            label_filename = "label-" + filename
            filename = filename.replace(':', '-') + ".csv"
            label_filename = label_filename.replace(':', '-') + ".csv"
            df[['id', 'label_0', 'label_1']].to_csv(os.path.join(args.out_dir, filename), index=False)
            # df[['id', 'label_pre']].to_csv(os.path.join(args.out_dir, label_filename), index=False)
            with open(os.path.join(args.out_dir, label_filename),'w',encoding='utf-8') as out:
                for i in range(df.shape[0]):
                    out.write("{}\t{}\n".format(df['id'][i],df['label_pre'][i]))

def do_inference(model, dataloader, device):
    model.eval()
    dataloader = iter(dataloader)
    eval_loss = 0
    scores_list = []
    inf_steps = 0
    steps = tqdm(list(range(len(dataloader))), total=len(dataloader))
    total_label = []
    for step in steps:
        data_batch = next(dataloader)
        total_label.append(data_batch.get("label").numpy())
        for k, v in data_batch.items():
            data_batch[k] = v.to(device)
        with torch.no_grad():  ##runing inference with in the no_grad() context
            tmp_eval_loss = model(batch = data_batch, feed_labels=True)
            logits = model(batch = data_batch)
        eval_loss += tmp_eval_loss.mean().item()
        logits = logits.detach().cpu().numpy()
        scores_list.append(logits)
        inf_steps += 1
        steps.set_description("eval loss {}:".format(eval_loss/inf_steps))
    total_label = np.concatenate(total_label,axis=0)
    scores = np.concatenate(scores_list, axis=0)##concatenate each batch scores
    return scores, eval_loss/inf_steps, total_label

def main():
    parser = argparse.ArgumentParser()
    ##required parameter
    parser.add_argument("--memo", default='running glue baseline Five Fold', type=str, required=False)
    parser.add_argument("--model_name", type=str, default='bert_Li', required=False)
    parser.add_argument("--data_dir", type=str, default="./data/nlp_new_train{}", required=False)
    parser.add_argument("--out_dir", type=str, default="./Five_adv2_Limodel_bert", required=False)
    parser.add_argument("--pretrained_model_path", type=str, default="./bert-base-chinese", required=False)
    parser.add_argument("--to_resume_model",type=bool,default=False, required=False)
    parser.add_argument("--resume_model_path", type=str, default="./Five_adv2_Limodel_bert", required=False)
    parser.add_argument("--num_labels", type=int,default=2, required=False)
    ##other parameters
    parser.add_argument("--output_hidden_states", type=bool, default=True, required=False)
    parser.add_argument("--do_kfold", action="store_true", default=True, required=False)
    parser.add_argument("--do_ensemble", action="store_true", default=False, required=False)
    parser.add_argument("--do_train", action="store_true",default=True, required=False)
    parser.add_argument("--do_eval", action="store_true",default=True, required=False)
    parser.add_argument("--test", action="store_true", default=True, required=False)
    parser.add_argument("--folds",type=int,default=5,required=False)
    parser.add_argument("--epochs", type=int, default=4, required=False)
    parser.add_argument("--weight", default=[1.0, 1.0], type=list, required=False, help='the weight for crossentropy')
    parser.add_argument("--ensemble_models",default=["bert_SPRNN", "bert_RCNN", "bert_RNN", "bert_CNN", "bert_Li"],required=False,help='the ensemble model names')
    parser.add_argument("--weight_list", default=[[1.0,1.0],[1.0,1.0],[1.0,1.0],[1.0,1.0],[1.0,1.0]],type=list,required=False,help='weight list used in the ensemble mode')
    parser.add_argument("--batch_size", type=int, default=32, required=False)
    parser.add_argument("--eval_batch_size", default=64, type=int, required=False)
    parser.add_argument("--max_seq_len",type=int, default=150,required=False)
    parser.add_argument("--title_seq_len", type=int, default=71, required=False)
    parser.add_argument("--content_seq_len", type=int, default=71, required=False)
    parser.add_argument("--no_cuda", default=False, action="store_true", required=False)
    parser.add_argument("--log_dir", default=None, type=str, required=False)
    parser.add_argument("--dev_loss",default=0,type=float,required=False)
    parser.add_argument("--seed",default=42,type=int,required=False)
    parser.add_argument("--do_lower_case", action="store_true", default=False, required=False)
    parser.add_argument("--optimize_steps", type= int, default=20000, required=False)
    parser.add_argument("--learning_rate", default=5e-5, type=float,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument("--weight_decay", default=0, type=float,
                        help="L2 regularization.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float,
                        help="Max gradient norm.")
    # parser.add_argument("--max_grad_norm", default=1.0, type=float,
    #                     help="Max gradient norm.")
    # parser.add_argument("--num_train_epochs", default=12.0, type=float,
    #                     help="Total number of training epochs to perform.")
    # parser.add_argument("--max_steps", default=-1, type=int,
    #                     help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
    parser.add_argument("--eval_steps", default=200, type=int,required=False,
                        help="")
    parser.add_argument("--lstm_hidden_size", default=512, type=int,
                        help="")
    parser.add_argument("--lstm_layers", default=1, type=int,
                        help="")
    parser.add_argument("--lstm_dropout", default=0.5, type=float,
                        help="")
    parser.add_argument("--linear_hidden_size", default=1024, type=float,
                        help="")
    parser.add_argument("--kernel_sizes", default=[2,3,4,5], type=list,
                        help="set the kernel sizes for cnn model")
    parser.add_argument("--out_channels", default=256, type=int,
                        help="set the out channel for cnn model")
    parser.add_argument("--kmax", default=2, type=float,
                        help="set the features from kmax")
    parser.add_argument("--meta_path", default=None, type=str, required=False,
                        help="Path to pre-trained model or shortcut name selected in the list:")

    # parser.add_argument("--report_steps", default=-1, type=int,
    #                     help="")
    # parser.add_argument("--warmup_steps", default=0, type=int,
    #                     help="Linear warmup over warmup_steps.")
    # parser.add_argument("--split_num", default=3, type=int,
    #                     help="text split")
    # parser.add_argument('--logging_steps', type=int, default=50,
    #                     help="Log every X updates steps.")
    args = parser.parse_args()
    args.n_gpus = torch.cuda.device_count()
    if not os.path.isdir(args.out_dir):
        os.mkdir(args.out_dir) #prepare output directory
    if args.do_ensemble:
        stacker = LogisticRegression(random_state=0, solver='lbfgs',multi_class='multinomial',class_weight={0:5.0, 1:1.0, 2:1.0})
        stacking_models(args,stacker)
        return
    if args.do_kfold:
        kfold_train(args)
        return
    if args.do_train:
        train(args)
        args.do_train = False###
    if args.do_eval:
        predict(args, is_eval=True)  ##output the dev set result
    if args.test:
        predict(args)

if __name__ == "__main__":
    # with launch_ipdb_on_exception():
    os.environ['CUDA_VISIBLE_DEVICES']='1'
    main()


import numpy
import os

from bayes_common import *

def file_parse(filename):
    with open(filename) as datafile:
        words = text_parse(datafile.read())
    return words

def test_parse_on_path(pathname):
    return [file_parse(os.path.join(pathname, filename))
        for filename in os.listdir(pathname)]

def run_evaluate(spam_docs, normal_docs, test_each_count):

    datasets = dispatch_training_and_test_dataset(
        spam_docs, normal_docs, test_each_count)

    vocabs = create_vocab_list(spam_docs + normal_docs)

    return evaluate(vocabs,
        datasets['test']['docs'], datasets['test']['classes'],
        datasets['training']['docs'], datasets['training']['classes'])

if __name__ == '__main__':
    from minitest import *

    with test_case("email spam"):
        with test("file_parse"):
            words = file_parse('ham/6.dataset')
            len(words).must_equal(198)
            words[:4].must_equal(['hello', 'since', 'you', 'are'])
            words[195:].must_equal(['changes', 'google', 'groups'])

        with test("run_evaluate"):
            test_each_count = 5
            run_evaluate(test_parse_on_path('spam'),
                test_parse_on_path('ham'),
                test_each_count).must_equal((1,10))


from django.db import models
from datetime import date
import pandas as pd

from routines.utils import BootstrapColors
from initsys.models import Usr, Direccion
from .models_opcs import *

def UsuarioCliente():
    return list(
        item['idusuario']
        for item in Cliente.objects.all().values('idusuario'))

def UsuarioNoCliente():
    dfUsr = pd.DataFrame(
        list(Usr.objects.all().values('idusuario')), columns=['idusuario'])
    return dfUsr[~dfUsr.idusuario.isin(UsuarioCliente())]['idusuario'].to_list()

def UsrResponsables():
    resps = list()
    for pk in UsuarioNoCliente():
        item = Usr.objects.filter(pk=pk)[0]
        resps.append((item.pk, f"{item}"), )
    return resps

class TaxonomiaExpediente(models.Model):
    idtaxonomia = models.AutoField(primary_key=True)
    nombre = models.CharField(max_length=50)
    color = models.CharField(
        max_length=50, blank=True, choices=BootstrapColors,
        default=BootstrapColors[0][0])
    descripcion = models.TextField(
        blank=True, verbose_name="Descripción")
    mostrar_en_panel = models.BooleanField(default=False, blank=True)
    padre = models.ForeignKey(
        to="self",
        on_delete=models.SET_NULL,
        null=True, blank=True,
        related_name="hijos")
    created_by = models.ForeignKey(
        Usr, on_delete=models.SET_NULL,
        null=True, blank=True, related_name="+")
    created_at = models.DateTimeField(auto_now_add=True)
    updated_by = models.ForeignKey(
        Usr, on_delete=models.SET_NULL,
        null=True, blank=True, related_name="+")
    updated_at = models.DateTimeField(auto_now=True)

    class Meta:
        ordering = ["nombre"]

    def __str__(self):
        return "{}".format(self.nombre).strip()

    def __unicode__(self):
        return self.__str__()

class Cliente(Usr):
    idcliente = models.AutoField(primary_key=True)
    fecha_nacimiento = models.DateField(
        verbose_name="Fecha de nacimiento", blank=True, default=date.today)
    CURP = models.CharField(max_length=18, blank=True)
    RFC = models.CharField(max_length=13, blank=True)
    NSS = models.CharField(max_length=15, blank=True)
    estado_civil = models.CharField(
        max_length=15,
        choices=ESTADOS_CIVILES,
        default=ESTADO_CIVIL_CASADO)
    conyuge = models.CharField(max_length=150, blank=True)
    clinica = models.CharField(
        max_length=150, blank=True, verbose_name="Clínica")
    subdelegacion = models.CharField(
        max_length=150, blank=True, verbose_name="Subdelegación")
    empresa = models.CharField(max_length=150, blank=True)
    domicilicio = models.ForeignKey(
        Direccion, on_delete=models.SET_NULL,
        null=True, blank=True, related_name="cliente")
    telefono_oficina = models.CharField(
        max_length=10, blank=True, verbose_name="Teléfono de oficina")
    otro_telefono = models.CharField(
        max_length=10, blank=True, verbose_name="Otro teléfono")
    afore_actual = models.CharField(max_length=50, blank=True)
    fecha_afore_actual = models.DateField(blank=True, default=date.today)
    tipo = models.ForeignKey(
        TaxonomiaExpediente,
        on_delete=models.PROTECT,
        related_name='clientes')
    observaciones = models.TextField(blank=True)
    obs_semanas_cotizadas = models.TextField(
        blank=True, verbose_name="Semanas Cotizadas")
    obs_homonimia = models.TextField(
        blank=True, verbose_name="Homonimia")
    obs_duplicidad = models.TextField(
        blank=True, verbose_name="Duplicidad")
    responsable = models.ForeignKey(
        Usr,
        on_delete=models.SET_NULL,
        related_name="clientes_asignados",
        blank=True,
        null=True,
        verbose_name="Ejecutivo",
        #limit_choices_to={'idusuario__in':UsuarioNoCliente}
        )
    gestor = models.ForeignKey(
        Usr,
        on_delete=models.SET_NULL,
        related_name="clientes_gestionados",
        blank=True,
        null=True,
        verbose_name="Gestor",
        #limit_choices_to={'idusuario__in':UsuarioNoCliente}
        )

    @property
    def edad(self):
        anios = date.today().year - self.fecha_nacimiento.year
        if date.today().month < self.fecha_nacimiento.month:
            anios -= 1
        elif (date.today().month == self.fecha_nacimiento.month
                and date.today().day < self.fecha_nacimiento.day):
            anios -= 1
        return anios

    class Meta:
        ordering = ["last_name", "apellido_materno", "first_name"]

    def __str__(self):
        return "{} {} {} - {}".format(
            self.last_name,
            self.apellido_materno,
            self.first_name,
            self.pk,
            ).strip()

    def __unicode__(self):
        return self.__str__()


# -*- coding: utf-8 -*-

""" Sahana Eden Auth Model

    @copyright: 2009-2012 (c) Sahana Software Foundation
    @license: MIT

    Permission is hereby granted, free of charge, to any person
    obtaining a copy of this software and associated documentation
    files (the "Software"), to deal in the Software without
    restriction, including without limitation the rights to use,
    copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following
    conditions:

    The above copyright notice and this permission notice shall be
    included in all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
    OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
    HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
    WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
    OTHER DEALINGS IN THE SOFTWARE.
"""

__all__ = ["S3AuthModel"]

from gluon import *
from gluon.storage import Storage
from ..s3 import *

# =============================================================================
class S3AuthModel(S3Model):

    names = ["auth_organisation"]

    def model(self):

        T = current.T

        # =============================================================================
        # Domain table
        #
        # When users register their email address is checked against this list.
        # If the Domain matches, then they are automatically assigned to the
        # Organization.
        # If there is no Approvals email then the user is automatically approved.
        # If there is an Approvals email then the approval request goes to this
        # address
        # If a user registers for an Organization & the domain doesn't match (or
        # isn't listed) then the approver gets the request

        if current.deployment_settings.get_auth_registration_requests_organisation():
            ORG_HELP = T("If this field is populated then a user who specifies this Organization when signing up will be assigned as a Staff of this Organization unless their domain doesn't match the domain field.")
        else:
            ORG_HELP = T("If this field is populated then a user with the Domain specified will automatically be assigned as a Staff of this Organization")

        DOMAIN_HELP = T("If a user verifies that they own an Email Address with this domain, the Approver field is used to determine whether & by whom further approval is required.")
        APPROVER_HELP = T("The Email Address to which approval requests are sent (normally this would be a Group mail rather than an individual). If the field is blank then requests are approved automatically if the domain matches.")

        tablename = "auth_organisation"
        table = self.define_table(tablename,
                                  self.org_organisation_id(
                                        comment=DIV(_class="tooltip",
                                                    _title="%s|%s" % (current.messages.ORGANISATION,
                                                                      ORG_HELP))),
                                  Field("domain",
                                        label=T("Domain"),
                                        comment=DIV(_class="tooltip",
                                                    _title="%s|%s" % (T("Domain"),
                                                                      DOMAIN_HELP))),
                                  Field("approver",
                                        label=T("Approver"),
                                        requires=IS_NULL_OR(IS_EMAIL()),
                                        comment=DIV(_class="tooltip",
                                                    _title="%s|%s" % (T("Approver"),
                                                                      APPROVER_HELP))),
                                  s3_comments(),
                                  *s3_meta_fields())

        # ---------------------------------------------------------------------
        # Pass variables back to global scope (s3db.*)
        #
        return Storage()

# END =========================================================================


import logging
import sys

from modules import Configuration

logging.basicConfig(level=logging.DEBUG)

class SubstitutionDecipher:
    """
    Decipher the file being passed through.
    """

    def __init__(self, file):
        self.logger = logging.getLogger(__name__)

        self.configuration = Configuration()
        self.config = self.configuration.load()

        if file:
            self.file = file
        else:
            self.logger.critical(f'You have not provided a file to decipher.')
            sys.exit(1)

    def _get_unfixed_characters(self):
        """
        We need to know how many unfixed characters there are,
        so this function will load the config and generate a figure
        based on the content of the config file key 'fixed-cipher-text'.
        :return:
        """

        self.logger.debug('Calculating Amount Of Unfixed Characters')
        self.un_fixed_characters = []

        # As the key 'fixed-cipher-text' is a list containing dictionaries,
        # we need to loop over it to enable us to check each item and run a
        # comparative check.
        for item in self.config['fixed-cipher-text']:
            # Let's grab the index of the item in this list,
            # incase we need to update or change the item later.
            index = self.config['fixed-cipher-text'].index(item)
            # This next line is rather complex, but it
            # basically just outputs the key in the dictionary.
            key = [key for key in item.keys()][0]
            # We are specifically looking for a False value in the list,
            # hence the negative check.
            if not item[str(key)]:
                # If a match is found, stick it in a new list inside of a
                # dictionary with it's index, so we can adjust it later on.
                self.un_fixed_characters.append(dict(item=item, index=index))

        # Output how many items are in the list of
        # unfixed characters.
        self.logger.info(f'Found {len(self.un_fixed_characters)} Unfixed Characters.')
        return(len(self.un_fixed_characters))


import cgi
from flask import render_template, abort, request
from jinja2 import TemplateNotFound
from twilio import twiml
from twilio.rest import TwilioRestClient
from app import app, redis_db, socketio

TWILIO_ACCOUNT_SID = app.config['TWILIO_ACCOUNT_SID']
TWILIO_AUTH_TOKEN = app.config['TWILIO_AUTH_TOKEN']
TWILIO_NUMBER = app.config['TWILIO_NUMBER']

client = TwilioRestClient(account=TWILIO_ACCOUNT_SID,
						  token=TWILIO_AUTH_TOKEN)

@app.route('/<presentation_name>/', methods=['GET'])
def landing(presentation_name):
    try:
        return render_template(presentation_name + '.html', telephone=TWILIO_NUMBER)
    except TemplateNotFound:
        abort(404)

@app.route('/presentation/twilio/webhook/', methods=['POST'])
def twilio_callback():
    to = request.form.get('To', '')
    from_ = request.form.get('From', '')
    message = request.form.get('Body', '').lower()
    if to == TWILIO_NUMBER:
        redis_db.incr(cgi.escape(message))
        socketio.emit('msg', {'div': cgi.escape(message),
                              'val': redis_db.get(message)},
                               namespace='/presentation')
    #resp = twiml.Response()
    #resp.message('Merci/Cheers.')
    return str(resp)


import pandas as pd
from level2_utils import *

# paths
nonbids_data_path = "/Users/benjaminsmith/Google Drive/oregon/data/DEV/nonbids_data/"
ml_data_folderpath = nonbids_data_path + "fMRI/ml"
dev_scripts_path = '/Users/benjaminsmith/Google Drive/oregon/code/DEV_scripts'
ml_scripting_path = dev_scripts_path + "/fMRI/ml"

train_betas_with_data = get_data_for_confirmed_train_subjs(
    beta_glob="/Users/benjaminsmith/Google Drive/oregon/data/DEV/nonbids_data/fMRI/fx/models/SST/wave1/posterror_conditions/sub-DEV*/beta_000[1-2].nii",
    nonbids_data_path=nonbids_data_path,
    ml_data_folderpath=ml_data_folderpath,
    ml_scripting_path=ml_scripting_path
)

subject_list = train_betas_with_data.SID.unique()
subject_list.sort()
# we're going to write matlab code in python...
# it's clumsy but effective.
for s_i, s in enumerate(subject_list):
    #for each subject, add a line to the code
    s_index = s_i+1
    print("%" + s)
    print("matlabbatch{1}.spm.stats.factorial_design.des.pt.pair(" + str(s_index) + ").scans = {")
    beta_paths = train_betas_with_data.spm_l2_path_description[train_betas_with_data.SID==s]
    # we need specifically the beta_0001 and then the beta_0002. need to make sure they're in order.
    [re.match("beta\_0001", bp) for bp in beta_paths]

    #ensure our betas are all in order
    for beta_to_match in ['beta_0001','beta_0002']:
        #get the matching path
        beta_path_match = [bp for bp in beta_paths if re.search(beta_to_match,bp) is not None][0]
        #should throw an error if the list is empty.
        #print it
        print("\t" + beta_path_match)
    print("};")

#OK SO WE HAVE GOT THESE, BUT WE REALLY NEED SOMETHING DIFFERENT, BECAUSE WE'RE NOT JUST GRABBING BETA_0002S. WE'RE GRABBING
#BETA_0001.NII AND BETA_0002.NII
#HOWEVER WE STILL PROBABLY WANNA MERGE IN INDIVIDUAL DIFFERNCES FOR MORE ANALYSES.


# -*- coding: utf-8 -*-
import pdb
import time
import json
import sys
import os
import glob

# Author: Preston Law
# 12/01/2018

# This script counts the number of tweets in all_tweets.json and prints
# out the range of tweet dates.

# This looks for the last tweet collection file, finds the lastest tweet id, and finds all tweets after that.

def custom_load_tweets(fp, date_range=None):
    tweets = []
    for line in fp:
        tweet = json.loads(line)
        tweets.append(tweet)
        date_str = my_tweet['tweet_date']
        date_list = date_str.split('-')
        year = int(date_list[0])
        month = int(date_list[1])
        date_list = date_list[2].split('T')
        day = int(date_list[0])
        tmp = month * 100 + day
        if date_range is None:
            date_range = [tmp,tmp]
        else:
            date_range[0] = min(date_range[0], tmp)
            date_range[1] = max(date_range[1], tmp)

    print(date_range)
    return tweets, date_range

def get_number_of_tweets_in_file(filename, date_range):
    month_dict = {'Jan': 1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9, 'Oct':10, 'Nov':11, 'Dec':12}
    print(filename)

    with open(filename, 'r') as fp:
        tweets = json.load(fp)

        for tweet in tweets:
            date_str = tweet['tweet_date']
            date_list = date_str.split('-')
            year = int(date_list[0])
            month = int(date_list[1])
            date_list = date_list[2].split('T')
            day = int(date_list[0])
            tmp = month * 100 + day

            if date_range is None:
                date_range = [tmp,tmp]
            else:
                date_range[0] = min(date_range[0], tmp)
                date_range[1] = max(date_range[1], tmp)

    return len(tweets), date_range

date_range = None
total = 0
filename = 'all_tweets.json'
count, date_range = get_number_of_tweets_in_file(filename, date_range)
total += count

print(total)
print(date_range)


#!/usr/bin/env python
import cgi
import urlparse
try:
    import json
except:
    import simplejson as json
import urllib
import logging
import tornado.httpserver
import tornado.ioloop
import tornado.web
import tornado.auth
import tornado.httpclient

AsyncHTTPClient = tornado.httpclient.AsyncHTTPClient

# hack to make pycurl not to verify peer certificate
# TODO this is dangerous, should find a better way
def fake_curl_setup_request():
    import pycurl
    orig = tornado.httpclient._curl_setup_request
    def _(curl, request, buffer, headers):
        orig(curl, request, buffer, headers)
        curl.setopt(pycurl.SSL_VERIFYHOST, 0)
        curl.setopt(pycurl.SSL_VERIFYPEER, 0)
    return _
tornado.httpclient._curl_setup_request = fake_curl_setup_request()

class Auth9Mixin(tornado.auth.OAuth2Mixin):
    _OAUTH_BASE_URL = "https://accounts.net9.org/api"
    _OAUTH_ACCESS_TOKEN_URL = _OAUTH_BASE_URL + "/access_token?"
    _OAUTH_AUTHORIZE_URL = _OAUTH_BASE_URL + "/authorize?"
    _OAUTH_NO_CALLBACKS = False

    def get_authenticated_user(self, redirect_uri, client_id, client_secret, code, callback, extra_fields = None):
        http = tornado.httpclient.AsyncHTTPClient()
        args = {
                "redirect_uri": redirect_uri,
                "code": code,
                "client_id": client_id,
                "client_secret": client_secret
                }
        if extra_fields and not isinstance(extra_fields, (set, frozenset)):
            extra_fields = set(extra_fields)

        http.fetch(self._oauth_request_token_url(**args),
                self.async_callback(self._on_access_token, redirect_uri, client_id, client_secret, callback, extra_fields))

    def _on_access_token(self, redirect_uri, client_id, client_secret, callback, extra_fields, response):
        if response.error:
            logging.warning('Net9 auth error: %s' % str(response))
            callback(None)
            return
        session = json.loads(response.body)

        self.net9_request(path="/userinfo", callback=self.async_callback(self._on_get_user_info, callback, session, extra_fields), access_token=session["access_token"])

    def _on_get_user_info(self, callback, session, extra_fields, user):
       redirect_url = self.get_cookie("_net9_auth_next_url", "/")
       if user is None:
            callback(None, redirect_url)
            return
       self.clear_cookie("_net9_auth_next_url")
       callback(user, redirect_url)

    def net9_request(self, path, callback, access_token=None, post_args=None, **args):
        url = self._OAUTH_BASE_URL + path
        all_args = {}
        if access_token:
            all_args["access_token"] = access_token
            all_args.update(args)
            all_args.update(post_args or {})
        if all_args:
            url += "?" + urllib.urlencode(all_args)
        callback = self.async_callback(self._on_net9_request, callback)
        http = tornado.httpclient.AsyncHTTPClient()
        if post_args is not None:
            http.fetch(url, method="POST", body=urllib.urlencode(post_args),
        callback=callback)
        else:
            http.fetch(url, callback=callback)

    def _on_net9_request(self, callback, response):
        if response.error:
            logging.warning("Error response %s fetching %s", response.error, response.request.url)
            callback(None)
            return
        callback(tornado.escape.json_decode(response.body))

class LoginHandler(tornado.web.RequestHandler, Auth9Mixin):
    @tornado.web.asynchronous
    def get(self):
        next_url = self.get_argument("next", "/")
        absolute_login_url = self.get_login_url()
        if not urlparse.urlsplit(absolute_login_url).scheme:
            absolute_login_url = self.request.protocol + "://" + self.request.host + absolute_login_url
        if not urlparse.urlsplit(next_url).scheme:
            next_url = self.request.protocol + "://" + self.request.host + next_url
        # this cookie will be cleared when authentication is finished
        self.set_cookie("_net9_auth_next_url", next_url)
        if self.get_argument("error", ""):
            logging.warning("error: %s" % self.get_argument("error"))
            self.write("error:" + self.get_argument("error"))
            self.finish()
        elif self.get_argument("code", False):
            self.get_authenticated_user(
              redirect_uri=absolute_login_url,
              client_id=self.settings["client_id"],
              client_secret=self.settings["client_secret"],
              code=self.get_argument("code"),
              callback=self.async_callback(self.on_login))
        else:
            self.authorize_redirect(redirect_uri=absolute_login_url, client_id=self.settings["client_id"])

    def on_login(self, user, redirect_url):
        pass


from setuptools import setup, find_packages
import httpscreenshot

install_requires = [
    'selenium',
    'requests',
    'm2crypto',
    'pillow']

setup(name='httpscreenshot',
      version=httpscreenshot.__version__,
      description=httpscreenshot.__doc__.strip(),
      url='https://github.com/breenmachine/httpscreenshot',
      author=httpscreenshot.__author__,
      author_email='unknown',
      license=httpscreenshot.__license__,
      packages=find_packages(),
      install_requires=install_requires,
      entry_points={
          'console_scripts': [
              'httpscreenshot = httpscreenshot.__main__:main',
          ],
      },
      zip_safe=False)