from django.db import models
from django.utils import timezone
from datetime import datetime, timedelta
from django import forms

#===========Project_Information=============================

class ProjectInformation(models.Model):
    class Meta:
        verbose_name = 'Project Information Record'

    RESPONSIBLE_PARTY = (
        ('Stanley Njoku', 'Stanley Njoku'),
        ('Beverly Everhart', 'Beverly Everhart'),
        ('Lenka Barrettova', 'Lenka Barrettova'),
        ('Chris Bradshaw', 'Chris Bradshaw'),
        ('Connie Gordon', 'Connie Gordon'),
        ('Xonie Lloyd', 'Xonie Lloyd'),
        ('Jeffrey Hamilton', 'Jeffrey Hamilton'),
    )
    person_Responsible = models.CharField(max_length=150, choices= RESPONSIBLE_PARTY)

# Project Category Type Dropdown Selection

    PROJECT_CATEGORY =(
        ('Safety','Safety'),
        ('Environment','Environment'),
        ('Fee',' Fee',),
        ('Regulatory', 'Regulatory'),
        ('Report','Report')
    )
    category = models.CharField(max_length=150, choices = PROJECT_CATEGORY)

# Project Type Dropdown Selection
    PROJECT_TYPE = (
        ('Hazardous Waste Management', 'Hazardous Waste Management'),
        ('Confined Space', 'Confined Space'),
        ('Emergency Response Management', ' Emergency Response Management',),
        ('Storm Water', 'Storm Water'),
        ('Hazardous Management Business Plan', 'Hazardous Management Business Plan'),
        ('Respiratory Fit Testing', 'Respiratory Fit Testing'),
        ('Worker Compensation', 'Worker Compensation'),
        ('Ergonomics', 'Ergonomics'),
        ('Respiratory Fit Testing', 'Respiratory Fit Testing'),
        ('Job Hazard Analysis', 'Job Hazard Analysis'),
        ('Lockout/tagout Procedures', 'Lockout/tagout Procedures'),
        ('Machine Safety', 'Machine Safety'),
        ('Chemical Hazard Communication', 'Chemical Hazard Communication'),
        ('Accident Prevention Programs', 'Accident Prevention Programs'),

    )
    Type= models.CharField(max_length =150, choices=PROJECT_TYPE)

    # RADIO BUTTON

    PROJECT_STATUS = (
        ('Not Started', 'Not Started'),
        ('In Progress', 'In Progress'),
        ('Pending Review', 'Pending Review',),
        ('On Hold', 'On Hold'),
        ('Approved', 'Approved'),
    )
    status = models.CharField(max_length=150, choices=PROJECT_STATUS)

    #Add Additional Field Lines
    date_added = models.DateTimeField(default=datetime.now)
    estimated_completion = models.DateField()
    comment = models.TextField()

#============Modifying Table to Read information ======================
    def __str__(self):
       return'person_Responsible:{}, Category: {}, Type:{}, Status:{}'.format(self.person_Responsible, self.category, self.Type, self.status)

#=================Manifest Class==============================================

class Manifest(models.Model):
    Name= models.CharField(max_length=200)
    Manifest_Tracking_Number = models.CharField(max_length=200,)
    EPA_ID_Location = models.CharField(max_length=200)
    Building_Address = models.CharField(max_length=200)

    VENDOR_NAMES = (
        ('Clean Harbors', 'Clean Harbors'),
        ('Ingenium', 'Ingenium'),
        ('Veolia', 'Veolia'),

     )
    Vendor_Name = models.CharField(max_length=150, choices= VENDOR_NAMES)

    LOCATION_EPAID = (
         ('B2500 - CAL000342024', 'B2500 - CAL000342024'),
         ('B6000 - CAD983606872', 'B6000 - CAD983606872'),
         ('4000  - CAD981991276', '4000  - CAD981991276'),
         ('B2000 - CAD983606872', 'B2000 - CAD983606872'),
         ('B925  - CAR000158816', 'B925  - CAR000158816'),
         ('B3110 - CAD981453871', 'B3110 - CAD981453871'),
         ('B487 - CAL000248918', 'B487 - CAL000248918'),
         ('B5731 - CAL000346207', 'B5731 - CAL000346207'),
         ('B5500 - CAL000095938', 'B5500 - CAL000095938'),
         ('Select One', 'Select One'),
     )
    EPA_ID_Location = models.CharField(max_length=100, choices=LOCATION_EPAID, default='Select One')

    SHIPPING_TYPE = (
        ('Weekly Shipment', 'Weekly Shipment'),
        ('Monthly Shipment', 'Monthly Shipment'),
        ('90-day Shipment', '90-day Shipment',),
        ('180-day/270-day Shipment', '180-day/270-day Shipment'),
        ('Urgent Shipment', 'Urgent Shipment'),

    )
    Type= models.CharField(max_length =150, choices=SHIPPING_TYPE)

    CATEGORIES= (
         ('Drum Shipment', 'Drum Shipment'),
         ('Labpack', 'Labpack'),
         ('Bulk Shipment','Bulk Shipment'),

     )
    category = models.CharField(max_length=150, choices=CATEGORIES)

    Weight_Shipped = models.CharField(max_length=150)

    WEIGHT_UNITS= (
         ('Gallons (Liquid Only)', 'Gallons (Liquid Only)'),
         ('Kilograms', 'Kilograms'),
         ('Liters Liquid Only','Liters Liquid Only'),
         ('Metric Tons', 'Metric Tons'),
         ('Cubic Meter', 'Cubic Meter'),
         ('Pounds', 'Pounds'),
         ('Tons', 'Tons'),
         ('Cubic Yards', 'Cubic Yards'),
         ('Select One', 'Select One'),
     )
    weight_Unit = models.CharField(max_length=150, choices=WEIGHT_UNITS, default='Select One')

    def due_Date():
        return datetime.today() + timedelta(days=30)
    Date_Generator_Sgned = models.DateField(default=due_Date)

    Date_to_DTSC = models.DateField(default=due_Date)

    # def DateField():
     # date_added = DateField.DateField.strptime(start_Date,"%m/%d/%y")
     # estimated_completion = date_added+DateField.timedelta(days=30)

    MANIFEST_VENDOR = (
        ('Yes', 'Yes'),
        ('No', 'No'),
    )
    Manifest_Return_from_Vendor = models.CharField(max_length=150, choices=MANIFEST_VENDOR)

    MANIFEST_SIGNATURE = (
        ('Stanley Njoku', 'Stanley Njoku'),
        ('Beverly Everhart', 'Beverly Everhart'),
        ('Lenka Barrettova', 'Lenka Barrettova'),
        ('Chris Bradshaw', 'Chris Bradshaw'),
        ('Connie Gordon', 'Connie Gordon'),
        ('Kevin Tennyson', 'Kevin Tennyson'),
    )
    Manifest_Signatory = models.CharField(max_length=150, choices= MANIFEST_SIGNATURE)

    MANIFEST_FILING_STATUS = (
        ('Not Started', 'Not Started'),
        ('In Progress', 'In Progress'),
        ('Pending Review', 'Pending Review',),
        ('On Hold', 'On Hold'),
        ('Approved', 'Approved'),
    )
    status = models.CharField(max_length=150, choices=MANIFEST_FILING_STATUS)

    comment = models.TextField()

# ==========Read Manifest======================================================

    def __str__(self):
       return'Manifest_Tracking_Number:{}, EPA_ID_Location:{}, date_to_DTSC :{}, Manifest_Return_from_Vendor:{}'.format(self.Manifest_Tracking_Number, self.EPA_ID_Location, self.Date_to_DTSC, self.Manifest_Return_from_Vendor)

# ==========Read Manifest ENDS HERE=====================================================

#=================Compliance Class==============================================

class Compliance(models.Model):
    RESPONSIBLE_INDIVIDUAL = (
        ('Stanley Njoku', 'Stanley Njoku'),
        ('Beverly Everhart', 'Beverly Everhart'),
        ('Lenka Barrettova', 'Lenka Barrettova'),
        ('Chris Bradshaw', 'Chris Bradshaw'),
        ('Connie Gordon', 'Connie Gordon'),
        ('Kevin Tennyson', 'Kevin Tennyson'),
    )
    Individual_Responsible = models.CharField(max_length=150, choices= RESPONSIBLE_INDIVIDUAL)

    agency = models.CharField(max_length=200)

    description = models.CharField(max_length=30)

    COMPLIANCE_CATEGORY =(
        ('Safety','Safety'),
        ('Environment','Environment'),
        ('Fee',' Fee',),
        ('Regulatory', 'Regulatory'),
        ('Report','Report')
    )
    category = models.CharField(max_length=150, choices = COMPLIANCE_CATEGORY)

    AGENCY_TYPE = (
        ('County', 'County'),
        ('Local', 'Local'),
        ('State', 'State',),
        ('Federal', 'Federal'),
        ('EPA', 'EPA'),
        ('OSHA', 'OSHA'),
    )
    Type = models.CharField(max_length=150, choices=AGENCY_TYPE)

    def due_Date():
       return datetime.today() + timedelta(days=30)

    start_Date = models.DateField(default=datetime.today)
    due_Date = models.DateField(default=due_Date)
    comment = models.TextField()

    COMPLIANCE_STATUS = (
        ('Not Started', 'Not Started'),
        ('In Progress', 'In Progress'),
        ('Pending Review', 'Pending Review',),
        ('On Hold', 'On Hold'),
        ('Approved', 'Approved'),
    )
    status = models.CharField(max_length=150, choices=COMPLIANCE_STATUS)

# Modifying Table to Read information ======================

    def __str__(self):
       return'Agency: {}, Description: {}, Type: {}, Start_Date:{}, Due_Date:{}'.format(self.agency, self.description, self.Type, self.start_Date, self.due_Date)

#=================Dashbaord==============================================

def index(request):
    return render(request,'management/index.html')
    num_compliance=Compliance.objects.all().count()
    num_manifest=Manifest.objects.all().count()

    return render(
        request,
        'management/index.html',
        context={'num_manifest':num_manifest,'num_compliance':num_compliance},

      )

    #=================EPAS===============================================================


from sqlalchemy import Column, MetaData, Table, Integer, String, DateTime
from sqlalchemy.sql import func
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.dialects.postgresql import JSON

Base = declarative_base()

class VcmsFeaturePklFile(Base):
    __tablename__ = 'feature_pkl_files'

    sn = Column(Integer, primary_key=True)
    company_sn = Column(Integer, default=0)
    service_sn = Column(Integer, default=0)
    branch_sn = Column(Integer, default=0)
    pkl_key = Column(String)
    pkl_update_send_mark = Column(Integer, default=0)
    pkl_update_send_time = Column(DateTime, nullable=True)
    pkl_update_finish_mark = Column(Integer, default=0)
    pkl_update_finish_time = Column(DateTime, nullable=True)
    output_pkl_mark = Column(Integer, default=0)
    output_pkl_time = Column(DateTime, nullable=True)
    ct_user_sn = Column(Integer, default=0)
    ut_user_sn = Column(Integer, default=0)
    enabled = Column(Integer, default=1)
    ct = Column(DateTime, default=func.now())
    ut = Column(DateTime, nullable=True, onupdate=func.now())
    deprecated = Column(Integer, default=0)

    def __init__(self,company_sn = 0,
                      service_sn = 0,
                      branch_sn = 0,
                      pkl_key = None,
                      pkl_update_send_mark = 0,
                      pkl_update_send_time = None,
                      pkl_update_finish_mark = 0,
                      pkl_update_finish_time = None,
                      output_pkl_mark = 0,
                      output_pkl_time = None,
                      ct_user_sn = 0,
                      ut_user_sn = 0,
                      enabled = 1,
                      ct = None,
                      ut = None,
                      deprecated = 0) :
        self.company_sn = company_sn
        self.service_sn = service_sn
        self.branch_sn = branch_sn
        self.pkl_key = pkl_key
        self.pkl_update_send_mark = pkl_update_send_mark
        self.pkl_update_send_time = pkl_update_send_time
        self.pkl_update_finish_mark = pkl_update_finish_mark
        self.pkl_update_finish_time = pkl_update_finish_time
        self.ct_user_sn = ct_user_sn
        self.ut_user_sn = ut_user_sn
        self.enabled = enabled
        self.ct = ct
        self.ut = ut
        self.deprecated = deprecated


# pp_session.py

import logging
from datetime import datetime
from enum import Enum
import pandas as pd

from typing import Optional
from binance import enums as k_binance

from src.pp_market import Market
from src.pp_order import Order, OrderStatus
from src.pp_account_balance import AccountBalance
from src.xb_pt_calculator import get_pt_values
from src.pp_pending_orders_book import PendingOrdersBook
from src.pp_traded_orders_book import TradedOrdersBook
from src.pp_strategy_manager import StrategyManager
from src.pp_balance_manager import BalanceManager
from src.pp_concentrator import ConcentratorManager

log = logging.getLogger('log')

K_MINIMUM_DISTANCE_FOR_PLACEMENT = 35.0  # order activation distance
K_MAX_DISTANCE_FOR_REMAINING_PLACED = 100.0
K_MINIMUM_SHIFT_STEP = 15  # mp shift applied to equidistant point
K_MAX_SHIFT = 50.0
K_ORDER_PRICE_BUFFER = 5.0  # not used
K_AUGMENTED_FEE = 10 / 100

K_MIN_CYCLES_FOR_FIRST_SPLIT = 100  # the rationale for this parameter is to give time to complete (b1, s1)
K_DISTANCE_FOR_FIRST_CHILDREN = 100  # 150
K_DISTANCE_INTER_FIRST_CHILDREN = 25.0  # 50
K_DISTANCE_FIRST_COMPENSATION = 150  # 200.0
K_GAP_FIRST_COMPENSATION = 35  # 50.0

# K_SPAN_FOR_CONCENTRATION = 500
K_DISTANCE_FOR_CONCENTRATION = 150
K_GAP_CONCENTRATION = 200
K_INTERDISTANCE_AFTER_CONCENTRATION = 50.0

# one placement per cycle control flag
K_ONE_PLACE_PER_CYCLE_MODE = True

K_INITIAL_PT_TO_CREATE = 1

# pt creation
PT_CREATED_COUNT_MAX = 100  # max number of pt created per session
PT_CMP_CYCLE_COUNT = 30  # approximately secs (cmp update elapsed time)

PT_NET_AMOUNT_BALANCE = 0.00002  # 0.000020
PT_S1_AMOUNT = 0.023  # 0.022
PT_BUY_FEE = 0.08 / 100
PT_SELL_FEE = 0.08 / 100
PT_GROSS_EUR_BALANCE = 0.0

COMPENSATION_GAP = 500.0  # applied gap for compensated orders

class QuitMode(Enum):
    CANCEL_ALL_PLACED = 1
    PLACE_ALL_PENDING = 2

class Session:
    def __init__(self, client_mode: str):

        self.market = Market(
            symbol_ticker_callback=self.symbol_ticker_callback,
            order_traded_callback=self.order_traded_callback,
            account_balance_callback=self.account_balance_callback,
            client_mode=client_mode
        )

        # ********** managers **********
        self.bm = BalanceManager(market=self.market)
        self.pob = PendingOrdersBook(orders=[])
        self.tob = TradedOrdersBook()

        self.cm = ConcentratorManager(pob=self.pob, tob=self.tob)

        self.sm = StrategyManager(pob=self.pob, cm=self.cm, bm=self.bm)

        # *********** concentrator **********

        self.cmps = []
        self.cycles_serie = []
        self.orders_book_depth = []
        self.orders_book_span = []

        self.session_id = f'S_{datetime.now().strftime("%Y%m%d_%H%M")}'
        self.pt_created_count = 0
        self.buy_count = 0
        self.sell_count = 0
        self.cmp_count = 0

        # self.dashboard = None

        self.new_pt_permission_granted = True

        self.cycles_from_last_trade = 0

        # get filters that will be checked before placing an order
        self.symbol_filters = self.market.get_symbol_info(symbol='BTCEUR')

        self.market.start_sockets()

        self.last_cmp = self.market.get_cmp('BTCEUR')

        self.ticker_count = 0

        self.partial_traded_orders_count = 0

    # ********** dashboard callback functions **********

    def get_all_orders_dataframe(self) -> pd.DataFrame:
        # get list with all orders: pending (monitor + placed) & traded (completed + pending_pt_id)
        all_orders = self.pob.get_pending_orders() + self.tob.get_all_traded_orders()
        # create dataframe
        df = pd.DataFrame([order.__dict__ for order in all_orders])
        # delete status column because it returns a tuple and raises an error in the dash callback
        df1 = df.drop(columns='status', axis=1)
        return df1

    def get_all_orders_dataframe_with_cmp(self) -> pd.DataFrame:
        df = self.get_all_orders_dataframe()
        # create cmp order-like and add to dataframe
        cmp_order = dict(pt_id='CMP', status_name='cmp', price=self.last_cmp)
        df1 = df.append(other=cmp_order, ignore_index=True)
        return df1

    # ********** Binance socket callback functions **********

    def symbol_ticker_callback(self, cmp: float) -> None:
        # 0.1: create first pt
        if self.ticker_count == 0 and cmp > 20000.0:
            self.create_new_pt(cmp=cmp)

        # 0.2: update cmp count to control timely pt creation
        self.cmp_count += 1
        self.ticker_count += 1

        # these two lists will be used to plot
        self.cmps.append(cmp)
        self.cycles_serie.append(self.cmp_count)

        self.last_cmp = cmp
        self.cycles_from_last_trade += 1

        # 2. loop through placed orders and move to monitor list if isolated
        self.check_placed_list_for_move_back(cmp=cmp)

        # strategy manager and update of trades needed for new pt
        self.partial_traded_orders_count += self.sm.assess_strategy_actions(cmp=cmp)

        # 4. loop through monitoring orders and place to Binance when appropriate
        self.check_monitor_list_for_placing(cmp=cmp)

        # 5. check inactivity & liquidity
        self.check_inactivity(cmp=cmp)

    def check_inactivity(self, cmp):
        if self.cycles_from_last_trade > 125:  # TODO: magic number (5')
            if self.bm.is_s1_below_buffer():
                # force BUY
                self.sm.force_buy(cmp=cmp)
            elif self.bm.is_s2_below_buffer():
                # force SELL
                self.sm.force_sell(cmp=cmp)
            else:
                # create new pt
                self.create_new_pt(cmp=cmp)  # direct to create_new_pt(), not to assess_new_pt()

            self.cycles_from_last_trade = 0  # equivalent to trading but without a trade

    def check_placed_list_for_move_back(self, cmp: float):
        for order in self.pob.placed:
            if order.is_isolated(cmp=cmp, max_dist=K_MAX_DISTANCE_FOR_REMAINING_PLACED):
                self.pob.place_back_order(order=order)
                # cancel order in Binance
                self.market.cancel_orders(orders=[order])

    def check_monitor_list_for_placing(self, cmp: float):
        new_placement_allowed = True
        sorted_orders = sorted(self.pob.monitor, key=lambda x: x.get_abs_distance(cmp=cmp))
        for order in sorted_orders:
            order.cycles_count += 1
            if new_placement_allowed and order.is_ready_for_placement(
                    cmp=cmp,
                    min_dist=K_MINIMUM_DISTANCE_FOR_PLACEMENT):
                # check balance
                if self.bm.is_balance_enough(order=order):
                    new_placement_allowed = self._process_place_order(order=order)

    def _process_place_order(self, order: Order) -> bool:
        new_placement_allowed = True
        self.pob.place_order(order=order)
        is_order_placed, new_status = self._place_order(order=order)
        if is_order_placed:
            # 2. placed: (s: PLACED, t: pending_orders, l: placed)
            order.set_status(status=OrderStatus.PLACED)
            # to control one new placement per cycle mode
            if K_ONE_PLACE_PER_CYCLE_MODE:
                new_placement_allowed = False
        else:
            self.pob.place_back_order(order=order)
            log.critical(f'for unknown reason the order has not been placed: {order}')
        return new_placement_allowed

    def order_traded_callback(self, uid: str, order_price: float, bnb_commission: float) -> None:
        print(f'********** ORDER TRADED:    price: {order_price} [EUR] - commission: {bnb_commission} [BNB]')
        # get the order by uid
        for order in self.pob.placed:
            if order.uid == uid:
                print(f'********** order traded: {order}')
                # set the cycle in which the order has been traded
                order.traded_cycle = self.cmp_count
                # reset counter
                self.cycles_from_last_trade = 0
                # update buy & sell count
                if order.k_side == k_binance.SIDE_BUY:
                    self.buy_count += 1
                else:
                    self.sell_count += 1
                # set commission and price
                order.set_bnb_commission(
                    commission=bnb_commission,
                    bnbbtc_rate=self.market.get_cmp(symbol='BNBBTC'))
                order.price = order_price
                # change status
                order.set_status(status=OrderStatus.TRADED)
                # remove from placed list
                self.pob.placed.remove(order)
                # add to traded list (once removed from placed list) depending on whether is pt_id completed or not
                if self.pob.has_completed_pt_id(order=order):
                    # completed
                    self.tob.add_completed(order=order)
                else:
                    self.tob.add_pending(order=order)

                # update counter for next pt
                self.partial_traded_orders_count += 1
                # check whether a new pt is allowed or not
                if self.pt_created_count < PT_CREATED_COUNT_MAX and self.partial_traded_orders_count >= 0:
                    self.create_new_pt(cmp=self.last_cmp)
                else:
                    log.info('no new pt created after the last traded order')
                # since the traded orders has been identified, do not check more orders
                break

    def account_balance_callback(self, ab: AccountBalance) -> None:
        # update of current balance from Binance
        self.bm.update_current(last_ab=ab)

    # ********** check methods **********
    def _place_order(self, order) -> (bool, Optional[str]):
        order_placed = False
        status_received = None
        # place order
        d = self.market.place_order(order=order)
        if d:
            order_placed = True
            order.set_binance_id(new_id=d.get('binance_id'))
            status_received = d.get('status')
            # log.debug(f'********** ORDER PLACED **********      msg: {d}')
        else:
            log.critical(f'error placing {order}')
        return order_placed, status_received

    # ********** new perfect trade related **********
    def create_new_pt(self, cmp: float):
        # get parameters
        dp = dict(
            mp=cmp,
            nab=PT_NET_AMOUNT_BALANCE,
            s1_qty=PT_S1_AMOUNT,
            buy_fee=PT_BUY_FEE,
            sell_fee=PT_SELL_FEE,
            geb=PT_GROSS_EUR_BALANCE)

        # create new orders
        b1, s1 = self.get_b1s1(dynamic_parameters=dp)

        if b1 and s1:
            # add orders to list
            self.pob.monitor.append(b1)
            self.pob.monitor.append(s1)

            # ********** update control variables **********
            # increase created counter
            self.pt_created_count += 1
            # set pt_id based on created counter
            pt_id = f'{self.pt_created_count:03}'
            b1.pt_id = pt_id
            s1.pt_id = pt_id
            # set number of trades needed for next pt creation
            self.partial_traded_orders_count -= 2
        else:
            log.critical('the pt (b1, s1) can not be created:')
            log.critical(f'b1: {b1}')
            log.critical(f's1: {s1}')
            print('\n********** CRITICAL ERROR CREATING PT **********\n')

    def get_b1s1(self,
                 dynamic_parameters: dict,
                 ) -> (Optional[Order], Optional[Order]):
        b1 = None
        s1 = None

        order_id = 'NA'

        # get perfect trade
        b1_qty, b1_price, s1_price, g = get_pt_values(**dynamic_parameters)
        s1_qty = dynamic_parameters.get('s1_qty')

        # check filters before creating order
        if Order.is_filter_passed(filters=self.symbol_filters, qty=b1_qty, price=b1_price):
            # create orders
            b1 = Order(
                session_id=self.session_id,
                order_id=order_id,
                pt_id='PENDING',
                k_side=k_binance.SIDE_BUY,
                price=b1_price,
                amount=b1_qty,
                name='b1'
            )
        else:
            log.critical(f'trying to create an order that do not meet limits: {dynamic_parameters}')

        if Order.is_filter_passed(filters=self.symbol_filters, qty=s1_qty, price=s1_price):
            s1 = Order(
                session_id=self.session_id,
                order_id=order_id,
                pt_id='PENDING',
                k_side=k_binance.SIDE_SELL,
                price=s1_price,
                amount=s1_qty,
                name='s1'
            )
        else:
            log.critical(f'trying to create an order that do not meet limits: {dynamic_parameters}')

        return b1, s1

    def quit(self, quit_mode: QuitMode):
        # action depending upon quit mode
        if quit_mode == QuitMode.CANCEL_ALL_PLACED:
            print('********** CANCELLING ALL PLACED ORDERS **********')
            self.market.cancel_orders(self.pob.placed)
        elif quit_mode == QuitMode.PLACE_ALL_PENDING:
            print('********** PLACE ALL PENDING ORDERS **********')
            for order in self.pob.monitor:
                self.market.place_order(order)

        # check for correct cancellation of all orders
        btc_bal = self.market.get_asset_balance(asset='BTC',
                                                tag='check for zero locked')
        eur_bal = self.market.get_asset_balance(asset='EUR',
                                                tag='check for zero locked')
        if btc_bal.locked != 0 or eur_bal.locked != 0:
            log.critical('after cancellation of all orders, locked balance should be 0')
            log.critical(btc_bal)
            log.critical(eur_bal)
        else:
            log.info(f'LOCKED BALANCE CHECK CORRECT: btc_balance: {btc_bal} - eur_balance: {eur_bal}')

        self.market.stop()


class Dictionary:
    '''
    Creating a Dictionary
    '''

    def create_dictionary(self):
        my_dict ={"name": "john", "age": 24, "address": "vizag", "eduction": "masters"}
        return my_dict

if __name__ == "__main__":
    print(Dictionary().__doc__)
    print(Dictionary().create_dictionary())

11 - Dictionaries/Dictionary Practice/01. Creating a Dictionary.py


import os
os.chdir("/home")

import numpy as np
from src.data.obtain import json_read, json_write

def persist(e):
    """
    Convert a sklearn estimator into a JSON object for persistence

    Parameters
    ----------
    e: Estimator
        An sklearn estimator

    Returns
    -------
    persist: dict
        A dictionary containing a copy of the data in 'e'
        in JSON friendly formats
    """
    d = {}
    for k,v in e.__dict__.items():
        d[k] = v.tolist() if isinstance(v, np.ndarray) else v

    return d

def recover(path='', estimator=''):
    """
    Build Estimator from JSON

    Parameters
    ----------
    path: string
        Location of the JSON file

    Returns
    -------
    est: sklearn Estimator
        The sklearn estimator with attributes updated
    """
    recovered_params = {}

    for k, v in json_read(path).items():
        if k[-1] == "_":
            recovered_params[k] = np.array(v)
        else:
            recovered_params[k] = v

    for item in recovered_params.items():
        k, v = item[0], item[1]
        setattr(estimator, k, v)

    return estimator


"""
Browser (:mod:`pydent.browser`)
=================================

.. versionadded:: 0.1
    Browser class created

.. currentmodule:: pydent.browser

Browser class for searching and cacheing results.
"""
import re
from collections import OrderedDict
from difflib import get_close_matches
from pprint import pformat
from typing import Callable
from typing import Dict
from typing import List
from typing import Union

import networkx as nx

from pydent import models as pydent_models
from pydent.base import ModelBase
from pydent.exceptions import ForbiddenRequestError
from pydent.exceptions import TridentBaseException
from pydent.interfaces import QueryInterface
from pydent.interfaces import QueryInterfaceABC
from pydent.marshaller import ModelRegistry
from pydent.models import Sample
from pydent.relationships import BaseRelationship
from pydent.sessionabc import SessionABC
from pydent.utils import logger
from pydent.utils.logging_helpers import did_you_mean

# TODO: browser documentation
# TODO: examples in sphinx
# TODO: methods to help pull relevant data from plans (user specifies types of data to
#       pull, and trident should pull and cache in the most efficient way possible)

class BrowserException(TridentBaseException):
    """Generic browser exception."""

class Browser(QueryInterfaceABC):
    """A class for browsing models and Aquarium inventory."""

    # TODO: ability to block model callbacks to enforce cache

    INTERFACE_CLASS = QueryInterface
    ACCEPTED_GET_RELATION_TYPES = [
        "HasOne",
        "HasMany",
        "HasManyThrough",
        "HasManyGeneric",
    ]

    def __init__(self, session: SessionABC, inherit_models: bool = False):
        """Instantiates a new browser from a AqSession instance.

        .. versionchanged:: 0.1.5a7
            'inherit_models' argument will inherit the sessions model_cache
            (default: False)

        :param session: a session instance
        :param inherit_models: if True, the browser will inherit the cache in the
            provided session's browser model_cache
        :type session: SessionABC
        """
        self.session = session
        self._list_models_fxn = self.sample_list
        self.use_cache = True
        self.model = Sample
        self.model_list_cache = {}
        self.model_cache = {}
        self.log = logger(name="Browser@{}".format(session.url))
        if session.browser and inherit_models:
            self.update_cache(session.browser.models)

    @property
    def model_name(self):
        return self.model.__name__

    # TODO: change session interface (find, where, etc.) to use cache IF use_
    #       cache = True
    # TODO: where and find queries can sort through models much more quickly than
    #       Aquarium, but can fallback to Aq

    @property
    def models(self):
        models = []
        for v in self.model_cache.values():
            models += list(v.values())
        return models

    def set_model(self, model_name):
        """Sets the default model of this browser."""
        self.model = ModelRegistry.get_model(model_name)
        if model_name == "Sample":
            self._list_models_fxn = self.sample_list
        else:
            self._list_models_fxn = self._generic_list_models

    def interface(self, model_class=None):
        """Returns a new model query interface.

        :param model_class:
        :type model_class: basestring
        :return: Interface
        :rtype: QueryInterface
        """
        if model_class is None:
            model_class = self.model_name
        return self.session.model_interface(
            model_class, interface_class=self.INTERFACE_CLASS
        )

    def _generic_list_models(self, **query):
        models = self.where(query, self.model_name)
        return ["{}: {}".format(m.id, m.name) for m in models]

    def sample_list(self, sample_type_id=None):
        """Returns a sample list."""
        path = "sample_list"
        if sample_type_id is not None:
            path += "/" + str(sample_type_id)
        return self.session.utils.aqhttp.get(path)

    def clear(self):
        """Clears the model cache."""
        self.model_list_cache = {}
        self.model_cache = {}

    def list_models(self, *args, **kwargs):
        def get_models():
            return self._list_models_fxn(*args, **kwargs)

        if self.use_cache:
            models_cache = self.model_list_cache.get("models", {})
            if self.model_name in models_cache:
                model_list = models_cache[self.model_name]
            else:
                model_list = get_models()
        else:
            model_list = get_models()
        self.model_list_cache.setdefault("models", {})
        self.model_list_cache["models"][self.model_name] = model_list[:]
        return model_list

    def where(
        self,
        query,
        model_class=None,
        primary_key="id",
        sample_type=None,
        methods=None,
        opts: Dict = None,
        page_size: int = None,
        include: Dict = None,
    ):
        """Perform a 'where' query. If models are found in the browser cache,
        those are returned, else new http queries are made to find the models.

        :param query: query as a dictionary
        :param model_class: model class to use (str)
        :param primary_key: which primary key to use (default: 'id')
        :param sample_type: optional sample_type short cut for finding samples
        :param kwargs: other kwargs
        :return: returned model list
        """
        if model_class is None:
            model_class = self.model_name
        if sample_type is not None:
            sample_type_id = self.find_by_name(sample_type, "SampleType").id
            query.update({"sample_type_id": sample_type_id})
        if self.use_cache and not methods and page_size is None:
            return self.cached_where(
                query,
                model_class,
                primary_key=primary_key,
                include=include,
                methods=methods,
                opts=opts,
            )
        else:
            return self.server_where(
                query,
                model_class,
                opts=opts,
                include=include,
                methods=methods,
                page_size=page_size,
            )

    def __query_helper(
        self,
        fname,
        query,
        model_class,
        sample_type=None,
        opts=None,
        params=None,
        as_single=False,
    ):
        """Builds a custom query for the browser.

        :param fname: the function name
        :type fname: basestring
        :param query: the query
        :type query: dict
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param opts: options to send to the function
        :type opts: dict
        :param params: additionaly keyword arguments to send to the function
        :type params: dict
        :param as_single: if True, will return the first model of the array or None if
            array is empty
        :type as_single: bool
        :return: Aquarium model or list of Aquarium models
        :rtype: ModelBase | list
        """
        if model_class is None:
            model_class = self.model_name
        if query is None:
            query = dict()
        if params is None:
            params = dict()
        if sample_type is not None:
            query.update(
                {"sample_type_id": self.find_by_name(sample_type, "SampleType").id}
            )
        interface = self.interface(model_class)
        fxn = getattr(interface, fname)
        if fname == "all":
            models = fxn(opts=opts, **params)
        else:
            models = fxn(query=query, opts=opts, **params)
        if as_single:
            models = [models]
        return self.update_cache(models).get(model_class, [])

    def one(self, model_class=None, sample_type=None, query=None, opts=None):
        """Finds one instance of a model (or returns None)

        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :param opts: additional options
        :type opts: dict
        :return:
        :rtype:
        """
        models = self.__query_helper(
            "one", query, model_class, sample_type, opts=opts, as_single=True
        )
        if not models:
            return None
        return models[0]

    def last(self, num=1, model_class=None, sample_type=None, query=None):
        """Finds last models. Will NOT return cached models.

        :param num: number of models to return
        :type num: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :return:
        :rtype:
        """
        return self.__query_helper(
            "last", query, model_class, sample_type, params=dict(num=num)
        )

    def first(self, num=1, model_class=None, sample_type=None, query=None):
        """Finds first models. Will NOT return cached models.

        :param num: number of models to return
        :type num: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param sample_type: optional sample_type name
        :type sample_type: basestring
        :param query: additional query to filter models
        :type query: dict
        :return:
        :rtype:
        """
        return self.__query_helper(
            "first", query, model_class, sample_type, params=dict(num=num)
        )

    def find(self, model_id, model_class=None):
        """Finds a model by id. Will returned cached model if possible.

        :param model_id: model_id
        :type model_id: int
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :return:
        :rtype:
        """
        if model_class is None:
            model_class = self.model_name
        if self.use_cache:
            return self.cached_find(model_class, model_id)
        return self.interface(model_class).find(model_id)

    def find_by_name(self, name, model_class=None, primary_key="id"):
        """Find model by name. Will return cached model if possible.

        :param name: name of the model
        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param primary_key:
        :return:
        """
        models = self.where({"name": name}, model_class, primary_key=primary_key)
        if not models:
            return None
        return models[0]

    def all(self, model_class=None, opts=None):
        """Return all models of a model_class.

        :param model_class: the name of the model class (e.g. "Sample")
        :type model_class: basestring
        :param opts:
        :type opts:
        :return:
        :rtype:
        """
        return self.__query_helper("all", query={}, model_class=model_class, opts=opts)

    @staticmethod
    def _match_query(query, model_dict):
        """Matches a query against a model dictionary.

        :param query: query dictionary
        :type query: dict
        :param model_dict: model dictionary (e.g. model.__dict__)
        :type model_dict: dict
        :return: whether the model matches the query
        :rtype: bool
        """
        for query_key in query:
            if query_key not in model_dict:
                return False
            query_val = query[query_key]
            model_val = model_dict[query_key]
            if isinstance(query_val, list):
                if model_val not in query_val:
                    return False
            elif model_val != query_val:
                return False
        return {k: model_dict[k] for k in query}

    @classmethod
    def _find_matches(cls, query, models):
        found = []
        found_queries = []
        for m in models:
            match = cls._match_query(query, m._get_data())
            if match:
                found.append(m)
                found_queries.append(match)
        return found, found_queries

    def update_cache(
        self,
        models: List[ModelBase],
        recursive: bool = True,
        update_session: bool = False,
    ) -> Dict[str, List[ModelBase]]:
        """Update the browser's model cache from a list of models.

        .. versionchanged:: 0.1.5a8
            Added 'update_session' kwarg which will change the model's
            session to the browser's session.

        .. warning::
            If `update_session=True`, the model's session
            will change and this may have unintended consequences.

        :param models: list of models
        :param recursive: if True, recursively collect all models contained in
            the relationships and use those to update the cache as well.
        :param update_session: if True, attach the browser's session to the models
        :return: the updated dictionary broken down by model class name
        """
        assert isinstance(models, list)
        if recursive:
            memo = {}
            ModelBase._flatten_deserialized_data(models, memo)
            models = list(memo.values())
        if update_session:
            for m in models:
                m._session = self.session
        return self._group_models_and_update_cache(models)

    # TODO: do we really want to simply overwrite the dictionary or update the models?
    def _update_model_cache_helper(
        self, modelname: str, modeldict: Dict
    ) -> List[ModelBase]:
        """Updates the browser's model cache with models from the provided
        model dict."""
        self.log.info(
            "CACHE updated cached with {} {} models".format(len(modeldict), modelname)
        )
        self.model_cache.setdefault(modelname, {})

        model_cache_dict = self.model_cache[modelname]
        for mid in modeldict:
            model = modeldict[mid]
            if mid in model_cache_dict:
                cached_model = model_cache_dict[mid]
                vars(cached_model).update(vars(model))
            else:
                model_cache_dict[mid] = model
        return [model_cache_dict[mid] for mid in modeldict]

    def _group_models_and_update_cache(self, models):
        grouped_by_type = {}
        for model in models:
            classname = model.__class__.__name__
            arr = grouped_by_type.setdefault(classname, [])
            if model is not None:
                arr.append(model)

        result = {}
        for clstype in grouped_by_type:
            result[clstype] = self._update_model_cache_helper(
                clstype, {m._primary_key: m for m in grouped_by_type[clstype]}
            )
        return result

    # TODO: support unsaved models as well
    def cached_find(self, model_class, id):
        if isinstance(id, list):
            return self.cached_where({"id": id}, model_class)
        cached_models = self.model_cache.get(model_class, {})
        found_model = cached_models.get(id, None)
        if found_model is None:
            found_model = self.interface(model_class).find(id)
        else:
            self.log.info(
                "CACHE found {} model with id={} in cache".format(model_class, id)
            )
        if found_model is None:
            return None
        return self._update_model_cache_helper(
            model_class, {found_model.id: found_model}
        )[0]

    def server_where(self, query, model, opts, include, methods, page_size):
        return self.interface(model).where(
            query, opts=opts, methods=methods, include=include, page_size=page_size
        )

    def cached_where(
        self,
        query,
        model,
        primary_key="id",
        methods: Dict = None,
        include: Dict = None,
        opts: Dict = None,
    ):
        if isinstance(query, str):
            server_models = self.server_where(
                query,
                model,
                opts=opts,
                methods=methods,
                include=include,
                page_size=None,
            )
            found_dict = {}
        elif [] in query.values():
            return []
        else:
            cached_models = self.model_cache.get(model, {})
            found, found_queries = self._find_matches(
                query, list(cached_models.values())
            )
            found_dict = {f.id: f for f in found}

            # TODO: this code is broken, remaining query
            remaining_query = dict(query)
            if primary_key in query:
                found_ids = [q[primary_key] for q in found_queries]
                query_id_list = query[primary_key]
                if isinstance(query_id_list, str) or isinstance(query_id_list, int):
                    query_id_list = [query_id_list]
                remaining_ids = list(set(query_id_list).difference(set(found_ids)))
                remaining_query[primary_key] = remaining_ids
            self.log.info(
                "CACHE found {num} {model} models in cache using query {query}".format(
                    num=len(found_dict), model=model, query=self.log.pprint_data(query)
                )
            )

            # TODO: this code may be sketchy... here {'id': []}, really means we found
            #       all of the models..
            if primary_key in remaining_query and not remaining_query[primary_key]:
                return list(found_dict.values())
            server_models = self.interface(model).where(remaining_query, opts=opts)

        models_dict = OrderedDict({s.id: s for s in server_models})
        models_dict.update(found_dict)

        returned = self._update_model_cache_helper(model, models_dict)
        if opts:
            if opts.get("reverse", True) is False:
                returned = returned[::-1]
            if opts.get("limit", None):
                returned = returned[-opts["limit"] :]
        return returned

    def _search_helper(self, pattern, filter_fxn, sample_type=None, **query):
        sample_type_id = None
        if sample_type is not None:
            if isinstance(sample_type, pydent_models.SampleType):
                sample_type_id = sample_type.id
            else:
                sample_type_id = self.find_by_name(sample_type, "SampleType").id

        if sample_type_id is not None:
            query.update({"sample_type_id": sample_type_id})

        model_list = self.list_models()
        self.log.info(
            "SEARCH found {} total models of type {}".format(
                len(model_list), self.model_name
            )
        )
        matches = filter_fxn(pattern, model_list)

        if not matches:
            return []

        if query:
            query.update({"id": matches})
            filtered = self.where(query, self.model_name)
        else:
            filtered = self.interface().find(matches)
        self.log.info(
            "SEARCH filtered to {} total models of type {}".format(
                len(filtered), self.model_name
            )
        )

        if self.use_cache:
            self.update_cache(filtered)
        return filtered

    def search(self, pattern, ignore_case=True, sample_type=None, **query):
        """Performs a regular expression search of Samples.

        :param pattern: regular expression pattern
        :type pattern: basestring
        :param ignore_case: whether to ignore case for regex search (default: True)
        :type ignore_case: bool
        :param sample_type: sample_type_name to filter samples (optional)
        :type sample_type: basestring
        :param query: additional query parameters to filter by
        :type query: dict
        :return: list of samples
        :rtype: list
        """

        def regex_filter(pattern, sample_list):
            matches = []
            for name in sample_list:
                args = []
                if ignore_case:
                    args.append(re.IGNORECASE)
                m = re.search(pattern, name, *args)
                if m is not None:
                    matches.append(name)
            return matches

        return self._search_helper(
            pattern, regex_filter, sample_type=sample_type, **query
        )

    def search_description(
        self, pattern, samples=None, sample_type=None, ignore_case=True
    ):
        """Search samples by their description.

        :param pattern: regex pattern
        :type pattern: basestring
        :param samples: samples to search. If left blank, a search to find samples will
            be performed
        :type samples: list
        :param sample_type: restrict to a particular sample type
        :type sample_type: name
        :param ignore_case: ignore case for regular pattern search (default True)
        :type ignore_case: bool
        :return: list of samples
        :rtype: list
        """
        matches = []
        args = []
        if samples is None:
            samples = self.search(".*", sample_type=sample_type)
        if ignore_case:
            args.append(re.IGNORECASE)
        for sample in samples:
            description = sample.description
            if description is None:
                continue
            if re.search(pattern, sample.description, *args):
                matches.append(sample)
        return matches

    def close_matches(self, pattern, sample_type=None, **query):
        """Finds samples whose names closely match the pattern.

        :param pattern: regular expression pattern
        :type pattern: basestring
        :param sample_type: sample_type_name to filter samples (optional)
        :type sample_type: basestring
        :param query: additional query parameters to filter by
        :type query: dict
        :return: list of samples
        :rtype: list
        """
        return self._search_helper(
            pattern, get_close_matches, sample_type=sample_type, **query
        )

    def list_field_values(self, model_ids, **query):
        """Lists sample field values.

        May supply an additional query to filter :class:`FieldValue`.
        """
        query.update({"parent_class": self.model_name, "parent_id": model_ids})
        return self.where(query, "FieldValue")

    @staticmethod
    def _group_by_attribute(models, attribute):
        """Group models by the given attribute."""
        d = {}
        for s in models:
            arr = d.setdefault(getattr(s, attribute), [])
            arr.append(s)
        return d

    def new_sample(self, sample_type, name, description, project, properties=None):
        st = self.find_by_name(sample_type, "SampleType", primary_key="name")
        if st._get_deserialized_data().get("field_types", None) is None:
            fts = self.where(
                {"parent_class": "SampleType", "parent_id": st.id}, "FieldType"
            )
            st.field_types = fts
        return st.new_sample(name, description, project, properties=properties)

    # TODO: handle not-yet existant samples using rid
    def _retrieve_has_many_or_has_one(
        self, models, relationship_name, relation=None, strict=True
    ):
        """Performs exactly 1 query to fullfill some relationship for a list of
        models."""
        if not models:
            return []
        models = models[:]
        if relation is None:
            relation = models[0].get_relationships()[relationship_name]

        ref = relation.ref  # sample_id
        attr = relation.attr  # id
        model_class2 = relation.nested

        # todo: collect existing fullfilled relationship
        # todo: partition, then collect callback
        # todo: how to handle when model_attr is absent?, or just raise error?

        retrieve_query = relation.build_query(models)
        retrieved_models = self.where(retrieve_query, model_class2)
        self.log.info(
            "RETRIEVE retrieved {num} {cls} models using query {query}".format(
                num=len(retrieved_models),
                cls=model_class2,
                query=self.log.pprint_data(retrieve_query),
            )
        )

        if not retrieved_models:
            return []

        if relation.QUERY_TYPE == "query":
            model_dict = {m1.id: list() for m1 in models}
            for model in retrieved_models:
                model_ref = getattr(model, ref)
                if model_ref is not None:
                    if not strict:
                        model_dict.setdefault(model_ref, []).append(model)
                    else:
                        model_dict[model_ref].append(model)
                else:
                    self.log.error(
                        "RETRIEVE ref: {ref} {model_ref}, attr: {attr}".format(
                            ref=ref, attr=attr, model_ref=model_ref
                        )
                    )
        elif relation.QUERY_TYPE == "by_id":
            retrieved_dict = {getattr(m2, attr): m2 for m2 in retrieved_models}
            model_dict = {m1.id: None for m1 in models}
            missing_models = []
            for model in models:
                model_attr = getattr(model, attr)
                model_ref = getattr(model, ref)
                if model_ref is not None:
                    if model_attr is not None:
                        if model_ref not in retrieved_dict:
                            self.log.info(strict)
                        else:
                            model_dict[model_attr] = retrieved_dict[model_ref]
                    else:
                        self.log.error(
                            "attr: {attr}={model_attr}, ref: {ref}={model_ref}".format(
                                ref=ref,
                                attr=attr,
                                model_attr=model_attr,
                                model_ref=model_ref,
                            )
                        )
                    if model_ref not in retrieved_dict:
                        missing_models.append(model_ref)
            if missing_models:
                self.log.error(
                    "INCONSISTENT AQUARIUM DATABASE - There where {models} missing {cls} "
                    "models "
                    "from the Aquarium database, which were ignored by trident. "
                    "This happens when models are deleted from Aquarium which "
                    "results in an inconsistent server database. Trident was unable "
                    "resolve the following relationships which returned no models "
                    "from the server: "
                    "{cls}.where({attr}={missing})".format(
                        models=len(missing_models),
                        cls=model_class2,
                        missing=missing_models,
                        attr=attr,
                    )
                )
        else:
            raise BrowserException(
                "QUERY_TYPE '{}' for relation '{}' not recognized.".format(
                    relation.QUERY_TYPE, relationship_name
                )
            )
        for model in models:
            found_models = model_dict[getattr(model, attr)]
            setattr(model, relationship_name, found_models)

        return retrieved_models

    def _retrieve_has_many_through(
        self, models: List[ModelBase], relationship_name: str, strict: bool = True
    ):
        """Performs exactly 2 queries to establish a HasManyThrough
        relationship."""
        relation = models[0].get_relationships()[relationship_name]
        association_relation = models[0].get_relationships()[
            relation.through_model_attr
        ]
        attr = relation.attr
        ref = association_relation.ref

        # find other key
        association_class = ModelRegistry.get_model(association_relation.nested)
        association_relationships = association_class.get_relationships()
        other_ref = None
        for r in association_relationships:
            ar = association_relationships[r]
            if ar.nested == relation.nested:
                other_ref = r

        associations = self._retrieve_has_many_or_has_one(
            models, relation.through_model_attr, strict=strict
        )
        self._retrieve_has_many_or_has_one(associations, other_ref, strict=strict)

        associations_by_mid = {}
        for a in associations:
            associations_by_mid.setdefault(getattr(a, ref), []).append(a)

        all_models = []
        for m in models:
            mid = getattr(m, attr)
            if mid in associations_by_mid:
                associations = associations_by_mid[mid]
                _models = [getattr(a, other_ref) for a in associations]
                setattr(m, relationship_name, _models)
                all_models += _models
            else:
                setattr(m, relationship_name, None)
        return list(set(all_models))

    @staticmethod
    def _get_relation_from_model(model, relationship_name, strict):
        relationships = model.get_relationships()
        relation = relationships.get(relationship_name, None)
        if relation is None:
            if strict:
                msg = did_you_mean(relationship_name, list(relationships))
                if not msg:
                    msg = "Available relations: {}".format(pformat(list(relationships)))
                raise BrowserException(
                    "Relation '{}' not found in relationships for {}.\nHint: {}".format(
                        relationship_name, type(model), msg
                    )
                )
        return relation

    def retrieve(
        self,
        models: List[ModelBase],
        relationship_name: str,
        relation: BaseRelationship = None,
        strict: bool = True,
        force_refresh: bool = False,
    ) -> List[ModelBase]:
        """Retrieves a model relationship for the list of models. Compared to a
        `for` loop, `retrieve` is >10X faster for most queries.

        .. code-block:: python

            # FAST code example
            samples = browser.search(".*mCherry.*")
            items = browser.retrieve(samples, 'items')
            for s in samples:
                sample_items = s.items
                # do something with items

        .. code-block:: python

            # very SLOW code example. DO NOT USE.
            samples = browser.search(".*mCherry.*")
            for s in samples:
                sample_items = s.items
                # do something with itmes

        :param models: list of models to retrieve the attribute
        :type models: list
        :param relationship_name: name of the attribute to retrieve
        :type relationship_name: basestring
        :param relation: the relation to retrieve (operational)
        :type relation: pydent.relationships.Relation
        :param strict: wither to ignore database inconsistencies
        :type strict: bool
        :return: list of models retrieved
        :rtype: list
        """
        if not models:
            return []
        self.log.info('RETRIEVE retrieving "{}"'.format(relationship_name))
        model_classes = {m.__class__.__name__ for m in models}
        assert (
            len(model_classes) == 1
        ), "Models must be all of the same BaseModel, but found {}".format(
            model_classes
        )
        if relation is None:
            relation = self._get_relation_from_model(
                models[0], relationship_name, strict
            )
            if relation is None:
                return []
        else:
            if relationship_name in models[0].get_relationships():
                raise BrowserException(
                    'Cannot add new relationship "{}" because it already exists in the '
                    "model definition".format(relationship_name)
                )
        # TODO: add relationship handler for Many and One
        if relation.__class__.__name__ not in self.ACCEPTED_GET_RELATION_TYPES:
            raise BrowserException(
                'retrieve is not supported for the "{}" relationship'.format(
                    relation.__class__.__name__
                )
            )
        self.log.info("RETRIEVE {}: {}".format(relationship_name, relation))

        if not force_refresh:
            needs_refresh = [
                m for m in models if not m.is_deserialized(relationship_name)
            ]
            no_refresh = [m for m in models if m.is_deserialized(relationship_name)]
        else:
            needs_refresh = models
            no_refresh = []

        if needs_refresh:
            if hasattr(relation, "through_model_attr"):
                found_models = self._retrieve_has_many_through(
                    needs_refresh, relationship_name, strict=strict
                )
            else:
                found_models = self._retrieve_has_many_or_has_one(
                    needs_refresh, relationship_name, relation, strict=strict
                )
        else:
            found_models = []

        self.log.info(
            'RETRIEVE retrieved {} for "{}"'.format(
                len(found_models), relationship_name
            )
        )
        for model in no_refresh:
            val = getattr(model, relationship_name)
            if isinstance(val, list):
                found_models += val
            else:
                found_models.append(val)
        return list(set(found_models))

    def recursive_retrieve(
        self,
        models: List[ModelBase],
        relations: Union[str, List[BaseRelationship], Dict],
        strict: bool = True,
        force_refresh: bool = False,
    ):
        """Efficiently retrieve a model relationship recursively from an
        iterable. The relations_dict iterable may be either a list or a
        dictionary. For example, the following will collect all of the
        field_values and their incoming and outgoing wires, the connecting
        field_values, and finally those FieldValues' operations.

        .. code-block::

            relation_dict = {
                "field_values": {
                    "wires_as_dest": {
                        "source": "operation",
                        "destination": "operation"
                    },
                    "wires_as_source": {
                        "source": "operation",
                        "destination": "operation"
                    },
                }
            }
            browser.retrieve(operations, relation_dict)

        :param models: models to retrieve from
        :type models: list
        :param relations: the relation to retrieve. May be a string (by \
            attribute name), a list, or a dict.
        :type relations: list|dict|basestring
        :param strict: wither to ignore database inconsistencies
        :param force_refresh:
        :type force_refresh: bool
        :return: dictionary of all models retrieved grouped by the attribute \
            name that retrieved them.
        :rtype: dictionary
        """
        self.log.info("RETRIEVE recursively retrieving {}".format(relations))
        if isinstance(relations, str):
            self.log.info('RETRIEVE retrieving "{}"'.format(relations))
            return {
                relations: self.retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
            }
        elif (
            isinstance(relations, list)
            or isinstance(relations, set)
            or isinstance(relations, dict)
            or isinstance(relations, tuple)
        ):
            models_by_attr = {}
            for relation_name in relations:
                models_by_attr.setdefault(relation_name, [])
                new_models = self.retrieve(
                    models, relation_name, strict=strict, force_refresh=force_refresh
                )
                models_by_attr[relation_name] += new_models
                if isinstance(relations, dict):
                    _models_by_attr = self.recursive_retrieve(
                        new_models,
                        dict(relations).pop(relation_name),
                        strict=strict,
                        force_refresh=force_refresh,
                    )
                    for attr in _models_by_attr:
                        _models = _models_by_attr[attr]
                        if attr in models_by_attr:
                            models_by_attr[attr] += _models
                        else:
                            models_by_attr[attr] = _models_by_attr[attr]
            return models_by_attr
        elif not strict:
            return []
        else:
            raise BrowserException(
                "Type {} for is not recognized for recursive_retrieve".format(
                    type(relations)
                )
            )

    @classmethod
    def sample_network(
        cls,
        samples: List[Sample],
        reverse: bool = False,
        g: nx.DiGraph = None,
        get_models: Callable = None,
        cache_func: Callable = None,
        key_func: Callable = None,
    ) -> nx.DiGraph:
        """Build a DAG of :class:`Samples <pydent.models.Sample>` from their.

        :class:`FieldValues <pydent.models.FieldValue>`.

        .. versionadded:: 0.1.5a7
            method added

        .. versionchanged:: 0.1.5a16
            added optional get_models, cache_func, key_func arguments

        :param samples: list of samples
        :param reverse: whether to reverse the edges of the final graph
        :param g: the graph
        :return:
        """

        if not get_models:

            def get_models(model):
                for fv in model.field_values:
                    if fv.sample and issubclass(type(fv.sample), ModelBase):
                        yield fv.sample, {"field_value_name": fv.name}

        if not cache_func:

            def cache_func(models):
                sess = models[0].session
                sess.browser.get(models, {"field_values": "sample"})

        return cls.relationship_network(
            samples,
            get_models=get_models,
            cache_func=cache_func,
            reverse=reverse,
            g=g,
            key_func=key_func,
        )

    @classmethod
    def relationship_network(
        cls,
        models: List[ModelBase],
        get_models: Callable,
        cache_func: Callable = None,
        key_func: Callable = None,
        reverse: bool = False,
        g: nx.DiGraph = None,
        strict_cache: bool = True,
    ):
        """Build a DAG of related models based on some relationships. By
        default are built from a model using (model.__class__.__name__,
        model._primary_key)

        .. versionadded:: 0.1.5a7
            method added

        .. seealso::
            Usage example :meth:`sample_network <pydent.browser.Browser.sample_network>`

        :param models: list of models
        :param get_models: A function that takes in a single instance of a Model
            and returns an iterable of Tuples of (model, data),
            which is used to build edges.
        :param key_func: An optional function that takes in a single instance of
            a Model and returns a key and some node data
        :param cache_func: A function that takes in a list of models and caches
            some results.
        :param g: optional nx.DiGraph
        :param reverse: whether to reverse the edge list
        :param strict_cache: if True, if a request occurs after the cache step,
            a ForbiddenRequestException will be raised.
        :return: the relationship graph
        """

        if key_func is None:

            def key_func(m):
                return (m.__class__.__name__, m._primary_key), {}

        if g is None:
            g = nx.DiGraph()

        models = list(models)

        if not models:
            return g

        for m in models:
            key, ndata = key_func(m)
            g.add_node(key, attr_dict=ndata)

        visited = list(g.nodes)

        if cache_func:
            cache_func(models)

        new_models = []
        new_edges = []

        if strict_cache:
            kwargs = {"using_requests": False, "session_swap": True}
        else:
            kwargs = {}

        with models[0].session(**kwargs):
            try:
                for m1 in models:
                    for m2, data in get_models(m1):
                        if key_func(m2)[0] not in visited:
                            new_models.append(m2)
                        new_edges.append((key_func(m2)[0], key_func(m1)[0], data))
            except ForbiddenRequestError as e:
                msg = (
                    "An exception occurred while strict_cache == True.\n"
                    "This is most likely due to the cache_func not being thorough.\n"
                    "{}".format(str(e))
                )
                raise e.__class__(msg)

        for n1, n2, edata in new_edges:
            if reverse:
                g.add_edge(n2, n1, attr_dict=edata)
            else:
                g.add_edge(n1, n2, attr_dict=edata)

        return cls.relationship_network(
            new_models,
            get_models,
            cache_func,
            key_func=key_func,
            g=g,
            reverse=reverse,
            strict_cache=strict_cache,
        )

    # def relationship_network(self, models, get_models, cache_func, g=None):
    #     """
    #
    #
    #     .. versionadded:: 0.1.5a7
    #
    #
    #     :param models:
    #     :param get_models:
    #     :param g:
    #     :return:
    #     """
    #
    #     def model_to_key(m):
    #         return (m.__class__.__name__, m.id)
    #
    #     if g is None:
    #         g = nx.DiGraph()
    #     for m in models:
    #         g.add_node(model_to_key(m))
    #
    #     cache_func(models)
    #
    #     for m in models:
    #         next_models = get_models(m)

    def get(
        self,
        models: List[ModelBase],
        relations: List[BaseRelationship] = None,
        query: dict = None,
        strict: bool = True,
        force_refresh: bool = False,
    ) -> Union[Dict[str, List[ModelBase]], List[ModelBase]]:
        """

        :param models:
        :param relations:
        :param query:
        :param strict:
        :param force_refresh:
        :return:
        """
        if isinstance(models, ModelBase):
            models = [models]
        elif isinstance(models, str):
            models = list(self.model_cache.get(models, {}).values())
            if query:
                models, _ = self._find_matches(query, models)
        if relations:
            if isinstance(relations, str):
                return self.retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
            else:
                return self.recursive_retrieve(
                    models, relations, strict=strict, force_refresh=force_refresh
                )
        else:
            return models

    def export_samples_to_csv(self, samples, out):
        """Exports the samples to a csv (for Aquarium import)

        :param samples: list of samples
        :type samples: list
        :param out: output path of csv file
        :type out: basestring
        :return: pandas dataframe used for the csv
        :rtype: pandas.DataFrame
        """
        df = self.samples_to_df(samples)
        df.to_csv(out)
        return df

    def samples_to_rows(self, samples, sample_resolver=None):
        """Return row of dictionaries containing sample information and their
        properties. Can be imported into a pandas DataFrame:

        .. code-block:: python

            import pandas

            df = pandas.DataFrame(samples_to_rows(samples))

        :param samples: samples, all of same sample type
        :type samples: list
        :param sample_resolver: callable to resolve a sample object if the \
            field value property contains a sample. \
            Defaults to return the sample name.
        :type sample_resolver: callable
        :return: list of dictionaries containing sample info and properties
        :rtype: list
        """

        assert (
            len({s.sample_type_id for s in samples}) == 1
        ), "Samples be the of the same SampleType"
        sample_type = samples[0].sample_type

        # self.recursive_retrieve(samples, {
        #     "field_values": "sample",
        #     "sample_type": {
        #         "field_types"
        #     }
        # })
        if sample_resolver is None:

            def default_resolver(sample):
                if isinstance(sample, pydent_models.Sample):
                    return sample.name
                if isinstance(sample, list):
                    resolved_samples = []
                    for s in sample:
                        if isinstance(s, pydent_models.Sample):
                            resolved_samples.append(s.name)
                        else:
                            resolved_samples.append(s)
                    sample = resolved_samples
                return sample

            sample_resolver = default_resolver
        rows = []
        for s in samples:
            row = {
                sample_type.name: s.name,
                "Description": s.description,
                "Project": s.project,
            }
            props = {k: sample_resolver(v) for k, v in s.properties.items()}
            row.update(props)
            rows.append(row)
        return rows

    # TODO: add to change log
    @staticmethod
    def _inspect_header(model, name=None, _id="id"):
        if name is not None:
            name = eval("model.{}".format(name))
        else:
            name = ""
        return "{model_class}: {_id} {name}".format(
            model_class=model.__class__.__name__,
            name=name,
            _id=eval("model.{}".format(_id)),
        )

    # TODO: add to change log
    @staticmethod
    def _inspect_rows(model, attributes, indent=2):
        rows = []
        for attr in attributes:
            rows.append(
                "{attr}: {val}".format(attr=attr, val=eval("model.{}".format(attr)))
            )
        sep = "\n" + " " * indent
        s = sep + sep.join(rows)
        return s

    # TODO: add to change log
    @classmethod
    def inspect(cls, model, header, attributes):
        model_header = cls._inspect_header(model, **header)
        rows = cls._inspect_rows(model, attributes)
        return "{header}{rows}".format(header=model_header, rows=rows)


from os import listdir
from os.path import isfile, join

def new_node(idd, label, value, r, g, b):
    node = ''' <node id="%s" label='%s'>
                  <attvalues>
                    <attvalue for="modularity_class" value="0"></attvalue>
                   </attvalues>
                   <viz:size value="%s"></viz:size>
                   <viz:color r="%s" g="%s" b="%s"></viz:color>
                 </node>''' % (idd, label, value, r, g, b)
    return node

def new_edge(idd, source, target, weight):
    edge = '''
          <edge id="%s" source="%s" target="%s" weight="%s">
                   <attvalues></attvalues>
                 </edge>''' % (idd, source, target, weight)
    return edge

onlyfiles = [f for f in listdir('./_posts') if isfile(join('./_posts/',f)) ]
categories = set()
posts = []
for f in onlyfiles:
    reader = open(join('./_posts/',f), 'r',encoding='utf-8')
    content = reader.readlines()
    post = {}
    for line in content:
        if line[:10] == 'categories':
            categories.add(line[12:].rstrip())
            post['categories'] = line[12:].rstrip()
        if line[:5] == 'title':
            post['title'] = line[7:].rstrip()
    posts.append(post)
print(posts)

front = open('./assets/data/front.gexf', 'r',encoding='utf-8').read()
end = open('./assets/data/end.gexf', 'r',encoding='utf-8').read()

graph_output = open('./assets/data/graph.gexf', 'w',encoding='utf-8')
graph_output.write(front)
node_index = 0
cate_count = {}
name2index = {}
relation = []
for post in posts:
    if post['categories'] not in cate_count:
        cate_count[post['categories']] = 0
    cate_count[post['categories']] += 7
    graph_output.write(new_node(node_index, post['title'], '7', '0', '0', '0'))
    name2index[post['title']] = node_index
    relation.append((post['title'], post['categories']))
    node_index += 1

for cate in categories:
    output = open('./_post_categories/'+cate+'.markdown', 'w',encoding='utf-8')
    output.write('--- \n')
    output.write('name: ' + cate + '\n')
    output.write('--- \n')
    graph_output.write(new_node(node_index, cate, str(cate_count[cate]), '100', '100', '100'))
    name2index[cate] = node_index
    relation.append((cate, 'root'))
    node_index += 1
graph_output.write(new_node(node_index, 'root', str(sum([cate_count[key] for key in cate_count])), '200', '200', '200'))
name2index['root'] = node_index

graph_output.write('''\n  </nodes>
                       <edges>\n''')

edge_index = 0
for rel in relation:
    graph_output.write(new_edge(edge_index, name2index[rel[0]], name2index[rel[1] ], 1))
    edge_index += 1

graph_output.write(end)


import pandas as pd
from pandas.core.frame import DataFrame
from decimal import Decimal

datas = pd.read_csv('Labeling_v3.2.csv').to_dict('records')

def get_txcount(data):
    mint = int(data['mint_count'])
    burn = int(data['burn_count'])
    swap = int(data['swap_count'])
    return mint + burn + swap

def get_holding_ratio(data):
    return Decimal(data['timestamp_creator_LP_amount']) / Decimal(data['total_LP_amount'])

def get_Lock_ratio(data):
    Lock_amount = Decimal(data['LP_Creator_amount']) - Decimal(data['timestamp_creator_LP_amount'])
    return Lock_amount / Decimal(data['total_LP_amount'])

def get_burn_ratio(data):
    return Decimal(data['burn_amount'] / data['current_token_total_supply'])

def get_token_holding_ratio(data):
    return Decimal(data['timestamp_creator_token_amount'] ) / Decimal(data['current_token_total_supply'])

creator_list = []
for data in datas:
    creator_list.append(data['receiver'])

result = []
for data in datas:
    dataset = {}
    try:
        dataset['id'] = data['id']
        dataset['Label'] = data['is_rugpull']
#        dataset['txcount'] = get_txcount(data)
        dataset['mint_count_per_week'] = data['mint_count'] / ((int(data['active_period']) / (60* 60 * 24 * 7)) + 1)
#        dataset['swap_count_per_week'] = data['swap_count'] / ((int(data['active_period']) / 60* 60 * 24 * 7) + 1)
        dataset['burn_count_per_week'] = data['burn_count'] / ((int(data['active_period']) / (60* 60 * 24 * 7)) + 1)
        dataset['mint_ratio'] = float(data['mint_ratio'])
        dataset['swap_ratio'] = float(data['swap_ratio'])
        dataset['burn_ratio'] = float(data['burn_ratio'])
        dataset['mint_mean_period'] = float(data['mint_mean_period'])
        dataset['swap_mean_period'] = float(data['swap_mean_period'])
        dataset['burn_mean_period'] = float(data['burn_mean_period'])
        dataset['swapIn_per_week'] = data['swapIn'] /((int(data['active_period']) / (60* 60 * 24 * 7)) + 1)
        dataset['swapOut_per_week'] = data['swapOut'] / ((int(data['active_period']) / (60* 60 * 24 * 7)) + 1)
        dataset['swap_rate'] = float(data['swap_rate'])
        dataset['LP_avg'] = data['LP_avg']
        dataset['LP_stdev'] = data['LP_stdev']
        dataset['LPCreator_holding_ratio'] = float(get_holding_ratio(data))
        dataset['Lock_ratio'] = float(get_Lock_ratio(data))
        dataset['token_burn_ratio'] = float(get_burn_ratio(data))
        dataset['Creator_token_holding_ratio'] = float(get_token_holding_ratio(data))
        dataset['number_of_token_creation_of_Creator'] = creator_list.count(data['receiver'])
    except Exception as e:
        print(e)
        continue
    result.append(dataset)
len(result)
pd.DataFrame(result).to_csv('Dataset_v1.9.csv',encoding='utf-8-sig',index=False)

2. Feature/4. Labeling File/Labeling_to_Dataset.py


from os import path
from setuptools import setup, find_packages

here = path.abspath(path.dirname(__file__))

with open(path.join(here, 'README.md'), encoding='utf-8') as readme_file:
    readme = readme_file.read()

requirements = [
    'click'
]

test_requirements = [
]

setup(
    name='intercom',
    version='0.9',
    description="Distance finder for intercom",
    long_description=readme,
    author="Batuhan Ceylan",
    author_email='batuhan@batuhanceylan.com',
    packages=find_packages(exclude=['tests']),
    entry_points={
        'console_scripts': [
            'intercom=intercom.intercom:intercom',
            ],
        },
    include_package_data=True,
    install_requires=requirements,
    license="MIT",
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.8',
    ],
    test_suite='tests',
    tests_require=test_requirements,
)


import sqlite3
import re
import jwt
import base64

from flask import request, make_response
from utils.util import f, secret_key

from flask_restful import Resource, reqparse
from flask_jwt_extended import jwt_required, get_jwt_identity

from models.user import UserModel
from models.card import CardModel

class Card(Resource):
    parser = reqparse.RequestParser()
    parser.add_argument("card_type",
                        type=str,
                        required=True,
                        help='card_number is missing')
    parser.add_argument("card_no",
                        type=str,
                        required=True,
                        help='card_number is missing')
    parser.add_argument("cvv",
                        type=str,
                        required=True,
                        help='cvv is missing')
    parser.add_argument("account_holder",
                        type=str,
                        required=True,
                        help='id is missing')
    parser.add_argument("phone_number",
                        type=str,
                        required=True,
                        help='id is missing')
    parser.add_argument("expiry_date",
                        type=str,
                        required=True,
                        help='expiry date is missing')

    @classmethod
    @jwt_required()
    def get(cls, username):
        try:
            if not check(username):
                return {'message': 'invalid username or password'}, 401
            username = username.lower()

            identity = get_jwt_identity()['identity']
            user = UserModel.find_by_username(username)
            user_identity = UserModel.find_by_userid(identity)

            if user:
                if user.id != user_identity.id or user.username != user_identity.username:
                    return {'message': 'invalid token'}, 401

                # bad query
                # query = "SELECT * FROM cards WHERE id in (SELECT id FROM users WHERE username='" + username + "')"

                query = "SELECT * FROM cards WHERE id in (SELECT id FROM users WHERE username=?)"

                connection = sqlite3.connect('data.db')
                cursor = connection.cursor()

                cards = []
                for _id, card_type, card_number, cvv, account_holder, phone_number, expiry_date in cursor.execute(query,
                                                                                (
                                                                                username,)):
                    cards.append(
                        {
                            "id": _id,
                            "card_type": f.decrypt(card_type).decode(),
                            "card_no": f.decrypt(card_number).decode(),
                            "cvv": f.decrypt(cvv).decode(),
                            "account_holder": f.decrypt(account_holder).decode(),
                            "phone_number": f.decrypt(phone_number).decode(),
                            "expiry_date": f.decrypt(expiry_date).decode()
                        }
                    )
                return {"username": username, "cards": cards}, 200
            else:
                return {'message': 'no user found'}, 404

        except ErrorException:
            return {'message': 'some error has occurred'}

    @classmethod
    @jwt_required()
    def post(cls, username):

        try:
            identity = get_jwt_identity()['identity']
            verify_csrf_token = request.headers['X-CSRFToken']

            token = request.cookies.get('token')

            if token_check(token) == -1:
                return {'message': 'invalid token in token check'}, 401

            data = jwt.decode(token, secret_key, algorithms=['HS256'])

            sub = data.get('sub')

            if sub is None:
                return {'message': 'invalid token'}, 401

            csrf_token = sub.get('csrf_token')

            if csrf_token is None:
                return {'message': 'token missing'}, 401

            if csrf_token != verify_csrf_token:
                return {'message': 'login required'}, 401

            if not check(username):
                return {'message': 'invalid username or password'}, 401

            username = username.lower()
            user = UserModel.find_by_username(username)
            user_identity = UserModel.find_by_userid(identity)

            if user:
                if user.id != user_identity.id or user.username != user_identity.username:
                    return {'message': 'invalid token'}, 401

                data = Card.parser.parse_args()

                _id = user.id
                card_type = f.encrypt(data['card_type'].encode())
                card_no = f.encrypt(data['card_no'].encode())
                cvv = f.encrypt(data['cvv'].encode())
                account_holder = f.encrypt(data['account_holder'].encode())
                phone_number = f.encrypt(data['phone_number'].encode())
                expiry_date = f.encrypt(data['expiry_date'].encode())

                # check if card with the provided number exists or not in the database
                if CardModel.find_by_card_number(card_no):
                    return {'message': 'Card already exists'}

                query = "INSERT INTO cards VALUES (?, ?, ?, ?, ?, ?, ?)"
                connection = sqlite3.connect('data.db')
                cursor = connection.cursor()
                cursor.execute(query, (_id, card_type, card_no, cvv, account_holder, phone_number, expiry_date))

                connection.commit()
                connection.close()
                response = make_response(
                    {'message': 'card added'},
                    201
                )
                response.headers.add('Access-Control-Allow-Credentials', True)
                return response
            else:
                return {'message': 'no user exist'}, 404
        except TokenInvalidException:
            return {'message': 'token invalid or not provided'}, 400

class TokenInvalidException(Exception):
    pass

class ErrorException(Exception):
    pass

# checking for any special character
def check(string):
    regex = re.compile("['@_!#$%^&*()<>?/|}{~:]")
    if regex.search(string) is None:
        return True

    else:
        return False

def token_check(token):
    try:
        # 1. verify that the token has three components
        components = token.split('.')
        signature = components[2]
        if len(components) != 3:
            return -1

        # 2. check algorithm
        header = jwt.get_unverified_header(token)
        algorithm = header.get('alg')
        if algorithm is None or algorithm == 'RS256':
            return -1

        # verifying the signature
        payload = jwt.decode(token, secret_key, algorithms=['HS256'])

        encoded_jwt = jwt.encode(
            payload,
            secret_key,
            algorithm='HS256',
            headers=header
        )

        if encoded_jwt != token:
            return -1
        return 1
    except Exception as e:
        print(e.args)
        return -1

def is_base64(s):
    try:
        return base64.b64encode(base64.b64decode(s)) == s
    except Exception:
        return False


'''
Created on 2019年4月8日

@author: bkd
'''
from traceback import format_exception
import sys
from PyQt5.QtWidgets import QMessageBox

class global_exception_hander:
    def new_except_hook(self,etype, evalue, tb):
        print(''.join(format_exception(etype, evalue, tb)))
        QMessageBox.information(None,
                                      str('error'),
                                      ''.join(format_exception(etype, evalue, tb)))
        sys.exit()

#     注册全局异常处理类
    def patch_excepthook(self):
        sys.excepthook = self.new_except_hook


#!/usr/bin/env python
# -*- coding: utf-8-*-
import xml.etree.ElementTree as ET
import tarfile
import zipfile
import re
from pathlib import Path
import shutil
import os
import pymysql
import sys
import traceback
import subprocess
import getpass
import Archive as archive

if __name__ in '__main__':

    print('処理対象の年度:範囲指定2012-2015')
    in_year = input('>> ')
    target_years = []
    if re.match(r'\d{4}-\d{4}',in_year):
        temp = in_year.split('-')
        target_years = range(int(temp[0]),int(temp[1])+1)
    elif re.match(r'\d{4}', in_year):
        target_years = [in_year]
    else:
        print('入力が不正です')
        sys.exit()

    print('公報種類を入力してください(A,A5,S,S5,T,T5)')
    in_filetype = input('>> ')
    if in_filetype not in ['A','A5','S','S5','T','T5']:
        print('入力が不正です')
        sys.exit()

    for y in target_years:
        target_year = str(y)
        root_dir = "/mnt/Drobo/XML_files/" + in_filetype + "/" +  target_year
        files = os.listdir(root_dir)
        file_list = files

        str_f_list = sorted(file_list, reverse=True)

        for f_name in str_f_list:
            print(f_name)
            tar_path = Path(root_dir + "/" + f_name)
            path_list = list(tar_path.glob("**/*.xml"))

            for pl in path_list:

                try:
                    xml_path = str(pl)
                    ret = archive.xml_elements(xml_path)
                    pub_nr = ret['publn_nr']
                    print(pub_nr)
                    print(xml_path)

                    #DB登録
                    sql_element = archive.generate_insert_sql(ret)
                    archive.register_xml_elements(sql_element)
                except (pymysql.IntegrityError, pymysql.err.IntegrityError):
                    print('already exist')
                    continue
                except (pymysql.OperationalError, pymysql.ProgrammingError, pymysql.InternalError) as error:
                    # データベースエラー　エラーログファイル生成、終了　処理は継続
                    except_str = traceback.format_exc()
                    print(except_str)
                    message = "--------------------------------------------------\n"
                    message += "XML_PATH:" + xml_path + "\n"
                    message += "ERROR_MSG:" + except_str + "\n"
                    message += "--------------------------------------------------\n"
                    f = open('publn_error_log.txt', 'a')
                    f.write(message)
                    f.close()
                except:
                    # データベースエラー　エラーログファイル生成、終了　処理は継続
                    except_str = traceback.format_exc()
                    print(except_str)
                    message = "--------------------------------------------------\n"
                    message += "XML_PATH:" + xml_path + "\n"
                    message += "ERROR_MSG:" + except_str + "\n"
                    message += "--------------------------------------------------\n"
                    f = open('publn_error_log.txt', 'a')
                    f.write(message)
                    f.close()
                    sys.exit()


from asyncio import tasks
from airflow import DAG
from datetime import datetime, timedelta
from airflow.operators.bash_operator import BashOperator

default_args = {
    'owner': 'airflow',
    'retries': 2,
    'retry_delay': timedelta(minutes=2)
}

with DAG(
    dag_id='my_first_dag',
    default_args=default_args,
    description='My first DAG is in testing mode',
    start_date=datetime(2022, 8, 3, 2),
    schedule_interval='@daily'

) as dag:
    task1 = BashOperator(
        task_id='task_first',
        bash_command="echo Hello World"
    )

    task2 = BashOperator(
        task_id='task_second',
        bash_command="echo Hello People"
    )

    task3 = BashOperator(
        task_id='task_three',
        bash_command="echo I am partner of task_first"
    )

    task4 = BashOperator(
        task_id='task_four',
        bash_command="echo I am partner of task_second"
    )

    task5 = BashOperator(
        task_id='task_five',
        bash_command="python /opt/airflow/landings/hello_landing.py"

    )

    # Task dependencies method 1
    # task1.set_downstream(task2)
    # task1.set_downstream(task3)
    # task2.set_upstream(task4)

    # Task dependencies method 2
    task1 >> task2 >> task3
    task2 >> task4
    task1 >> task5

    # Task dependencies method 3
    # task1 >> [task2, task3]
    # task2 >> [task4]


# Importing necessary files
import os
import discord
from dotenv import load_dotenv
import logging.handlers

# .env
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
GUILD = os.getenv('GUILD_ID')

# Set up logging
logger = logging.getLogger('discord')
logger.setLevel(logging.DEBUG)
logging.getLogger('discord.http').setLevel(logging.INFO)

handler = logging.handlers.RotatingFileHandler(
    filename='discord.log',
    encoding='utf-8',
    maxBytes=32 * 1024 * 1024,  # 32 MiB
    backupCount=5,  # Rotate through 5 files
)
dt_fmt = '%Y-%m-%d %H:%M:%S'
formatter = logging.Formatter('[{asctime}] [{levelname:<8}] {name}: {message}', dt_fmt, style='{')
handler.setFormatter(formatter)
logger.addHandler(handler)

# Intents are like permissions; groups of events
intents = discord.Intents.default() # use the default set of intents
intents.message_content = True # allow the message_content events into our app

# load a new discord client with the intents we specified above
client = discord.Client(intents=intents)

# when the on_ready event is triggered,
@client.event
async def on_ready():
    for guild in client.guilds:
        if guild.name == GUILD:
            break

    print(
        f'BOT({client.user}) Successfully Connected To Guild: {guild.name} (id: {guild.id})'
    )

# when the on_message event is triggered,
@client.event
async def on_message(message):
    if message.author == client.user: # stop the bot from messaging itself
        return

    if message.content.startswith('$hello'): # basic $hello command
        await message.channel.send('Hello!') # returns "Hello!"

client.run(TOKEN, log_handler=None) # run the discord client/bot using our client token (on dev portal)