import numpy as np
from contextlib import ExitStack
import graphicle as gcl
import torch
import dgl
from pathlib import Path

from torch.utils.data import Dataset, DataLoader

from heparchy.read.hdf import HdfReader

#from manifold.poincare import PoincareBall
class ParticleDataset(Dataset):
    '''Particles shower dataset.'''

    def __init__(
        self,
        sig_path: Path,
        bkg_path: Path,
        num_samples: int =-1,
        open_at_init: bool = False,
    ) -> None:
        self.sig_path = sig_path
        self.bkg_path = bkg_path
        self._stack = ExitStack()
        self.num_samples = num_samples
        #self.indices = range(int(self.num_samples / 2.))
        self.indices = range(int(self.__len__() / 2.))
        if open_at_init is True:
            self._open_file()

    def __len__(self) -> int:
        with HdfReader(self.sig_path) as hep_file:
            sig_size = len(hep_file["signal"])
        with HdfReader(self.bkg_path) as hep_file:
            bkg_size = len(hep_file["background"])

        return min(self.num_samples, min(sig_size, bkg_size) * 2)

    def __getitem__(self, idx: int) -> tuple[dgl.DGLGraph, torch.Tensor]:
        half_samples = int(self.__len__() / 2.)
        if idx < half_samples:
            event = self._sig_events[self.indices[idx]]
            label = torch.tensor(0)

        else:
            event = self._bkg_events[self.indices[idx - half_samples]]
            label = torch.tensor(1)

        graph = self._generate_graph(event)
        return (graph, label)

    def _generate_graph(self, event):
        coordinates = torch.tensor(event.custom['tree_pmu']).to(torch.float32)
        features = torch.tensor(event.custom['tree_lund']).to(torch.float32)
        edges = torch.tensor(event.custom['tree_edges'])
        edges = torch.cat((edges, edges.flip(1)), dim=0)

        graph = dgl.graph((edges[:,0], edges[:,1]))
        graph.ndata['coordinates'] = coordinates
        graph.ndata['features'] = features
        pmu = gcl.MomentumArray(coordinates)
        graph.ndata['pmu'] = torch.tensor(np.array([
            pmu.eta, pmu.phi, pmu.pt, pmu.mass])).transpose(0, 1).to(torch.float32)

        return graph

    def _open_file(self) -> None:
        self._stack = ExitStack()
        sig_file = self._stack.enter_context(HdfReader(self.sig_path))
        bkg_file = self._stack.enter_context(HdfReader(self.bkg_path))

        self._sig_events = sig_file["signal"]
        self._bkg_events = bkg_file["background"]

"""
class ParticleDataset(Dataset):
    '''Particles shower dataset.'''

    def __init__(self, path) -> None:
        self.__files = glob.glob(path + '/*.hdf5')
        self.__ranges = [(-1, -1)]

        stack = ExitStack()
        for file in self.__files:
            file_obj = stack.enter_context(HdfReader(path=file))
            try:
                process = file_obj['signal']
                self.__process_name = 'signal'
                self.label = 1
            except KeyError:
                process = file_obj['background']
                self.__process_name = 'background'
                self.label = 0
            ini = self.__ranges[-1][1] + 1
            fin = ini + len(process) - 1
            self.__ranges.append((ini, fin))
        stack.close()

        _dtype = [('ini', 'i4'), ('fin', 'i4')]
        self.__ranges = np.array(self.__ranges[1:], dtype=_dtype)

    def __len__(self):
        return self.__ranges['fin'][-1]

    def __getitem__(self, idx):
        _file_idx = int(np.where(
            np.logical_and(np.less_equal(self.__ranges['ini'], idx),
                            np.greater_equal(self.__ranges['fin'], idx)))[0])

        _event_idx = idx - self.__ranges[_file_idx]['ini']
        with HdfReader(path=self.__files[_file_idx]) as hep_file:
            process = hep_file[self.__process_name]
            _event = process[_event_idx]

            k = 0
            mask = _event.masks["final"]
            pmu = _event.pmu
            edges = _event.edges

        G = nx.Graph()
        G.add_edges_from(edges)
        nodes = np.array(G.nodes())
        mapping = {nodes[i]: i for i in range(len(nodes))}
        G = nx.relabel_nodes(G, mapping)
        edges = torch.tensor(list(G.edges))

        X = torch.tensor(
                np.transpose([pmu['x'],pmu['y'],pmu['z'],pmu['e'] ]),
                dtype=torch.float32
        )
        y = torch.tensor(self.label)

        data = Data(
            x = X,
            edge_index = edges.t().contiguous(),
            y = y,
        )
        return data
"""


from typing import List
import logging
import os

import numpy as np
from fastapi import FastAPI
from pydantic.main import BaseModel
from pydantic import BaseSettings
from fastapi.middleware.cors import CORSMiddleware

from utils.features import Datapoint

from model.boosting import LGBMModel

from twilio.rest import Client
from dotenv import load_dotenv

load_dotenv()  # load environment variables

logging.basicConfig(format='%(levelname)s--%(asctime)s--%(filename)s--%(message)s', level=logging.DEBUG)
logger = logging.getLogger(__name__)

TWILIO_ACCOUNT_SID = os.environ["TWILIO_ACCOUNT_SID"]
TWILIO_AUTH_TOKEN = os.environ["TWILIO_AUTH_TOKEN"]
twilio_api = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)

base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

class Settings(BaseSettings):
    model_dir : str = os.path.join(base_dir, 'model/checkpoints/lgbm')

app = FastAPI()
settings = Settings()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

config = {
    "model_output_path": settings.model_dir,
    "featurizer_output_path": settings.model_dir,
    "params" : {}
}

model = LGBMModel(config)

class SMS(BaseModel):
    text: str

class Predictions(BaseModel):
    label: float
    proba: List[float]

@app.post("/api/spam-sms", response_model=Predictions)
def predict(sms: SMS):
    datapoint = Datapoint(**{"sms": sms.text, "sms_length": int(float("nan"))})
    proba = model.predict([datapoint])
    label = np.argmax(proba, axis=1)
    prediction = Predictions(label=label[0], proba=list(proba[0]))
    logger.info(prediction)
    return prediction


import logging
import sys

import aiohttp_jinja2
import jinja2
from aiohttp import web

from source.routes import setup_routes, setup_static_routes

async def init_app(argv=None):

    # init server
    app = web.Application()

    # setup Jinja2 template renderer
    aiohttp_jinja2.setup(
        app, loader=jinja2.PackageLoader('source', 'templates'))

    # setup views and routes
    setup_static_routes(app)
    setup_routes(app)

    return app

def main(argv):
    logging.basicConfig(level=logging.DEBUG)

    app = init_app(argv)

    web.run_app(app,
                host='localhost',
                port=8080)

if __name__ == '__main__':
    main(sys.argv[1:])


from __future__ import annotations

from abc import ABC, abstractmethod
from typing import TYPE_CHECKING

if TYPE_CHECKING:  # pragma: no cover
    from pathlib import Path

    from ..game import Game

class Formatter(ABC):
    """Base class for Game output.

    To implement your own `Formatter`, subclass this class.
    """

    @abstractmethod
    def show(self, game: Game, *args, **kwargs) -> str:
        """Return a string representation of the game."""

    @abstractmethod
    def save(
        self,
        game: Game,
        path: str | Path,
        format: str = "PDF",
        solution: bool = False,
        *args,
        **kwargs,
    ) -> Path:
        """Save the current puzzle to a file.

        Args:
            game (Game): Parent `WordSearch` puzzle.
            path (str | Path): File save path.
            format (str, optional): Type of file to save ("CSV", "JSON", "PDF").
                Defaults to "PDF".
            solution (bool, optional): Include solution with the saved file.
                For CSV and JSON files, only placed word characters will be included.
                For PDF, a separate solution page will be included with word
                characters highlighted in red. Defaults to False.

        Returns:
            str: Final save path of the file.
        """

    @abstractmethod
    def write_csv_file(
        self,
        path: Path,
        game: Game,
        solution: bool = False,
        *args,
        **kwargs,
    ) -> Path:
        """Write current puzzle to CSV format at `path`.

        Args:
            path (Path): Path to write the file to.
            game (Game): Current Word Search puzzle.
            solution (bool, optional): Only include the puzzle solution.
            Defaults to False.

        Returns:
            Path: Final save path.
        """

    @abstractmethod
    def write_json_file(
        self,
        path: Path,
        game: Game,
        solution: bool = False,
        *args,
        **kwargs,
    ) -> Path:
        """Write current puzzle to JSON format at `path`.

        Args:
            path (Path): Path to write the file to.
            game (Game): Current Word Search puzzle.
            solution (bool, optional): Only include the puzzle solution.
            Defaults to False.

        Returns:
            Path: Final save path.
        """

    @abstractmethod
    def write_pdf_file(
        self,
        path: Path,
        game: Game,
        solution: bool = False,
        *args,
        **kwargs,
    ) -> Path:
        """Write current puzzle to PDF format at `path`.

        Args:
            path (Path): Path to write the file to.
            game (Game): Current Word Search puzzle.
            solution (bool, optional): Include the puzzle solution. Defaults to False.

        Raises:
            OSError: File could not be written.

        Returns:
            Path: Final save path.
        """


import os
import time as t
from threading import Thread
import utils.tools as tools
import cv2

class VideoWriter:
    def __init__(self, im_width, im_height, folder, video_queue):
        self.video_queue = video_queue
        self.folder = folder
        self.im_width = im_width
        self.im_height = im_height
        self.writer = None
        self.file = None
        self.fps = None
        self.stopped = True
        self.thread = Thread(target=self.update, args=())

    def init(self, file_prefix="output", fps=30):
        filename = "{}_{}.avi".format(file_prefix, tools.get_timestamp_ms())
        self.file = os.path.join(self.folder, filename)
        self.fps = fps

        fourcc = cv2.VideoWriter_fourcc(*"MJPG")
        self.writer = cv2.VideoWriter(self.file, fourcc,
                                      float(self.fps), (self.im_width,
                                                        self.im_height))
        return self

    def start(self):
        self.stopped = False
        self.thread.start()
        print("[VIDEO WRITER] Thread for writing video started")
        return self

    def update(self):
        while True:
            # if self.stopped and self.video_queue.empty():
            if self.stopped:
                return
            # wait for element in queue
            try:
                image = self.video_queue.get_nowait()
            except Exception as e:
                t.sleep(0.02)
                continue

            self.writer.write(image)

    def stop(self):
        while not self.video_queue.empty():
            t.sleep(0.1)
        self.stopped = True
        self.writer.release()
        print('[VIDEO WRITER] Video written to file: {}'.format(self.file))

    def is_running(self):
        return not self.stopped

    def is_thread_alive(self):
        return self.thread.is_alive()

    def get_video_file_name(self):
        return self.file


import pandas as pd
import time
import os
import requests
from Analysis_Tools.Parsers.Historical import Data_Proccessing
from Analysis_Tools.Parsers.Historical import Game_Collection

'''
Each Week                                                        /
    -Get the data just as before excluding the unecessary data \/                                                 /
    -Need to track all the opponents each team plays and the week they play them in a dataframe for each season \/
    -Can take the existing Get Opponent Part already in use and add the opponent list to this new DF each week \/
    Team    Opp W1  Opp W2 ...
    ARZ     Det     Bal
                                                                                /
    -Proccess just as before up to the differing part and don't add opponent stats to dataframe yet (just the opponent name) \/
    -Make stats per game excluding (Basic Season, Betting and Result Stats) \/                                       /
    -Change every stat excluding (Year, Games, Spread and Result Stats) to a ranking from 1-#teams up to that week \/

    -For the big 6 stats:
        Need Teams' For and Against Rank as of the week
        Need Teams' Opponents up to the week and the week # they played
            Need these teams' for/agaisnt ranks as of the week they played - then take average
                From the Matchup DataFrame (above) determine the teams and order (week number) - skip blanks
                From the Curent Data frame (up to current week) locate each team and their rank for the given week
        Discount Teams For and Against Rank as per the Template Doc
        Need Opponents' For and Against Rank as of the week
        Need Opponents' Opponents up to the week and the week # they played
            Need these teams' for/agaisnt ranks as of the week they played - then take average
                From the Matchup DataFrame (above) determine the teams and order (week number) - skip blanks
                From the Curent Data frame (up to current week) locate each team and their rank for the given week
        Discount Opponents For and Against Rank as per the Template Doc
    -For all ranked stats:
        Take difference between Teams' for and Opponents Against (if for/against applies - else do Teams Rank - Opponents Rank) as described in template

'''

class NFL_DATA_COLLECTER():

    def __init__(self, project_path, mode='Historical', latest_week=16, max_season=2019):
        self.time = time
        #Dictionary of stat names to use in search on database, corresponding name in html table and name to be used in dataframe for the collumn
        #Some search names have muliple table collumns to be collected
        #What is important is the order of html table name and dataframe names matches so will rename the collumn to proper name
        self.Search_Stat_Dict = {
            #Only collecting Up-To-week data for Result Stats (week 1-5, week 1-6 etc)
            "Basic Season Stats":{'Search Name': ['points_diff'],
                            'Number of Headers': [4],
                            'Table Headers': ['Year', 'G', 'W-L%', 'PD'],
                            'My Headers': ['Year', 'Games Played', 'W-L%', 'Point Differential']
            },
            "Betting Stats":{'Search Name': ['vegas_line'],
                            'Number of Headers': [2],
                            'Table Headers': ['Spread', 'Over/Under'],
                            'My Headers': ['Spread', 'Over/Under']
            },
            "Basic Off/Def Stats":{'Search Name': ['pass_rating', 'pass_sacked_opp', 'rush_yds_per_att', 'rush_yds_per_att_opp', 'pass_yds', 'pass_yds_opp'],
                            'Number of Headers': [1, 1, 2, 2, 2, 2],
                            'Table Headers': ['Sk', 'Sk', 'Y/A', 'Yds', 'Y/A', 'Yds', 'Rate', 'Yds', 'Rate', 'Yds'],
                            'My Headers': ['Sacks Against', 'Sacks For', 'Yards per Rush For', 'Rushing Yards For', 'Yards per Rush Against', 'Rushing Yards Against', 'Passer Rating For', 'Passing Yards For', 'Passer Rating Against', 'Passing Yards Against']
            },
            "Advanced Stats":{'Search Name': ['time_of_poss', 'comb_penalties', 'third_down_pct', 'third_down_pct_opp', 'tot_yds_diff'],
                            'Number of Headers': [3, 2, 1, 1, 2],
                            'Table Headers': ['ToP', "Y/P", 'DY/P', 'Yds', 'OppYds', '3D%', 'Opp3D%', 'Tot', 'TO'],
                            'My Headers': ['Time of Poss', 'Yards For per Play', 'Yards Against per Play', 'Penalty Yards Committed', 'Penalty Yards Received', '3rd Down %_For', '3rd Down % Against', 'Total Yard Margin', 'Turnover Margin']
            },
            #Only collecting by-week data for Result Stats (week 1, week 2 etc)
            "Results Stats":{'Search Name': ['points_diff'],
                            'Number of Headers': [1],
                            'Table Headers': ['PD'],
                            'My Headers': ['Game Scoring Margin']
            }
        }
        self.mode = mode
        if self.mode == 'Historical':
            self.min_week = 2
            self.max_week = 16
            self.min_season = 2006
            self.last_season_max_week = latest_week
            self.max_season = max_season
            self.final_csv_name = 'All Seasons Results'
        elif self.mode == 'This Season':
            self.min_week = 2 #latest_week - 1
            self.max_week = latest_week
            self.last_season_max_week = latest_week
            self.max_season = max_season
            self.min_season = self.max_season
            self.final_csv_name = f'{self.max_season}/This Season Results'

        self.Read_Previous_Data = True
        #Define Years which Data is being collected for
        self.All_Search_Seasons = [n for n in range(self.min_season,self.max_season+1)]
        #Defien Weeks which Data is being collected for
        self.All_Search_Weeks = [n for n in range(self.min_week,self.max_week+1)] #Stats heading into Weeks 2 through 17 games

        #Lists and DFs
        self.All_Seasons_DF = pd.DataFrame()
        self.Season_DF = pd.DataFrame()
        self.Week_DF = pd.DataFrame()
        self.Opponent_Tracking_DF = pd.DataFrame()
        self.All_Seasons_DF_List = []
        self.All_Weeks_DF_List = []
        self.All_Cleaned_Weeks_DF_List = []
        self.Blank_Positions = []

        #Make save locations
        # self.project_path = 'E:/Project Stuff/Data Analysis Stuff/V5/NFL Data V1/Raw Data'
        self.project_path = project_path
        self.data_path = f'{project_path}/Raw Data'

    def Make_Folder(self, new_path):
        data_exists = False
        try:
            os.mkdir(new_path)
        except:
            print('folder exists')
            files = os.listdir(new_path)
            if len(files) > 1:
                data_exists = True
        return data_exists

    def Concat_and_Save(self, df_list, path):
        df = pd.concat(df_list) #Concat the list of dfs into a season df
        df.to_csv(path, index=False)
        return df

    def Track_All_Oppononents(self):
        self.Opponent_Tracking_DF[f'Week {self.week} Opp'] = self.Week_DF['Opponent'].tolist()

    def Get_DF(self):
        #Get the df from the website
        html = requests.get(self.URL_BASE).content
        df_list = pd.read_html(html, header=1, index_col=None)
        df = df_list[-1]
        #Sort the df by team name (so lines up all stats (since team names the same each season))
        df = df.sort_values(by=['Year', 'Tm'])
        #Cut off last row which is the headers again
        df = df.head(len(df['Tm'])-1)
        lastrow_df = df.tail(1)

        return df

    def Get_Teams_List(self, season):
        #Get teams in alphabetical order for the season
        df = self.Get_DF()
        team_list = df['Tm'].tolist()
        return team_list

    def Match_By_Teams(self, These_Teams):
        #Match team position to be the same for each week of the season
        self.Blank_Positions = []
        #print("starting matching")
        #print(These_Teams)
        for team_pos in range(0, len(self.All_Season_Teams)):
            try:
                if These_Teams[team_pos] != self.All_Season_Teams[team_pos]:
                    These_Teams.insert(team_pos, self.All_Season_Teams[team_pos])
                    self.Blank_Positions.append(team_pos)
            except IndexError:
                These_Teams.append(self.All_Season_Teams[team_pos])
                self.Blank_Positions.append(team_pos)
        # print(self.All_Season_Teams)
        # print(These_Teams)

    def Setup_This_Week_Stats(self):
        if self.season == self.max_season:
            This_Week_Game_Info = Game_Collection.This_Week_Parsing(self.season, self.week, self.min_week, self.All_Season_Teams, self.season_path)
            This_Week_Teams, Sched_Week_DF = This_Week_Game_Info.Setup_This_Week_Data()
            print(This_Week_Teams)
            if self.week == self.min_week:
                self.last_week_teams = This_Week_Teams
            #Get the location of missing teams relative to the list of all teams for the season
            # self.Match_By_Teams(This_Week_Teams)
        else:
            This_Week_Teams = self.All_Season_Teams
        #Add the team list for this week (should be same for every week of the season)
        print(This_Week_Teams)
        print('There')
        print(This_Week_Teams)
        self.Week_DF['Team'] = This_Week_Teams
        self.Week_DF.dropna(axis=0, how="any", inplace=True)
        self.Week_DF['Week'] = [self.week for i in range(len(self.Week_DF['Team'].tolist()))]
        #Get the information about this week's games
        if self.season == self.max_season:
            This_Week_Game_Info = Game_Collection.This_Week_Stats(self.last_week_teams, self.Week_DF, Sched_Week_DF)
            self.Week_DF = This_Week_Game_Info.Get_Info()
            self.last_week_teams = This_Week_Teams
        print('HERE')
        print(self.Week_DF)

    def Collect_Data(self, stat_type):
        #Get the df from the website
        if self.season == self.max_season and self.week == self.max_week and ("Results Stats" in stat_type or "Betting Stats" in stat_type):
            print("Dont Get DF")
        else:
            sorted_df = self.Get_DF()
            #Get df headers to looks for
            df_headers = list(sorted_df)
        #print(df_headers)
        next_df_header_pos = 0
        for header_pos in range(self.first_available_stat_header, self.first_available_stat_header + self.Search_Stat_Dict[stat_type]['Number of Headers'][self.search_stat_pos]):
            print(self.Search_Stat_Dict[stat_type]['Table Headers'][header_pos])
            #For Spreads and Results, Need to treat differently
            if self.season == self.max_season and self.week == self.max_week and ("Results Stats" in stat_type or "Betting Stats" in stat_type) and self.mode != 'Historical':
                #For spreads, will get data from seperate source
                if "Spread" in self.Search_Stat_Dict[stat_type]['My Headers'][header_pos]:
                    spread_collector = Game_Collection.Betting_Parsing(self.Week_DF, self.project_path)
                    latest_spreads = spread_collector.Get_Current_Spreads()
                    self.Week_DF[self.Search_Stat_Dict[stat_type]['My Headers'][header_pos]] = latest_spreads
                    print(self.Week_DF['Spread'].tolist())
                    # time.sleep(10)
                    # latest_spreads_path = f'{self.project_path}/Templates/Latest Week Spreads.csv'
                    # latest_spreads_df = pd.read_csv(latest_spreads_path)
                    # print(latest_spreads_df["Spread"].tolist())
                    # self.Week_DF[self.Search_Stat_Dict[stat_type]['My Headers'][header_pos]] = latest_spreads_df["Spread"].tolist()
                #For results, there are none yet so populate with 0s
                else:
                    self.Week_DF[self.Search_Stat_Dict[stat_type]['My Headers'][header_pos]] = [0 for i in range(len(self.All_Season_Teams))]
                print('last week')
            #For rest of stats get conventionally
            else:
                for df_header_pos in range(next_df_header_pos, len(df_headers)):
                    if self.Search_Stat_Dict[stat_type]['Table Headers'][header_pos] == df_headers[df_header_pos]:
                        self.listed_stat = sorted_df[self.Search_Stat_Dict[stat_type]['Table Headers'][header_pos]].tolist()
                        #Normalize the team list to all teams for the season
                        for pos in self.Blank_Positions:
                            self.listed_stat.insert(pos, "")
                        print(len(self.listed_stat))
                        print(self.listed_stat)
                        self.Week_DF[self.Search_Stat_Dict[stat_type]['My Headers'][header_pos]] = self.listed_stat
                        print(self.Week_DF.head())
                        break

    def Setup_Data_Search(self):
        for self.season in self.All_Search_Seasons:
            print(f'Season: {self.season}')
            if self.season == self.max_season:
                if self.mode == 'This Season':
                    self.max_week = self.last_season_max_week ##IF WANT THE LAST WEEK NOT TOO LOOK FOR SCORES (SINCE DON"T EXIST YET)
                elif self.mode == 'Historical':
                    self.max_week = self.last_season_max_week+1 ##IF WANT THE LAST WEEK TO LOOK FOR SCORES
                self.All_Search_Weeks = [n for n in range(self.min_week,self.max_week+1)] #Stats heading into Weeks 2 through 10 games
                if self.max_week == self.last_season_max_week+1: #Get rid of the last week in this case where we added it to the max week
                    del self.All_Search_Weeks[-1]
            #Make the folder for data
            self.season_path = f'{self.data_path}/{self.season}'
            self.Make_Folder(self.season_path)

            #Clear the season df and list of all weeks
            self.Season_DF = pd.DataFrame()# self.Season_DF.iloc[0:0]
            self.Opponent_Tracking_DF = pd.DataFrame()
            self.All_Weeks_DF_List = []
            self.All_Cleaned_Weeks_DF_List = []

            #Get all teams list for that season
            # self.URL_BASE = f'https://www.pro-football-reference.com/play-index/tgl_finder.cgi?request=1&match=single&year_min={self.season}&year_max={self.season}&game_type=R&game_num_min=0&game_num_max=99&week_num_min=1&week_num_max=17&temperature_gtlt=lt&league_id=NFL&c5val=1.0&order_by=points_diff'
            self.URL_BASE = f'https://stathead.com/football/tgl_finder.cgi?request=1&match=single&year_min={self.season}&year_max={self.season}&game_type=R&game_num_min=0&game_num_max=99&week_num_min=1&week_num_max=17&temperature_gtlt=lt&league_id=NFL&c5val=1.0&order_by=points_diff'
            self.All_Season_Teams = self.Get_Teams_List(self.season)

            #Add the Season Teams to the Opponent Tracker
            self.Opponent_Tracking_DF["Team"] = self.All_Season_Teams

            for self.week in self.All_Search_Weeks:
                print(f'Week: {self.week}')
                #Make the folder for data
                self.week_path = f'{self.season_path}/Week {self.week}'
                Data_Exists = self.Make_Folder(self.week_path)
                if Data_Exists and self.Read_Previous_Data == True and (self.season != self.max_season or self.week < self.last_season_max_week or self.mode == 'Historical'):
                    #Read csv that already exists
                    self.Week_DF = pd.read_csv(f'{self.week_path}/Week {self.week} Opponent Combined Data.csv')
                    self.All_Weeks_DF_List.append(self.Week_DF)
                    self.last_week_teams = self.Week_DF['Team'].tolist()
                else:
                    #Get Game Info for this week
                    #Clear the week_df
                    self.Week_DF = pd.DataFrame()
                    self.Setup_This_Week_Stats()
                    #Get Stat Info for this week
                    for stat_type in self.Search_Stat_Dict:
                        print(stat_type)
                        #Reset the first header to 0
                        self.search_stat_pos = 0

                        for search_name in range(0, len(self.Search_Stat_Dict[stat_type]['Search Name'])):
                            stat = self.Search_Stat_Dict[stat_type]['Search Name'][search_name]
                            print(stat)
                            #Define the headers to look at for this stat
                            if self.search_stat_pos == 0:
                                self.first_available_stat_header = 0
                            else:
                                #for pos in range(1, self.search_stat_pos):
                                self.first_available_stat_header += self.Search_Stat_Dict[stat_type]['Number of Headers'][self.search_stat_pos-1]

                            if "Results Stats" in stat_type or "Betting Stats" in stat_type: #Only that week
                                self.URL_BASE = f'https://www.pro-football-reference.com/play-index/tgl_finder.cgi?request=1&match=single&year_min={self.season}&year_max={self.season}&game_type=R&game_num_min=0&game_num_max=99&week_num_min={self.week}&week_num_max={self.week}&temperature_gtlt=lt&league_id=NFL&c5val=1.0&order_by={stat}'
                                week_using = self.week
                                # if self.season == 2019 and self.week == self.max_week:
                                #     week_using = self.week-1 #Not doing betting stats for last week yet
                                #     self.URL_BASE = f'https://www.pro-football-reference.com/play-index/tgl_finder.cgi?request=1&match=single&year_min={self.season}&year_max={self.season}&game_type=R&game_num_min=0&game_num_max=99&week_num_min={self.week-1}&week_num_max={self.week-1}&temperature_gtlt=lt&league_id=NFL&c5val=1.0&order_by={stat}'
                            else: #Week 1 to that Week
                                self.URL_BASE = f'https://www.pro-football-reference.com/play-index/tgl_finder.cgi?request=1&match=single&year_min={self.season}&year_max={self.season}&game_type=R&game_num_min=0&game_num_max=99&week_num_min=1&week_num_max={self.week-1}&temperature_gtlt=lt&league_id=NFL&c5val=1.0&order_by={stat}'
                                week_using = self.week-1
                            #Get the list of teams playing this week
                            # if self.season != 2019 or self.week != self.max_week:
                            if self.season == self.max_season and self.week == self.max_week and ("Results Stats" in stat_type or "Betting Stats" in stat_type):
                                print("Don't Match Teams")
                            else:
                                self.This_Stat_Teams = self.Get_Teams_List(self.season)
                                #Get the location of missing teams relative to the list of all teams for the season
                                self.Match_By_Teams(self.This_Stat_Teams)
                            #Add the team list for this week (should be same for every week of the season)
                            # self.Week_DF['Team'] = self.This_Stat_Teams
                            # self.Week_DF['Week'] = [self.week for i in range(len(self.This_Stat_Teams))]

                            #Collect the data for this week
                            self.Collect_Data(stat_type)
                            self.search_stat_pos+=1

                    # #Get the information about this week's games ###To Do: MIGHT NEED TO ADD THE self.Week_DF RETURN FOR IT TO ACTUALLY UPDATE THE DATA
                    if self.season != self.max_season:
                        #Get the information about this week's games
                        This_Week_Game_Info = Game_Collection.HISTORICAL_PARSING(self.Week_DF, self.This_Stat_Teams, self.All_Season_Teams, self.season, self.week)
                        This_Week_Game_Info.Do_Game_Stuff()
                    This_Week_Game_Info = Game_Collection.HISTORICAL_PARSING(self.Week_DF, self.This_Stat_Teams, self.All_Season_Teams, self.season, self.week)
                    This_Week_Game_Info.Do_Betting_Stuff()
                    #NEW GAME STATS HERE
                    # This_Week_Game_Info = Game_Collection.This_Week_Parsing(self.Week_DF, self.This_Stat_Teams, self.All_Season_Teams, self.season, self.week)
                    # This_Week_Game_Info.Do_Stuff()
                    #Proccess the data
                    Proccess_Stats = Data_Proccessing.PROCCESS_DATA(self.week, self.season, self.Week_DF, self.week_path, self.All_Cleaned_Weeks_DF_List, self.All_Weeks_DF_List)
                    self.All_Weeks_DF_List, self.All_Cleaned_Weeks_DF_List = Proccess_Stats.Do_Stuff()

                    # self.Track_All_Oppononents()
                    #To Do Still
                    # 3. Use this version to make model verison (remove even more columns)
                    # 4. Find new way to get Game Stats (Home Team, Opponent) For Current Week

            #Combine all weeks into a Season DF and save it
            season_excel_name = f'{self.season_path}/{self.season} Results.csv'
            self.Season_DF = self.Concat_and_Save(self.All_Weeks_DF_List, season_excel_name)
            self.All_Seasons_DF_List.append(self.Season_DF) #Put each season df in a list
            #Save the Season Opponents DF
            season_opp_excel_name = f'{self.season_path}/{self.season} Opponents.csv'
            self.Opponent_Tracking_DF.to_csv(season_opp_excel_name, index=False)

        project_excel_name = f'{self.data_path}/{self.final_csv_name}.csv'
        self.Concat_and_Save(self.All_Seasons_DF_List, project_excel_name)

    def Do_Stuff(self):
        self.Setup_Data_Search()

# NFL_DATA = NFL_DATA_COLLECTER(latest_week)
# NFL_DATA.Do_Stuff()


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  3 15:13:50 2019

@author: YQW
"""

def __partition(arr,left,right):
    v = arr[left]
    p = left;
    i = left + 1
    while True:
        if arr[i] < v:
            c        = arr[p+1];
            arr[p+1] = arr[i]
            arr[i]   = c
            p        = p + 1

        i = i + 1
        if i > right:
            break

    c         = arr[left];
    arr[left] = arr[p];
    arr[p]    = c;

    return p

def __quickSort(arr,left,right):
    if left >= right:
        return
    # partition
    pp = __partition(arr,left,right)
    # digui sort
    __quickSort(arr,left,pp-1)
    __quickSort(arr,pp+1,right)

def quickSort(arr):
    length = len(arr)
    __quickSort(arr,0,length-1)
    return

a = [1,4,5,3,2]
quickSort(a)
print(a)


import os
import numpy as np
import torch
import torch.nn as nn
import seaborn as sns
import matplotlib.pyplot as plt
import torch.nn.functional as F

# Vanilla GAN model for y|x
class GAN(object):
    def __init__(self, x_dim, y_dim, z_dim, H, batch_size, device,
                 d_lr=1e-3, g_lr=3e-4,
                 adam_b1=0.5,
                 adam_b2=0.999,
                 n_disc_batch=2):
        self.z_dim = z_dim
        self.input_dim = x_dim + z_dim
        self.batch_size = batch_size
        self.generator = nn.Sequential(
            nn.Linear(self.input_dim, H),
            # nn.BatchNorm1d(H),
            nn.LeakyReLU(0.1),
            nn.Linear(H, H),
            # nn.BatchNorm1d(H // 2),
            nn.LeakyReLU(0.1),
            nn.Linear(H, y_dim)
        ).to(device)

        self.discriminator = nn.Sequential(
            nn.Linear(x_dim + y_dim, H),
            # nn.BatchNorm1d(H),
            nn.LeakyReLU(0.1),
            nn.Linear(H, H),
            # nn.BatchNorm1d(H // 2),
            nn.LeakyReLU(0.1),
            nn.Linear(H, 1)
        ).to(device)

        self.scale_y = False
        self.n_disc_batch = n_disc_batch
        self.disc_optim = torch.optim.Adam(self.discriminator.parameters(), lr=d_lr, betas=(adam_b1, adam_b2))
        self.gen_optim = torch.optim.Adam(self.generator.parameters(), lr=g_lr, betas=(adam_b1, adam_b2))
        self.ad_loss = nn.BCEWithLogitsLoss()
        self.device = device

    def train_step(self, dataset, batch_size):

        for _ in range(self.n_disc_batch):
            x, y = dataset.sample(batch_size)
            v = torch.cat([x, y], 1)
            z = torch.randn(batch_size, self.z_dim).to(self.device)
            g_input = torch.cat([x, z], 1)
            gen_y = self.generator(g_input)
            w = torch.cat([x, gen_y], 1)

            disc_loss = self.ad_loss(self.discriminator(v), torch.ones(self.batch_size, 1).to(self.device)) + \
                        self.ad_loss(self.discriminator(w.detach()), torch.zeros(self.batch_size, 1).to(self.device))

            self.disc_optim.zero_grad()
            disc_loss.backward()
            self.disc_optim.step()

        x, _ = dataset.sample(batch_size)
        gen_z = torch.randn(batch_size, self.z_dim).to(self.device)
        gen_y = self.generator(torch.cat([x, gen_z], 1))
        w = torch.cat([x, gen_y], 1)
        gen_loss = self.ad_loss(self.discriminator(w), torch.ones(self.batch_size, 1).to(self.device))

        self.gen_optim.zero_grad()
        gen_loss.backward()
        self.gen_optim.step()

        return disc_loss.item(), gen_loss.item()

    def train(self, dataset, batch_size, n_epochs):
        if dataset.scale_y:
            self.scale_y = True
            self.y_scaler = dataset.y_scaler

        iter_per_epoch = dataset.n_samples // batch_size
        disc_stats = []
        gen_stats = []
        for epoch in range(n_epochs+1):

            ### Train ###
            disc_loss_epoch = []
            gen_loss_epoch = []
            for _ in range(iter_per_epoch):
                disc_loss, gen_loss = self.train_step(dataset, batch_size)
                disc_loss_epoch.append(disc_loss)
                gen_loss_epoch.append(gen_loss)

            disc_stats.append(np.mean(disc_loss_epoch))
            gen_stats.append(np.mean(gen_loss_epoch))

            ### Evaluation ###
            if epoch % 10 == 0:
                print(f"GAN: Epoch {epoch} disc_loss {disc_stats[-1]} gen_loss {gen_stats[-1]}")

    def decode(self, x, z=None):
        if z is None:
            z = torch.randn((x.shape[0], self.z_dim)).to(self.device)

        y = self.generator(torch.cat([x, z], 1))
        if self.scale_y:
            y = torch.from_numpy(self.y_scaler.inverse_transform(y.detach().cpu().numpy()))
        return y

    def eval_uncertainty(self, x, num_bins=50):
        ### Not Finished ###
        n = len(x)
        x = torch.repeat_interleave(x, num_bins, dim=0)
        gen_z = torch.randn(n * num_bins, self.z_dim)
        acts = self.generator(torch.cat([x, gen_z], 1)).view(n, num_bins, -1)
        return acts.detach().numpy()

    def sample(self, x):
        return self.decode(x)

    def eval(self):
        self.generator = self.generator.eval()
        self.discriminator = self.discriminator.eval()


import json
from channels.generic.websocket import WebsocketConsumer
from django.contrib.auth.models import User

from market.models import Profile, Favorites, Product

class FavoriteAddConsumer(WebsocketConsumer):
    def connect(self):
        print('Вошел!')
        self.accept()

    def disconnect(self, close_code):
        print('Вышел')

    # Receive message from WebSocket
    def receive(self, text_data):
        print('1')
        text_data_json = json.loads(text_data)
        pr_slug = text_data_json['product_slug']
        print(pr_slug)
        user = text_data_json['user']
        try:
            print('user')
            user = Profile.objects.get(user__username=user)
            fav = Favorites.objects.get(user=user)
            fav.products.add(Product.objects.get(slug=pr_slug))
            print(pr_slug)
            self.send(text_data=json.dumps({
                'pr_slug': pr_slug,
                'sus': 'true',
            }))
        except Exception as e:
            print(e)
            pass


"""
Provides utilities for running nussl algorithms that do not belong to
any specific algorithm or that are shared between algorithms.
"""

import warnings

import numpy as np
import torch
import random
from .. import musdb
import librosa
from . import constants

import os
from contextlib import contextmanager

def seed(random_seed, set_cudnn=False):
    """
    Seeds all random states in nussl with the same random seed
    for reproducibility. Seeds ``numpy``, ``random`` and ``torch``
    random generators.

    For full reproducibility, two further options must be set
    according to the torch documentation:

    https://pytorch.org/docs/stable/notes/randomness.html

    To do this, ``set_cudnn`` must be True. It defaults to
    False, since setting it to True results in a performance
    hit.

    Args:
        random_seed (int): integer corresponding to random seed to
        use.
        set_cudnn (bool): Whether or not to set cudnn into determinstic
        mode and off of benchmark mode. Defaults to False.
    """

    torch.manual_seed(random_seed)
    np.random.seed(random_seed)
    random.seed(random_seed)

    if set_cudnn:
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

def find_peak_indices(input_array, n_peaks, min_dist=None, do_min=False, threshold=0.5):
    """
    This function will find the indices of the peaks of an input n-dimensional numpy array.
    This can be configured to find max or min peak indices, distance between the peaks, and
    a lower bound, at which the algorithm will stop searching for peaks (or upper bound if
    searching for max). Used exactly the same as :func:`find_peak_values`.

    This function currently only accepts 1-D and 2-D numpy arrays.

    Notes:
        * This function only returns the indices of peaks. If you want to find peak values,
        use :func:`find_peak_values`.

        * min_dist can be an int or a tuple of length 2.
            If input_array is 1-D, min_dist must be an integer.
            If input_array is 2-D, min_dist can be an integer, in which case the minimum
            distance in both dimensions will be equal. min_dist can also be a tuple if
            you want each dimension to have a different minimum distance between peaks.
            In that case, the 0th value in the tuple represents the first dimension, and
            the 1st value represents the second dimension in the numpy array.

    Args:
        input_array: a 1- or 2- dimensional numpy array that will be inspected.
        n_peaks: (int) maximum number of peaks to find
        min_dist: (int) minimum distance between peaks. Default value: len(input_array) / 4
        do_min: (bool) if True, finds indices at minimum value instead of maximum
        threshold: (float) the value (scaled between 0.0 and 1.0)

    Returns:
        peak_indices: (list) list of the indices of the peak values

    """
    input_array = np.array(input_array, dtype=float)

    if input_array.ndim > 2:
        raise ValueError('Cannot find peak indices on data greater than 2 dimensions!')

    is_1d = input_array.ndim == 1
    zero_dist = zero_dist0 = zero_dist1 = None
    min_dist = len(input_array) // 4 if min_dist is None else min_dist

    if is_1d:
        zero_dist = min_dist
    else:
        if type(min_dist) is int:
            zero_dist0 = zero_dist1 = min_dist
        elif len(min_dist) == 1:
            zero_dist0 = zero_dist1 = min_dist[0]
        else:
            zero_dist0, zero_dist1 = min_dist

    # scale input_array between [0.0, 1.0]
    input_array -= np.min(input_array)
    input_array /= np.max(input_array)

    # flip sign if doing min
    input_array = -input_array if do_min else input_array

    # throw out everything below threshold
    input_array = np.multiply(input_array, (input_array >= threshold))

    # check to make sure we didn't throw everything out
    if np.size(np.nonzero(input_array)) == 0:
        raise ValueError('Threshold set incorrectly. No peaks above threshold.')
    if np.size(np.nonzero(input_array)) < n_peaks:
        warnings.warn('Threshold set such that there will be less peaks than n_peaks.')

    peak_indices = []
    for i in range(n_peaks):
        # np.unravel_index for 2D indices e.g., index 5 in a 3x3 array should be (1, 2)
        # Also, wrap in list for duck typing
        cur_peak_idx = list(np.unravel_index(
            np.argmax(input_array.flatten()), input_array.shape
        ))

        # zero out peak and its surroundings
        if is_1d:
            cur_peak_idx = cur_peak_idx[0]
            peak_indices.append(cur_peak_idx)
            lower, upper = _set_array_zero_indices(cur_peak_idx, zero_dist, len(input_array))
            input_array[lower:upper] = 0
        else:
            peak_indices.append(cur_peak_idx)
            lower0, upper0 = _set_array_zero_indices(cur_peak_idx[0], zero_dist0,
                                                     input_array.shape[0])
            lower1, upper1 = _set_array_zero_indices(cur_peak_idx[1], zero_dist1,
                                                     input_array.shape[1])
            input_array[lower0:upper0, lower1:upper1] = 0

        if np.sum(input_array) == 0.0:
            break

    return peak_indices

def _set_array_zero_indices(index, zero_distance, max_len):
    lower = index - zero_distance
    upper = index + zero_distance + 1
    lower = 0 if lower < 0 else lower
    upper = max_len if upper >= max_len else upper
    return int(lower), int(upper)

def complex_randn(shape):
    """
    Returns a complex-valued numpy array of random values with shape :param:`shape`.

    Args:
        shape (tuple): Tuple of ints that will be the shape of the resultant complex numpy array.

    Returns:
        (:obj:`np.ndarray`): a complex-valued numpy array of random values with shape `shape`
    """
    return np.random.randn(*shape) + 1j * np.random.randn(*shape)

def _get_axis(array, axis_num, i):
    """
    Will get index 'i' along axis 'axis_num' using np.take.

    Args:
        array (:obj:`np.ndarray`): Array to fetch axis of.
        axis_num (int): Axes to retrieve.
        i (int): Index to retrieve.

    Returns:
        The value at index :param:`i` along axis :param:`axis_num`
    """

    return np.take(array, i, axis_num)

def _slice_along_dim(data, dim, start, end):
    """
    Takes a slice of data along a dim between a start and an end. Agnostic to
    whether the data is a numpy array or a torch tensor.

    Args:
        data (np.ndarray or torch.Tensor): Data to slice.
        dim (int): Dimension along which to do the slicing.
        start (int): Start of the slice.
        end (int): End of the slice
    """
    if dim > 3:
        raise ValueError("Unsupported for dim > 4")
    if dim >= len(data.shape):
        raise ValueError(f"dim {dim} too high for data.shape {data.shape}!")

    if dim == 0:
        return data[start:end, ...]
    elif dim == 1:
        return data[:, start:end, ...]
    elif dim == 2:
        return data[:, :, start:end, ...]
    elif dim == 3:
        return data[:, :, :, start:end, ...]

def _format(string):
    """ Formats a class name correctly for checking function and class names.
        Strips all non-alphanumeric chars and makes lowercase.
    """
    return ''.join(list(filter(str.isalnum, string))).lower()

def musdb_track_to_audio_signals(track):
    """
    Converts a musdb track to a dictionary of AudioSignal objects.

    Args:
        track (musdb.audio_classes.MultiTrack): MultiTrasack object
            containing stems that will each be turned into AudioSignal
            objects.

    Returns:
        (2-tuple): tuple containing the mixture AudioSignal and a dictionary of
            the sources.
    """
    # lazy load to prevent circular imports
    from .audio_signal import AudioSignal

    mixture = AudioSignal(audio_data_array=track.audio, sample_rate=track.rate)
    mixture.path_to_input_file = track.name
    stems = track.stems
    sources = {}

    for k, v in sorted(track.sources.items(), key=lambda x: x[1].stem_id):
        sources[k] = AudioSignal(
            audio_data_array=stems[v.stem_id],
            sample_rate=track.rate
        )
        sources[k].path_to_input_file = f'musdb/{track.name}_{k}.wav'

    return mixture, sources

def audio_signals_to_musdb_track(mixture, sources_dict, targets_dict):
    """
    Converts :class:`AudioSignal` objects to ``musdb`` :class:`Track` objects that
    contain the mixture, the ground truth sources, and the targets for use with the ``mus_eval``
    implementation of BSS-Eval and ``musdb``.

    See Also:
        * More information on ``musdb``:  `Github<https://github.com/sigsep/sigsep-mus-db>`
            and `documentation<http://musdb.readthedocs.io/>`
        * More information on ``mus_eval``: `Github<https://github.com/sigsep/sigsep-mus-eval>`
            and `documentation<https://sigsep.github.io/sigsep-mus-eval/>`
        * :class:`BSSEvalV4` for *nussl*'s interface to BSS-Eval v4.

    Examples:
        .. code-block:: python
            :linenos:
            import nussl
            signal = nussl.AudioSignal(nussl.efz_utils.download_audio_file('HistoryRepeating.wav'))

            repet = nussl.Repet(signal)
            repet.run()

            bg, fg = repet.make_audio_signals()

            src_dict = {'vocals': fg, 'accompaniment': bg}
            target = nussl.core.constants.STEM_TARGET_DICT
            track = nussl.utils.audio_signals_to_musdb_track(signal, src_dict, target)

    Args:
        mixture (:class:`AudioSignal`): The :class:`AudioSignal` object that contains the mixture.
        sources_dict (dict): Dictionary where the keys are the labels for the sources and values
            are the associated :class:`AudioSignal` objects.
        targets_dict (dict): Dictionary where the keys are the labels for the sources (as above)
            and the values are weights.

    Returns:
        (:obj:`musdb.MultiTrack`) populated as specified by inputs.
    """
    verify_audio_signal_list_strict(list(sources_dict.values()) + [mixture])

    path = mixture.path_to_input_file if mixture.path_to_input_file else "None"
    fname = mixture.file_name if mixture.file_name else "None"
    track = musdb.audio_classes.MultiTrack(path=path, name=fname, is_wav=True)
    track.audio = mixture.audio_data.T
    track.rate = mixture.sample_rate

    stems = [track.audio]

    for name, target_srcs in list(targets_dict.items()):
        if name in sources_dict:
            stems.append(sources_dict[name].audio_data.T)

    track._stems = np.array(stems)
    return track

def verify_audio_signal_list_lax(audio_signal_list):
    """
    Verifies that an input (:param:`audio_signal_list`) is a list of :ref:`AudioSignal` objects.
    If not so, attempts to correct the list (if possible) and returns the corrected list.

    Args:
        audio_signal_list (list): List of :ref:`AudioSignal` objects

    Returns:
        audio_signal_list (list): Verified list of :ref:`AudioSignal` objects.

    """
    # Lazy load to prevent a circular reference upon initialization
    from .audio_signal import AudioSignal

    if isinstance(audio_signal_list, AudioSignal):
        audio_signal_list = [audio_signal_list]
    elif isinstance(audio_signal_list, list):
        if not all(isinstance(s, AudioSignal) for s in audio_signal_list):
            raise ValueError('All input objects must be AudioSignal objects!')
        if not all(s.has_data for s in audio_signal_list):
            raise ValueError('All AudioSignal objects in input list must have data!')
    else:
        raise ValueError(
            'audio_signal_list must be a list of or a single AudioSignal objects!')

    return audio_signal_list

def verify_audio_signal_list_strict(audio_signal_list):
    """
    Verifies that an input (:param:`audio_signal_list`) is a list of :ref:`AudioSignal` objects and
    that they all have the same sample rate and same number of channels. If not true,
    attempts to correct the list (if possible) and returns the corrected list.

    Args:
        audio_signal_list (list): List of :ref:`AudioSignal` objects

    Returns:
        audio_signal_list (list): Verified list of :ref:`AudioSignal` objects, that all have
        the same sample rate and number of channels.

    """
    audio_signal_list = verify_audio_signal_list_lax(audio_signal_list)

    if not all(audio_signal_list[0].sample_rate == s.sample_rate for s in audio_signal_list):
        raise ValueError('All input AudioSignal objects must have the same sample rate!')

    if not all(audio_signal_list[0].num_channels == s.num_channels for s in audio_signal_list):
        raise ValueError('All input AudioSignal objects must have the same number of channels!')

    if not all(audio_signal_list[0].signal_length == s.signal_length for s in audio_signal_list):
        raise ValueError('All input AudioSignal objects must have the same signal length!')

    return audio_signal_list

def visualize_gradient_flow(named_parameters, n_bins=50):
    """
    Visualize the gradient flow through the named parameters of a PyTorch model.

    Plots the gradients flowing through different layers in the net during training.
    Can be used for checking for possible gradient vanishing / exploding problems.

    Usage: Plug this function in Trainer class after loss.backwards() as
    "visualize_gradient_flow(self.model.named_parameters())" to visualize
    the gradient flow

    Args:
        named_parameters (generator): Generator object yielding name and parameters
          for each layer in a PyTorch model.
        n_bins (int): Number of bins to use for each histogram. Defaults to 50.
    """
    import matplotlib.pyplot as plt

    data = []

    for n, p in named_parameters:
        if p.requires_grad and "bias" not in n:
            if p.grad is not None:
                _data = p.grad.cpu().data.numpy().flatten()
                lower = np.percentile(_data, 10)
                upper = np.percentile(_data, 90)
                _data = _data[_data >= lower]
                _data = _data[_data <= upper]
                n = n.split('layers.')[-1]
                data.append((n, _data, np.abs(_data).mean()))

    _data = [d[1] for d in sorted(data, key=lambda x: x[-1])]
    _names = [d[0] for d in sorted(data, key=lambda x: x[-1])]

    plt.hist(_data, len(_data) * n_bins, histtype='step', fill=False,
             stacked=True, label=_names)

    plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2)

def visualize_spectrogram(audio_signal, ch=0, do_mono=False, x_axis='time',
                          y_axis='linear', **kwargs):
    """
    Wrapper around `librosa.display.specshow` for usage with AudioSignals.

    Args:
        audio_signal (AudioSignal): AudioSignal to plot
        ch (int, optional): Which channel to plot. Defaults to 0.
        do_mono (bool, optional): Make the AudioSignal mono. Defaults to False.
        x_axis (str, optional): x_axis argument to librosa.display.specshow. Defaults to 'time'.
        y_axis (str, optional): y_axis argument to librosa.display.specshow. Defaults to 'linear'.
        kwargs: Additional keyword arguments to librosa.display.specshow.
    """
    import librosa.display

    if do_mono:
        audio_signal = audio_signal.to_mono(overwrite=False)

    if y_axis == 'mel':
        # Monkey patch for https://github.com/librosa/librosa/issues/1240
        data = librosa.feature.melspectrogram(audio_signal.get_channel(ch),
                                              sr=audio_signal.sample_rate)
        kwargs.update({'fmax': audio_signal.sample_rate / 2.})
    else:
        data = np.abs(audio_signal.stft())[..., ch]

    data = librosa.amplitude_to_db(data, ref=np.max)
    librosa.display.specshow(data, x_axis=x_axis, y_axis=y_axis,
                             sr=audio_signal.sample_rate,
                             hop_length=audio_signal.stft_params.hop_length,
                             **kwargs)

def visualize_waveform(audio_signal, ch=0, do_mono=False, x_axis='time', **kwargs):
    """
    Wrapper around `librosa.display.waveplot` for usage with AudioSignals.

    Args:
        audio_signal (AudioSignal): AudioSignal to plot
        ch (int, optional): Which channel to plot. Defaults to 0.
        do_mono (bool, optional): Make the AudioSignal mono. Defaults to False.
        x_axis (str, optional): x_axis argument to librosa.display.waveplot. Defaults to 'time'.
        kwargs: Additional keyword arguments to librosa.display.waveplot.
    """
    import librosa.display
    import matplotlib.pyplot as plt

    if do_mono:
        audio_signal = audio_signal.to_mono(overwrite=False)

    data = np.asfortranarray(audio_signal.audio_data[ch])
    librosa.display.waveplot(data, sr=audio_signal.sample_rate, x_axis=x_axis, **kwargs)
    plt.ylabel('Amplitude')

def visualize_sources_as_waveform(audio_signals, ch=0, do_mono=False, x_axis='time',
                                  colors=None, alphas=None, show_legend=True, **kwargs):
    """
    Visualizes a dictionary or list of sources with overlapping waveforms with transparency.

    The labels of each source are either the key, if a dictionary, or the
    path to the input audio file, if a list.

    Args:
        audio_signals (list or dict): List or dictionary of audio signal objects to be
          plotted.
        ch (int, optional): Which channel to plot. Defaults to 0.
        do_mono (bool, optional): Make each AudioSignal mono. Defaults to False.
        x_axis (str, optional): x_axis argument to librosa.display.waveplot. Defaults to 'time'.
        colors (list, optional): Sequence of colors to use for each signal.
          Defaults to None, which uses the default matplotlib color cycle.
        alphas (list, optional): Sequence of alpha transparency to use for each signal.
          Defaults to None.
        kwargs: Additional keyword arguments to librosa.display.waveplot.
    """
    import matplotlib.pyplot as plt

    if isinstance(audio_signals, list):
        audio_signals = {
            f'{i}:{a.path_to_input_file}': a
            for i, a in enumerate(audio_signals)
        }

    sorted_keys = sorted(
        audio_signals.keys(),
        key=lambda k: audio_signals[k].rms().mean(),
        reverse=True
    )

    alphas = (
        np.linspace(0.25, .75, len(audio_signals))
        if alphas is None else alphas
    )
    colors = (
        plt.rcParams['axes.prop_cycle'].by_key()['color']
        if colors is None else colors
    )

    for i, key in enumerate(sorted_keys):
        val = audio_signals[key]
        color = colors[i % len(audio_signals)]
        visualize_waveform(val, ch=ch, do_mono=do_mono, x_axis=x_axis,
                           alpha=alphas[i % len(audio_signals)],
                           label=key, color=color)

    if show_legend:
        plt.legend(bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2)

def visualize_sources_as_masks(audio_signals, ch=0, do_mono=False, x_axis='time',
                               y_axis='linear', db_cutoff=-60, colors=None, alphas=None,
                               alpha_amount=1.0, show_legend=True, **kwargs):
    """
    Visualizes a dictionary or list of sources with overlapping waveforms with transparency.

    The labels of each source are either the key, if a dictionary, or the
    path to the input audio file, if a list.

    Args:
        audio_signals (list or dict): List or dictionary of audio signal objects to be
          plotted.
        ch (int, optional): Which channel to plot. Defaults to 0.
        do_mono (bool, optional): Make each AudioSignal mono. Defaults to False.
        x_axis (str, optional): x_axis argument to librosa.display.waveplot. Defaults to 'time'.
        colors (list, optional): Sequence of colors to use for each signal.
          Defaults to None, which uses the default matplotlib color cycle.
        alphas (list, optional): Sequence of alpha transparency to use for each signal.
          Defaults to None.
        kwargs: Additional keyword arguments to librosa.display.specshow.
    """
    import matplotlib.pyplot as plt
    import matplotlib
    import librosa.display
    from .. import datasets

    if isinstance(audio_signals, list):
        audio_signals = {
            f'{i}:{a.path_to_input_file}': a
            for i, a in enumerate(audio_signals)
        }

    if do_mono:
        for key in audio_signals:
            audio_signals[key] = audio_signals[key].to_mono()

    sorted_keys = sorted(
        audio_signals.keys(),
        key=lambda k: audio_signals[k].rms().mean(),
        reverse=True
    )

    source_names = sorted(list(audio_signals.keys()))
    mix = sum(audio_signals.values())
    data = {
        'mix': mix,
        'sources': audio_signals
    }
    data = datasets.transforms.PhaseSensitiveSpectrumApproximation()(data)

    colors = (
        plt.rcParams['axes.prop_cycle'].by_key()['color']
        if colors is None else colors
    )

    # construct each image with alpha values
    masks = data['source_magnitudes'] / (np.maximum(
            data['mix_magnitude'][..., None], data['source_magnitudes'])
            + constants.EPSILON
        )
    legend_elements = []

    silence_mask = librosa.amplitude_to_db(np.abs(mix.stft()), ref=np.max) > db_cutoff
    masks *= silence_mask[..., None]

    y_coords = librosa.display.__mesh_coords(y_axis, None, masks.shape[0],
        sr=mix.sample_rate, hop_length=mix.stft_params.hop_length)
    x_coords = librosa.display.__mesh_coords(x_axis, None, masks.shape[1],
        sr=mix.sample_rate, hop_length=mix.stft_params.hop_length)

    extent = [x_coords.min(), x_coords.max(), y_coords.min(), y_coords.max()]

    for j, key in enumerate(sorted_keys):
        i = source_names.index(key)
        mask = masks[..., ch, i]
        color = colors[j % len(colors)]

        cmap = matplotlib.colors.LinearSegmentedColormap.from_list(
            'custom', ['white', color])
        image = cmap(mask)
        image[:, :, -1] = mask ** alpha_amount
        plt.imshow(image, origin='lower', aspect='auto',
            interpolation='none', extent=extent)

        legend_elements.append(
            matplotlib.patches.Patch(facecolor=color, label=key))

    axes = librosa.display.__check_axes(None)

    axes.set_xlim(x_coords.min(), x_coords.max())
    axes.set_ylim(y_coords.min(), y_coords.max())

    # Set up axis scaling
    librosa.display.__scale_axes(axes, x_axis, 'x')
    librosa.display.__scale_axes(axes, y_axis, 'y')

    # Construct tickers and locators
    librosa.display.__decorate_axis(axes.xaxis, x_axis)
    librosa.display.__decorate_axis(axes.yaxis, y_axis)

    if show_legend:
        plt.legend(handles=legend_elements,
            bbox_to_anchor=(0., 1.02, 1., .102), loc=3, ncol=2)

@contextmanager
def _close_temp_files(tmpfiles):
    """
    Utility function for creating a context and closing all temporary files
    once the context is exited. For correct functionality, all temporary file
    handles created inside the context must be appended to the ```tmpfiles```
    list.

    This function is taken wholesale from Scaper.

    Args:
        tmpfiles (list): List of temporary file handles
    """
    def _close():
        for t in tmpfiles:
            try:
                t.close()
                os.unlink(t.name)
            except:
                pass
    try:
        yield
    except:
        _close()
        raise
    _close()


import sys
import heapq
input = sys.stdin.readline

T = int(input())

for _ in range(T):
    N = int(input())
    chapters = list(map(int, input().split()))
    heapq.heapify(chapters)

    answer = 0
    while(len(chapters) != 1):
        chapter1 = heapq.heappop(chapters)
        chapter2 = heapq.heappop(chapters)
        chapter3 = chapter1 + chapter2
        answer += chapter3
        heapq.heappush(chapters, chapter3)
    print(answer)


import traceback
from global_cfgs import Global_Cfgs
from scenario import Scenario
from UIs.console_UI import Console_UI

def main():
    cfgs = Global_Cfgs()
    scenario = Scenario(scenario_name=cfgs.get('scenario'))

    start_scene = cfgs.get('start_scene')
    try:
        for scene in iter(scenario):
            if start_scene is None or scene.scene_name.strip().lower() == start_scene.strip().lower():
                start_scene = None
                scene.run_scene()
            else:
                Console_UI().inform_user(f'Skip \'{scene.scene_name}\' - waiting for \'{start_scene}\'')

    except RuntimeError as error:
        Console_UI().warn_user(error)
        Console_UI().inform_user("\n\n Traceback: \n")

        traceback.print_exc()
    except KeyboardInterrupt:
        Console_UI().inform_user(f'\nInterrupted by ctrl+c - stopped ad "{scene.scene_name}"')
    else:
        Console_UI().inform_user("Done with all scenarios!")

    Console_UI().inform_user('To view results, checkout the tensorboard:')
    Console_UI().inform_user(f'tensorboard --logdir /media/max/HD_1_3TB/log/{cfgs.sub_log_path}/tensorboard')

if __name__ == "__main__":
    main()


import matplotlib.pyplot as mp

def price_comparison(amazon_data,flipkart_data):

    #seperating the data of the amazon product
    a_name=amazon_data[0]
    a_price=int(amazon_data[1])
    a_rating=amazon_data[2]
    a_details=amazon_data[3]

    #seperating the data for flipkart product
    f_name=flipkart_data[0]
    f_price=int(flipkart_data[1])
    f_rating=flipkart_data[2]
    f_details=flipkart_data[3]

    #differnece between the price
    if a_price>f_price:
        difference_in_prices=a_price-f_price
    else:
        difference_in_prices=f_price-a_price

    x_axis=[0,1,2]
    #x_axis.append(a_name)
    #x_axis.append(f_name)
    y_axis=[b_var]
    y_axis.append(a_price)
    y_axis.append(f_price)

    mp.bar(x_axis,y_axis)
    mp.show()


# -*- coding=utf-8 -*-

from __future__ import print_function
from __future__ import absolute_import
import keras
import keras.backend as K
from keras import regularizers
from keras.models import Sequential, Model
from keras.layers import *
from keras.activations import softmax
import tensorflow as tf
from model import BasicModel
from utils.utility import *
from layers.SparseFullyConnectedLayer import *

class DSSM(BasicModel):
    def __init__(self, config):
        super(DSSM, self).__init__(config)
        self.__name = 'DSSM'
        self.check_list = [ 'vocab_size', 'hidden_sizes', 'dropout_rate']
        self.setup(config)
        if not self.check():
            raise TypeError('[DSSM] parameter check wrong')
        print('[DSSM] init done', end='\n')

    def setup(self, config):
        if not isinstance(config, dict):
            raise TypeError('parameter config should be dict:', config)

        self.set_default('hidden_sizes', [300, 128])
        self.set_default('dropout_rate', 0.5)
        self.set_default('reg_rate', 0.0)
        self.config.update(config)

    def build(self):
        query = Input(name='query', shape=(self.config['vocab_size'],))#, sparse=True)
        show_layer_info('Input', query)
        doc = Input(name='doc', shape=(self.config['vocab_size'],))#, sparse=True)
        show_layer_info('Input', doc)

        def mlp_work(input_dim):
            seq = Sequential()
            #seq.add(SparseFullyConnectedLayer(self.config['hidden_sizes'][0], input_dim=input_dim, activation='relu'))
            num_hidden_layers = len(self.config['hidden_sizes'])
            if num_hidden_layers == 1:
                seq.add(Dense(self.config['hidden_sizes'][0], input_shape=(input_dim,), activity_regularizer=regularizers.l2(self.config['reg_rate'])))
            else:
                seq.add(Dense(self.config['hidden_sizes'][0], activation='tanh', input_shape=(input_dim,), activity_regularizer=regularizers.l2(self.config['reg_rate'])))
                for i in range(num_hidden_layers-2):
                    seq.add(Dense(self.config['hidden_sizes'][i+1], activation='tanh', activity_regularizer=regularizers.l2(self.config['reg_rate'])))
                    seq.add(Dropout(rate=self.config['dropout_rate']))
                seq.add(Dense(self.config['hidden_sizes'][num_hidden_layers-1], activity_regularizer=regularizers.l2(self.config['reg_rate'])))
                seq.add(Dropout(rate=self.config['dropout_rate']))
            return seq

        mlp = mlp_work(self.config['vocab_size'])
        rq = mlp(query)
        show_layer_info('MLP', rq)
        rd = mlp(doc)
        show_layer_info('MLP', rd)

        '''
        rep = Concatenate(axis=1) ([rq, rd])
        show_layer_info('Concatenate', rep)
        rep = Dropout(rate=self.config['dropout_rate'])(rep)
        show_layer_info('Dropout', rep)
        if self.config['target_mode'] == 'classification':
            out_ = Dense(2, activation='softmax')(rep)
        elif self.config['target_mode'] in ['regression', 'ranking']:
            out_ = Dense(1)(rep)
        show_layer_info('Dense', out_)
        '''
        out_ = Dot( axes= [1, 1], normalize=True)([rq, rd])
        show_layer_info('Dot', out_)
        if self.config['target_mode'] == 'classification':
            out_ = Dense(2, activation='softmax')(out_)
            show_layer_info('Dense', out_)

        model = Model(inputs=[query, doc], outputs=[out_])
        return model


from unittest import TestCase
from core import Statement, Literal, Variable, KnowledgeBase, InferenceRule, Quantor, Block

class TestCore(TestCase):
    def setUp(self):
        self.kb = KnowledgeBase()

    def testEquals(self):
        block = self.kb.root
        literal = self.kb.addEntity(Literal(10))
        statement = Statement("foo", {"x": literal})
        self.kb.addStatement(block, statement)

        literal2 = self.kb.addEntity(Literal(10))
        statement2 = Statement("foo", {"x": literal2})
        self.kb.addStatement(block, statement)
        self.assertTrue(statement.equals(self.kb, statement2))

    def testNotEquals(self):
        block = self.kb.root
        literal = self.kb.addEntity(Literal(10))
        statement = Statement("foo", {"x": literal})
        self.kb.addStatement(block, statement)
        self.assertFalse(self.kb[literal].equals(self.kb, statement))

        literal2 = self.kb.addEntity(Literal(10))
        statement2 = Statement("foo", {"x": literal2, "y": literal2})
        self.kb.addStatement(block, statement)
        self.assertFalse(statement.equals(self.kb, statement2))

        literal3 = self.kb.addEntity(Literal(5))
        statement3 = Statement("foo", {"x": literal3})
        self.kb.addStatement(block, statement)
        self.assertFalse(statement.equals(self.kb, statement3))

    def testSubstitute(self):
        # foo(a: x, b: x)[x -> 10] == foo(a: 10, b: 10)

        block = self.kb.root
        variable = Variable("x")
        statement = Statement("foo", {"a": variable, "b": variable})
        statementID = self.kb.addStatement(block, statement)

        literal = self.kb.addEntity(Literal(10))
        statement2 = statement.substitute(statementID, self.kb, {"x": literal})

        statement3 = self.kb.addStatement(block, Statement("foo", {"a": literal, "b": literal}))

        self.assertTrue(self.kb[statement2].equals(self.kb, self.kb[statement3]))

    def testUnify(self):
        # foo(l: bar(a: x), r: y) ~~ foo(l: z, r:z) == { y -> bar(a: x), z -> bar(a: x) }

        block = self.kb.root
        x = Variable("x")
        y = Variable("y")
        z = Variable("z")
        statement = Statement("bar", {"a": x})
        statementID = self.kb.addEntity(statement)
        statement2 = Statement("foo", {"l": statementID, "r": y})
        statement2ID = self.kb.addStatement(block, statement2)
        statement3 = Statement("foo", {"l": z, "r": z})
        statement3ID = self.kb.addStatement(block, statement3)

        self.assertTrue(statement2.unify(self.kb, statement3ID) == {"y": statementID, "z": statementID})

    def testQuery(self):
        block = self.kb.root

        x = Variable("x")
        literal = self.kb.addEntity(Literal(10))
        literal2 = self.kb.addEntity(Literal(15))
        statement = Statement("foo", {"a": literal})
        statement2 = Statement("foo", {"a": literal2})
        statementID = self.kb.addStatement(block, statement)
        statement2ID = self.kb.addStatement(block, statement2)

        pattern = Statement("foo", {"a": x})
        result = [r for r in self.kb.query(pattern)]
        self.assertTrue(len(result) == 2)
        self.assertTrue(result[0][0] == statementID)
        self.assertTrue(result[0][1]['x'] == literal)
        self.assertTrue(result[1][0] == statement2ID)
        self.assertTrue(result[1][1]['x'] == literal2)

    def testPatternInference(self):
        # Inference rule: foo(a: x), bar(b: x) -> conclusion()
        # Knowledge Base:
        #  foo(a: 10) #1
        #  bar(b: 15) #2
        #  bar(b: 10) #3
        # Expected result:
        #  env: {x -> 10}
        #  dependencies: [#1, #3]
        #  conclusion: conclusion()
        block = self.kb.root

        x = Variable("x")
        literal = self.kb.addEntity(Literal(10))
        literal2 = self.kb.addEntity(Literal(15))
        statement = Statement("foo", {"a": literal})
        statement2 = Statement("bar", {"b": literal2})
        statement3 = Statement("bar", {"b": literal})
        statementID = self.kb.addStatement(block, statement)
        statement2ID = self.kb.addStatement(block, statement2)
        statement3ID = self.kb.addStatement(block, statement3)

        pattern = Statement("foo", {"a": x})
        pattern2 = Statement("bar", {"b": x})
        conclusion = Statement("conclusion", {})
        patternID = self.kb.addEntity(pattern)
        pattern2ID = self.kb.addEntity(pattern2)
        conclusionID = self.kb.addEntity(conclusion)

        irule = InferenceRule(["x"], conclusionID, [patternID, pattern2ID])
        results = [result for result in irule.getInferences(self.kb)]
        self.assertTrue(results == [({'x': literal}, [statementID, statement3ID], conclusionID)])

    def testQuantor(self):
        block = self.kb.root

        quantorBlockID = self.kb.addBlock()
        statement = Statement("foo", {"1": Variable("x"), "2": Variable("y")})
        self.kb.addStatement(quantorBlockID, statement)
        quantor = Quantor("forall", {}, ["x"], quantorBlockID)
        statementID = self.kb.addStatement(block, quantor)
        ten = self.kb.addEntity(Literal(10))
        substitutedID = quantor.substitute(statementID, self.kb, {"x": ten, "y": ten})

        self.assertTrue(self.kb[self.kb[self.kb[substitutedID].block].statements[0]].parameters["1"].equals(Variable("x")))
        self.assertTrue(self.kb[self.kb[self.kb[self.kb[substitutedID].block].statements[0]].parameters["2"]].equals(self.kb, Literal(10)))


import time
from multiprocessing import Queue
from multiprocessing.managers import BaseManager

#从机类
class Slave:

    def __init__(self):
        # 派发出去的任务队列
        self.dispatched_task_queue = Queue()
        # 完成的任务队列
        self.finished_task_queue = Queue()

    def start(self):
        # 把派发任务队列和完成任务队列注册到网络上
        BaseManager.register('get_dispatched_task_queue')
        BaseManager.register('get_finished_task_queue')

        # 连接master
        server = '127.0.0.1'
        print('连接主机服务器 %s...' % server)
        manager = BaseManager(address=(server,8888), authkey='taimei'.encode("utf-8"))
        manager.connect()

        # 使用上面注册的方法获取队列
        dispatched_tasks = manager.get_dispatched_task_queue()
        finished_tasks = manager.get_finished_task_queue()

        # 运行作业并返回结果，这里只是模拟作业运行，所以返回的是接收到的作业
        while True:
            try:
                task = dispatched_tasks.get(timeout=1)
                print("正在执行任务..."+str(task))
                #这里省略掉处理任务的过程
                time.sleep(1)
                finished_tasks.put(task["name"]+" 任务已经执行完毕！")
            except Exception:
                continue

if __name__ == "__main__":
    slave = Slave()
    slave.start()


import sys
import builtins
builtins.sys = sys

def fixme(module):
    module = '%s.%s' % ( __name__ , module )
    __import__( module )
    module = eval(module)
    module.__file__ = '<fixes>'
    module.__name__ = module.__name__.rsplit('.',1)[-1]
    sys.modules[module.__name__] = module
    setattr( builtins, module.__name__, module )

if sys.version_info[:2]<(3,7):
    # running micropython for a sim
    builtins.__UPY__ = True
    import stupyde

    #order matters
    fixes = ('micropython','machine', 'struct', 'ctypes', 'contextlib','contextvars',)

else:
    # running cpython
    builtins.__UPY__ = False

    import traceback

    def print_exception(e, out=sys.stderr, **kw):
        kw["file"] = out
        traceback.print_exc(**kw)

    sys.print_exception = print_exception
    try:
        import pythons
        import pythons.fixes as fixes
        pythons.fixes = fixes
        fixme('utime')
        print(utime,sys.modules['utime'])
        fixes = ('micropython','machine','contextvars',)  #contextvars is unsupported on micropython
    except:
        print("pythons.fixes not found")
        fixes =()

for fix in fixes:
    fixme(fix)


from threading import Thread
from multiprocessing import Queue,Pipe
import queue
from GPUAllocator_thread.modelexecutor import ModelExecutor
import numpy as np
import time

def StartProcess(model_name:str,input_queue:Queue,run_signal: Pipe,done_signal: Pipe, output_queue:Queue,model_dict:dict):
    '''
    you can recv 1 by process_ok if test ok.

    model-dict example:
        {
            "0": "xxx.onnx",
            "1": "xxx.onnx"
        }

    input_shapes example:

        [
            {
                "type": "float32",
                "name": ...,
                "shape": ...
            },
            {
                ...
            }
        ]

    '''

    print("init %s executor..."%model_name)

    all_threads=[]
    task_time_record=queue.Queue()
    myexecutor=ModelExecutor(model_name)
    myexecutor.Init(model_dict)

    # # run test
    # if RunTest(myexecutor,run_signal,done_signal,test_ok,input_shapes,default_batchsize):
    #     print("test run model: %s ok."%(model_name))
    # else:
    #     print("run model: %s may meet some error, fail to finish in 3 minutes, process exit."%(model_name))
    #     return

    # data-in-thread
    def read_input():
        while True:
            myexecutor.AddRequest(input_queue.get(block=True))                          # block while no input
            task_time_record.put(time.time())
    data_in_thread=Thread(target=read_input)
    all_threads.append(data_in_thread)
    data_in_thread.start()

    # data-out-thread
    def read_output():
        while True:
            result=myexecutor.modelOutput.get(block=True)                               # block while no output
            output_queue.put((model_name,result,task_time_record.get(),time.time()))
    data_out_thread=Thread(target=read_output)
    all_threads.append(data_out_thread)
    data_out_thread.start()

    # deal-data thread
    deal_thread=Thread(target=myexecutor.RunCycle, args=(run_signal,done_signal))
    all_threads.append(deal_thread)
    deal_thread.start()

    # wait all thread done. In fact, they run with no end.
    print("init %s executor ok."%model_name)
    for mythread in all_threads:
        mythread.join()

    print("process to deal %s exit."%(model_name))

# def RunTest(myexecutor: ModelExecutor, run_signal: Pipe, done_signal: Pipe,test_ok:Pipe,input_shapes:dict,default_batchsize=15)->bool:
#     success=True
#     test_input={}
#     for input_shape in input_shapes:
#         shape=[v if v>=0 else default_batchsize for v in input_shape["shape"]]
#         test_input[input_shape["name"]]=np.array(np.random.randn(*shape),dtype=input_shape["type"])

#     myexecutor.AddRequest(test_input)
#     for _ in myexecutor.modelInferenceSessions:
#         run_signal.send(1)

#     try:
#         myexecutor.modelOutput.get(block=True,timeout=180)
#     except Exception as ex:
#         success=False
#     finally:
#         # clear done_signal created by test
#         count=myexecutor.todo
#         if count==0:
#             count=len(myexecutor.modelInferenceSessions)
#         for _ in range(count):
#             done_signal.recv()

#     if success:
#         test_ok.send(1)
#         test_ok.close()
#         return True
#     else:
#         test_ok.send(0)
#         test_ok.close()
#         return False


import requests
import requests_cache
from lxml import etree
import chaojiying
from requests.auth import HTTPBasicAuth
#下载验证码图片
requests_cache.install_cache()
requests_cache.clear()
headers = {
        'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
                       ' AppleWebKit/537.36 (KHTML, like Gecko)'
                       ' Chrome/109.0.0.0 Safari/537.36'
    }
url='https://www.qceit.org.cn/bos/default.html'
page_text=requests.get(url=url,headers=headers).text
tree=etree.HTML(page_text)
image_code_url='https://www.qceit.org.cn/calldata/calldata.aspx?g=638489f9-51a0-6798-658e-1ba5bad9b727&datatype=getverifycodeimage&d=20232322326'
image_code=requests.get(url=image_code_url,headers=headers).content
with open('codema.jpg','wb') as f:
    f.write(image_code)
chaojiying = chaojiying.Chaojiying_Client('18973022769', 'mark123456', '944571')	#用户中心>>软件ID 生成一个替换 96001
# im = open('img.png', 'rb').read()													#本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
image_code=chaojiying.PostPic(image_code, 1902)
print(image_code)

爬虫/11识别古诗文网页验证码/验证码识别.py


import pandas as pd
import openpyxl
import numpy as np
from openpyxl.utils.dataframe import dataframe_to_rows
from openpyxl.styles import Font
from openpyxl.styles import Alignment
from openpyxl.styles import Color, PatternFill, Font, Border
from openpyxl.styles import colors
import time
import datetime
from datetime import date

name_file_abiturs = 'data/abitur.xlsx'
name_file_person = 'data/person.xlsx'
path_to_end_folder_report = 'data'

df_abitur = pd.read_excel(name_file_abiturs,skiprows=3,usecols=['Абитуриент','Доп. статус','Состояние'])
df_person = pd.read_excel(name_file_person,sheet_name='Абитуриенты',skiprows=8,usecols=['ФИО','Нуждается в общежитии','Формирующее подр.','Направление, специальность, профессия','Сдан оригинал'])

wb = openpyxl.Workbook()
# Переименовываем лист
sheet = wb['Sheet']
sheet.title = 'Отчет'

# Соединяем оба датафрейма

cross_df = df_person.merge(df_abitur,how='inner',left_on='ФИО',right_on='Абитуриент')

# Преобразовываем да-нет в 1 или 0 для подсчетов
cross_df['Нуждается в общежитии'] =cross_df['Нуждается в общежитии'].apply(lambda x:0 if x =='нет' else 1)
cross_df['Сдан оригинал'] =cross_df['Сдан оригинал'].apply(lambda x:0 if x =='нет' else 1)
cross_df['Состояние'] =cross_df['Состояние'].apply(lambda x:1 if x =='Забрал документы' else 0)

# заменяем нан на пустые строки чтобы произвести поиск слова сирота;
cross_df['Доп. статус'].fillna('',inplace=True)
cross_df['Доп. статус'] = cross_df['Доп. статус'].apply(lambda x:1 if 'Сирота;' in x else 0)

cross_df['for_counting'] = 1

svod_df = pd.DataFrame.pivot_table(cross_df,index=['Формирующее подр.','Направление, специальность, профессия'],
                     values=['for_counting','Состояние','Сдан оригинал','Доп. статус','Нуждается в общежитии'],
                     aggfunc='sum')

svod_df.columns = ['Сдали всего','Сирот чел.','Нуждается в общежитии чел.','Сдано оригиналов','Забрали заявления']

svod_df['Итого'] = svod_df['Сдали всего'] - svod_df['Забрали заявления']

# Меняем местами столбцы
out_df = svod_df.reindex(columns=['Сдали всего','Забрали заявления','Итого','Сдано оригиналов','Сирот чел.','Нуждается в общежитии чел.'])

# разворачиваем столбец в строку
sum_row=out_df.sum(axis=0).to_frame().transpose()

# Добавляем колонки чтобы сделать из них мультинидекс .Ужасно решение но что есть то есть
sum_row['1'] ='Всего'
sum_row['2'] = ''

# Делем мультинидекс и объединяем датафреймы
sum_row.set_index(['1','2'],inplace=True)
all_out_df = pd.concat([out_df,sum_row],axis=0)

#Преобразовываем мультинидекс в колонки
finish_df=all_out_df.reset_index()

for r in dataframe_to_rows(finish_df,index =False,header=True):
    if len(r) != 1:
        wb['Отчет'].append(r)

# # Настраиваем выходной файл
wb['Отчет'].column_dimensions['A'].width =30
wb['Отчет'].column_dimensions['B'].width =50
wb['Отчет']['B2'].alignment = Alignment(wrap_text=True)
wb['Отчет'].column_dimensions['C'].width =20
wb['Отчет'].column_dimensions['D'].width =20
wb['Отчет'].column_dimensions['F'].width =20
wb['Отчет'].column_dimensions['G'].width =20
wb['Отчет'].column_dimensions['H'].width =30
wb['Отчет']['H1'].alignment = Alignment(wrap_text=True)

 # Получаем текущее время для того чтобы использовать в названии
t = time.localtime()
current_time = time.strftime('%H_%M_%S', t)
# Сохраняем итоговый файл
wb.save(f'{path_to_end_folder_report}/Ежедневный отчет приемной комиссии ГБПОУ БРИТ {current_time}.xlsx')


import pandas as pd
import re
from datetime import datetime as dt,timedelta
import time

xls = pd.ExcelFile("E:/week5input.xlsx")

temp = []
for sheet in xls.sheet_names:
    df = pd.read_excel(xls,sheet)
    df.insert(0,'Start Date',re.search('.*commencing (.*)', sheet).group(1).strip())
    temp.append(df)
final = pd.concat(temp)

final['Start Date'] = final.apply(lambda x: dt.strptime(re.sub('(?<=[0-9])(?:st|nd|rd|th)','',x['Start Date']), '%d %B %y'),axis=1)
final['Ture Date'] = final.apply(lambda x: x['Start Date']+timedelta(days=time.strptime(x['Date'], '%A').tm_wday),axis=1)

for check in [['Statement?','statement'],['Balance?','balance'],['Complaint','complain']]:
    final[check[0]] = final.apply(lambda x: 1 if bool(re.search(check[1], x['Notes'], re.IGNORECASE)) else 0 ,axis=1)

final['Policy Number'] = final.apply(lambda x: re.search('.*#([0-9]+).*',x['Notes']).group(1).strip() \
     if bool(re.search('#[0-9]+', x['Notes'], re.IGNORECASE)) else None ,axis=1)
final.dropna(inplace=True)
final.drop(['Date','Start Date','Notes'],axis=1,inplace = True)
final.reset_index(drop=True,inplace=True)


import sys, os
sys.path.append(os.pardir)
import numpy as np
from dataset.mnist import load_mnist
import pickle
from sigmoid import sigmoid
from softmax import softmax

def get_data():#mnist data 가져오기
    (x_train, t_train), (x_test, t_test) = \
        load_mnist(normalize=True, flatten=True, one_hot_label=False)
    return x_test, t_test

def init_network(): #pickle 로 저장된 가중치, 편향  불러오기
    with open("sample_weight.pkl", "rb") as f:
        network = pickle.load(f)

    return network

def predict(network, x):
    W1, W2, W3 = network['W1'], network['W2'], network['W3']
    b1, b2, b3 = network['b1'], network['b2'], network['b3']

    a1 = np.dot(x, W1) + b1
    z1 = sigmoid(a1)
    a2 = np.dot(z1, W2) + b2
    z2 = sigmoid(a2)
    a3 = np.dot(z2, W3) + b3

    y = softmax(a3)

x, t = get_data() # mnist data
network = init_network() # 가중치, 편향 세팅

accuracy_cnt = 0

for i in range(len(x)):
    y = predict(network, x[i])
    # 추론을 하면 결과값으로
    # 10개의 출력 값이 나온다.( 0 ~ 9 중 어떤 것에 가까운지 추론)

    p = np.argmax(y) # 확률이 가장 높은 원소의 인덱스를 얻는다.
    if p == t[i]:
        accuracy_cnt += 1

print("Accuracy:" + str(float(accuracy_cnt)/len(x)))


import typer
from artificery.database import Database
from artificery import artificer
from artificery import forge

# --------------------------------------- Constants ----------------------------------------------------------

GEMSTONE = "gemstone"

# --------------------------------------- Pymongo Setup ------------------------------------------------------

db = Database()

# --------------------------------------- Typer Setup --------------------------------------------------------

app = typer.Typer()

# --------------------------------------- Typer Commands -----------------------------------------------------

@app.command("add")
def add_gemstones(
    filename: str,
    debug: bool = typer.Option(False, help="print the results to test if working"),
):
    """
    Add gemstones to database from csv file
    """
    try:
        gem_list = artificer.make_items(filename, GEMSTONE)
        gems = forge.make_gemstones(gem_list)
        if debug:
            for gem in gems:
                typer.echo(gem)
        else:
            db.gemstones.insert_many(gems)
    except Exception as e:
        typer.echo("an exception occured", e)
    else:
        typer.echo(f"added gemstones to database from {filename}")

@app.command("add_one")
def add_gemstone(
    value: str,
    name: str,
    debug: bool = typer.Option(False, help="print the results to test if working"),
):
    """
    add single gemstone to database from command line
    """
    value = int(value)
    try:
        uncut_gem = artificer.make_item(GEMSTONE, name=name, value=value)
        gem = forge.make_gemstone(uncut_gem)
        if debug:
            typer.echo(gem)
        else:
            db.gemstones.insert_one(gem)
    except Exception as e:
        typer.echo("an exception occured", e)
    else:
        typer.echo(f"added {name} gemstone with value {value} to the database")

# --------------------------------------- Program Operations -------------------------------------------------
if __name__ == "__main__":
    # runs the app
    app()


# Create a class containing __init__(), __iter__() and __next__() methods and define them as per need.

class myclass:
    def __init__(self,*x):
        self.list_items = x

    def __iter__(self):
        self.a = 0
        return self

    def __next__(self):
        x = self.list_items[self.a]
        self.a += 1
        return x

# Creating an object for myclass
obj = myclass('apple','banana','kiwi')

# Creating an iterator for obj
myit = iter(obj)

# Iterating using next()
print(next(myit)) # apple
print(next(myit)) # banana
print(next(myit)) # kiwi


from multiprocessing.managers import BaseManager
from dash import html, dcc
from dash.dependencies import Input, Output, State
from datetime import date, datetime, timedelta
import dash_bootstrap_components as dbc
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import calendar
# from globals import *
from app import app

card_icon = {
    'color': 'white',
    'textAlign': 'center',
    'fontSize': 30,
    'margin': 'auto'
}

graph_margin = dict(l=25, r=25, t=25, b=0)

# =========  Layout  =========== #
layout = dbc.Col([
    # Primeira Linha
    dbc.Row([
        # Coluna Saldo Total
        dbc.Col([
            dbc.CardGroup([
                dbc.Card([
                    html.Legend('Saldo'),
                    html.H5('R$ 5000', id='p-saldo-dashboard', style={})
                ], style={'padding-left': '20px', 'padding-top': '10px'}),
                dbc.Card([
                    html.Div(className='fa fa-university', style=card_icon),
                ], color='warning', style={'maxWidth': 75, 'height': 100, 'margin-left': '-10px'})
            ])

        ], width=4),
        # Coluna Receita
        dbc.Col([
            dbc.CardGroup([
                dbc.Card([
                    html.Legend('Receita'),
                    html.H5('R$ 10000', id='p-receita-dashboards', style={})
                ], style={'padding-left': '20px', 'padding-top': '10px'}),
                dbc.Card([
                    html.Div(className='fa fa-smile-o', style=card_icon),
                ], color='success', style={'maxWidth': 75, 'height': 100, 'margin-left': '-10px'})
            ])

        ], width=4),
        # Coluna Despesa
        dbc.Col([
            dbc.CardGroup([
                dbc.Card([
                    html.Legend('Despesa'),
                    html.H5('R$ 5000', id='p-despesa-dashboards', style={})
                ], style={'padding-left': '20px', 'padding-top': '10px'}),
                dbc.Card([
                    html.Div(className='fa fa-meh-o', style=card_icon),
                ], color='danger', style={'maxWidth': 75, 'height': 100, 'margin-left': '-10px'})
            ])

        ], width=4)
    ], style={'margin': '10px'}),

    # Segunda Linha
    dbc.Row([
        dbc.Col([
            dbc.Card([
                html.Legend('Filtar Lançamentos', className='card-title'),
                html.Label('Categorias das Receitas'),
                html.Div(
                    dcc.Dropdown(
                        id='dropdown-receita',
                        clearable=False,
                        style={'width': '100%'},
                        persistence=True,
                        persistence_type='session',
                        multi=True
                    ),
                ),
                html.Label('Categorias das Despesas', style={'margin-top': '10px'}),
                html.Div(
                    dcc.Dropdown(
                        id='dropdown-despesa',
                        clearable=False,
                        style={'width': '100%'},
                        persistence=True,
                        persistence_type='session',
                        multi=True
                    ),
                ),
                html.Legend('Período da Análise', style={'margin-top': '10px'}),
                dcc.DatePickerRange(
                    month_format='Do MMM, YY',
                    end_date_placeholder_text='Data...',
                    start_date=datetime(2022,4,1).date(),
                    end_date=datetime.today() + timedelta(days=31),
                    updatemode='singledate',
                    id='date-picker-config',
                    style={'z-index': '100'}
                )

            ], style={'height': '100%', 'padding': '20px'})

        ],width=4),

        dbc.Col([
            dbc.Card([
                dcc.Graph(id='graph-line-dash', style={'height': '100%', 'padding':'10px'})
            ])
        ], width=8),

    ], style={'margin': '10px'}),

    # Terceira Linha
    dbc.Row([
        dbc.Col([dbc.Card(dcc.Graph(id='graph-time-dash', style={'padding':'10px'}))], width=6),
        dbc.Col([dbc.Card(dcc.Graph(id='graph-pie-dash-receita', style={'padding':'10px'}))], width=3),
        dbc.Col([dbc.Card(dcc.Graph(id='graph-pie-dash-despesa', style={'padding':'10px'}))], width=3),
    ])

])

# =========  Callbacks  =========== #
# RECEITA
@app.callback(
    [
        Output('dropdown-receita', 'options'),
        Output('dropdown-receita', 'value'),
        Output('p-receita-dashboards', 'children')
    ],
    Input('store-receitas', 'data')
)
def populate_dropdownvalues(data):
    df = pd.DataFrame(data)
    valor = df['Valor'].sum()
    val = df.Categoria.unique().tolist()

    return ([{'label': x, 'value': x} for x in val], val, f'R$ {valor}')

# DESPESA
@app.callback(
    [
        Output('dropdown-despesa', 'options'),
        Output('dropdown-despesa', 'value'),
        Output('p-despesa-dashboards', 'children')
    ],
    Input('store-despesas', 'data')
)
def populate_dropdownvalues(data):
    df = pd.DataFrame(data)
    valor = df['Valor'].sum()
    val = df.Categoria.unique().tolist()

    return ([{'label': x, 'value': x} for x in val], val, f'R$ {valor}')

# SALDO
@app.callback(
    Output('p-saldo-dashboard', 'children'),
    [
        Input('store-despesas', 'data'),
        Input('store-receitas', 'data')
    ]
)
def saldo_total(despesas, receitas):
    df_despesa = pd.DataFrame(despesas)
    df_receita = pd.DataFrame(receitas)

    valor = df_receita['Valor'].sum() - df_despesa['Valor'].sum()

    return f'R$ {valor}'

# PRIMEIRA FIGURA
@app.callback(
    Output('graph-line-dash', 'figure'),
    [
        Input('store-despesas', 'data'),
        Input('store-receitas', 'data'),
        Input('dropdown-despesa', 'value'),
        Input('dropdown-receita', 'value'),

    ]
)
def update_graph_line(data_despesa, data_receitas, cat_despesa, cat_receita):

    df_despesas = pd.DataFrame(data_despesa).set_index('Data')[['Valor']]
    df_receitas = pd.DataFrame(data_receitas).set_index('Data')[['Valor']]

    df_ds = df_despesas.groupby('Data').sum().rename(columns={'Valor': 'Despesas'})
    df_rc = df_receitas.groupby('Data').sum().rename(columns={'Valor': 'Receitas'})

    df_acum = df_ds.join(df_rc, how='outer').fillna(0)
    df_acum['Acumulo'] = df_acum['Receitas'] - df_acum['Despesas']
    df_acum['Acumulo'] = df_acum['Acumulo'].cumsum()

    fig = go.Figure()
    fig.add_trace(go.Scatter(name='Fluxo de Caixa', x = df_acum.index,
    y = df_acum['Acumulo'], mode='lines'))

    fig.update_layout(margin=graph_margin, height=400)
    fig.update_layout(paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(0,0,0,0)')

    return fig

@app.callback(
    Output('graph-time-dash', 'figure'),
    [
        Input('store-receitas', 'data'),
        Input('store-despesas', 'data'),
        Input('dropdown-receita', 'value'),
        Input('dropdown-despesa', 'value'),
        Input('date-picker-config', 'start_date'),
        Input('date-picker-config', 'end_date')
    ]
)
def update_graph_time(data_receita, data_despesa, drop_receita, drop_despesa, start_date, end_date):

    df_receita = pd.DataFrame(data_receita)
    df_despesa = pd.DataFrame(data_despesa)

    df_receita['Output'] = 'Receita'
    df_despesa['Output'] = 'Despesa'

    df_final = pd.concat([df_receita, df_despesa])
    df_final['Data'] = pd.to_datetime(df_final['Data'])

    start_date = pd.to_datetime(start_date)
    end_date = pd.to_datetime(end_date)

    df_final = df_final[(df_final['Data'] >= start_date) & (df_final['Data'] <= end_date)]
    df_final = df_final[(df_final['Categoria'].isin(drop_receita)) | (df_final['Categoria'].isin(drop_despesa))]

    fig = px.bar(df_final, x='Data', y='Valor', color='Output', barmode='group')

    fig.update_layout(margin=graph_margin, height=350)
    fig.update_layout(paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(0,0,0,0)')

    return fig

@app.callback(
    Output('graph-pie-dash-receita','figure'),
    [
        Input('store-receitas','data'),
        Input('dropdown-receita', 'value')
    ]
)
def update_pie_receita(data_receita, drop_receita):

    df = pd.DataFrame(data_receita)
    df = df[df['Categoria'].isin(drop_receita)]

    fig = px.pie(df, values=df.Valor, names=df.Categoria, hole=.2)
    fig.update_layout(title={'text': 'Receitas'})
    fig.update_layout(margin = graph_margin, height=350)
    fig.update_layout(paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(0,0,0,0)')

    return fig

@app.callback(
    Output('graph-pie-dash-despesa','figure'),
    [
        Input('store-despesas','data'),
        Input('dropdown-despesa', 'value')
    ]
)
def update_pie_receita(data_despesa, drop_despesa):

    df = pd.DataFrame(data_despesa)
    df = df[df['Categoria'].isin(drop_despesa)]

    fig = px.pie(df, values=df.Valor, names=df.Categoria, hole=.2)
    fig.update_layout(title={'text': 'despesas'})
    fig.update_layout(margin = graph_margin, height=350)
    fig.update_layout(paper_bgcolor='rgba(0,0,0,0)', plot_bgcolor='rgba(0,0,0,0)')

    return fig


import sys

def main():
    input = sys.stdin.readline
    L, H = map(int, input().split())
    N = int(input())
    for _ in range(N):
        A = int(input())
        if L <= A <= H:
            print(0)
        elif A < L:
            print(L - A)
        else:
            print(-1)

if __name__ == '__main__':
    main()