"""
A simple python script to demonstrate a Markov Decision Process problem
"""

import numpy as np

# Markov state, probability and reward definitions for the random walk problem
rw_states = ['-3', '-2', '-1', '0', '1', '2', '3']

rw_policy = {
    '-2': np.array([['l', 'r'],
                   [ 0.5, 0.5]]),
    '-1': np.array([['l', 'r'],
                   [ 0.5, 0.5]]),
    '0': np.array([['l', 'r'],
                   [ 0.5, 0.5]]),
    '1': np.array([['l', 'r'],
                   [ 0.5, 0.5]]),
    '2': np.array([['l', 'r'],
                   [ 0.5, 0.5]])
}

# NB: State-action pairs with no transition probabilities are assumed terminal
rw_state_action_probs = {
    '-2': {
        'l' : np.array([['-3'], [1]]),
        'r' : np.array([['-1'], [1]])
    },
    '-1': {
        'l' : np.array([['-2'], [1]]),
        'r' : np.array([['0'], [1]])
    },
    '0': {
        'l' : np.array([['-1'], [1]]),
        'r' : np.array([['1'], [1]])
    },
    '1': {
        'l' : np.array([['0'], [1]]),
        'r' : np.array([['2'], [1]])
    },
    '2': {
        'l' : np.array([['1'], [1]]),
        'r' : np.array([['3'], [1]])
    }
}

# NB: States with no transition probabilities are assumed terminal
rw_state_probs = {
    '-2': np.array([['-3', '-1'],
                   [ 0.5,  0.5]]),
    '-1': np.array([['-2', '0'],
                   [ 0.5,  0.5]]),
    '0': np.array([['-1', '1'],
                   [ 0.5,  0.5]]),
    '1': np.array([['0', '2'],
                   [ 0.5,  0.5]]),
    '2': np.array([['1', '3'],
                   [ 0.5,  0.5]])
}

# NB: States with no defined reward are assumed to have 0 reward
rw_rewards = {
    '-3': -1,
    '3': 1
}

"""
A simple Markov Process class
"""
class MarkovProcess(object):

    """
    Constructor
    """
    def __init__(self, states, state_transition_probabilities, initial_state=None):

        # Store states
        self.states = states

        # Store transition probabilities
        self.state_transition_probabilities = state_transition_probabilities

        # Store initial and current state
        self.initial_state = self.states[0]
        if initial_state is not None:
            self.initial_state = initial_state
        self.state = self.initial_state

        # Initialize number of steps taken
        self.num_steps = 0

        # Compute list of terminal states
        self.terminal_states = []
        for state in self.states:
            if state in self.state_transition_probabilities.keys():
                p = self.state_transition_probabilities[state]
                if len(p[0]) == 1 and p[0][0] == state:
                    # If a state transitions only to itself, it is a terminal state
                    self.terminal_states.append(state)
            else:
                # If a state has no probabilities defined, treat it as terminal
                self.terminal_states.append(state)

    def __repr__(self):
        states = '(' + ', '.join(self.states) + ')'
        probs = "(...)"

        return "Markov Process with S={}, Ps={}, S_t={:+d}".format(
            states,
            probs,
            int(self.state)
        )

    """
    Steps us to a new state
    """
    def step(self):

        # Transition to new state
        tmp = self.state_transition_probabilities[self.state]
        self.state = np.random.choice(a=list(tmp[0]), p=list(tmp[1]))

        # Increment step counter
        self.num_steps += 1

    """
    Lazy check to see if the process has reached a terminal state yet
    """
    def is_terminated(self):
        return self.state in self.terminal_states

    """
    Performs a full rollout of the given Markov Process
    """
    def rollout(self, verbose=False):
        # Perform a full rollout of the Markov Process
        while not self.is_terminated():
            self.step()
            if verbose is True:
                print(self)

    """
    Resets the Markov Process to it's initial state and return
    """
    def reset(self):
        self.state = self.initial_state
        self.num_steps = 0

"""
A simple Markov Reward Process class

TODO ajs 24/Jan/2017 Add ability to do future return estimation using discount
"""
class MarkovRewardProcess(MarkovProcess):

    """
    Constructor
    """
    def __init__(self, states, state_transition_probabilities, rewards=None, discount=1, initial_state=None):

        # Call Markov Process constructor
        super().__init__(states, state_transition_probabilities, initial_state)

        # Store states
        self.rewards = rewards

        # Store discount
        self.discount = discount

        # Initialize return
        self._return = 0

    def __repr__(self):
        states = '(' + ', '.join(self.states) + ')'
        probs = "(...)"
        rewards = "(...)"

        return "Markov Reward Process with S={}, Psa={}, R={}, g={:.2f}, S_t={:+d}, G_t={:+d}".format(
            states,
            probs,
            rewards,
            self.discount,
            int(self.state),
            self._return
        )

    """
    Steps us to a new state
    """
    def step(self):

        # Call Markov Process step
        super().step()

        # Check if we got any reward
        if self.rewards is not None:
            self._return += self.rewards.get(self.state, 0)

    """
    Resets the Markov Process to it's initial state and return
    """
    def reset(self):

        # Call Markov Process reset
        super().reset()

        self._return = 0

"""
A simple Markov Decision Process class
"""
class MarkovDecisionProcess(MarkovRewardProcess):

    """
    Constructor
    """
    def __init__(
        self,
        states,
        policy,
        state_action_transition_probabilities,
        rewards=None,
        discount=1,
        initial_state=None
        ):

        # TODO ajs 24/Jan/2017 Unroll the decision process under the given policy into a
        # reward process, then call Markov Reward Process constructor
        #super().__init__(states, transition_probabilities, initial_state)
        pass

    def __repr__(self):
        states = '(' + ', '.join(self.states) + ')'
        policy = "(...)"
        probs = "(...)"
        rewards = "(...)"

        return "Markov Decision Process with S={}, pi={}, Psa={}, R={}, g={:.2f}, S_t={:+d}, G_t={:+d}".format(
            states,
            policy,
            probs,
            rewards,
            self.discount,
            int(self.state),
            self._return
        )

    """
    Steps us to a new state
    """
    def step(self):
        # TODO: Sample policy, then let environment go
        pass

"""
Main function
"""
def main():

    #mp = MarkovProcess(rw_states, rw_state_probs, initial_state='0')
    mp = MarkovRewardProcess(rw_states, rw_state_probs, rewards=rw_rewards, initial_state='0')

    average_num_steps = 0
    average_return = 0
    N = 10
    for i in range(N):
        mp.reset()
        mp.rollout(True)
        average_num_steps += mp.num_steps
        average_return += mp._return

    average_num_steps /= N
    average_return /= N

    print(average_num_steps)
    print(average_return)

if __name__ == "__main__":
    main()


from botocore.exceptions import ClientError

from .aws_helper import create_s3_session
from .config import create_config

default_bucket_name = "no-name-bucket"
region = "ru-central1"
endpoint_url = "https://storage.yandexcloud.net"

def initialize():
    bucket_name = default_bucket_name
    aws_access_key_id = input("input key id: ")
    aws_secret_access_key = input("input secret key: ")
    bucket_name = input("bucket name: ")

    try:
        s3 = create_s3_session(aws_access_key_id, aws_secret_access_key, endpoint_url, region)
        s3.create_bucket(Bucket=bucket_name, ACL='public-read-write')
    except ClientError as clientError:
        if clientError.response["Error"]["Code"] != "BucketAlreadyOwnedByYou":
            raise clientError

    create_config(access_key=aws_access_key_id, secret_key=aws_secret_access_key, bucket_name=bucket_name)


"""
This file includes functions for manipulating and accessing the Appointment table in the MAPSDB database.
This table keeps track of the appointments that patients make with doctors
"""

from flask import Flask, request
import MapsDB

app = Flask(__name__)
db, ma = MapsDB.connect_db(app)

class Appointment(db.Model):
    """
    This class defines the data inside the appointment table. Columns include:
    id as a primary key, start_datetime, doctor_id, patient_id and medical_note.
    """
    id = db.Column(db.Integer, db.Sequence(
        'seq_reg_id', start=1, increment=1), primary_key=True)
    start_datetime = db.Column(db.DATETIME, unique=False)
    doctor_id = db.Column(db.Integer, unique=False)
    patient_id = db.Column(db.Integer, unique=False)
    medical_note = db.Column(db.String(400), unique=False, nullable=True)

    def __init__(self, start_datetime, doctor_id, patient_id, medical_note):
        """
        This function initialises the Appointment table.
        """
        self.start_datetime = start_datetime
        self.doctor_id = doctor_id
        self.patient_id = patient_id
        self.medical_note = medical_note

class AppointmentSchema(ma.Schema):
    """
    This function describes the schema for the Appointment table.
    """
    class Meta:
        # Fields to expose
        fields = ('id', 'start_datetime', 'doctor_id',
                  'patient_id', 'medical_note')

appointment_schema = AppointmentSchema()
appointments_schema = AppointmentSchema(many=True)

def add_appointment():
    """
    This function does not accept any parameters and will add an appointment to the Appoinments table
    getting the appointment information from a HTML form.
    """
    app = request.form['app_slot']
    start_datetime = app.split(',')[1]
    start_datetime = start_datetime.split('+')[0]
    doctor_id = app.split(',')[3]
    patient_id = app.split(',')[4]

    new_appointment = Appointment(start_datetime, doctor_id, patient_id, None)

    db.session.add(new_appointment)
    db.session.flush()
    db.session.commit()

def remove_appointment(start_datetime, doctor_id):
    """
    This function accepts an appointment id and will delete the appointment from the Appointment table.
    """
    appointment = Appointment.query.filter(Appointment.start_datetime.like(start_datetime),
                                            Appointment.doctor_id.like(doctor_id)
                                            ).first()
    db.session.delete(appointment)
    db.session.commit()

def add_medical_note(app_dt, doctor_id, medical_note):
    """
    This function accepts an appointment date, doctor id and a medical note and will add the supplied
    medical note the the Appointment table, using the appointment data and doctor id as keys to
    query the Appointment table.
    """

    app = Appointment.query.filter(Appointment.doctor_id.like(
        doctor_id), Appointment.start_datetime.like(app_dt)).first()
    app.medical_note = medical_note
    db.session.commit()


from zoo.evaluation.metrics.utils import (
    get_scenario_agents_data,
    save_evaluation_result_to_json,
)

class KinematicsEvaluation:
    def __init__(self, scenarios_data_path, result_path, agent_name_list):
        self.scenarios_data_path = scenarios_data_path
        self.result_path = result_path
        self.agent_name_list = agent_name_list

    def run_evaluation(self):
        scenario_agents_data = get_scenario_agents_data(
            self.scenarios_data_path, self.agent_name_list
        )
        kinematics_score = {}
        for scenario_name, data in scenario_agents_data.items():
            agent_result = self.kinematics_result(data)
            kinematics_score[scenario_name] = agent_result
        save_evaluation_result_to_json(kinematics_score, self.result_path, "kinematics")

    def kinematics_result(self, json_curve_list):
        pos_curve_list = []
        speed_curve_list = []
        max_acceleration = 8
        max_speed = 34
        for json_data in json_curve_list:
            pos_curve_list.append(json_data["cartesian_pos_list"])
            speed_curve_list.append(json_data["speed_list"])
        kinematics_result_dict = {}
        for index, agent_name in enumerate(self.agent_name_list):
            agent_speed = speed_curve_list[index]
            agent_acceleration = [
                (agent_speed[i + 1] - agent_speed[i]) / 0.1
                for i in range(len(agent_speed) - 1)
            ]
            temp_result_list = []
            for step, speed in enumerate(agent_speed):
                if step == 0 and abs(speed) > max_speed:
                    temp_result_list.append(step)
                elif step != 0 and abs(speed) > max_speed >= abs(agent_speed[step - 1]):
                    temp_result_list.append(step)
            for step, accel in enumerate(agent_acceleration):
                if step == 0 and abs(accel) > max_acceleration:
                    temp_result_list.append(step)
                elif step != 0 and abs(accel) > max_acceleration >= abs(
                    agent_acceleration[step - 1]
                ):
                    temp_result_list.append(step)
            kinematics_result_dict[agent_name] = sorted(list(set(temp_result_list)))
        return kinematics_result_dict


import pathlib

def injector_dual_wag(well, group, operate, monitor, geometry, perf
        , completion, open_time, on_time, wag_start, layerclump, output_folder):

    from well.scripts.frames.inje_dual_wag import Inje_Dual_Wag

    w = Inje_Dual_Wag(well, group)

    w.get_incomp('W','*WATER')
    w.get_incomp('G','*GAS')

    for ope in operate:
        w.get_operate(*ope)

    for mon in monitor:
        w.get_monitor(*mon)

    w.get_geometry(*geometry)
    w.get_perf(perf)

    for com in (completion):
        w.get_completion(com)

    if on_time:
        w.get_on_time(on_time)

    if open_time:
        w.get_open(*open_time)

    for lay in layerclump:
        w.get_layerclump(lay)

    if wag_start:
        w.get_wag(*wag_start)

    w.build()
    w.write(pathlib.Path(output_folder) / '{}.inc'.format(well))


from flask import Flask, request, jsonify, render_template
import os
import dialogflow
import requests
import json
import time
#import pusher
from twilio.twiml.messaging_response import MessagingResponse

app = Flask(__name__)

# initialize Pusher

@app.route('/')
def index():
    return render_template('index.html')

@app.route("/receive_message", methods=['GET', 'POST'])
def message_reply():
    print (request.values)
    body = request.values.get('Body', None) #https://www.twilio.com/docs/sms/tutorials/how-to-receive-and-reply-python
    project_id = os.getenv('DIALOGFLOW_PROJECT_ID')
    session_id = request.values.get('From', None)
    fulfillment_text = detect_intent_texts(project_id, session_id, body, 'en')

    resp = MessagingResponse()

    resp.message(fulfillment_text)
    time.sleep(0.5)
    return str(resp)

@app.route('/get_movie_detail', methods=['POST'])
def get_movie_detail():
    data = request.get_json(silent=True)

    try:
        movie = data['queryResult']['parameters']['movie']
        api_key = os.getenv('OMDB_API_KEY')

        movie_detail = requests.get('http://www.omdbapi.com/?t={0}&apikey={1}'.format(movie, api_key)).content
        movie_detail = json.loads(movie_detail)

        response =  """
            Title : {0}
            Released: {1}
            Actors: {2}
            Plot: {3}
        """.format(movie_detail['Title'], movie_detail['Released'], movie_detail['Actors'], movie_detail['Plot'])
    except:
        response = "Could not get movie detail at the moment, please try again"

    reply = { "fulfillmentText": response }

    return jsonify(reply)

def detect_intent_texts(project_id, session_id, text, language_code):
    session_client = dialogflow.SessionsClient()
    session = session_client.session_path(project_id, session_id)

    if text:
        text_input = dialogflow.types.TextInput(
            text=text, language_code=language_code)
        query_input = dialogflow.types.QueryInput(text=text_input)
        response = session_client.detect_intent(
            session=session, query_input=query_input)

        return response.query_result.fulfillment_text

@app.route('/send_message', methods=['POST'])
def send_message():
    try:
        socketId = request.form['socketId']
    except KeyError:
        socketId = ''

    message = request.form['message']
    project_id = os.getenv('DIALOGFLOW_PROJECT_ID')
    fulfillment_text = detect_intent_texts(project_id, "unique", message, 'en')
    response_text = { "message":  fulfillment_text }

    return jsonify(response_text)

# run Flask app
if __name__ == '__main__':
    app.run(host="0.0.0.0", port=8000, debug=True, threaded=True)


import os

dir_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
f = open(dir_path + "\\list\\result.txt", "r")
count = 0

content = f.read()
content = content.split('\n')
#content.sort()

for x in content:
  count += 1
  print('{0}: {1}'.format(count,x))
  if ( count < 0):
    continue

  os.system('start chrome "https://www.google.com/search?q='+ x.replace(' ','+') +'&tbm=isch"')
  input("Press Enter to continue...")

f.close()
print('Done')

print('\nHere is the result.')
dir_path = r"C:\Users\tlchong\Downloads"

list = os.listdir(dir_path)
count = 0
for x in list:
	count += 1
	f = open(os.path.sep.join([dir_path,x]), "r")
	content = f.read()
	content = content.split('\n')

	lvl = 'S'
	if (len(content) > 200):
		lvl = 'M'
	if (len(content) > 500):
		lvl = 'L'
	print("{}. {} - {} - {}".format(count, lvl, len(content), x[:-4]))
	f.close()

input("\npress enter to continue...")


#!/usr/bin/env python
# encoding:utf8

"""
    unittests
    =========

    :created: 2015 by Jens Diemer - www.jensdiemer.de
    :copyleft: 2015 by the DragonPy team, see AUTHORS for more details.
    :license: GNU GPL v3 or above, see LICENSE for more details.
"""

from __future__ import absolute_import, division, print_function

import unittest
from pygments.token import Token

from dragonlib.dragon32.pygments_lexer import BasicLexer

class TestHighlighting(unittest.TestCase):
    def test(self):
        lexer = BasicLexer()
        listing='10 PRINT"FOO"'
        tokensource = lexer.get_tokens(listing)
        self.assertEqual(list(tokensource), [
            (Token.Name.Label, '10'),
            (Token.Text, ' '),
            (Token.Name.Builtin, 'PRINT'),
            (Token.Literal.String, '"FOO"'),
            (Token.Text, '\n')
        ])


#brute-forces IKE group names
#github.com/the-jcksn

import os
import argparse
from subprocess import PIPE, Popen
from termcolor import colored

def cmdline(command):
        process = Popen(
                args=command,
                stdout=PIPE,
                shell=True
        )
        return process.communicate()[0]

parser = argparse.ArgumentParser()
parser.add_argument('-w', default='none', dest='wordlist', help='Provide a wordlist of Group Names for bruteforcing',type=str)
parser.add_argument('-ip', default='none', dest='ip', help='Provide target IP address',type=str)
args = parser.parse_args()

if not 'SUDO_UID' in os.environ.keys():
        print('This script requires super user privs, please run again with sudo')
        quit()
if args.ip == 'none':
        print('No target IP specified with -\'ip\' - Quitting')
        quit()
if args.wordlist == 'none':
        print('No wordlist provided with \'-w\' - Quitting')
        quit()

valid = []
attempt = 'empty'
ikescan = 'sudo ike-scan -M -A -n ' + attempt + ' ' + args.ip
wordlist = args.wordlist

wl_length = 0
with open(wordlist, 'r') as lengthcheck:
        for line in lengthcheck:
                wl_length += 1
print('\nStarting Ike-Brute...')
print('\nChecking',wl_length,'group names against',args.ip,':\n')

with open(wordlist, 'r') as input:
        for line in input:
                line = line[:-1]
                attempt = line
                output = cmdline(ikescan)
                string = '1 returned notify'
                string2 = '1 returned handshake'
                output_text = str(output)
                if string in output_text:
                        print(line, colored(' - failed','red'))
                if string2 in output_text:
                        print(colored(line,'green'),colored('- success!','green'))
                        valid.append(line)
if len(valid) < 1:
        print('\nNo valid Group names found')
else:
        print(colored('\nValid Group Names:','green'))
        print('------------------')
        for line in valid:
                print(line)


txt = "hello, and welcome to my world"
txt1 = "For only {price:.2f} dollars!"
txt2= "H\te\tl\tl\to"
txt3= "50800"
a1 = txt.capitalize()
a2= txt.casefold()
a3= txt.center(20)
a4= txt.count(txt)
a5= txt.encode()
a6= txt.endswith(".")
a7 =txt2.expandtabs(2)
a8= txt.find(txt)
a9 = txt.index(txt)
a10= txt.isalnum()
a11= txt.isalpha()
a12= txt.isascii()
a13= txt3.isdecimal()
a14= txt3.isdigit()
a15= txt.isidentifier()
a16= txt.islower()
a17= txt.upper()
a18= txt.rfind("world")
a19= txt.startswith("Hello")
a20= txt.splitlines()
a21= txt.replace("world", "place")
a22= txt.title()
print (a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21,a22)
print(txt1.format(price = 49))


'''
Created on 12.04.2018

@author: hm
'''

import re
import traceback
import os.path
import tempfile
import posix
import fnmatch

import base.FileHelper
import base.MemoryLogger

class UnitTestAppInfo:
    def __init__(self, usage):
        self._usage = usage

class UnitTestCase:
    def __init__(self):
        self._userId = posix.getuid()
        self._isRoot = self._userId == 0
        self._inTestSuite = False
        self._errors = 0
        self._summary = None
        self._name = type(self).__name__
        self._asserts = 0
        self._application = UnitTestAppInfo(self.usage)
        self._logger = base.MemoryLogger.MemoryLogger(True)
        self._silentLogger = base.MemoryLogger.MemoryLogger(False)
        base.FileHelper.setLogger(self._logger)

    def _finish(self):
        '''The last action of a test case.
        Should be overwritten by sub classes
        '''

    def _describeDifference(self, str1, str2, prefix = ''):
        '''Logs the difference of two strings.
        @param str1: first string to compare
        @param str2: second string to compare
        @param prefix: '' or a prefix
        @return: a string describing the difference
        '''
        count1 = len(str1)
        count2 = len(str2)
        ixDiff = -1
        for ix in range(min(count1, count2)):
            if str1[ix] != str2[ix]:
                ixDiff = ix
                break
        if ixDiff != -1:
            rc = '+++ {:s}different at pos {:d}: {:s}/{:s}\n'.format(prefix, ixDiff + 1, str1[ixDiff:ixDiff+5], str2[ixDiff:ixDiff+5])
        elif count1 > count2:
            rc = '+++ expected is longer:\n'
        else:
            rc = '+++ expected is shorter:\n'
        rc += str1 + '\n' + str2
        if ixDiff != -1:
            rc += '\n' + ('=' * ixDiff) + '^'
        return rc

    def assertDirExists(self, current):
        '''Tests whether a given directory exists.
        @param current: the directory to inspect
        @returns: True: the directory exists
        '''
        self._asserts += 1
        rc = True
        if not os.path.exists(current):
            rc = self.error('directory does not exist: ' + current)
        elif not os.path.isdir(current):
            rc = self.error('file {} exists but it is not a directory'.format(current))
        return rc

    def assertIsEqual(self, expected, current):
        '''Tests whether two values are equal.
        @param expected: the expected value
        @param current: the value to test
        @return: True: values are equal
        '''
        rc = False
        self._asserts += 1
        if type(expected) != type(current):
            self.error('different types: ' + str(type(expected)) + ' / ' + str(type(current)))
        else:
            rc = expected == current
            if not rc:
                if type(expected) is str:
                    if expected.find('\n') < 0:
                        self.error(self._describeDifference(expected, current))
                    else:
                        listExpected = expected.splitlines()
                        listCurrent = current.splitlines()
                        ixDiff = -1
                        for ix in range(min(len(listExpected), len(listCurrent))):
                            if listExpected[ix] != listCurrent[ix]:
                                self.error(self._describeDifference(listExpected[ix], listCurrent[ix], 'in line {:d}: '.format(ix + 1)))
                                ixDiff = ix
                                break
                        if ixDiff == -1 and len(listExpected) != len(listCurrent):
                            if len(listExpected) < len(listCurrent):
                                msg = 'expected has fewer lines: {:d}/{:d}\n'.format(len(listExpected), len(listCurrent)) + listCurrent[len(listExpected)]
                            else:
                                msg = 'expected has more lines: {:d}/{:d}\n'.format(len(listExpected), len(listCurrent)) + listExpected[len(listCurrent)]
                            self.error(msg)
                elif type(expected) is int:
                    self.error('different: {:d}/{:d} [0x{:x}/0x{:x}]'.format(expected, current, expected, current))
                else:
                    self.error('different: {:s} / {:s}'.format(str(expected), str(current)))
        return rc

    def assertFalse(self, current):
        '''Tests whether a value is False.
        @param current: value to test
        @returns: True: the value is False
        '''
        rc = True
        self._asserts += 1
        if current != False:
            rc = self.error('+++ False expected, found: ' + str(current))
        return rc

    def assertFileContent(self, expectedContent, currentFile):
        '''Compares a given string with an expected file content.
        @param currentFile: the name of the file with the expected content, e.g. 'data/abc.xyz'
        @param expectedContent: the content to compare
        @returns: True: file content is the expected
        '''
        rc = True
        self._asserts += 1
        full = currentFile if currentFile.startswith(os.sep) else os.path.dirname(__file__) + os.sep + currentFile
        if not os.path.exists(full):
            rc =self.error('missing file: ' + currentFile)
        else:
            with open(full, 'r') as fp:
                current = fp.read()
            rc = self.assertIsEqual(expectedContent, current)
            if not rc:
                tempFile = self.tempFile(os.path.basename(currentFile))
                with open(tempFile, "w") as fp:
                    fp.write(expectedContent)
                self.log('meld {} {}'.format(tempFile, full))
        return rc

    def assertFileContains(self, expected, currentFile):
        '''Tests whether a given file contains a given content.
        @param expected: content to search. May be a string or a re.RegExpr instance
        @param currentFile: file to inspect
        @returns: True: the file contains the expected string
        '''
        rc = True
        self._asserts += 1
        if not os.path.isfile(currentFile):
            rc = self.error('missing file ' + currentFile)
        else:
            found = False
            with open(currentFile) as fp:
                # lineNo = 0
                for line in fp:
                    #lineNo += 1
                    #if lineNo == 126:
                    #    lineNo = 126
                    if type(expected) == str:
                        if line.find(expected) >= 0:
                            found = True
                            break
                    else:
                        if expected.search(line) is not None:
                            found = True
                            break
            if not found:
                text = expected if type(expected) == str else expected.pattern
                rc = self.error('missing content {:s} in {:s}'.format(text[0:20], currentFile))
        return rc

    def assertFileExists(self, filename):
        '''Tests whether a given file exists.
        @param filename: the name of the file to test
        @returns: True: the file exists
        '''
        rc = True
        self._asserts += 1
        if not os.path.exists(filename):
            rc = self.error('file does not exist: ' + filename)
        return rc

    def assertFileNotContains(self, unexpectedContent, currentFile):
        '''Tests whether a given file contains not a given content.
        @param unexpectedContent: content to search
        @param currentFile: file to inspect
        '''
        rc = True
        self._asserts += 1
        if not os.path.isfile(currentFile):
            rc = self.error('missing file ' + currentFile)
        else:
            found = False
            with open(currentFile) as fp:
                for line in fp:
                    if line.find(unexpectedContent) >= 0:
                        found = True
                        break
            if found:
                rc = self.error('unexpected content {:s} in {:s}'.format(unexpectedContent[0:20], currentFile))
        return rc

    def assertFileNotExists(self, filename):
        '''Tests whether a given file does not exist.
        @param filename: the name of the file to test
        @returns: True: the file does not exist
        '''
        rc = True
        self._asserts += 1
        if os.path.exists(filename):
            rc = self.error('file exists: ' + filename)
        return rc

    def assertMatches(self, expectedRegExpr, current, flags=0):
        '''Tests whether a string matches a given regular expression.
        @param expectedRegExpr: regular expression
        @param current: string to test
        @param flags: flags for re.match, e.g. re.IGNORECASE or re.MULTILINE
        @returns: True: the value matches the expected
        '''
        rc = True
        self._asserts += 1
        if not re.search(expectedRegExpr, current, flags):
            rc = self.error('+++ does not match\n' + expectedRegExpr + '\n' + current)
        return rc

    def assertNodeExists(self, path, nodePattern):
        '''Tests whether at least one file exists in a given path with a given node pattern.
        @param path: the directory to inspect
        @param nodePattern: a pattern with unix wildcards to inspect
        @return True: node found
        '''
        nodes = os.listdir(path)
        found = False
        for node in nodes:
            if fnmatch.fnmatch(node, nodePattern):
                found = True
                break
        if not found:
            self.error('node {} not found in {}'.format(nodePattern, path))
        return found

    def assertNone(self, current):
        '''Tests whether a value is None.
        @param current: value to test
        @returns: True: the value is None
        '''
        rc = True
        self._asserts += 1
        if current is not None:
            rc = self.error('+++ None expected, found: ' + str(current))
        return rc

    def assertNotNone(self, current):
        '''Tests whether a value is not None.
        @param current: value to test
        @returns: True: the value is not None
        '''
        self._asserts += 1
        if current is None:
            self.error('+++ unexpected None found')

    def assertTrue(self, current):
        '''Tests whether a value is True.
        @param current: value to test
        @returns: True: the value is True
        '''
        rc = True
        self._asserts += 1
        if current != True:
            rc = self.error('+++ True expected, found: ' + str(current))
        return rc

    def clearDirectory(self, path):
        '''Removes all files and subdirs in a given directory.
        @param path: name of the directory
        '''
        base.FileHelper.clearDirectory(path)

    def ensureDirectory(self, directory):
        '''Ensures that the given directory exists.
        @param directory: the complete name
        @return: None: could not create the directory
            otherwise: the directory's name
        '''
        rc = base.FileHelper.ensureDirectory(directory)
        return rc

    def ensureFileDoesNotExist(self, filename):
        '''Ensures that a file does not exist.
        @param filename: the file to delete if it exists.
        '''
        base.FileHelper.ensureFileDoesNotExist(filename)

    def error(self, message):
        '''Displays an error with backtrace.
        @param message: error message
        @returns: False
        '''
        self._errors += 1
        print(message)
        info = traceback.extract_stack()
        # ignore runtime methods:
        while len(info) > 1 and (info[0].filename.find('/pysrc/') > 0 or info[0]._line.startswith('tester.run()')
                or info[0].filename.find('UnitTestCase.py') > 0 and info[0]._line.startswith('method()')):
            del info[0]
        # ignore UnitTest internals:
        length = len(info)
        while length > 1 and info[length - 1].filename.find('UnitTestCase.py') > 0:
            del info[length - 1]
            length -= 1
        for entry in info:
            print('{:s}:{:d} {:s}'.format(entry.filename, entry.lineno, entry.line))
        return False

    def getSummary(self):
        '''Return the summary message.
        @return the summary message
        '''
        return self._summary

    def log(self, message):
        '''Displays a message
        @param message: message
        '''
        print(message)

    def run(self):
        '''Searches the methods starting with 'test' and call them.
        '''
        for item in self.__dir__():
            if item.startswith('test'):
                method = getattr(self, item)
                print('= ' + item)
                method()
        self._summary = '=== unit {:s}: {:d} assert(s) with {:d} error(s)'.format(self._name, self._asserts, self._errors)
        print(self._summary)
        self._finish()

    def setInTestSuite(self, value):
        '''Sets the attribute.
        @param value: the new value of _inTestSuite
        '''
        self._inTestSuite = value

    def tempDir(self, node, subdir = None):
        '''Builds the name of a directory and ensures that the directory exists.
        @param node: the directory's name (without path)
        @param subdir: None or the name of a directory inside the temp dir
        @return: None: cannot create directory
                otherwise: the name of an existing temporary directory (with path)
        '''
        rc = tempfile.gettempdir()
        if subdir is not None:
            rc += os.sep + subdir
        if node is not None:
            if rc[-1] != os.sep:
                rc += os.sep
            rc += node
        rc = self.ensureDirectory(rc)
        return rc

    def tempFile(self, node, subdir = None, subdir2 = None):
        '''Builds the name of a temporary file and ensures that the parent directories exist.
        @param node: the file's name (without path)
        @param subdir: None or the name of a directory inside the temp dir
        @param subdir2: None or the name of a directory inside subdir
        @return: the name of a temporary file (with path), e.g. /tmp/subdir/subdir2/node
        '''
        rc = tempfile.gettempdir() + os.sep
        if subdir is not None:
            rc += subdir
            rc = self.ensureDirectory(rc)
            if subdir[-1] != os.sep:
                rc += os.sep
            if subdir2 is not None:
                rc += subdir2
                self.ensureDirectory(rc)
                if subdir2[-1] != os.sep:
                    rc += os.sep
        rc += node
        return rc

    def usage(self, message):
        print(message)
        self.assertFalse(True)

    def xtestMyself(self):
        self.assertIsEqual(3, 4)
        self.assertIsEqual('Hello', 'Hallo')
        self.assertIsEqual('abc\nhallo', 'abc\nhello')
        self.assertNone('not none')
        self.assertNotNone(None)

if __name__ == '__main__':
    tester = UnitTestCase()
    tester.run()


# -*- coding: utf-8 -*-
"""
基于twisted实现的HTTPS代理服务; 支持HTTP/HTTPS协议

SSL/TLS requests follow a different logic when a browser uses a proxy.
Instead of performing the SSL/TLS handshake at the start,
web browser will send a CONNECT request to the proxy
to make sure that the proxy can connect to the target server.
Once the browser receives confirmation, it then initiates the SSL/TLS session
and continues with the request.
"""
import sys
from urllib import parse as urlparse

import twisted.web.http
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.web.proxy import Proxy, ProxyRequest
from twisted.python import log
from twisted.python.versions import Version
import twisted.web.http

class ConnectProxyRequest(ProxyRequest):
    """HTTP ProxyRequest handler (factory) that supports CONNECT"""
    connectedProtocol = None

    def process(self):
        if self.method == b'CONNECT':
            self.processConnectRequest()
        else:
            ProxyRequest.process(self)

    def fail(self, message, body):
        self.setResponseCode(501, message)
        self.responseHeaders.addRawHeader("Content-Type", "text/html")
        self.write(body.encode())
        self.finish()

    def split_host_port(self, hostport, default_port):
        port = default_port
        parts = hostport.split(b':', 1)
        if len(parts) == 2:
            try:
                port = int(parts[1])
            except ValueError:
                pass
        return parts[0], port

    def processConnectRequest(self):
        parsed = urlparse.urlparse(self.uri)
        default_port = self.ports.get(parsed.scheme)
        host, port = self.split_host_port(parsed.netloc or parsed.path,
                                          default_port)
        if port is None:
            self.fail("Bad CONNECT Request",
                      "Unable to parse port from URI: %s" % repr(self.uri))
            return

        clientFactory = ConnectProxyClientFactory(host, port, self)

        # TODO provide an API to set proxy connect timeouts
        self.reactor.connectTCP(host, port, clientFactory)

class ConnectProxy(Proxy):
    """HTTP Server Protocol that supports CONNECT"""
    requestFactory = ConnectProxyRequest
    connectedRemote = None

    def requestDone(self, request):
        if request.method == b'CONNECT' and self.connectedRemote is not None:
            self.connectedRemote.connectedClient = self
            if twisted.version >= Version(twisted.__name__, 16, 3, 0):
                self._handlingRequest = False
                # self._producer.resumeProducing()
                if self._savedTimeOut:
                    self.setTimeout(self._savedTimeOut)
                data = b''.join(self._dataBuffer)
                self._dataBuffer = []
                self.setLineMode(data)
        else:
            Proxy.requestDone(self, request)

    def connectionLost(self, reason):
        if self.connectedRemote is not None:
            self.connectedRemote.transport.loseConnection()
        Proxy.connectionLost(self, reason)

    def dataReceived(self, data):
        if self.connectedRemote is None:
            Proxy.dataReceived(self, data)
        else:
            # Once proxy is connected, forward all bytes received
            # from the original client to the remote server.
            self.connectedRemote.transport.write(data)

class ConnectProxyClient(Protocol):
    connectedClient = None

    def connectionMade(self):
        self.factory.request.channel.connectedRemote = self
        self.factory.request.setResponseCode(200, b"CONNECT OK")
        self.factory.request.setHeader('X-Connected-IP',
                                       self.transport.realAddress[0])
        self.factory.request.setHeader('Content-Length', '0')
        self.factory.request.finish()

    def connectionLost(self, reason):
        if self.connectedClient is not None:
            self.connectedClient.transport.loseConnection()

    def dataReceived(self, data):
        if self.connectedClient is not None:
            # Forward all bytes from the remote server back to the
            # original connected client
            self.connectedClient.transport.write(data)
        else:
            log.msg("UNEXPECTED DATA RECEIVED:", data)

class ConnectProxyClientFactory(ClientFactory):
    protocol = ConnectProxyClient

    def __init__(self, host, port, request):
        self.request = request
        self.host = host
        self.port = port

    def clientConnectionFailed(self, connector, reason):
        print(reason)
        self.request.fail(b"Gateway Error", str(reason))

class HTTPSProxyServer(object):
    def __init__(self, listen_port: int = 10443):
        self.listen_port = listen_port

    def start(self):
        log.startLogging(sys.stderr)
        factory = twisted.web.http.HTTPFactory()
        factory.protocol = ConnectProxy
        twisted.internet.reactor.listenTCP(self.listen_port, factory)
        twisted.internet.reactor.run()

if __name__ == "__main__":
    p = HTTPSProxyServer()
    p.start()


from os import getenv
from typing import Any, List, Mapping, Tuple

from airbyte_cdk.logger import AirbyteLogger
from airbyte_cdk.models import SyncMode
from airbyte_cdk.sources import AbstractSource
from airbyte_cdk.sources.streams import Stream
from source_amazon_seller_partner.auth import AWSAuthenticator
from source_amazon_seller_partner.constants import get_marketplaces
from source_amazon_seller_partner.streams import (
    BrandAnalyticsAlternatePurchaseReports,
    BrandAnalyticsItemComparisonReports,
    BrandAnalyticsMarketBasketReports,
    BrandAnalyticsRepeatPurchaseReports,
    BrandAnalyticsSearchTermsReports,
    FbaAfnInventoryByCountryReports,
    FbaAfnInventoryReports,
    FbaCustomerReturnsReports,
    FbaEstimatedFbaFeesTxtReport,
    FbaFulfillmentCustomerShipmentPromotionReport,
    FbaInventoryPlaningReport,
    FbaMyiUnsuppressedInventoryReport,
    FbaOrdersReports,
    FbaReimbursementsReports,
    FbaReplacementsReports,
    FbaShipmentsReports,
    FbaSnsForecastReport,
    FbaSnsPerformanceReport,
    FbaStorageFeesReports,
    FlatFileActionableOrderDataShipping,
    FlatFileArchivedOrdersDataByOrderDate,
    FlatFileOpenListingsReports,
    FlatFileOrdersReports,
    FlatFileOrdersReportsByLastUpdate,
    FlatFileReturnsDataByReturnDate,
    FlatFileSettlementV2Reports,
    FulfilledShipmentsReports,
    GetXmlBrowseTreeData,
    LedgerDetailedViewReports,
    LedgerSummaryViewReport,
    ListFinancialEventGroups,
    ListFinancialEvents,
    MerchantCancelledListingsReport,
    MerchantListingsFypReport,
    MerchantListingsInactiveData,
    MerchantListingsReport,
    MerchantListingsReportBackCompat,
    MerchantListingsReports,
    OrderItems,
    OrderReportDataShipping,
    Orders,
    RestockInventoryReports,
    SellerAnalyticsSalesAndTrafficReports,
    SellerFeedbackReports,
    StrandedInventoryUiReport,
    VendorDirectFulfillmentShipping,
    VendorInventoryReports,
    VendorSalesReports,
    XmlAllOrdersDataByOrderDataGeneral,
)

class SourceAmazonSellerPartner(AbstractSource):
    def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]:
        endpoint, marketplace_id, _ = get_marketplaces(config.get("aws_environment"))[config.get("region")]
        auth = AWSAuthenticator(
            token_refresh_endpoint="https://api.amazon.com/auth/o2/token",
            client_id=config.get("lwa_app_id"),
            client_secret=config.get("lwa_client_secret"),
            refresh_token=config.get("refresh_token"),
            host=endpoint.replace("https://", ""),
            refresh_access_token_headers={"Content-Type": "application/x-www-form-urlencoded"},
        )
        stream_kwargs = {
            "url_base": endpoint,
            "authenticator": auth,
            "replication_start_date": config.get("replication_start_date"),
            "marketplace_id": marketplace_id,
            "period_in_days": config.get("period_in_days", 90),
            "report_options": config.get("report_options"),
            "replication_end_date": config.get("replication_end_date"),
            "advanced_stream_options": config.get("advanced_stream_options"),
        }
        return stream_kwargs

    def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]:
        """
        Check connection to Amazon SP API by requesting the Orders endpoint
        This endpoint is not available for vendor-only Seller accounts,
        the Orders endpoint will then return a 403 error
        Therefore made an exception for 403 errors (when vendor-only accounts).
        When no access, a 401 error is given.
        Validate if response has the expected error code and body.
        Show error message in case of request exception or unexpected response.
        """
        try:
            stream_kwargs = self._get_stream_kwargs(config)
            orders_stream = Orders(**stream_kwargs)
            next(orders_stream.read_records(sync_mode=SyncMode.full_refresh))

            return True, None
        except Exception as e:
            # Validate Orders stream without data
            if isinstance(e, StopIteration):
                return True, None

            # Additional check, since Vendor-only accounts within Amazon Seller API
            # will not pass the test without this exception
            if "403 Client Error" in str(e):
                stream_to_check = VendorSalesReports(**stream_kwargs)
                next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh))
                return True, None

            return False, e

    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
        """
        :param config: A Mapping of the user input configuration as defined in the connector spec.
        """
        stream_kwargs = self._get_stream_kwargs(config)
        streams = [
            FbaCustomerReturnsReports(**stream_kwargs),
            FbaAfnInventoryReports(**stream_kwargs),
            FbaAfnInventoryByCountryReports(**stream_kwargs),
            FbaOrdersReports(**stream_kwargs),
            FbaShipmentsReports(**stream_kwargs),
            FbaReplacementsReports(**stream_kwargs),
            FbaStorageFeesReports(**stream_kwargs),
            RestockInventoryReports(**stream_kwargs),
            FlatFileActionableOrderDataShipping(**stream_kwargs),
            FlatFileOpenListingsReports(**stream_kwargs),
            FlatFileOrdersReports(**stream_kwargs),
            FlatFileOrdersReportsByLastUpdate(**stream_kwargs),
            FlatFileSettlementV2Reports(**stream_kwargs),
            FulfilledShipmentsReports(**stream_kwargs),
            MerchantListingsReports(**stream_kwargs),
            VendorDirectFulfillmentShipping(**stream_kwargs),
            Orders(**stream_kwargs),
            OrderItems(**stream_kwargs),
            OrderReportDataShipping(**stream_kwargs),
            SellerFeedbackReports(**stream_kwargs),
            GetXmlBrowseTreeData(**stream_kwargs),
            ListFinancialEventGroups(**stream_kwargs),
            ListFinancialEvents(**stream_kwargs),
            LedgerDetailedViewReports(**stream_kwargs),
            FbaEstimatedFbaFeesTxtReport(**stream_kwargs),
            FbaFulfillmentCustomerShipmentPromotionReport(**stream_kwargs),
            FbaMyiUnsuppressedInventoryReport(**stream_kwargs),
            MerchantCancelledListingsReport(**stream_kwargs),
            MerchantListingsReport(**stream_kwargs),
            MerchantListingsReportBackCompat(**stream_kwargs),
            MerchantListingsInactiveData(**stream_kwargs),
            StrandedInventoryUiReport(**stream_kwargs),
            XmlAllOrdersDataByOrderDataGeneral(**stream_kwargs),
            MerchantListingsFypReport(**stream_kwargs),
            FbaSnsForecastReport(**stream_kwargs),
            FbaSnsPerformanceReport(**stream_kwargs),
            FlatFileArchivedOrdersDataByOrderDate(**stream_kwargs),
            FlatFileReturnsDataByReturnDate(**stream_kwargs),
            FbaInventoryPlaningReport(**stream_kwargs),
            LedgerSummaryViewReport(**stream_kwargs),
            FbaReimbursementsReports(**stream_kwargs),
        ]
        # TODO: Remove after Brand Analytics will be enabled in CLOUD:
        #  https://github.com/airbytehq/airbyte/issues/32353
        if getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD":
            brand_analytics_reports = [
                BrandAnalyticsMarketBasketReports(**stream_kwargs),
                BrandAnalyticsSearchTermsReports(**stream_kwargs),
                BrandAnalyticsRepeatPurchaseReports(**stream_kwargs),
                BrandAnalyticsAlternatePurchaseReports(**stream_kwargs),
                BrandAnalyticsItemComparisonReports(**stream_kwargs),
                SellerAnalyticsSalesAndTrafficReports(**stream_kwargs),
                VendorSalesReports(**stream_kwargs),
                VendorInventoryReports(**stream_kwargs),
            ]
            streams += brand_analytics_reports
        return streams


class myclass2:
    num=0
    def __init__(self):
        myclass2.num+=1
        self.num=myclass2.num
        print("Создается классa myclass2. Общее количество созданий объектов этого класса",myclass2.num)
    def __del__(self):
        print("Удален объект", self.num)

def myf(n):
    """Из предыдущего задания"""
    global myclass2
    a=myclass2()
    b=a
    for i in range(n-1):
        a.next=myclass2()
        a=a.next
    return b

A=myf(5)
print(A.__class__)
print(A.next.__dict__)
print(A.next.next.next.__class__)

class myclass1:
    pass

def del_obj(num:"Номер удаляемого объекта",obj:"Объект из которого производиться удаление",vaule:"Имя переменной"):
    """Напишите программу, в которой создается цепочка объектов. Предложите метод или функцию, которые позволяют вставить новый объект
    в уже существующую цепочку, а также метод или функцию, которые позволяют удалить объект из цепочки (так, чтобы оставшиеся объекты образовали цепочку)"""
    if num!=2:
        for i in obj.__dict__:
            if i==vaule:
                del_obj(num-1,obj.__dict__[i],vaule)
    else:
        for i in obj.__dict__:
            if i==vaule:
                c=obj.__dict__[i]
                obj.__dict__[i]=c.__dict__[i]
    return obj

B=del_obj(3,A,"next")
print(B.num)
print(B.next.num)
print(B.next.next.num)
print(B.next.next.next.num)
# Не успел понять почему по окончанию все объекты удаляются


import pytest

from werkzeug.test import EnvironBuilder
from werkzeug.wrappers import Request

from .keys import APIKeys

@pytest.fixture()
def key_file(tmpdir):
  key_file = tmpdir.join("keys.csv")
  key_file.write(
    'key,domains\n'
    'wildcard,\n'
    '# comment,with,commas,\n'
    'multi,example.org;example.com\n'
    'single,test.local'
  )
  yield key_file.strpath

@pytest.mark.parametrize("key,referrer,permitted", [
  ["single", None, True],
  ["single", "", True],
  ["single", "--", True],
  ["single", "http", True],
  ["single", "http://", True],
  ["single", "test.local", True],
  ["single", "http://test.local", True],
  ["single", "https://test.local", True],
  ["single", "http://sub.test.local/path?arg=1", True],
  ["single", "http://sub.test.local", True],
  ["single", "https://subtest.local", False],
  ["single", "http://sub.test.local.com", False],
  ["single", "https://1.2.3.4", False],
  ["multi", "https://example.org", True],
  ["multi", "https://example.com", True],
  ["multi", "https://example.net", False],
  ["wildcard", "https://example.net", True],
  ["wildcard", "https://1.2.3.4", True],

])
def test_api_key(key_file, key, referrer, permitted):
  a = APIKeys(key_file)
  headers = []
  if referrer:
    headers.append(('Referer', referrer))
  builder = EnvironBuilder(method='GET', query_string={'key': key}, headers=headers)
  req = Request(builder.get_environ())
  assert a.is_permitted(req) == permitted


import pandas as pd
import numpy as np
import os

import caliperreader as cr

import hatchet.graphframe
from hatchet.node import Node
from hatchet.graph import Graph
from hatchet.frame import Frame
from hatchet.util.timer import Timer

class CaliperNativeReader:
    """Read in a native `.cali` file using Caliper's python reader."""

    def __init__(self, filename_or_caliperreader, native, string_attributes):
        """Read in a native cali using Caliper's python reader.

        Args:
            filename_or_caliperreader (str or CaliperReader): name of a `cali` file OR
                a CaliperReader object
            native (bool): use native metric names or user-readable metric names
            string_attributes (str or list): Adds existing string attributes from within the caliper file to the dataframe
        """
        self.filename_or_caliperreader = filename_or_caliperreader
        self.filename_ext = ""
        self.use_native_metric_names = native
        self.string_attributes = string_attributes

        self.df_nodes = {}
        self.metric_cols = []
        self.record_data_cols = []
        self.node_dicts = []
        self.callpath_to_node = {}
        self.idx_to_node = {}
        self.callpath_to_idx = {}
        self.global_nid = 0
        self.node_ordering = False

        self.default_metric = None

        self.timer = Timer()

        if isinstance(self.filename_or_caliperreader, str):
            _, self.filename_ext = os.path.splitext(filename_or_caliperreader)

        if isinstance(self.string_attributes, str):
            self.string_attributes = [self.string_attributes]

    def read_metrics(self, ctx="path"):
        all_metrics = []
        records = self.filename_or_caliperreader.records

        # read metadata from the caliper reader
        for record in records:
            node_dict = {}
            if ctx in record:
                # only parse records that have spot.channel=regionprofile or no
                # spot.channel attribute
                if (
                    "spot.channel" in record
                    and record["spot.channel"] == "regionprofile"
                ) or "spot.channel" not in record:
                    # get the node label and callpath for the record
                    if isinstance(record[ctx], list):
                        # specify how to parse cupti records
                        if "cupti.activity.kind" in record:
                            if record["cupti.activity.kind"] == "kernel":
                                node_label = record["cupti.kernel.name"]
                                node_callpath = tuple(record[ctx] + [node_label])
                            elif record["cupti.activity.kind"] == "memcpy":
                                node_label = record["cupti.activity.kind"]
                                node_callpath = tuple(record[ctx] + [node_label])
                        else:
                            node_label = record[ctx][-1]
                            node_callpath = tuple(record[ctx])
                    else:
                        node_label = record[ctx]
                        node_callpath = tuple([record[ctx]])

                    if "spot.channel" in record:
                        node_dict["spot.channel"] = record["spot.channel"]

                    # get node nid based on callpath
                    node_dict["nid"] = self.callpath_to_idx.get(node_callpath)

                    for item in record.keys():
                        if item != ctx:
                            if (
                                self.filename_or_caliperreader.attribute(
                                    item
                                ).attribute_type()
                                == "double"
                            ):
                                node_dict[item] = float(record[item])
                                if item not in self.record_data_cols:
                                    self.record_data_cols.append(item)
                            elif (
                                self.filename_or_caliperreader.attribute(
                                    item
                                ).attribute_type()
                                == "int"
                            ):
                                node_dict[item] = int(record[item])
                                if item not in self.record_data_cols:
                                    self.record_data_cols.append(item)
                            elif (
                                self.filename_or_caliperreader.attribute(
                                    item
                                ).attribute_type()
                                == "string"
                            ):
                                if item in self.string_attributes:
                                    node_dict[item] = record[item]
                                    if item not in self.record_data_cols:
                                        self.record_data_cols.append(item)

                    all_metrics.append(node_dict)

        # make list of metric columns
        for col in self.record_data_cols:
            if self.filename_or_caliperreader.attribute(col).is_value():
                self.metric_cols.append(col)

        df_metrics = pd.DataFrame.from_dict(data=all_metrics)
        df_new = df_metrics.groupby(df_metrics["nid"]).aggregate("first").reset_index()
        return df_new

    def create_graph(self, ctx="path"):
        list_roots = []

        def _create_parent(child_node, parent_callpath):
            """We may encounter a parent node in the callpath before we see it
            as a child node. In this case, we need to create a hatchet node for
            the parent.

            This function recursively creates parent nodes in a callpath
            until it reaches the already existing parent in that callpath.
            """
            parent_node = self.callpath_to_node.get(parent_callpath)

            if parent_node:
                # return if arrives at the parent
                parent_node.add_child(child_node)
                child_node.add_parent(parent_node)
                return
            else:
                # else create the parent and add parent/child

                # if root node, end recursive call to create parent nodes
                if not parent_callpath:
                    list_roots.append(child_node)

                    node_dict = dict(
                        {
                            "name": child_node.frame["name"],
                            "node": child_node,
                            "nid": self.global_nid,
                        }
                    )

                    self.idx_to_node[self.global_nid] = node_dict
                    self.global_nid += 1
                else:
                    grandparent_callpath = parent_callpath[:-1]
                    parent_name = parent_callpath[-1]

                    parent_node = Node(
                        Frame({"type": "function", "name": parent_name}), None
                    )

                    self.callpath_to_node[parent_callpath] = parent_node
                    self.callpath_to_idx[parent_callpath] = self.global_nid

                    node_dict = dict(
                        {
                            "name": parent_name,
                            "node": parent_node,
                            "nid": self.global_nid,
                        },
                    )
                    self.idx_to_node[self.global_nid] = node_dict
                    self.global_nid += 1

                    parent_node.add_child(child_node)
                    child_node.add_parent(parent_node)
                    _create_parent(parent_node, grandparent_callpath)

        parent_hnode = None
        records = self.filename_or_caliperreader.records
        order = -1

        for record in records:
            node_label = ""
            if ctx in record:
                if (
                    "spot.channel" in record
                    and record["spot.channel"] == "regionprofile"
                ) or "spot.channel" not in record:
                    # if it's a list, then it's a callpath
                    if isinstance(record[ctx], list):
                        # specify how to parse cupti records
                        if "cupti.activity.kind" in record:
                            if record["cupti.activity.kind"] == "kernel":
                                node_label = record["cupti.kernel.name"]
                                node_callpath = tuple(record[ctx] + [node_label])
                                parent_callpath = node_callpath[:-1]
                                node_type = "kernel"
                            elif record["cupti.activity.kind"] == "memcpy":
                                node_label = record["cupti.activity.kind"]
                                node_callpath = tuple(record[ctx] + [node_label])
                                parent_callpath = node_callpath[:-1]
                                node_type = "memcpy"
                            else:
                                Exception("Haven't seen this activity kind yet")
                        else:
                            node_label = record[ctx][-1]
                            node_callpath = tuple(record[ctx])
                            parent_callpath = node_callpath[:-1]
                            node_type = "function"

                        hnode = self.callpath_to_node.get(node_callpath)

                        if not hnode:
                            # set the _hatchet_nid by the node order column if it exists, else -1
                            if "min#min#aggregate.slot" in record:
                                self.node_ordering = True
                                order = record["min#min#aggregate.slot"]
                            frame = Frame({"type": node_type, "name": node_label})
                            order = int(order)
                            hnode = Node(frame, hnid=order)
                            self.callpath_to_node[node_callpath] = hnode

                            # get parent from node callpath
                            parent_hnode = self.callpath_to_node.get(parent_callpath)

                            # create parent if it doesn't exist
                            # else if parent already exists, add child-parent
                            if not parent_hnode:
                                _create_parent(hnode, parent_callpath)
                            else:
                                parent_hnode.add_child(hnode)
                                hnode.add_parent(parent_hnode)

                            self.callpath_to_idx[node_callpath] = self.global_nid
                            node_dict = dict(
                                {
                                    "name": node_label,
                                    "node": hnode,
                                    "nid": self.global_nid,
                                },
                            )
                            self.idx_to_node[self.global_nid] = node_dict
                            self.global_nid += 1

                    # if it's a string, then it's a root
                    else:
                        root_label = record[ctx]
                        root_callpath = tuple([root_label])

                        if root_callpath not in self.callpath_to_node:
                            # create the root since it doesn't exist
                            frame = Frame({"type": "function", "name": root_label})
                            graph_root = Node(frame, None)

                            # store callpaths to identify the root
                            self.callpath_to_node[root_callpath] = graph_root
                            self.callpath_to_idx[root_callpath] = self.global_nid
                            list_roots.append(graph_root)

                            node_dict = dict(
                                {
                                    "name": root_label,
                                    "node": graph_root,
                                    "nid": self.global_nid,
                                }
                            )

                            self.idx_to_node[self.global_nid] = node_dict
                            self.global_nid += 1

        return list_roots

    def _parse_metadata(self, mdata):
        """Convert Caliper Metadata values into correct Python objects.

        Args:
            mdata (dict[str: str]): metadata to convert

        Return:
            (dict[str: str]): modified metadata
        """
        parsed_mdata = {}
        for k, v in mdata.items():
            # If the value is an int, convert it to an int.
            try:
                parsed_mdata[k] = int(v)
            except ValueError:
                # If the value is a float, convert it to a float
                try:
                    parsed_mdata[k] = float(v)
                except ValueError:
                    # If the value is a list or tuple, convert it to a list or
                    # tuple
                    if v.startswith("[") and v.endswith("]"):
                        parsed_mdata[k] = [
                            elem.strip() for elem in v.strip("][").split(",")
                        ]
                    elif v.startswith("(") and v.endswith(")"):
                        parsed_mdata[k] = [
                            elem.strip() for elem in v.strip(")(").split(",")
                        ]
                    # If the value is a string, just save it as-is
                    else:
                        parsed_mdata[k] = v
        return parsed_mdata

    def read(self):
        """Read the caliper records to extract the calling context tree."""
        if isinstance(self.filename_or_caliperreader, str):
            if self.filename_ext != ".cali":
                raise ValueError("from_caliperreader() needs a .cali file")
            else:
                cali_file = self.filename_or_caliperreader
                self.filename_or_caliperreader = cr.CaliperReader()
                self.filename_or_caliperreader.read(cali_file)

        with self.timer.phase("graph construction"):
            list_roots = self.create_graph()
        self.df_nodes = pd.DataFrame(data=list(self.idx_to_node.values()))

        # create a graph object once all the nodes have been added
        graph = Graph(list_roots)
        if self.node_ordering:
            graph.node_ordering = True
        graph.enumerate_traverse()

        with self.timer.phase("read metrics"):
            df_fixed_data = self.read_metrics()

        metrics = pd.DataFrame.from_dict(data=df_fixed_data)

        # add missing intermediate nodes to the df_fixed_data dataframe
        if "mpi.rank" in df_fixed_data.columns:
            num_ranks = metrics["mpi.rank"].max() + 1
            rank_list = range(0, num_ranks)

        # create a standard dict to be used for filling all missing rows
        default_metric_dict = {}
        for idx, col in enumerate(self.record_data_cols):
            if self.filename_or_caliperreader.attribute(col).is_value():
                default_metric_dict[list(self.record_data_cols)[idx]] = 0
            else:
                default_metric_dict[list(self.record_data_cols)[idx]] = None
        default_metric_dict["nid"] = np.nan

        # create a list of dicts, one dict for each missing row
        missing_nodes = []
        for iteridx, row in self.df_nodes.iterrows():
            # check if df_nodes row exists in df_fixed_data
            metric_rows = df_fixed_data.loc[metrics["nid"] == row["nid"]]
            if "mpi.rank" not in self.metric_cols:
                if metric_rows.empty:
                    # add a single row
                    node_dict = dict(default_metric_dict)
                    missing_nodes.append(node_dict)
            else:
                if metric_rows.empty:
                    # add a row per MPI rank
                    for rank in rank_list:
                        node_dict = dict(default_metric_dict)
                        node_dict["nid"] = row["nid"]
                        node_dict["mpi.rank"] = rank
                        missing_nodes.append(node_dict)
                elif len(metric_rows) < num_ranks:
                    # add a row for each missing MPI rank
                    present_ranks = metric_rows["mpi.rank"].values
                    missing_ranks = [x for x in rank_list if x not in present_ranks]
                    for rank in missing_ranks:
                        node_dict = dict(default_metric_dict)
                        node_dict["nid"] = row["nid"]
                        node_dict["mpi.rank"] = rank
                        missing_nodes.append(node_dict)

        df_missing = pd.DataFrame.from_dict(data=missing_nodes)
        df_metrics = pd.concat([df_fixed_data, df_missing], sort=False)

        # rename columns to user-readable metric names (i.e., aliases)
        if not self.use_native_metric_names:
            for col in df_metrics.columns:
                if col == "nid":
                    continue
                alias = self.filename_or_caliperreader.attribute(col).get(
                    "attribute.alias"
                )
                if alias:
                    # update column name in metrics dataframe
                    df_metrics.rename(columns={col: alias}, inplace=True)

                    # also update list of metric columns
                    self.metric_cols = [
                        alias if item == col else item for item in self.metric_cols
                    ]

        # dict mapping old to new column names to make columns consistent with
        # other readers
        old_to_new = {
            "mpi.rank": "rank",
            "module#cali.sampler.pc": "module",
            "sum#time.duration": "time",
            "sum#avg#sum#time.duration": "time",
            "inclusive#sum#time.duration": "time (inc)",
            "sum#avg#inclusive#sum#time.duration": "time (inc)",
        }

        # change column names
        new_cols = []
        for col in df_metrics.columns:
            if col in old_to_new:
                new_cols.append(old_to_new[col])
            else:
                new_cols.append(col)
        df_metrics.columns = new_cols

        # create list of exclusive and inclusive metric columns
        exc_metrics = []
        inc_metrics = []
        for column in self.metric_cols:
            # ignore rank as an exc or inc metric
            if column == "mpi.rank":
                continue

            # add new column names to list of metrics if inc or inclusive in
            # old column names
            if "(inc)" in column or "inclusive" in column:
                if column in old_to_new:
                    column = old_to_new[column]
                inc_metrics.append(column)
            else:
                if column in old_to_new:
                    column = old_to_new[column]
                exc_metrics.append(column)

        with self.timer.phase("data frame"):
            # merge the metrics and node dataframes on the nid column
            dataframe = pd.merge(df_metrics, self.df_nodes, on="nid")
            dataframe["nid"] = dataframe["nid"].astype(pd.Int64Dtype())

            # set the index to be a MultiIndex
            indices = ["node"]
            if "rank" in dataframe.columns:
                indices.append("rank")
            dataframe.set_index(indices, inplace=True)
            dataframe.sort_index(inplace=True)

        # set the default metric
        if self.default_metric is None:
            if "time (inc)" in dataframe.columns:
                self.default_metric = "time"
            elif "avg#inclusive#sum#time.duration" in dataframe.columns:
                self.default_metric = "avg#inclusive#sum#time.duration"
            elif len(inc_metrics) > 0:
                self.default_metric = inc_metrics[0]
            elif len(exc_metrics) > 0:
                self.default_metric = exc_metrics[0]

        metadata = self.filename_or_caliperreader.globals
        parsed_metadata = self._parse_metadata(metadata)

        return hatchet.graphframe.GraphFrame(
            graph,
            dataframe,
            exc_metrics,
            inc_metrics,
            self.default_metric,
            metadata=parsed_metadata,
        )


from bpclass import BPneuron, Loss
import matplotlib.pyplot as plt
import numpy as np
from random import random
import matplotlib.pyplot as plt
from IPython.display import clear_output

class test_network:
    def __init__(self, I, H, O, W01_size, W12_size):
        # 重みとバイアスの定義
        W01 = W01_size*np.random.rand(I, H)-W01_size/2
        W12 = W12_size*np.random.rand(H, O)-W12_size/2
        b1 = np.zeros(H)
        b2 = np.zeros(O)

        # モデルの生成
        self.Seccond_layer = BPneuron(W01, b1)
        self.Third_layer = BPneuron(W12, b2)
        self.test_loss_layer = Loss()

        self.loss_memo = []
        self.dx_memo = np.empty(0)
        self.bo_memo = np.empty(0)
        self.dbo_memo = np.empty(0)
        self.W12_memo = np.empty(0)

    def traning(self, test_data, target_data, epoch):
        # 学習に使う配列の決定
        for i in range(epoch):
            test_number = np.random.randint(target_data.shape[0])
            x = np.array([test_data[test_number]])
            t = np.array([target_data[test_number]])

            out1, container1 = self.Seccond_layer.forward(x)
            out2, container2 = self.Third_layer.forward(out1)
            W, b, x, y, z = container1
            W, b, x, y, z = container2

            loss = self.test_loss_layer.forward(out2, t)
            dout = self.test_loss_layer.backward()
            if i % 500 == 0:
                print("loss:", dout, "epoch:", i)
                print("_________")
                print("b:", b)
                print("_________")

            if i % 5000 == 0:
                clear_output()

            self.loss_memo.append(dout[0][0])
            dx, containe = self.Third_layer.backward(dout)
            dinput, containe = self.Seccond_layer.backward(dx)
            self.dx_memo = np.append(self.dx_memo, dx)
            self.bo_memo = np.append(self.bo_memo, b)
            self.W12_memo = np.append(self.W12_memo, W)

        return self.loss_memo

    def dx(self):
        return self.dx_memo

    def bo(self):
        return self.bo_memo

    def W_12(self):
        return self.W12_memo


import os
import time
import signal

from tinkoff.invest import Client

import modules.bonds_divergence as bonds_divergence

TOKEN = os.environ["INVEST_TOKEN"]

running = True

def signal_handler(signum, frame):
    global running
    running = False

def main():

    signal.signal(signal.SIGTERM, signal_handler)
    signal.signal(signal.SIGINT, signal_handler)

    with Client(TOKEN) as client:

        div = bonds_divergence.divergence(client)
        print(div)

        # loop
        counter = 0
        while(running):
            if counter % 5 == 0:
                div = bonds_divergence.divergence(client)
                print(div)

            time.sleep(1)
            counter += 1

if __name__ == "__main__":
    main()


#!/usr/bin/env python

import argparse
from collections import defaultdict
import sys

def print_docs(docs):
    not_found = []

    for k in docs:
        en_url, fr_url = k

        if 'en_text' not in docs[k]:
            not_found.append("not found(en): {0}".format(en_url))
            continue

        if 'fr_text' not in docs[k]:
            not_found.append("not found(fr): {0}".format(fr_url))
            continue

        en_text, fr_text = docs[k]['en_text'], docs[k]['fr_text']
        print("{0}\t{1}\t{2}\t{3}\t".format(en_url, fr_url, en_text, fr_text))

    if len(not_found):
        sys.stderr.write("Number of documents without matches: {0}\n".format(len(not_found)))
        for n in not_found:
            sys.stderr.write("{0}\n".format(n))

def load_docs(matches_filepath, threshold):
    map_e2f = {}
    map_f2e = {}

    docs = defaultdict(dict)

    with open(matches_filepath, 'r') as f_mapping:
        for line in f_mapping:
            score, e, f = line.strip().split('\t')
            if float(score) < float(threshold):
                continue

            map_e2f[e] = f
            map_f2e[f] = e

    for line in sys.stdin:
        lang, format, encoding, url, html, text = line.strip().split('\t')

        if url in map_e2f:
            key = (url, map_e2f[url])
            docs[key]['en_text'] = text

        elif url in map_f2e:
            key = (map_f2e[url], url)
            docs[key]['fr_text'] = text

    return docs

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--matches', help='path to the file with matched documents', required=True)
    parser.add_argument('--threshold', help='documents with lower TF-IDF score will be skipped', default=0.1, type=float, required=False)

    args = parser.parse_args()

    docs = load_docs(args.matches, args.threshold)
    print_docs(docs)


import time

def timer(fn):
    """ it'll return a modified function with a time tracker """

    def modified_fn(*args, **kwargs):
        start_time = time.time()
        result = fn(*args, **kwargs)
        end_time = time.time()
        seconds = end_time - start_time
        print(f'TIME_SPENT: {fn.__name__} took {seconds:.2f} seconds')
        return result

    return modified_fn


from nltk import word_tokenize
from nltk.stem import PorterStemmer
import numpy as np
import nltk

#Package with pre-trained tokenizer
nltk.download('punkt')

stemmer = PorterStemmer()

def tokenize(sentence: str) -> list:
    '''
    Get tokenized words from a sentence
    '''
    return word_tokenize(sentence)

def stem(word: str) -> str:
    '''
    Get words their root form
    '''
    return stemmer.stem(word.lower())

def bag_of_words(tokenized_sentence: list, all_words: list):
    '''
    tokenized_sentence = ['hello', 'how', 'are', 'you']
    all_words = ['hi', 'hello', 'I', 'you', 'bye', 'thank', 'cool']
    returned_value = [   0,     1,      0,    1,     0,      0,       0  ]
    '''
    tokenized_sentence = [stem(word) for word in tokenized_sentence]

    bag = np.zeros(len(all_words), dtype=np.float32)
    for index, word in enumerate(all_words):
        if word in tokenized_sentence:
            bag[index] = 1.0

    return bag

tokenized_sentence = ['hello', 'how', 'are', 'you']
all_words = ['hi', 'hello', 'I', 'you', 'bye', 'thank', 'cool']

bag = bag_of_words(tokenized_sentence, all_words)
# print(bag)


# ex10-final example of PY4E

fname=input("Enter file name: ")
fhandle=open(fname)

hrs=list()
temp=list()
counts=dict()

for line in fhandle:
    if line.startswith("From "):
        words=line.split()
        time=words[5].split(":")
        hrs.append(time[0])

for hour in hrs:
    counts[hour]=counts.get(hour,0)+1

for k,v in counts.items():
    temp.append((k,v))

for k,v in sorted(temp):
    print(k,v)


import sys
import threading
import gc
import types
from contextlib import contextmanager
from enum import Enum
from typing import Optional, Callable

from blinker import Signal

from streamlit import config
from streamlit import magic
from streamlit import source_util
from streamlit import util
from streamlit.error_util import handle_uncaught_app_exception
from streamlit.in_memory_file_manager import in_memory_file_manager
from streamlit.proto.ForwardMsg_pb2 import ForwardMsg
from streamlit.script_run_context import ScriptRunContext, add_script_run_ctx
from streamlit.script_run_context import get_script_run_ctx
from streamlit.script_request_queue import ScriptRequest, ScriptRequestQueue, RerunData
from streamlit.session_data import SessionData
from streamlit.state import (
    SessionState,
    SCRIPT_RUN_WITHOUT_ERRORS_KEY,
)
from streamlit.logger import get_logger
from streamlit.proto.ClientState_pb2 import ClientState

from streamlit.uploaded_file_manager import UploadedFileManager

LOGGER = get_logger(__name__)

class ScriptRunnerEvent(Enum):
    # The script started running.
    SCRIPT_STARTED = "SCRIPT_STARTED"

    # The script run stopped because of a compile error.
    SCRIPT_STOPPED_WITH_COMPILE_ERROR = "SCRIPT_STOPPED_WITH_COMPILE_ERROR"

    # The script run stopped because it ran to completion, or was
    # interrupted by the user.
    SCRIPT_STOPPED_WITH_SUCCESS = "SCRIPT_STOPPED_WITH_SUCCESS"

    # The ScriptRunner is done processing the ScriptEventQueue and
    # is shut down.
    SHUTDOWN = "SHUTDOWN"

"""
Note [Threading]
There are two kinds of threads in Streamlit, the main thread and script threads.
The main thread is started by invoking the Streamlit CLI, and bootstraps the
framework and runs the Tornado webserver.
A script thread is created by a ScriptRunner when it starts. The script thread
is where the ScriptRunner executes, including running the user script itself,
processing messages to/from the frontend, and all the Streamlit library function
calls in the user script.
It is possible for the user script to spawn its own threads, which could call
Streamlit functions. We restrict the ScriptRunner's execution control to the
script thread. Calling Streamlit functions from other threads is unlikely to
work correctly due to lack of ScriptRunContext, so we may add a guard against
it in the future.
"""

class ScriptRunner:
    def __init__(
        self,
        session_id: str,
        session_data: SessionData,
        enqueue_forward_msg: Callable[[ForwardMsg], None],
        client_state: ClientState,
        request_queue: ScriptRequestQueue,
        session_state: SessionState,
        uploaded_file_mgr: UploadedFileManager,
    ):
        """Initialize the ScriptRunner.

        (The ScriptRunner won't start executing until start() is called.)

        Parameters
        ----------
        session_id : str
            The AppSession's id.

        session_data : SessionData
            The AppSession's session data.

        enqueue_forward_msg : Callable
            Function to call to send a ForwardMsg to the frontend.
            (When not running a unit test, this will be the enqueue function
            of the AppSession instance that created this ScriptRunner.)

        client_state : ClientState
            The current state from the client (widgets and query params).

        request_queue : ScriptRequestQueue
            The queue that the AppSession is publishing ScriptRequests to.
            ScriptRunner will continue running until the queue is empty,
            and then shut down.

        uploaded_file_mgr : UploadedFileManager
            The File manager to store the data uploaded by the file_uploader widget.

        """
        self._session_id = session_id
        self._session_data = session_data
        self._enqueue_forward_msg = enqueue_forward_msg
        self._request_queue = request_queue
        self._uploaded_file_mgr = uploaded_file_mgr

        self._client_state = client_state
        self._session_state: SessionState = session_state
        self._session_state.set_widgets_from_proto(client_state.widget_states)

        self.on_event = Signal(
            doc="""Emitted when a ScriptRunnerEvent occurs.

            This signal is *not* emitted on the same thread that the
            ScriptRunner was created on.

            Parameters
            ----------
            event : ScriptRunnerEvent

            exception : BaseException | None
                Our compile error. Set only for the
                SCRIPT_STOPPED_WITH_COMPILE_ERROR event.

            widget_states : streamlit.proto.WidgetStates_pb2.WidgetStates | None
                The ScriptRunner's final WidgetStates. Set only for the
                SHUTDOWN event.
            """
        )

        # Set to true when we process a SHUTDOWN request
        self._shutdown_requested = False

        # Set to true while we're executing. Used by
        # _maybe_handle_execution_control_request.
        self._execing = False

        # This is initialized in start()
        self._script_thread: Optional[threading.Thread] = None

    def __repr__(self) -> str:
        return util.repr_(self)

    def start(self) -> None:
        """Start a new thread to process the ScriptEventQueue.

        This must be called only once.

        """
        if self._script_thread is not None:
            raise Exception("ScriptRunner was already started")

        self._script_thread = threading.Thread(
            target=self._run_script_thread,
            name="ScriptRunner.scriptThread",
        )
        self._script_thread.start()

    def _get_script_run_ctx(self) -> ScriptRunContext:
        """Get the ScriptRunContext for the current thread.

        Returns
        -------
        ScriptRunContext
            The ScriptRunContext for the current thread.

        Raises
        ------
        AssertionError
            If called outside of a ScriptRunner thread.
        RuntimeError
            If there is no ScriptRunContext for the current thread.

        """
        assert self._is_in_script_thread()

        ctx = get_script_run_ctx()
        if ctx is None:
            # This should never be possible on the script_runner thread.
            raise RuntimeError(
                "ScriptRunner thread has a null ScriptRunContext. Something has gone very wrong!"
            )
        return ctx

    def _run_script_thread(self) -> None:
        """The entry point for the script thread.

        Processes the ScriptRequestQueue, which will at least contain the RERUN
        request that will trigger the first script-run.

        When the ScriptRequestQueue is empty, or when a SHUTDOWN request is
        dequeued, this function will exit and its thread will terminate.
        """
        assert self._is_in_script_thread()

        LOGGER.debug("Beginning script thread")

        # Create and attach the thread's ScriptRunContext
        ctx = ScriptRunContext(
            session_id=self._session_id,
            enqueue=self._enqueue,
            query_string=self._client_state.query_string,
            session_state=self._session_state,
            uploaded_file_mgr=self._uploaded_file_mgr,
        )
        add_script_run_ctx(threading.current_thread(), ctx)

        while not self._shutdown_requested and self._request_queue.has_request:
            request, data = self._request_queue.dequeue()
            if request == ScriptRequest.STOP:
                LOGGER.debug("Ignoring STOP request while not running")
            elif request == ScriptRequest.SHUTDOWN:
                LOGGER.debug("Shutting down")
                self._shutdown_requested = True
            elif request == ScriptRequest.RERUN:
                self._run_script(data)
            else:
                raise RuntimeError("Unrecognized ScriptRequest: %s" % request)

        # Send a SHUTDOWN event before exiting. This includes the widget values
        # as they existed after our last successful script run, which the
        # AppSession will pass on to the next ScriptRunner that gets
        # created.
        client_state = ClientState()
        client_state.query_string = ctx.query_string
        widget_states = self._session_state.as_widget_states()
        client_state.widget_states.widgets.extend(widget_states)
        self.on_event.send(ScriptRunnerEvent.SHUTDOWN, client_state=client_state)

    def _is_in_script_thread(self) -> bool:
        """True if the calling function is running in the script thread"""
        return self._script_thread == threading.current_thread()

    def _enqueue(self, msg: ForwardMsg) -> None:
        """Enqueue a ForwardMsg to our browser queue.
        This private function is called by ScriptRunContext only.

        It may be called from the script thread OR the main thread.
        """
        # Whenever we enqueue a ForwardMsg, we also handle any pending
        # execution control request. This means that a script can be
        # cleanly interrupted and stopped inside most `st.foo` calls.
        #
        # (If "runner.installTracer" is true, then we'll actually be
        # handling these requests in a callback called after every Python
        # instruction instead.)
        if not config.get_option("runner.installTracer"):
            self._maybe_handle_execution_control_request()

        # Pass the message up to our associated AppSession.
        self._enqueue_forward_msg(msg)

    def _maybe_handle_execution_control_request(self) -> None:
        if not self._is_in_script_thread():
            # We can only handle execution_control_request if we're on the
            # script execution thread. However, it's possible for deltas to
            # be enqueued (and, therefore, for this function to be called)
            # in separate threads, so we check for that here.
            return

        if not self._execing:
            # If the _execing flag is not set, we're not actually inside
            # an exec() call. This happens when our script exec() completes,
            # we change our state to STOPPED, and a statechange-listener
            # enqueues a new ForwardEvent
            return

        # Pop the next request from our queue.
        request, data = self._request_queue.dequeue()
        if request is None:
            return

        LOGGER.debug("Received ScriptRequest: %s", request)
        if request == ScriptRequest.STOP:
            raise StopException()
        elif request == ScriptRequest.SHUTDOWN:
            self._shutdown_requested = True
            raise StopException()
        elif request == ScriptRequest.RERUN:
            raise RerunException(data)
        else:
            raise RuntimeError("Unrecognized ScriptRequest: %s" % request)

    def _install_tracer(self) -> None:
        """Install function that runs before each line of the script."""

        def trace_calls(frame, event, arg):
            self._maybe_handle_execution_control_request()
            return trace_calls

        # Python interpreters are not required to implement sys.settrace.
        if hasattr(sys, "settrace"):
            sys.settrace(trace_calls)

    @contextmanager
    def _set_execing_flag(self):
        """A context for setting the ScriptRunner._execing flag.

        Used by _maybe_handle_execution_control_request to ensure that
        we only handle requests while we're inside an exec() call
        """
        if self._execing:
            raise RuntimeError("Nested set_execing_flag call")
        self._execing = True
        try:
            yield
        finally:
            self._execing = False

    def _run_script(self, rerun_data: RerunData) -> None:
        """Run our script.

        Parameters
        ----------
        rerun_data: RerunData
            The RerunData to use.

        """
        assert self._is_in_script_thread()

        LOGGER.debug("Running script %s", rerun_data)

        # Reset DeltaGenerators, widgets, media files.
        in_memory_file_manager.clear_session_files()

        ctx = self._get_script_run_ctx()
        ctx.reset(query_string=rerun_data.query_string)

        self.on_event.send(ScriptRunnerEvent.SCRIPT_STARTED)

        # Compile the script. Any errors thrown here will be surfaced
        # to the user via a modal dialog in the frontend, and won't result
        # in their previous script elements disappearing.

        try:
            with source_util.open_python_file(self._session_data.main_script_path) as f:
                filebody = f.read()

            if config.get_option("runner.magicEnabled"):
                filebody = magic.add_magic(
                    filebody, self._session_data.main_script_path
                )

            code = compile(
                filebody,
                # Pass in the file path so it can show up in exceptions.
                self._session_data.main_script_path,
                # We're compiling entire blocks of Python, so we need "exec"
                # mode (as opposed to "eval" or "single").
                mode="exec",
                # Don't inherit any flags or "future" statements.
                flags=0,
                dont_inherit=1,
                # Use the default optimization options.
                optimize=-1,
            )

        except BaseException as e:
            # We got a compile error. Send an error event and bail immediately.
            LOGGER.debug("Fatal script error: %s" % e)
            self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = False
            self.on_event.send(
                ScriptRunnerEvent.SCRIPT_STOPPED_WITH_COMPILE_ERROR, exception=e
            )
            return

        # If we get here, we've successfully compiled our script. The next step
        # is to run it. Errors thrown during execution will be shown to the
        # user as ExceptionElements.

        if config.get_option("runner.installTracer"):
            self._install_tracer()

        # This will be set to a RerunData instance if our execution
        # is interrupted by a RerunException.
        rerun_with_data = None

        try:
            # Create fake module. This gives us a name global namespace to
            # execute the code in.
            module = _new_module("__main__")

            # Install the fake module as the __main__ module. This allows
            # the pickle module to work inside the user's code, since it now
            # can know the module where the pickled objects stem from.
            # IMPORTANT: This means we can't use "if __name__ == '__main__'" in
            # our code, as it will point to the wrong module!!!
            sys.modules["__main__"] = module

            # Add special variables to the module's globals dict.
            # Note: The following is a requirement for the CodeHasher to
            # work correctly. The CodeHasher is scoped to
            # files contained in the directory of __main__.__file__, which we
            # assume is the main script directory.
            module.__dict__["__file__"] = self._session_data.main_script_path

            with modified_sys_path(self._session_data), self._set_execing_flag():
                # Run callbacks for widgets whose values have changed.
                if rerun_data.widget_states is not None:
                    # Update the WidgetManager with the new widget_states.
                    # The old states, used to skip callbacks if values
                    # haven't changed, are also preserved in the
                    # WidgetManager.
                    self._session_state.compact_state()
                    self._session_state.set_widgets_from_proto(rerun_data.widget_states)

                    self._session_state.call_callbacks()

                ctx.on_script_start()
                exec(code, module.__dict__)
                self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = True
        except RerunException as e:
            rerun_with_data = e.rerun_data

        except StopException:
            pass

        except BaseException as e:
            self._session_state[SCRIPT_RUN_WITHOUT_ERRORS_KEY] = False
            handle_uncaught_app_exception(e)

        finally:
            self._on_script_finished(ctx)

        # Use _log_if_error() to make sure we never ever ever stop running the
        # script without meaning to.
        _log_if_error(_clean_problem_modules)

        if rerun_with_data is not None:
            self._run_script(rerun_with_data)

    def _on_script_finished(self, ctx: ScriptRunContext) -> None:
        """Called when our script finishes executing, even if it finished
        early with an exception. We perform post-run cleanup here.
        """
        self._session_state.reset_triggers()
        self._session_state.cull_nonexistent(ctx.widget_ids_this_run)
        # Signal that the script has finished. (We use SCRIPT_STOPPED_WITH_SUCCESS
        # even if we were stopped with an exception.)
        self.on_event.send(ScriptRunnerEvent.SCRIPT_STOPPED_WITH_SUCCESS)
        # Delete expired files now that the script has run and files in use
        # are marked as active.
        in_memory_file_manager.del_expired_files()

        # Force garbage collection to run, to help avoid memory use building up
        # This is usually not an issue, but sometimes GC takes time to kick in and
        # causes apps to go over resource limits, and forcing it to run between
        # script runs is low cost, since we aren't doing much work anyway.
        if config.get_option("runner.postScriptGC"):
            gc.collect(2)

class ScriptControlException(BaseException):
    """Base exception for ScriptRunner."""

    pass

class StopException(ScriptControlException):
    """Silently stop the execution of the user's script."""

    pass

class RerunException(ScriptControlException):
    """Silently stop and rerun the user's script."""

    def __init__(self, rerun_data: RerunData):
        """Construct a RerunException

        Parameters
        ----------
        rerun_data : RerunData
            The RerunData that should be used to rerun the script
        """
        self.rerun_data = rerun_data

    def __repr__(self) -> str:
        return util.repr_(self)

def _clean_problem_modules() -> None:
    """Some modules are stateful, so we have to clear their state."""

    if "keras" in sys.modules:
        try:
            keras = sys.modules["keras"]
            keras.backend.clear_session()
        except:
            pass

    if "matplotlib.pyplot" in sys.modules:
        try:
            plt = sys.modules["matplotlib.pyplot"]
            plt.close("all")
        except:
            pass

def _new_module(name: str) -> types.ModuleType:
    """Create a new module with the given name."""
    return types.ModuleType(name)

# Code modified from IPython (BSD license)
# Source: https://github.com/ipython/ipython/blob/master/IPython/utils/syspathcontext.py#L42
class modified_sys_path:
    """A context for prepending a directory to sys.path for a second."""

    def __init__(self, session_data: SessionData):
        self._session_data = session_data
        self._added_path = False

    def __repr__(self) -> str:
        return util.repr_(self)

    def __enter__(self):
        if self._session_data.main_script_path not in sys.path:
            sys.path.insert(0, self._session_data.main_script_path)
            self._added_path = True

    def __exit__(self, type, value, traceback):
        if self._added_path:
            try:
                sys.path.remove(self._session_data.main_script_path)
            except ValueError:
                pass

        # Returning False causes any exceptions to be re-raised.
        return False

# The reason this is not a decorator is because we want to make it clear at the
# calling location that this function is being used.
def _log_if_error(fn: Callable[[], None]) -> None:
    try:
        fn()
    except Exception as e:
        LOGGER.warning(e)