import os
import psycopg2
from flask import Flask
from flask import render_template_string
from flask import render_template
import csv
import pandas as pd
import folium
from folium.plugins import HeatMapWithTime
from flask import Markup

app = Flask(__name__)
# # Menu(app=app)

# # DATABASE_URL will contain the database connection string: HEROKU
# app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL', '')
# app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
# # Connects to the database using the app config
# # db = SQLAlchemy(app)

@app.route("/")
def timelapse():
    # use the csv for now to generate the map

    # session = Session(engine)
    # histData = session.query(airQuality.AQI, airQuality.Defining_Parameter, airQuality.Date ).\
    #     join(sites, airQuality.site_no == sites.site_no). \
    #     join(counties, counties.county_code == sites.County_Code). \
    #     filter(airQuality.Date >= '2010-01-01' and airQuality.Date <= '2020-10-31' and \
    #         airQuality.Defining_Parameter == 'PM2.5')
    file = "Data/all_data.csv"
    param = "Ozone"
    # get the data
    histData = pd.read_csv(file, parse_dates=['Date'])

    histData['year'] = (histData['Date'].dt.year)
    histData['month'] = (histData['Date'].dt.month)
    histData['day'] = 1
    histData['newDate'] = pd.to_datetime(histData[['year', 'month', 'day']])

    #Reorder columns
    columns = ['newDate', 'Latitude', 'Longitude', 'AQI', 'Defining_Parameter','county_name']
    reorderData = histData[columns]

    # Get only the data for the parameter
    paramData = reorderData.loc[reorderData['Defining_Parameter'] == param]

    sortedhistData = paramData.sort_values(by=['newDate'])

    newhistData = sortedhistData.reset_index()
    newhistData.drop(columns=['index'], inplace=True)

    mapData=newhistData[["newDate", "Latitude", "Longitude", "AQI", 'Defining_Parameter']]
    # Create list of lists, with the "key" of the list being the date, and the "value" being all measurements for that date.
    # 1. Need list of date
    mapTime = mapData["newDate"].sort_values().unique()

    # 2. create the lists
    mapData_list = []

    # Weight must be between 0 and 1.  Divide AQI by max value to get correct data
    minAQI = mapData["AQI"].min()
    maxAQI = mapData['AQI'].max()

    mapData["AQI_adj"] = mapData["AQI"]/maxAQI

    for date in mapTime:
        mapData_list.append(mapData.loc[mapData['newDate'] == date, ["Latitude", "Longitude", "AQI_adj"]].groupby(["Latitude", "Longitude"]).mean().reset_index().values.tolist())

    # create the base map
    start_coords = (36.7378, -119.7871)
    folium_map = folium.Map(location=start_coords, zoom_start=6, tiles='Stamen Terrain')

    #add the Heat Map from the data
    #HeatMap(data=mapData, radius=20).add_to(folium_map)
    HeatMapWithTime(data=mapData_list, index=mapTime, radius=20, auto_play=True, max_opacity=0.3,
                       gradient = {0.2: '#FBD973',
                            0.4: '#fa782f',
                            0.75: '#F16578',
                            .9: '#782890'}).add_to(folium_map)

    # display base map with HeatMap
    _ = folium_map._repr_html_()

    map_id = Markup(folium_map.get_root().html.render())
    hdr_txt = Markup(folium_map.get_root().header.render())
    script_txt = Markup(folium_map.get_root().script.render())

    return render_template(
        'timelapse.html',
        map_id = map_id,
        hdr_txt=hdr_txt,
        script_txt = script_txt,
        title="Timelapse Heat Map - January 2010 - October 2020"
    )
@app.route("/yearlyvpop")
def yearlyvpop():
    return render_template("yearlyvpop.html")

# @app.after_request
# def add_header(r):
#     """
#     Add headers to both force latest IE rendering engine or Chrome Frame,
#     and also to cache the rendered page for 10 minutes.
#     """
#     r.headers["Cache-Control"] = "no-cache, no-store, must-revalidate"
#     r.headers["Pragma"] = "no-cache"
#     r.headers["Expires"] = "0"
#     r.headers['Cache-Control'] = 'public, max-age=0'
#     return r

# def tmpl_show_menu():
#     return render_template_string(
#         """
#         {%- for item in current_menu.children %}
#             {% if item.active %}*{% endif %}{{ item.text }}
#         {% endfor -%}
#         """)

# @app.route('/')
# @register_menu(app, '.', 'Home')
# def index():
#     return tmpl_show_menu()

# @app.route('/first')
# @register_menu(app, '.first', 'First', order=0)
# def first():
#     return tmpl_show_menu()

# @app.route('/second')
# @register_menu(app, '.second', 'Second', order=1)
# def second():
#     return tmpl_show_menu()

if __name__ == '__main__':
    app.run(debug=True)


from typing import Dict
import requests
import authorization_v2
import pandas as pd
import json

"""
This script updates the OS of the list of IP's in the Server's tab.

Usage:
1. Modify the input.csv  file to reflect your desired entries.
2. If you change the name of the input file, then edit this variable:  input_csv_filename = "input.csv"
3. Modify the headers (Encoded credentials to base 64)
4. Execute

Author: John Takacs
Email: john.takacs@gmail.com
Creation date: 2023-02-23
Developer note: This is code from my personal lab for my professional development. This should be considered example
code only.

Revision History:
-----------------
2023-02-23:
    Initial creation of the script.

"""

# Important filename variable
input_csv_filename = "input.csv"

def update_server_group_iplist(host: str, port: str, site_name: str, server_group_name: str, ip_address: str,
                               body: Dict[str, str], headers: Dict[str, str]) -> None:
    """
    Updates the OS of a server group IP address.

    Args:
    - host (str): the server host name or IP address
    - port (str): the server port number
    - site_name (str): the name of the site
    - server_group_name (str): the name of the server group
    - ip_address (str): the IP address of the server to update
    - body (Dict[str, str]): the request body containing the updated OS type
    - headers (Dict[str, str]): the request headers

    Returns: None
    """
    # https://docs.imperva.com/bundle/v14.7-dam-api-reference-guide/page/61821.htm
    # URL must match https://{host:port}/SecureSphere/api/v1/conf/serverGroups/{siteName}/{serverGroupName}/servers/{ip}
    url: str = f"https://{host}:{port}/SecureSphere/api/v1/conf/serverGroups/{site_name}/{server_group_name}/servers/{ip_address}"

    response = requests.put(url, json=body, headers=headers, verify=False)

    if response.status_code == 200:
        print(f"Successfully updated OS for IP address: {ip_address}")
    else:
        print("Error Code:  " + str(response.status_code))
        print(f"Failed to update OS for IP address : {ip_address}\nHere is the error message: {response.text}")

def read_csv(debug: bool) -> Dict:
    """
    Reads a CSV file and returns a dictionary containing the file data.
    Args:
    - debug (bool): if True, print additional debug information

    Returns:
    - data (Dict): a dictionary containing the file data
    """
    # Read CSV file
    data = pd.read_csv(input_csv_filename)

    # Convert DataFrame to JSON string
    json_string = data.to_json(orient='records')
    # Convert data to dictionary
    data = json.loads(json_string)
    if debug:
        print(type(data))
        assert isinstance(data, object)
        print(data)
        # Print JSON string
        print("This is json_string")
        print(json_string)
        print("This is data")
        print(type(data))
        print("This is data[0]")
        print(data[0])
        print(type(data[0]))
    # body=data[0]
    # comment = {"comment": "Some comment"}
    # body = comment
    return data

'''
This script reads data from a CSV file and uses it to update the OS of the protected IP lists in SecureSphere.
It first logs in to SecureSphere using the `authorization` module and gets an authorization cookie,
then uses this cookie to make API requests to update the OS.

Steps
Updates the OS of the list of IP's in the Server's tab by:
1. Iterating through a dictionary of server data; and
2. Calling the 'update_server_group_iplist' function for each server

The 'debug' flag controls whether additional debug information is printed.

Args: None
Returns: None
Raises: None

Usage:
$ python update_os_connection_ip_list_v2.py
'''
if __name__ == '__main__':
    debug = False
    data = read_csv(debug)
    # Get the session_id via get_cookie()
    my_response = authorization.authorization()
    my_cookies = authorization.get_cookie(my_response, debug)
    headers: Dict[str, str] = {"Content-Type": "application/json", "Authorization": "Basic YtW4",
                               'Cookie': my_cookies}
    if my_response.status_code == 200:
        # The request was successful
        print("\nThe initial login request was successful\nWe will now begin updating the OS Server Group"
              " IP's")
        # Iterate through the dictionary
        assert isinstance(data, object)
        count = len(data)
        print(count)
        rownum = 1
        for row in data:
            print(f"\nUpdating row {str(rownum)}:  {row['ip-address']}")
            if debug:
                print(f"This is the row {row}")
            host: str = row['MX-IP']
            port: str = row['MX-port']
            site_name: str = row['site']
            server_group_name: str = row['server_group_name']
            service_name: str = row['service_name']
            connection_name: str = row['connection_name']
            ip_address: str = row['ip-address']
            gateway_group_name: str = row['gateway_group_name']
            os_type = row['OS-type']
            assert isinstance(os_type, object)
            body = {'OS-type': os_type}
            print(body)
            update_server_group_iplist(host, port, site_name, server_group_name, ip_address, body, headers)
            rownum = rownum + 1

    else:
        # The request was not successful
        print(f'Request failed with status code {my_response.status_code}')


from apps.libraries.general.cache import CacheMixin
from .models import Prohibition

class ProhibitionManager(CacheMixin):
    key_list = (
        'get_list',
    )
    key_id_list = ()
    model = Prohibition

    @classmethod
    def get_list(cls):
        """
        禁止語句リストを取得
        :return: 禁止語句リスト
        :rtype: list of Prohibition
        """
        key_name = 'get_list'
        data_list = cls.get_cache(key_name)

        if not data_list:
            data_list = list(cls.model.objects.all())
            if data_list:
                cls.set_cache(key_name, data_list)

        return data_list

    @classmethod
    def check(cls, text):
        data_list = cls.get_list()

        for data in data_list:
            if data.text in text:
                return True

        return False


"""Models for Blogly."""
from flask_sqlalchemy import SQLAlchemy

db = SQLAlchemy()

DEFAULT_IMAGE_URL = "https://i.stack.imgur.com/l60Hf.png"

def connect_db(app):
    '''connect to database'''
    db.app = app
    db.init_app(app)

class User(db.Model):
    '''User'''
    __tablename__ = "users"

    id = db.Column(db.Integer,
                   primary_key=True,
                   autoincrement=True)
    first_name = db.Column(db.String(20),
                           nullable=False)
    last_name = db.Column(db.String(20),
                          nullable=False)
    image_url = db.Column(db.String,
                        nullable=False,
                        default=DEFAULT_IMAGE_URL)

    def __repr__(self):
        rep = f'<User: {self.first_name} {self.last_name}, id={self.id} >'
        return rep

class Post(db.Model):
    '''post'''
    __tablename__ = "posts"

    id = db.Column(db.Integer,
                   primary_key=True,
                   autoincrement=True)
    title = db.Column(db.String(20),
                           nullable=False)
    content = db.Column(db.String,
                          nullable=False)
    created_at = db.Column(db.DateTime,
                            nullable=False,
                            default=db.func.now())
    user_id=db.Column(db.Integer,
                        db.ForeignKey('users.id'))

    user = db.relationship('User', backref='posts')

    def __repr__(self):
        rep = f'''<Post: {self.title} {self.created_at} {self.user_id},
         id={self.id} >'''
        return rep


from tigramite.independence_tests.gpdc import GPDC
from fpcmci.CPrinter import CPLevel
from fpcmci.FPCMCI import FPCMCI
from fpcmci.preprocessing.data import Data
from fpcmci.selection_methods.TE import TE, TEestimator
from fpcmci.basics.constants import LabelType
from time import time
from datetime import timedelta
import pandas as pd

DATA_PATH = r'/home/lucacastri/git/tiago_ws/src/hrsi/tiago_postprocess/tiago_postprocess_bringup/data'
NUM_DATASET = 16
ACTOR = "greta"
INTERVENTION = ["noaction", "decrease", "increase"]
FILE_EXT = ".csv"

if __name__ == '__main__':

    li = []

    all_files = [DATA_PATH + "/" + ACTOR + "_" + i + "_" + str(d) + "_causal_notheta" + FILE_EXT for i in INTERVENTION for d in range(NUM_DATASET)]
    colnames = [r"r_{vh1}", r"r_{vh2}", r"r_{vt}", r"r_v", r"d_{rg}", r"t_{rg}", r"h_v", r"risk", r"d_{rh}"]

    for filename in all_files:
        try:
            df = pd.read_csv(filename, index_col = [0], names=colnames, header = 0)
            li.append(df)
        except:
            continue
    frame = pd.concat(li, axis = 0, ignore_index = True)

    f_alpha = 0.1
    pcmci_alpha = 0.05
    min_lag = 1
    max_lag = 1

    df = Data(frame)
    df.plot_timeseries()
    start = time()
    fpcmci = FPCMCI(df,
                    f_alpha = f_alpha,
                    pcmci_alpha = pcmci_alpha,
                    min_lag = min_lag,
                    max_lag = max_lag,
                    sel_method = TE(TEestimator.Gaussian),
                    val_condtest = GPDC(significance = 'analytic', gp_params = None),
                    verbosity = CPLevel.DEBUG,
                    neglect_only_autodep = True,
                    resfolder = ACTOR + "_" + "_".join(INTERVENTION) + "_ttc_uninoise_03trg_0175risk_05alpha")

    fpcmci_res, causal_model = fpcmci.run()
    elapsed_FPCMCI = time() - start
    print(str(timedelta(seconds = elapsed_FPCMCI)))

    colors = {r"r_{vh1}" : "orangered",
              r"r_{vh2}" : "orangered",
              r"r_{vt}" : "orangered",
              r"r_v" : "orangered",
              r"d_{rg}": "dodgerblue",
              r"t_{rg}": "dodgerblue",
              r"h_v": "dodgerblue",
              r"risk": "dodgerblue",
              r"d_{rh}" : "dodgerblue"}
    nodes_color = {'$' + key + '$': colors[key] for key in fpcmci_res}
    fpcmci.dag(label_type = LabelType.NoLabels, node_layout = 'circular', node_color = nodes_color)


class Ship:
    def __init__(self):
        # direction in degrees/ North = 0, East = 90, south = 180, West = 270
        self._heading = 90
        self._northing = 0
        self._easting = 0

        self._actions = {
            'N' : self._move_north,
            'S' : self._move_south,
            'E' : self._move_east,
            'W' : self._move_west,
            'L' : self._rotate_left,
            'R' : self._rotate_right,
            'F' : self._move_forward
        }

    def take_action(self, instruction):
        action = instruction[0]
        value = int(instruction[1:])
        assert action in self._actions
        self._actions[action](value)

    def manhattan_distance(self):
        return abs(self._easting) + abs(self._northing)

    def _move_north(self, value):
        self._northing += value

    def _move_south(self, value):
        self._northing -= value

    def _move_east(self, value):
        self._easting += value

    def _move_west(self, value):
        self._easting -= value

    def _rotate_left(self, value):
        self._heading -= value
        self._heading %= 360 # make sure we stay on the 0-360 degree range
        assert self._heading % 90 == 0, 'Puzzle suggest that we should only point north, west, south, or east'

    def _rotate_right(self, value):
        self._heading += value
        self._heading %= 360  # make sure we stay on the 0-360 degree range
        assert self._heading % 90 == 0, 'Puzzle suggest that we should only point north, west, south, or east'

    def _move_forward(self, value):
        assert self._heading % 90 == 0, 'Puzzle suggest that we should only point north, west, south, or east'
        if self._heading == 0:
            self._move_north(value)
        elif self._heading == 90:
            self._move_east(value)
        elif self._heading == 180:
            self._move_south(value)
        else:
            assert self._heading == 270
            self._move_west(value)

s = Ship()
'''
F10
N3
F7
R90
F11
These instructions would be handled as follows:
'''
# F10 would move the ship 10 units east (because the ship starts by facing east) to east 10, north 0.
s.take_action('F10')
assert s._easting == 10
assert s._northing == 0
# N3 would move the ship 3 units north to east 10, north 3.
s.take_action('N3')
assert s._easting == 10
assert s._northing == 3
# F7 would move the ship another 7 units east (because the ship is still facing east) to east 17, north 3.
s.take_action('F7')
assert s._easting == 17
assert s._northing == 3
# R90 would cause the ship to turn right by 90 degrees and face south; it remains at east 17, north 3.
s.take_action('R90')
assert s._easting == 17
assert s._northing == 3
# F11 would move the ship 11 units south to east 17, south 8.
s.take_action('F11')
assert s._easting == 17
assert s._northing == -8
# At the end of these instructions, the ship's Manhattan distance (sum of the absolute values of its east/west position and its north/south position) from its starting position is 17 + 8 = 25.
assert s.manhattan_distance() == 25

def puzzle1(instructions):
    ship = Ship()
    for instruction in instructions:
        ship.take_action(instruction)
    return ship.manhattan_distance()

print(f'The solution to puzzle1 is {puzzle1(open("input.txt").readlines())}')

class ShipNew:
    def __init__(self):
        self._northing = 0
        self._easting = 0
        self._waypoint_northing = 1
        self._waypoint_easting = 10

        self._actions = {
            'N' : self._move_waypoint_north,
            'S' : self._move_waypoint_south,
            'E' : self._move_waypoint_east,
            'W' : self._move_waypoint_west,
            'L' : self._rotate_waypoint_left,
            'R' : self._rotate_waypoint_right,
            'F' : self._move_to_waypoint
        }

    def take_action(self, instruction):
        action = instruction[0]
        value = int(instruction[1:])
        assert action in self._actions
        self._actions[action](value)

    def manhattan_distance(self):
        return abs(self._easting) + abs(self._northing)

    def _move_waypoint_north(self, value):
        self._waypoint_northing += value

    def _move_waypoint_south(self, value):
        self._waypoint_northing -= value

    def _move_waypoint_east(self, value):
        self._waypoint_easting += value

    def _move_waypoint_west(self, value):
        self._waypoint_easting -= value

    def _rotate_waypoint_left(self, value):
        assert value % 90 == 0, 'puzzle suggests we only rotate a multitude of 90 degrees'
        for i in range(value // 90):
            self._rotate_90_degrees_waypoint_counter_clockwise()

    def _rotate_waypoint_right(self, value):
        assert value % 90 == 0, 'puzzle suggests we only rotate a multitude of 90 degrees'
        for i in range(value // 90):
            self._rotate_90_waypoint_degrees_clockwise()

    def _rotate_90_waypoint_degrees_clockwise(self):
        northing = -1 * self._waypoint_easting
        self._waypoint_easting = self._waypoint_northing
        self._waypoint_northing = northing

    def _rotate_90_degrees_waypoint_counter_clockwise(self):
        northing = self._waypoint_easting
        self._waypoint_easting = -1 * self._waypoint_northing
        self._waypoint_northing = northing

    def _move_to_waypoint(self, value):
        self._northing += self._waypoint_northing * value
        self._easting += self._waypoint_easting * value

# For example, using the same instructions as above:
s = ShipNew()
# F10 moves the ship to the waypoint 10 times (a total of 100 units east and 10 units north), leaving the ship at east 100, north 10. The waypoint stays 10 units east and 1 unit north of the ship.
s.take_action('F10')
assert s._northing == 10
assert s._easting == 100
assert s._waypoint_northing == 1
assert s._waypoint_easting == 10
# N3 moves the waypoint 3 units north to 10 units east and 4 units north of the ship. The ship remains at east 100, north 10.
s.take_action('N3')
assert s._northing == 10
assert s._easting == 100
assert s._waypoint_northing == 4
assert s._waypoint_easting == 10
# F7 moves the ship to the waypoint 7 times (a total of 70 units east and 28 units north), leaving the ship at east 170, north 38. The waypoint stays 10 units east and 4 units north of the ship.
s.take_action('F7')
assert s._northing == 38
assert s._easting == 170
assert s._waypoint_northing == 4
assert s._waypoint_easting == 10
# R90 rotates the waypoint around the ship clockwise 90 degrees, moving it to 4 units east and 10 units south of the ship. The ship remains at east 170, north 38.
s.take_action('R90')
assert s._northing == 38
assert s._easting == 170
assert s._waypoint_northing == -10
assert s._waypoint_easting == 4

# F11 moves the ship to the waypoint 11 times (a total of 44 units east and 110 units south), leaving the ship at east 214, south 72. The waypoint stays 4 units east and 10 units south of the ship.
s.take_action('F11')
assert s._northing == -72
assert s._easting == 214
assert s._waypoint_northing == -10
assert s._waypoint_easting == 4
# After these operations, the ship's Manhattan distance from its starting position is 214 + 72 = 286.
assert s.manhattan_distance() == 286

def puzzle2(instructions):
    ship = ShipNew()
    for instruction in instructions:
        ship.take_action(instruction)
    return ship.manhattan_distance()

print(f'The solution to puzzle2 is {puzzle2(open("input.txt").readlines())}')


from aries_cloudagent.tests import mock
from unittest import IsolatedAsyncioTestCase

from ......core.in_memory import InMemoryProfile
from ......messaging.decorators.attach_decorator import AttachDecorator
from ......messaging.models.base_record import BaseExchangeRecord, BaseExchangeSchema

from ...message_types import ATTACHMENT_FORMAT, PRES_20_PROPOSAL
from ...messages.pres_format import V20PresFormat
from ...messages.pres_proposal import V20PresProposal

from .. import pres_exchange as test_module
from ..pres_exchange import V20PresExRecord

S_ID = "NcYxiDXkpYi6ov5FcYDi1e:2:vidya:1.0"
CD_ID = f"NcYxiDXkpYi6ov5FcYDi1e:3:CL:{S_ID}:tag1"
INDY_PROOF_REQ = {
    "name": "proof-req",
    "version": "1.0",
    "nonce": "12345",
    "requested_attributes": {
        "0_player_uuid": {
            "name": "player",
            "restrictions": [
                {
                    "cred_def_id": f"{CD_ID}",
                    "attr::player::value": "Richie Knucklez",
                }
            ],
            "non_revoked": {
                "from": 1234567890,
                "to": 1234567890,
            },
        },
        "0_screencapture_uuid": {
            "name": "screenCapture",
            "restrictions": [{"cred_def_id": f"{CD_ID}"}],
            "non_revoked": {
                "from": 1234567890,
                "to": 1234567890,
            },
        },
    },
    "requested_predicates": {
        "0_highscore_GE_uuid": {
            "name": "highScore",
            "p_type": ">=",
            "p_value": 1000000,
            "restrictions": [{"cred_def_id": f"{CD_ID}"}],
            "non_revoked": {
                "from": 1234567890,
                "to": 1234567890,
            },
        }
    },
}

class BasexRecordImpl(BaseExchangeRecord):
    class Meta:
        schema_class = "BasexRecordImplSchema"

    RECORD_TYPE = "record"

class BasexRecordImplSchema(BaseExchangeSchema):
    class Meta:
        model_class = BasexRecordImpl

class TestRecord(IsolatedAsyncioTestCase):
    async def test_record(self):
        pres_proposal = V20PresProposal(
            comment="Hello World",
            formats=[
                V20PresFormat(
                    attach_id="indy",
                    format_=ATTACHMENT_FORMAT[PRES_20_PROPOSAL][
                        V20PresFormat.Format.INDY.api
                    ],
                )
            ],
            proposals_attach=[
                AttachDecorator.data_base64(INDY_PROOF_REQ, ident="indy")
            ],
        )
        record = V20PresExRecord(
            pres_ex_id="pxid",
            thread_id="thid",
            connection_id="conn_id",
            initiator="init",
            role="role",
            state="state",
            verified="false",
            auto_present=True,
            error_msg="error",
            auto_remove=True,
        )
        record.pres_proposal = pres_proposal  # cover setter

        assert record.pres_ex_id == "pxid"

        assert record.record_value == {
            "connection_id": "conn_id",
            "initiator": "init",
            "role": "role",
            "state": "state",
            "pres_proposal": pres_proposal.serialize(),
            "verified": "false",
            "verified_msgs": None,
            "auto_present": True,
            "auto_verify": False,
            "error_msg": "error",
            "trace": False,
            "auto_remove": True,
        }

        bx_record = BasexRecordImpl()
        assert record != bx_record

    async def test_save_error_state(self):
        session = InMemoryProfile.test_session()
        record = V20PresExRecord(state=None)
        assert record._last_state is None
        await record.save_error_state(session)  # cover short circuit

        record.state = V20PresExRecord.STATE_PROPOSAL_RECEIVED
        await record.save(session)

        with mock.patch.object(
            record, "save", mock.CoroutineMock()
        ) as mock_save, mock.patch.object(
            test_module.LOGGER, "exception", mock.MagicMock()
        ) as mock_log_exc:
            mock_save.side_effect = test_module.StorageError()
            await record.save_error_state(session, reason="testing")
            mock_log_exc.assert_called_once()


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pandas as pd
import nltk,string
from gensim import corpora

data = pd.read_csv("news_cnn.csv",header=None, usecols=[1,2],names=["sentiment","review"])
data.head()
len(data)
#data['sentiment'].replace( [1,2],[0,1],inplace=True)

# if your computer does not have enough resource
# reduce the dataset
#data=data.loc[0:8000]

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

import numpy as np

# set the maximum number of words to be used
MAX_NB_WORDS=4700

# set sentence/document length
MAX_DOC_LEN=200

# get a Keras tokenizer
# https://keras.io/preprocessing/text/
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(data["review"])

# get the mapping between word and its index
voc=tokenizer.word_index

# convert each document to a list of word index as a sequence
sequences = tokenizer.texts_to_sequences(data["review"])

# total number of words
total_nb_words=len(tokenizer.word_counts)
print(total_nb_words)

# put word and its counts into a data frame
word_counts=pd.DataFrame(\
            tokenizer.word_counts.items(), \
            columns=['word','count'])
word_counts.head(3)

# get histogram of word counts
# after reset index, "index" column
# is the word frequency
# "count" column gives how many words appear at
# a specific frequency
df=word_counts['count'].value_counts().reset_index()
df.head(3)

# convert absolute counts to precentage
df['percent']=df['count']/len(tokenizer.word_counts)
# get cumulative percentage
df['cumsum']=df['percent'].cumsum()
df.head(5)

# plot the chart
# chart shows >90% words appear in less than 50 times
# if you like to include only words occur more than 50 times
# then MAX_NB_WORDS = 10% * total_nb_words
plt.bar(df["index"].iloc[0:50], df["percent"].iloc[0:50])
plt.plot(df["index"].iloc[0:50], df['cumsum'].iloc[0:50], c='green')

plt.xlabel('Word Frequency')
plt.ylabel('Percentage')
plt.show()

# create a series based on the length of all sentences
sen_len=pd.Series([len(item) for item in sequences])

# create histogram of sentence length
# the "index" is the sentence length
# "counts" is the count of sentences at a length
df=sen_len.value_counts().reset_index().sort_values(by='index')
df.columns=['index','counts']
df.head(3)

# sort by sentence length
# get percentage and cumulative percentage

df=df.sort_values(by='index')
df['percent']=df['counts']/len(sen_len)
df['cumsum']=df['percent'].cumsum()
df.head(3)

# From the plot, 90% sentences have length<500
# so it makes sense to set MAX_DOC_LEN=4~500
plt.plot(df["index"], df['cumsum'], c='green')

plt.xlabel('Sentence Length')
plt.ylabel('Percentage')
plt.show()


"""
Ekam - The command runner for masochists
"""

from sys import stderr
from pprint import pprint

WHITESPACE = " \t\n\r\f"

TYPES = [
    "number",     # 0: value
    "string",     # 1: value
    "list",       # 2: array
    "identifier", # 3: name
    "verb",       # 4: name
    "nil",        # 5: nil/null
    "alias",      # 6: recipe name
    "character"   # 7: for strings
]

# Helper functiens
def e    (t, v): return { "t": t, "v": v }
def en   (v):    return e(0, v)
def es   (v):    return e(1, v)
def el   (v):    return e(2, v)
def ei   (v):    return e(3, v)
def ev   (v):    return e(4, v)
def ea   (v):    return e(6, v)
def ec   (v):    return e(7, v)
def btoi (v):    return en(1) if v else en(0) # Boolean to o-integer
def error(v):    print("Error: ", v, file=stderr)
def warn (v):    print("Warn: ", v, file=stderr)

TRUE  = en(1)
FALSE = en(0)

symbol = lambda c: c.isascii() and (c not in WHITESPACE) and (not c.isalpha()) and (not c.isnumeric())

def consume(cond, code, pos):
    "Consume a token in `code` until the conditien `cond` is false"
    prev_pos = pos
    pos += 1
    while pos < len(code) and cond(code[pos]):
        pos += 1
    return code[prev_pos:pos], pos

def consume_string(code: str, pos: int):
    "Consume string literals"
    lst = []
    while pos < len(code) and code[pos] != "'":
        if code[pos] == "\\":
            lst.append(ec(rf"{code[pos + 1]}"))
            pos += 2
        elif code[pos] == "{":
            ident, pos = consume_string_interpolation(code, pos + 1)
            lst.append(ei(ident))
        else:
            lst.append(ec(code[pos]))
            pos += 1
    return lst, pos

def consume_string_interpolation(code, pos):
    "Parse string interpolations"
    prev_pos = pos
    pos += 1
    while pos < len(code):
        # When it is the end of string interpolation, we skips those two }s
        # and exit
        if code[pos] == "}":
            pos += 1
            break
        pos += 1
    return code[prev_pos:pos-1], pos

def identifier(code, pos):
    "Consume an identifier"
    prev_pos = pos
    pos += 1
    if code[pos].isalpha():
        while pos < len(code) and (code[pos].isalpha() or code[pos].isnumeric() or code[pos] in "_-"):
            pos += 1
    return code[prev_pos:pos], pos

def parseVal(code, pos):
    if code[pos] == "'":
        val, pos = consume_string(code, pos + 1)
        return es(val), pos + 1
    elif code[pos] == "$":
        ident, pos = identifier(code, pos + 1)
        return ei(ident), pos
    elif code[pos] == '[':
        pos += 1
        lst = []
        while code[pos] != ']':
            val, pos = parseVal(code, pos)
            if val != None:
                lst.append(val)
        return el(lst), pos + 1
    elif code[pos] == "#":
        _, pos = consume(lambda x: x != "\n", code, pos)
        return None, pos
    elif symbol(code[pos]):
        sym, pos = consume(lambda x: symbol(x), code, pos)
        return ev(sym), pos
    elif code[pos] in WHITESPACE:
        return None, pos + 1
    return None, pos + 1

def parse(code):
    pos = 0
    lst = []
    while pos < len(code):
        val, pos = parseVal(code, pos)
        lst.append(val)
    lst = list(filter(lambda x: x != None, lst))
    return lst

def format(o):
    "Format an object"
    if isinstance(o, list):
        return format(o[0])
    if o["t"] == 2:                  # Lists
        return [format(obj) for obj in o["v"]]
    elif o["t"] == 3 or o["t"] == 4: # Identifiers/Verbs
        return f'{o["v"]}'
    else:                            # Numbers/Strings/Other literals
        return o["v"]

def eval_token(env, token):
    """Evaluate a single token"""
    # Here we exclude out verbs because those will be handled by the main eval() function
    if token["t"] == 3: # Variable
        return env[token["v"]]
    elif token["t"] == 2: # List
        return [eval_token(env, e) for e in token["v"]]
    elif token["t"] == 1: # String
        # We will parse the string by parsing each part of
        # the string list one by one
        res = ""
        string_lst = token["v"]
        for part in string_lst:
            #print(part)
            res += eval_token(env, part)
        return res
    else:
        return token["v"]

def eval(tokens):
    """Evaluate the code from tokens"""
    i = 0
    # Envirenments: Centaining variables, aliases, and recipes
    env = {}
    while i < len(tokens):
        # If the token is a verb, then we can parse the commands
        # based on them
        if tokens[i]["t"] == 4:
            # Set
            if tokens[i]["v"] == "<-":
                value, name = eval_token(env, tokens[i + 1]), tokens[i + 2]
                env[name["v"]] = value
                i += 3
            # Alias
            elif tokens[i]["v"] == "->":
                alias, name = tokens[i + 1], tokens[i + 2]
                env[alias["v"]] = ea(name["v"])
                i += 3
            # Recipe
            elif tokens[i]["v"] == ":":
                name, args, cmds = tokens[i + 1], eval_token(env, tokens[i + 2]), eval_token(env, tokens[i + 3])
                env[name["v"]] = (args, cmds)
                i += 4
        else:
            return eval_token(env, tokens[i])
    return env

def run(code, tree=False, quiet=False):
    """Run the code, and return tokens and the environment"""
    parsed = parse(code)
    env = eval(parsed)
    if not quiet:
        if tree:
            print("Parsed: ", end="")
            pprint(parsed)
        else:
            print(f"Parsed: {parsed}")
        print("Envirenments: ", end="")
        pprint(env)
    return parsed, env


from . import models, __version__
from collections.abc import MutableMapping
import pprint
import pdb

import logging
logger = logging.getLogger(__name__)

# MutableMapping should be enough, but it causes problems with py2
DICTCLASSES = (MutableMapping, dict, models.BaseModel)

def check_template(indict, template):
    if isinstance(template, DICTCLASSES) and isinstance(indict, DICTCLASSES):
        for k, v in template.items():
            if k not in indict:
                raise models.Error('{} not in {}'.format(k, indict))
            check_template(indict[k], v)
    elif isinstance(template, list) and isinstance(indict, list):
        for e in template:
            for i in indict:
                try:
                    check_template(i, e)
                    break
                except models.Error as ex:
                    # raise
                    continue
            else:
                raise models.Error(('Element not found.'
                                   '\nExpected: {}\nIn: {}').format(pprint.pformat(e),
                                                                    pprint.pformat(indict)))
    elif isinstance(template, float) and isinstance(indict, float):
        diff = abs(indict - template)
        if (diff > 0) and diff/(abs(indict+template)) > 0.05:
            raise models.Error('Differences greater than 10% found.\n')
    else:
        if indict != template:
            raise models.Error(('Differences found.\n'
                                '\tExpected: {}\n'
                                '\tFound: {}').format(pprint.pformat(template),
                                                      pprint.pformat(indict)))

def convert_dictionary(original, mappings):
    result = {}
    for key, value in original.items():
        if key in mappings:
            key = mappings[key]
        result[key] = value
    return result

def easy_load(app=None, plugin_list=None, plugin_folder=None, **kwargs):
    '''
    Run a server with a specific plugin.
    '''

    from flask import Flask
    from .extensions import Senpy

    if not app:
        app = Flask(__name__)
    sp = Senpy(app, plugin_folder=plugin_folder, **kwargs)
    if not plugin_list:
        from . import plugins
        import __main__
        plugin_list = plugins.from_module(__main__)
    for plugin in plugin_list:
        sp.add_plugin(plugin)
    sp.install_deps()
    sp.activate_all()
    return sp, app

def easy_test(plugin_list=None, debug=True):
    logger.setLevel(logging.DEBUG)
    logging.getLogger().setLevel(logging.INFO)
    try:
        if not plugin_list:
            import __main__
            logger.info('Loading classes from {}'.format(__main__))
            from . import plugins
            plugin_list = plugins.from_module(__main__)
        plugin_list = list(plugin_list)
        for plug in plugin_list:
            plug.test()
            plug.log.info('My tests passed!')
        logger.info('All tests passed for {} plugins!'.format(len(plugin_list)))
    except Exception:
        if not debug:
            raise
        pdb.post_mortem()

def easy(host='0.0.0.0', port=5000, debug=False, **kwargs):
    '''
    Run a server with a specific plugin.
    '''
    logging.getLogger().setLevel(logging.DEBUG)
    logging.getLogger('senpy').setLevel(logging.INFO)
    sp, app = easy_load(**kwargs)
    easy_test(sp.plugins())
    app.debug = debug
    import time
    logger.info(time.time())
    logger.info('Senpy version {}'.format(__version__))
    logger.info('Server running on port %s:%d. Ctrl+C to quit' % (host,
                                                                  port))
    app.debug = debug
    app.run(host,
            port,
            debug=app.debug)


import os
import logging
import glob

import yaml
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

from mathtools import utils

logger = logging.getLogger(__name__)

def plotLabels(fn, label_seq):
    fig, axis = plt.subplots(1, figsize=(12, 9))

    axis.plot(label_seq)

    plt.tight_layout()
    plt.savefig(fn)
    plt.close()

def writeLabels(fn, label_seq, vocab):
    seg_label_idxs, seg_durs = utils.computeSegments(label_seq)

    seg_durs = np.array(seg_durs)
    seg_ends = np.cumsum(seg_durs) - 1
    seg_starts = np.array([0] + (seg_ends + 1)[:-1].tolist())
    seg_labels = tuple(str(vocab[i]) for i in seg_label_idxs)
    d = {
        'start': seg_starts,
        'end': seg_ends,
        'label': seg_labels
    }
    pd.DataFrame(d).to_csv(fn, index=False)

def parseActions(assembly_actions, num_frames, vocab):
    def makeJoint(part1, part2):
        return tuple(sorted([part1, part2]))

    def updateAssembly(assembly, joint):
        return tuple(sorted(cur_assembly + (joint,)))

    assembly_index_seq = np.zeros(num_frames, dtype=int)

    cur_assembly = tuple()
    prev_start = -1
    prev_end = -1
    for i, row in assembly_actions.iterrows():
        if row.start != prev_start or row.end != prev_end:
            cur_assembly_index = utils.getIndex(cur_assembly, vocab)
            assembly_index_seq[prev_end:row.end + 1] = cur_assembly_index
            prev_start = row.start
            prev_end = row.end

        if row.action == 'connect':
            joint = makeJoint(row.part1, row.part2)
            cur_assembly = updateAssembly(cur_assembly, joint)
        elif row.action == 'pin':
            continue
        else:
            raise ValueError()
    cur_assembly_index = utils.getIndex(cur_assembly, vocab)
    assembly_index_seq[prev_end:] = cur_assembly_index
    return assembly_index_seq

def main(out_dir=None, data_dir=None, labels_dir=None):
    out_dir = os.path.expanduser(out_dir)
    data_dir = os.path.expanduser(data_dir)
    labels_dir = os.path.expanduser(labels_dir)

    logger = utils.setupRootLogger(filename=os.path.join(out_dir, 'log.txt'))

    fig_dir = os.path.join(out_dir, 'figures')
    if not os.path.exists(fig_dir):
        os.makedirs(fig_dir)

    out_data_dir = os.path.join(out_dir, 'data')
    if not os.path.exists(out_data_dir):
        os.makedirs(out_data_dir)

    filenames = [
        utils.stripExtension(fn)
        for fn in glob.glob(os.path.join(labels_dir, '*.csv'))
    ]

    metadata = utils.loadMetadata(data_dir)
    metadata['seq_id'] = metadata.index
    metadata = metadata.set_index('dir_name', drop=False).loc[filenames].set_index('seq_id')

    seq_ids = np.sort(metadata.index.to_numpy())
    logger.info(f"Loaded {len(seq_ids)} sequences from {labels_dir}")

    vocab = []
    for i, seq_id in enumerate(seq_ids):
        seq_id_str = f"seq={seq_id}"
        seq_dir_name = metadata['dir_name'].loc[seq_id]
        labels_fn = os.path.join(labels_dir, f'{seq_dir_name}.csv')
        event_labels = utils.loadVariable(f'{seq_id_str}_labels', data_dir)

        assembly_actions = pd.read_csv(labels_fn)
        label_seq = parseActions(assembly_actions, event_labels.shape[0], vocab)
        utils.saveVariable(label_seq, f'{seq_id_str}_label-seq', out_data_dir)

        plotLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.png'), label_seq)
        writeLabels(os.path.join(fig_dir, f'{seq_id_str}_labels.csv'), label_seq, vocab)

    utils.saveMetadata(metadata, out_data_dir)
    utils.saveVariable(vocab, 'vocab', out_data_dir)

if __name__ == "__main__":
    # Parse command-line args and config file
    cl_args = utils.parse_args(main)
    config, config_fn = utils.parse_config(cl_args, script_name=__file__)

    # Create output directory, instantiate log file and write config options
    out_dir = os.path.expanduser(config['out_dir'])
    if not os.path.exists(out_dir):
        os.makedirs(out_dir)
    with open(os.path.join(out_dir, config_fn), 'w') as outfile:
        yaml.dump(config, outfile)
    utils.copyFile(__file__, out_dir)

    main(**config)


import json
import urllib.request, urllib.parse, urllib.error
import xml.etree.ElementTree as ET
import ssl

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

url = input('Enter location: ')
html = urllib.request.urlopen(url, context=ctx)
data = html.read().decode()

print('Retrieving', url)
print('Retrieved', len(data), 'characters')

info = json.loads(data)
#type: how json converts info in python,in this case it was in dict
#print(type(info))
#to convert info in list
#print(type(info['comments']))

# len = How many dicts in list/info['comments']
print('User count:', len(info['comments']))

num=list()
for item in info['comments']:
    x = (int(item['count']))
    num.append(x)

print(sum(num))


import os

from google.cloud import storage, vision
from google.oauth2 import service_account
import requests

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
from django.conf import settings

# constants
KEY_FILE = 'key.json'

# authentication
def create_storage_client():
    credentials = service_account.Credentials.from_service_account_file(os.path.join(BASE_DIR, KEY_FILE))
    client = storage.Client(project=settings.GOOGLE_CLOUD_PROJECT_ID, credentials=credentials)
    return client

def create_product_search_client():
    credentials = service_account.Credentials.from_service_account_file(os.path.join(BASE_DIR, KEY_FILE))
    client = vision.ProductSearchClient(credentials=credentials)
    return client

def create_image_annotator_client():
    credentials = service_account.Credentials.from_service_account_file(os.path.join(BASE_DIR, KEY_FILE))
    client = vision.ImageAnnotatorClient(credentials=credentials)
    return client

# storage part
def list_blobs():
    client = create_storage_client()
    blobs = client.list_blobs(settings.GOOGLE_STORAGE_BUCKET_NAME)
    return blobs

def upload_file(file_name, storage_path):
    client = create_storage_client()
    bucket = client.get_bucket(settings.GOOGLE_STORAGE_BUCKET_NAME)
    new_blob = bucket.blob(storage_path)
    new_blob.upload_from_filename(file_name)
    return new_blob

# product search part
def create_product_set(product_set_id, product_set_display_name):
    client = create_product_search_client()
    location_path = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)

    new_product_set = vision.types.ProductSet(display_name=product_set_display_name)
    response = client.create_product_set(
        parent=location_path,
        product_set=new_product_set,
        product_set_id=product_set_id
    )
    return response

def create_product_add_to_product_set(product_set_path, product_display_name, product_category):
    client = create_product_search_client()
    location_path = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)

    new_product = vision.types.Product(
        display_name=product_display_name,
        product_category=product_category,
    )

    new_product_response = client.create_product(
        parent=location_path,
        product=new_product,
    )

    client.add_product_to_product_set(name=product_set_path, product=new_product_response.name)

    return new_product_response

def create_reference_image(product_path, gcs_uri, boundingPolys=None):
    create_ref_image_api_url = 'https://vision.googleapis.com/v1/{}/referenceImages?key={}'.format(product_path, settings.GOOGLE_CLOUD_API_KEY)
    if boundingPolys is not None:
        r = requests.post(
            create_ref_image_api_url, json={
                "uri": gcs_uri,
                "boundingPolys": [
                    {
                        "normalizedVertices": boundingPolys,
                    }
                ]
            }
        )
        return r.json()
    else:
        r = requests.post(
            create_ref_image_api_url, json={
                "uri": gcs_uri,
            }
        )
        return r.json()

def create_multiple_reference_images(csv_gcs_uri):
    client = create_product_search_client()
    location_path = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)

    gcs_source = vision.types.ImportProductSetsGcsSource(csv_file_uri=csv_gcs_uri)
    input_config = vision.types.ImportProductSetsInputConfig(gcs_source=gcs_source)

    response = client.import_product_sets(parent=location_path, input_config=input_config)

    print('Processing operation name: {}'.format(response.operation.name))
    # synchronous check of operation status
    result = response.result()
    print('Processing done.')

    for i, status in enumerate(result.statuses):
        print('Status of processing line {} of the csv: {}'.format(i, status))
        # Check the status of reference image
        # `0` is the code for OK in google.rpc.Code.
        if status.code == 0:
            reference_image = result.reference_images[i]
            print(reference_image)
        else:
            print('Status code not OK: {}'.format(status.message))

def list_product_sets():
    client = create_product_search_client()
    location_path = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)

    product_sets = client.list_product_sets(parent=location_path)
    return product_sets

def get_product_set(product_set_path):
    client = create_product_search_client()
    product_set = client.get_product_set(name=product_set_path)
    return product_set

def list_products_in_product_set(product_set_path):
    client = create_product_search_client()
    products = client.list_products_in_product_set(name=product_set_path)
    return products

def get_product(product_path):
    client = create_product_search_client()
    product = client.get_product(name=product_path)
    return product

def list_reference_images(product_path):
    client = create_product_search_client()
    reference_images = client.list_reference_images(parent=product_path)
    return reference_images

def get_reference_image(reference_image_path):
    client = create_product_search_client()
    image = client.get_reference_image(name=reference_image_path)
    return image

def remove_product_from_product_set(product_path, product_set_path):
    client = create_product_search_client()
    client.remove_product_from_product_set(
        name=product_set_path,
        product=product_path
    )
    return True

def delete_product_set(product_set_path):
    client = create_product_search_client()
    client.delete_product_set(name=product_set_path)
    return True

def delete_product(product_path):
    client = create_product_search_client()
    client.delete_product(name=product_path)
    return True

def delete_all_products(force):
    client = create_product_search_client()
    parent = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)
    operation = client.purge_products(
        parent=parent,
        delete_orphan_products=True,
        force=force
    )
    operation.result(timeout=300)

def delete_reference_image(image_path):
    client = create_product_search_client()
    client.delete_reference_image(name=image_path)

    return True

def delete_all_products_in_product_set(product_set_id, force):
    client = create_product_search_client()
    parent = client.location_path(project=settings.GOOGLE_CLOUD_PROJECT_ID, location=settings.GOOGLE_CLOUD_LOCATION_ID)
    product_set_purge_config = vision.types.ProductSetPurgeConfig(product_set_id=product_set_id)
    operation = client.purge_products(
        parent=parent,
        product_set_purge_config=product_set_purge_config,
        force=force
    )
    operation.result(timeout=300)

def get_similar_product_uri(product_set_path, image_uri):
    product_search_client = create_product_search_client()
    image_annotator_client = create_image_annotator_client()

    image_source = vision.types.ImageSource(image_uri=image_uri)
    image = vision.types.Image(source=image_source)

    product_search_params = vision.types.ProductSearchParams(
        product_set=product_set_path,
        product_categories=['apparel-v2', 'homegoods-v2', 'toys-v2', 'packagedgoods-v1', 'general-v1'],
        # filter=filter
    )
    image_context = vision.types.ImageContext(product_search_params=product_search_params)

    response = image_annotator_client.product_search(image, image_context=image_context)

    index_time = response.product_search_results.index_time
    print('Product set index time:')
    print('  seconds: {}'.format(index_time.seconds))
    print('  nanos: {}\n'.format(index_time.nanos))

    results = response.product_search_results.results

    print('Search results:')
    for result in results:
        product = result.product

        print('Score(Confidence): {}'.format(result.score))
        print('Image name: {}'.format(result.image))
        print('Image URI:', product_search_client.get_reference_image(name=result.image).uri)

        print('Product name: {}'.format(product.name))
        print('Product display name: {}'.format(product.display_name))
        print('Product description: {}\n'.format(product.description))
        print('Product labels: {}\n'.format(product.product_labels))


# -*- coding: utf-8 -*-
'''
Created on 07.12.2010

@author: Sergey Prokhorov <me@seriyps.ru>
'''
import hashlib
import urllib
import json

__all__=['vkException',
         'vkApiOAuth',
         'vkApiOld']

class vkException(Exception):

    def __init__(self, code, msg, url, response):
        self.code=code
        self.msg=msg
        self.response=response
        self.url=url
        Exception.__init__(self, code, msg, url, response)

def Download(url, params):
    qstring=urllib.urlencode(params)
    return urllib.urlopen("?".join((url, qstring))).read()

class _vkApiBase(object):

    def __init__(self):
        self._prefix=[]

    """Support for api.namespace.method(**kwargs) calls below"""
    def __getattr__(self, name):
        if ("(" in name) or (name in self._prefix):
            return self#protection from ipython introspection calls
        self._prefix.append(name)
        return self

    def __call__(self, **kwargs):
        if self._prefix:
            method=".".join(self._prefix)
            self._prefix=[]
        else:
            method=kwargs.pop("method")
        return self._load(method, **kwargs)

class vkApiOAuth(_vkApiBase):
    """Vkontakte.ru (vk.com) API wrapper for Gwibber
    == Usage ==
    Initialization:
        api=vk_api(access_token)
        "access_token" can be retrieved from call to http://api.vkontakte.ru/oauth/authorize
        (see http://vkontakte.ru/developers.php?o=-1&p=%C0%E2%F2%EE%F0%E8%E7%E0%F6%E8%FF%20%EA%EB%E8%E5%ED%F2%F1%EA%E8%F5%20%EF%F0%E8%EB%EE%E6%E5%ED%E8%E9)
    Make calls to simple methods (has no dots in name, like "getProfiles", "isAppUser"):
        res=api.method(arg_name1=arg1, arg_name2=arg2)
        eg res=api.getProfiles(uids="535397,1", fields="uid,first_name,last_name")
    Make calls to namespaced methods (has dots in name, like "wall.post", "newsfeed.get"):
        res=api.namespace.method(arg_name1=arg1, arg_name2=arg2)
        eg res=api.wall.post(message="New wall post from API")
    Alternative call syntax (better performance):
        res=api._load('namespace.method', arg_name1=arg1, arg_name2=arg2)
        eg res=api._get('wall.post', message="New wall post from API")
    """

    api_url='https://api.vkontakte.ru/method/%s'

    def __init__(self, access_token, downloader=None):
        self._access_token=access_token
        self._prefix=[]
        self._downloader=downloader or Download
        super(vkApiOAuth, self).__init__()

    def _load(self, method, **params):
        """Make call by method name and arguments"""
        params["access_token"]=self._access_token
        url=self.api_url%method
        res=json.loads(self._downloader(url, params))

        if res.has_key("error"):
            raise vkException(res["error"]["error_code"],
                              res["error"]["error_msg"],
                              "%s?%s"%(url, urllib.urlencode(params)),
                              res)
        return res

class vkApiOld(_vkApiBase):
    """Vkontakte.ru (vk.com) API wrapper
    == Usage ==
    Initialization:
        api=vkApi(session, secret, mid, app_id)
        "session", "secret", "mid" can be retrieved from call to http://vkontakte.ru/login.php
        "app_id" is an application id from application administrative panel
        (see http://vk.com/developers.php?o=-17680044&p=Desktop+Application+Authorization)
    Make calls to simple methods (has no dots in name, like "getProfiles", "isAppUser"):
        res=api.method(arg_name1=arg1, arg_name2=arg2)
        eg res=api.getProfiles(uids="535397,1", fields="uid,first_name,last_name")
    Make calls to namespaced methods (has dots in name, like "wall.post", "newsfeed.get"):
        res=api.namespace.method(arg_name1=arg1, arg_name2=arg2)
        eg res=api.wall.post(message="New wall post from API")
    Alternative call syntax (better performance):
        res=api._load('namespace.method', arg_name1=arg1, arg_name2=arg2)
        eg res=api._load('wall.post', message="New wall post from API")
    """

    api_url='http://api.vkontakte.ru/api.php'
    version='3.0'

    def __init__(self, sid, secret, mid, app_id,
				downloader=None):
        self._session=sid
        self._secret=secret
        self._mid=mid
        self._app_id=app_id
        self._prefix=[]
        self._downloader=downloader or Download
        super(vkApiOld, self).__init__()

    def _make_sig(self, params):
        """Create request signature"""
        params="".join("%s=%s"%(k, v) for k, v in sorted(params.items()))
        sig_str=unicode(self._mid)+unicode(params)+unicode(self._secret)
        return hashlib.md5(sig_str).hexdigest()

    def _load(self, method, **kwargs):
        """Make call by method name and arguments"""
        for k, v in kwargs.items():#convert lists/tuples to comma-separated string
            if isinstance(v, (tuple, list)):
                kwargs[k]=",".join(str(e) for e in v)#convert each element to str and join by comma

        params=dict(kwargs, **{'api_id': str(self._app_id),
                               'method': method,
                               'v': self.version,
                               'format': 'JSON'})
        params["sig"]=self._make_sig(params)
        params["sid"]=self._session

        res=json.loads(self._downloader(self.api_url, params))

        if res.has_key("error"):
            raise vkException(res["error"]["error_code"],
                              res["error"]["error_msg"],
                              "%s?%s"%(self.api_url, urllib.urlencode(params)),
                              res)
        return res


"""
This module contains additional test methods used to test AOC 2018 solutions
against example inputs. This test module is configured to use the pytest
library.
"""

from src.solutions import day08

def test_day08_part1():
    """
    Solution test method for AOC 2018 Day 8 Part 1.
    """
    input_data = day08.process_input_file(
        filepath="./input/example/day08_ex01.txt")
    solution = day08.solve_part1(input_data)
    assert solution == 138

def test_day08_part2():
    """
    Solution test method for AOC 2018 Day 8 Part 2.
    """
    input_data = day08.process_input_file(
        filepath="./input/example/day08_ex01.txt")
    solution = day08.solve_part2(input_data)
    assert solution == 66


from PIL import Image
from web.db.models import *
import cStringIO

class Pixels:
    def get_pixels_from_img(self, picture):
        image = Image.open(cStringIO.StringIO(picture.image))
        self.width, self.height = image.size

        self.pixels = []
        for i in xrange(self.width):
            for j in xrange(self.height):
                pixel = image.getpixel((i, j))
                self.pixels.append({
                    'x': i,
                    'y': j,
                    'color': pixel,
                })

    def to_hash(self):
        return {
            'width': self.width,
            'height': self.height,
            'pixels': self.pixels,
        }


import sys
from PyQt5 import QtGui, QtWidgets
from config import *
from PyQt5.QtCore import *
import time
#TODO: self.logTextBox should offer API to append information
class WorkingDemo(QThread):
    #TODO: what this syntax it is!? I can't put sinOut into __init__
    sinOut = pyqtSignal(str)
    def __init__(self):
        super().__init__()
        self.m = 0

    def run(self):
        while True:
            self.m+=1
            print(self.m)
            self.sinOut.emit(str(self.m))
            time.sleep(0.1)

class Window(object):
    def __init__(self):
        self.app = QtWidgets.QApplication(sys.argv)
        w = QtWidgets.QWidget()
        w.setGeometry(100,100,200,50)
        w.setWindowTitle("P2P Demo")
        self.window = w

        self.initIP(w)
        self.initPort(w)
        self.initLog(w)
        self.initSubmit(w)

        w.resize(WINDOW_WIDTH, WINDOW_HEIGHT)
        w.show()
    def run(self):
        self.window.show()
        return self.app.exec_()
    def submitOnClick(self):
        if self.IPValidator(self.ip) and self.PortValidator(self.port):
            print('success', self.ip, self.port)
            self.task = WorkingDemo()
            self.task.sinOut.connect(self.append)
            self.task.start()
        else:
            if self.task:
                self.task.terminate()

    def initIP(self, window):
        self.initIPAddressTextbox(window)
        self.initIPLabel(window)
    def initPort(self, window):
        self.initPortTexbox(window)
        self.initPortLabel(window)
    def initLog(self, window):
        self.initLogTexbox(window)
        self.initLogLabel(window)
        self.logIndex = 0
    def initSubmit(self, window):
        self.submitButton = QtWidgets.QPushButton(window)
        self.submitButton.setText("submit")
        self.submitButton.move(SUBMIT_LEFT_MARGIN, SUBMIT_BUTTON_TOP_MARGIN)
        self.submitButton.clicked.connect(self.submitOnClick)

    def initLogTexbox(self, window):
        self.LogTextbox = QtWidgets.QPlainTextEdit(window)
        self.LogTextbox.move(LOG_LEFT_MARGIN, MARGIN)
        self.LogTextbox.resize(LOG_WIDTH, LOG_HEIGH)
        self.LogTextbox.setReadOnly(True)
    def initLogLabel(self, window):
        self.LogLabel = QtWidgets.QLabel(window)
        self.LogLabel.setText("Log")
        self.LogLabel.move(LOG_LABEL_LEFT_MARGIN, 10)

    def initIPAddressTextbox(self, window):
        self.IPTextbox = QtWidgets.QLineEdit(window)
        self.IPTextbox.move(LEFT_MARGIN + LABEL_WIDTH, MARGIN)
        self.IPTextbox.textChanged.connect(self.editIP)
    def initPortTexbox(self, window):
        self.portTexbox = QtWidgets.QLineEdit(window)
        self.portTexbox.move(LEFT_MARGIN + LABEL_WIDTH, MARGIN + LABEL_HEIGHT)
        self.portTexbox.textChanged.connect(self.editPort)
    def initIPLabel(self,window):
        self.IPLabel = QtWidgets.QLabel(window)
        self.IPLabel.setText('IP')
        self.IPLabel.move(LEFT_MARGIN, MARGIN + WORD_OFFSET)
    def initPortLabel(self,window):
        self.IPLabel = QtWidgets.QLabel(window)
        self.IPLabel.move(LEFT_MARGIN, MARGIN + WORD_OFFSET + LABEL_HEIGHT)
        self.IPLabel.setText('Port')

    def editIP(self, newIP):
        self.ip = newIP
    def editPort(self, newPort):
        self.port = newPort

    def IPValidator(self, ip):
        ips = ip.split('.')
        if (len(ips) != 4):
            print("Error '.' in IP")
            return False
        try:
            for part in ips:
                ipart = int(part)
                if (ipart <= 0 or ipart >= 256):
                    return False
        except:
            print("Error in parcing IP")
            return False
        return True
    def PortValidator(self, port):
        try:
            iport = int(port)
            if (iport <= 1024 or iport >= 65536):
                print("port {} not in range(1024, 65535)".format(iport))
                return False
        except:
            print("Port {} not int".format(port))
            return False
        return True
    def append(self, txt):
        self.logIndex += 1
        self.LogTextbox.setPlainText(self.LogTextbox.toPlainText()  + txt + "\n")
        self.LogTextbox.verticalScrollBar().setValue(self.logIndex)
if __name__ == '__main__':
    window = Window()
    # Should return the exit code of QApplication
    sys.exit(window.run())


import os.path as osp

import torch
import torch.nn as nn
import torch.nn.functional as F

class DecoderBaseModel(nn.Module):
    def __init__(self):
        super().__init__()

    def load_weights(self, load_m_dir=None, load_m_file_name='decoder.pth',):
        if load_m_dir is not None:
            dec_path = osp.join(load_m_dir, load_m_file_name)
            param = torch.load(dec_path)
            self.load_state_dict(param)
            print(f'[info] {load_m_file_name} loaded !')

    def save(self, save_f_path='_logs/test/decoder.pth',):
        torch.save(self.state_dict(), save_f_path)

###########################################
##                 Decoder               ##
###########################################

class DecoderLSTM(DecoderBaseModel):
    def __init__(self, emb_size, hidden_size, output_size, pad_token=-1):
        super().__init__()

        self.embedding = nn.Embedding(output_size, emb_size, padding_idx=pad_token)
        self.lstm = nn.LSTMCell(emb_size, hidden_size)
        self.out_w = nn.Linear(hidden_size, output_size)

    def forward(self, input, hidden):
        """
        :param input: (b)
        :param hidden: ((b,h), (b,h))
        :return: (b,o), (b,h)
        """
        embedded = self.embedding(input)  # (b) -> (b,e)
        decoder_output, hidden = self.lstm(embedded, hidden)  # (b,e),((b,h),(b,h)) -> (b,h),((b,h),(b,h))
        output = self.out_w(decoder_output)  # (b,h) -> (b,o)
        output = F.log_softmax(output, dim=1)

        return output, hidden  # (b,o), (b,h)

class DecoderGRU(DecoderBaseModel):
    def __init__(self, hidden_size, output_size):
        super().__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, input, hidden):
        output = self.embedding(input).view(1, 1, -1)
        output = F.relu(output)
        output, hidden = self.gru(output, hidden)
        output = self.softmax(self.out(output[0]))
        return output, hidden

    def initHidden(self, device):
        return torch.zeros(1, 1, self.hidden_size, device=self.device)

###########################################
##          Attention  Decoder           ##
###########################################

class AttnDecoderLSTM1(DecoderBaseModel):
    def __init__(self, emb_size, hidden_size, output_size, device, max_length=20):
        super().__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size
        self.MAX_LENGTH = max_length

        self.embedding = nn.Embedding(output_size, emb_size)
        self.attn = nn.Linear(emb_size+2*hidden_size, max_length)
        self.attn_combine = nn.Linear(emb_size+2*hidden_size, hidden_size)

        self.lstm = nn.LSTMCell(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)

        self.device = device

    def forward(self, input, hidden, encoder_outputs):
        """
        :param input: (b)
        :param hidden: ((b, h), (b, h))
        :param encoder_outputs: (il, b, 2h)
        :return: (b,o), ((b,h),(b,h)), (b,il)
        """
        input_length = encoder_outputs.shape[0]
        #padding
        encoder_outputs = torch.cat([
            encoder_outputs,
            torch.zeros(
                self.MAX_LENGTH - input_length,
                encoder_outputs.shape[1],
                encoder_outputs.shape[2],
                device=self.device
            )
        ], dim=0)  # (il,b,2h), (ml-il,b,2h) -> (ml,b,2h)
        drop_encoder_outputs = F.dropout(encoder_outputs, p=0.1, training=self.training)

        # embedding
        embedded = self.embedding(input)  # (b) -> (b,e)
        embedded = F.dropout(embedded, p=0.5, training=self.training)

        emb_hidden = torch.cat([embedded, hidden[0], hidden[1]], dim=1)  # (b,e),((b,h),(b,h)) -> (b,e+2h)

        attn_weights = self.attn(emb_hidden)  # (b,e+2h) -> (b,ml)
        attn_weights = F.softmax(attn_weights, dim=1)

        attn_applied = torch.bmm(
            attn_weights.unsqueeze(1),  # (b, 1, ml)
            drop_encoder_outputs.transpose(0, 1)  # (b, ml, 2h)
        )  # -> (b, 1, 2h)

        attn_applied = F.dropout(attn_applied, p=0.1, training=self.training)
        output = torch.cat((embedded, attn_applied.squeeze(1)), 1)  # ((b,e),(b,2h)) -> (b,e+2h)
        output = self.attn_combine(output)  # (b,e+2h) -> (b,h)
        output = F.dropout(output, p=0.5, training=self.training)

        output = F.relu(output)
        hidden = self.lstm(output, hidden)  # (b,h),((b,h),(b,h)) -> (b,h)((b,h),(b,h))

        output = F.log_softmax(self.out(hidden[0]), dim=1)  # (b,h) -> (b,o)
        return output, hidden, attn_weights  # (b,o),(b,h),(b,il)

class AttnDecoderLSTM2(DecoderBaseModel):
    def __init__(self, emb_size, hidden_size, attn_size, output_size, device,
                        pad_token=-1, max_length=10):
        super().__init__()
        self.hidden_size = hidden_size
        self.output_size = output_size

        self.embedding = nn.Embedding(output_size, emb_size, padding_idx=pad_token)
        self.lstm = nn.LSTMCell(emb_size, hidden_size)

        self.score_w = nn.Linear(2*hidden_size, 2*hidden_size)
        self.attn_w = nn.Linear(4*hidden_size, attn_size)
        self.out_w = nn.Linear(attn_size, output_size)

        self.device = device

    def forward(self, input, hidden, encoder_outputs):
        """
        :param: input: (b)
        :param: hidden: ((b,h),(b,h))
        :param: encoder_outputs: (il,b,2h)

        :return: (b,o), ((b,h),(b,h)), (b,il)
        """

        embedded = self.embedding(input)  # (b) -> (b,e)
        embedded = F.dropout(embedded, p=0.5, training=self.training)

        hidden = self.lstm(embedded, hidden)  # (b,e),((b,h),(b,h)) -> ((b,h),(b,h))
        decoder_output = torch.cat(hidden, dim=1)  # ((b,h),(b,h)) -> (b,2h)
        decoder_output = F.dropout(decoder_output, p=0.5, training=self.training)

        # score
        score = self.score_w(decoder_output)  # (b,2h) -> (b,2h)
        scores = torch.bmm(
            encoder_outputs.transpose(0, 1),  # (b,il,2h)
            score.unsqueeze(2)  # (b,2h,1)
        )  # (b,il,1)
        attn_weights = F.softmax(scores, dim=1)  # (b,il,1)

        # context
        context = torch.bmm(
            attn_weights.transpose(1, 2),  # (b,1,il)
            encoder_outputs.transpose(0, 1)  # (b,il,2h)
        )  # (b,1,2h)
        context = context.squeeze(1)  # (b,1,2h) -> (b,2h)

        concat = torch.cat((context, decoder_output), dim=1)  # ((b,2h),(b,2h)) -> (b,4h)
        #concat = F.dropout(concat, p=0.5, training=self.training)

        attentional = self.attn_w(concat)  # (b,4h) -> (b,a)
        attentional = torch.tanh(attentional)
        #attentional = F.dropout(attentional, p=0.5, training=self.training)

        output = self.out_w(attentional)  # (b,a) -> (b,o)
        output = F.log_softmax(output, dim=1)

        return output, hidden, attn_weights.squeeze(2)  # (b,o), ((b,h),(b,h)), (b,il)

class AttnDecoderGRU(DecoderBaseModel):
    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=10):
        super(AttnDecoderGRU, self).__init__()

        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout_p = dropout_p
        self.MAX_LENGTH = max_length
        print('[Info] MAX_LENGTH : ', self.MAX_LENGTH)

        self.embedding = nn.Embedding(self.output_size, self.hidden_size)
        self.attn = nn.Linear(self.hidden_size * 2, self.MAX_LENGTH)
        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)
        self.dropout = nn.Dropout(self.dropout_p)
        self.gru = nn.GRU(self.hidden_size, self.hidden_size)
        self.out = nn.Linear(self.hidden_size, self.output_size)

    def forward(self, input, hidden, enc_outputs):
        embedded = self.embedding(input).view(1, 1, -1)
        embedded = self.dropout(embedded)

        attn_weights = F.softmax(
            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)
        attn_applied = torch.bmm(attn_weights.unsqueeze(0),
                                 enc_outputs.unsqueeze(0))

        output = torch.cat((embedded[0], attn_applied[0]), 1)
        output = self.attn_combine(output).unsqueeze(0)

        output = F.relu(output)
        output, hidden = self.gru(output, hidden)

        output = F.log_softmax(self.out(output[0]), dim=1)
        return output, hidden, attn_weights

    def initHidden(self, device):
        return torch.zeros(1, 1, self.hidden_size, device=device)


import numpy as np
from maddux.environment import Environment
from maddux.objects import Ball, Target
from maddux.robots import simple_human_arm

def arm_ball_animation():
    """Animate an arm holding a ball, moving around to an arbitrary spot."""

    # Create a human arm
    q0 = np.array([0.5, 0.2, 0, 0.5, 1.5])
    human_arm = simple_human_arm(2.0, 2.0, q0, np.array([2.0, 2.0, 0.0]))

    # Create a ball the arm can hold
    ball = Ball(np.array([0.0, 0.0, 0.0]), 0.15)
    human_arm.hold(ball)

    # Declare our environment
    env = Environment([5.0, 5.0, 5.0], dynamic_objects=[ball],
                      robot=human_arm)

    # Find joint config to random location show we can show arm moving
    # while holding the ball
    position = human_arm.ikine([3.0, 2.0, 3.0])

    # Animate for 3 seconds
    env.animate(3.0)

if __name__ == '__main__':
    arm_ball_animation()


class Solution(object):
    #This solution actually outputs all result
    #Could be much faster since the problem only needs the count.
    def findStrobogrammatic(self, n, lower = None, upper = None):
        """
        :type n: int
        :rtype: List[str]
        """
        if n == 1 and lower != None and upper != None:
            return [i for i in ["1","8","0"] if int(i) >= int(lower) and int(i) <= int(upper)  ]
        if lower == upper and upper == "0": return ["0"]
        pairDict = {"1": "1", "8": "8", "6": "9", "9":"6", "0":"0"}
        solutionList = [""]
        if n%2 == 1:
            solutionList = ["1","8","0"]
            n-= 1
        mid = n/2
        while mid > 0 :
            newSol = []
            for key, val in pairDict.iteritems():
                if key == '0' and mid == 1: continue
                for s in solutionList:
                    result = key+s+val
                    if mid == 1:
                        if lower != None and int(result) < int(lower): continue
                        if upper != None and int(result) > int(upper): continue
                    newSol.append(result)
            solutionList = newSol
            mid -= 1
        return solutionList

    def strobogrammaticInRange(self, low, high):
        """
        :type low: str
        :type high: str
        :rtype: int
        """
        lenLow  = len(low)
        lenHigh = len(high)
        if lenLow < lenHigh:
            result = self.findStrobogrammatic(lenLow, low, None) + self.findStrobogrammatic(lenHigh, None, high)
            for i in range (lenLow+1, lenHigh):
                result += self.findStrobogrammatic(i)
        else:
            result = self.findStrobogrammatic(lenLow, low, high)
        #print result
        return len(result)


from django.urls import path
from django.conf import settings
from . import views
from django.conf.urls.static import static

urlpatterns = [
    path('', views.plant_list, name='plant_list'),
    path('plant/<slug:slug>/', views.plant_detail, name='plant_detail'),
    path('about/', views.about, name='about'),
    path('contact/', views.contact, name='contact'),
    path('search/', views.search_results, name='search_results'),
    path('native_plants/', views.native_plants_list, name='native_plants_list'),
    path('non_native_plants/', views.non_native_plants_list, name='non_native_plants_list'),
    path('invasive_plants/', views.invasive_plants_list, name='invasive_plants_list'),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


import logging
import os
from typing import Optional, Union

from mvt.common.utils import convert_unix_to_iso

from ..base import IOSExtraction

class Filesystem(IOSExtraction):
    """This module extracts creation and modification date of files from a
    full file-system dump.
    """

    def __init__(
        self,
        file_path: Optional[str] = None,
        target_path: Optional[str] = None,
        results_path: Optional[str] = None,
        module_options: Optional[dict] = None,
        log: logging.Logger = logging.getLogger(__name__),
        results: Optional[list] = None,
    ) -> None:
        super().__init__(
            file_path=file_path,
            target_path=target_path,
            results_path=results_path,
            module_options=module_options,
            log=log,
            results=results,
        )

    def serialize(self, record: dict) -> Union[dict, list]:
        return {
            "timestamp": record["modified"],
            "module": self.__class__.__name__,
            "event": "entry_modified",
            "data": record["path"],
        }

    def check_indicators(self) -> None:
        if not self.indicators:
            return

        for result in self.results:
            if "path" not in result:
                continue

            ioc = self.indicators.check_file_path(result["path"])
            if ioc:
                result["matched_indicator"] = ioc
                self.detected.append(result)

            # If we are instructed to run fast, we skip the rest.
            if self.module_options.get("fast_mode", None):
                continue

            ioc = self.indicators.check_file_path_process(result["path"])
            if ioc:
                result["matched_indicator"] = ioc
                self.detected.append(result)

    def run(self) -> None:
        for root, dirs, files in os.walk(self.target_path):
            for dir_name in dirs:
                try:
                    dir_path = os.path.join(root, dir_name)
                    result = {
                        "path": os.path.relpath(dir_path, self.target_path),
                        "modified": convert_unix_to_iso(os.stat(dir_path).st_mtime),
                    }
                except Exception:
                    continue
                else:
                    self.results.append(result)

            for file_name in files:
                try:
                    file_path = os.path.join(root, file_name)
                    result = {
                        "path": os.path.relpath(file_path, self.target_path),
                        "modified": convert_unix_to_iso(os.stat(file_path).st_mtime),
                    }
                except Exception:
                    continue
                else:
                    self.results.append(result)


# python std-pkg
import os
import math
import random
import warnings
from pathlib import Path
from typing import Callable
from functools import partial

# 3rd pkgs
import numpy as np
from PIL import Image

import torch
from torch.utils.data import DataLoader, Dataset

# import guard, we only expose the following functions!!
__all__ = ['load_data', 'collate_fn']

global DS_DICT  # hoisting.. see javascript hoisting

def load_data(
    data_dir:str = 'None',
    cache_dir:str = 'None',
    cache_file_callbk:Callable = None,
    resize_size:tuple = None,
    subset_type:str = 'train',
    ret_dataset:bool = True,
    random_flip:bool = True,
    data_ld_kwargs:dict = None,
    name_qry:str = '*/*'
):
    if 'cityscape' in data_dir.lower():
        ds_mode = 'cityscape'
    elif 'kitti' in data_dir.lower():
        ds_mode = 'kitti'
    else:
        raise ValueError(f"Currently we don't support dataset by given {data_dir}")

    # given dir chking..
    data_dir, cache_dir = Path(data_dir), Path(cache_dir)
    # cache mode, early return !!
    if cache_dir.exists():
        warnings.warn(f"Apply cache file, cache file only return train subset.")
        return DS_DICT['cache'][ds_mode](cache_dir, cache_file_callbk)

    if not data_dir.exists():
        raise ValueError(f"data_dir '{data_dir}' is not exists!")

    if not subset_type in ['train', 'val', 'all']:
        raise ValueError(f"Doesn't support subset_type mode {subset_type}, it should be one of it ('train' or 'val' or 'all')!!")

    subset_type = ['train', 'val'] if subset_type == 'all' else [subset_type]
    all_ds = {}
    for subset in subset_type:
        all_ds[subset] = DS_DICT['ds'][ds_mode](data_dir, subset, resize_size, random_flip, True,name_qry)

    # wrap with torch dataloader ~
    if not ret_dataset:
        for subset in all_ds.keys():
            all_ds[subset] = DataLoader(all_ds[subset], collate_fn=collate_fn, **data_ld_kwargs)

    # integrate with previous version, ret dict while enable 'all' subset_type
    return all_ds[subset_type[0]] if len(subset_type) == 1 else all_ds

class Cityscape_ds(Dataset):
    def __init__(self, data_dir, subset, resize_size, random_flip=True, rnd_rng=True, name_qry='*/*'):
        super().__init__()
        self.resize_size = resize_size
        self.random_flip = random_flip
        self.rnd_rng = rnd_rng
        # Path object glob method return iterator, so we immediately turn it into list
        self.imgs_path = list( (data_dir / 'leftImg8bit' / subset).rglob(f'{name_qry}.png') )
        # get corresponding label(s)
        self.classes, self.instances, self.clr_instances = [], [], []
        #breakpoint()
        for lab_path in (data_dir / 'gtFine' / subset).rglob(f'{name_qry}.png'):
            if str(lab_path).endswith('_labelIds.png'):
                self.classes.append(lab_path)
            elif str(lab_path).endswith('_instanceIds.png'):
                self.instances.append(lab_path)
            elif str(lab_path).endswith('_color.png'):
                self.clr_instances.append(lab_path)
            else:
                warnings.warn(f"Unidentified file : {lab_path}")
        # sort lst to confirm the order
        self.imgs_path.sort() ; self.classes.sort() ; self.instances.sort() ; self.clr_instances.sort()

    def __pil_read(self, path, read_mode='RGB'):
        with Image.open(path) as meta_im:
            meta_im.load()
        return meta_im.convert(read_mode)

    def __ratio_resize(self, pil_im, resample_method):
        min_len = min(*pil_im.size)
        # directly resize wrt. scale : "ratio is kept"  or "input square image, H == W"
        if (self.resize_size % min_len == 0) or len(set(pil_im.size)) == 1:
            scale = self.resize_size / min_len
            resiz_pil_im = pil_im.resize(
                tuple(int(x * scale) for x in pil_im.size), resample=Image.BOX
            )

        ## two-stage resize to mimic the distortion of resize
        # 1. stage : keep ratio downsampling (fast)
        while min(*pil_im.size) >= 2 * self.resize_size:
            pil_im = pil_im.resize(
                tuple(x // 2 for x in pil_im.size), resample=Image.BOX
            )
        # 2. stage : bicubic style, carefully resize without keeping ratio
        scale = self.resize_size / min(*pil_im.size)
        resiz_pil_im = pil_im.resize(
            tuple(round(x * scale) for x in pil_im.size), resample=resample_method
        )
        return resiz_pil_im

    # confirm: crop_size is the same for all of input tensor (img, inst_im, ..., etc.)
    def __center_crop(self, pil_im, crop_size, rnd_rng=False):
        arr_im = np.array(pil_im)
        if 'int' in str(type(rnd_rng)):
            crop_x = rnd_rng
            return arr_im[:, crop_x: crop_x + crop_size[1]]
        elif rnd_rng:
            samp_rng = (arr_im.shape[1] - crop_size[1])
            crop_x = np.random.randint(0, samp_rng)
            return (arr_im[:, crop_x: crop_x + crop_size[1]], crop_x)
        else:
            crop_x = (arr_im.shape[1] - crop_size[1]) // 2
            return arr_im[:, crop_x: crop_x + crop_size[1]]

    def __getitem__(self, idx):
        # read img from pil format
        im_path = self.imgs_path[idx]
        img = self.__pil_read(im_path)

        # read labels from pil format
        cls_path, inst_path, clr_msk_path = self.classes[idx], self.instances[idx], self.clr_instances[idx]
        clr_msk_im = self.__pil_read(clr_msk_path)
        # read cls and instance mask with binary mode 'L'!!
        cls_im = self.__pil_read(cls_path, read_mode='L')
        inst_im = self.__pil_read(inst_path, read_mode='L')

        # resize img & labels
        # dwn img with better quality : https://pillow.readthedocs.io/en/stable/handbook/concepts.html#filters-comparison-table
        img = self.__ratio_resize(img, Image.BICUBIC)
        # labels resize with lower quality but better performance!
        clr_msk_im, cls_im, inst_im = \
            self.__ratio_resize(clr_msk_im, Image.NEAREST), self.__ratio_resize(cls_im, Image.NEAREST), self.__ratio_resize(inst_im, Image.NEAREST)

        # center-crop (hard-code 1080, 1440 currently..)
        sc = 1080 // self.resize_size
        crop_size = ( self.resize_size, 1440//sc )

        if self.rnd_rng:
            img, crop_x = self.__center_crop(img, crop_size, self.rnd_rng)
            fix_crop = partial(self.__center_crop, rnd_rng=crop_x)
            clr_msk_im, cls_im, inst_im = \
                fix_crop(clr_msk_im, crop_size), fix_crop(cls_im, crop_size), fix_crop(inst_im, crop_size)
        else:
            img = self.__center_crop(img, crop_size, False)
            clr_msk_im, cls_im, inst_im = \
                self.__center_crop(clr_msk_im, crop_size), self.__center_crop(cls_im, crop_size), self.__center_crop(inst_im, crop_size)

        if self.random_flip and random.random() < 0.5:
            img = img[:, ::-1].copy()
            clr_msk_im = clr_msk_im[:, ::-1].copy()
            cls_im = cls_im[:, ::-1].copy()
            inst_im = inst_im[:, ::-1].copy()

        img = img.astype(np.float32) / 127.5 - 1

        out_dict = {'path':im_path, 'label_ori':cls_im, 'label':cls_im[None, ...],
                    'instance':inst_im[None, ...], 'clr_instance':clr_msk_im[None, ...]}

        # switch to channel first format due to torch tensor..
        return {"pixel_values":np.transpose(img, [2, 0, 1]), "label":out_dict}

    def __len__(self):
        return len(self.imgs_path)

class Cityscape_cache(Dataset):
    VAE_SCALE = 0.18215

    def __init__(self, cache_dir, cache_file_callbk=None):
        self.cache_path = list( (cache_dir).glob('train/*/*.pt') )
        self.cache_file_callbk = cache_file_callbk

    def __getitem__(self, idx):
        vae_cache = torch.load(self.cache_path[idx])
        # customized callback to deal with cache file
        if self.cache_file_callbk:
            return self.cache_file_callbk(vae_cache)

        # default procedure to load the cache file for VAE & VQVAE..
        if isinstance(vae_cache['x'], dict):
            mean, std = vae_cache['x']['mean'], vae_cache['x']['std']
            # normal_ make more efficient with std=1, mean=0 (exactly as randn)
            # https://pytorch.org/docs/stable/generated/torch.randn.html
            sample = torch.cuda.FloatTensor(**mean.shape).normal_(mean=0, std=1)
            x = mean + std * sample
            x = x * Cityscape_cache.VAE_SCALE
        elif isinstance(vae_cache['x'], list):
            ret = random.randint(0, len(vae_cache['x'])-1)
            x = vae_cache['x'][ret] * Cityscape_cache.VAE_SCALE
            vae_cache['label']["segmap"] = vae_cache['label']["segmap"][ret]
        else:
            x = vae_cache['x']
            print(x.shape)
        return {"pixel_values": x, "label": vae_cache['label']}

    def __len__(self):
        return len(self.cache_path)

class Kitti_cache:
    VAE_SCALE = 0.18215

    def __init__(self, cache_dir, cache_file_callbk=None):
        self.cache_path = list( (cache_dir).glob('train/*/*.pt') )
        self.cache_file_callbk = cache_file_callbk

    def __getitem__(self, idx):
        vae_cache = torch.load(self.cache_path[idx])
        # customized callback to deal with cache file
        if self.cache_file_callbk:
            return self.cache_file_callbk(vae_cache)

        # default procedure to load the cache file for VAE & VQVAE..
        if isinstance(vae_cache['x'], dict):
            mean, std = vae_cache['x']['mean'], vae_cache['x']['std']
            # normal_ make more efficient with std=1, mean=0 (exactly as randn)
            # https://pytorch.org/docs/stable/generated/torch.randn.html
            sample = torch.cuda.FloatTensor(**mean.shape).normal_(mean=0, std=1)
            x = mean + std * sample
            x = x * Kitti_cache.VAE_SCALE
        elif isinstance(vae_cache['x'], list):
            ret = random.randint(0, len(vae_cache['x'])-1)
            x = vae_cache['x'][ret] * Kitti_cache.VAE_SCALE
        else:
            x = vae_cache['x']
            print(x.shape)
        return {"pixel_values": x, "label": vae_cache['label']}

    def __len__(self):
        return len(self.cache_path)

class Kitti_ds:
    def __init__(self, data_dir, subset, resize_size, random_flip=True, rnd_rng=True, interval=15):

        def grap_path(data_dir):
            # Path object glob method return iterator, so we immediately turn it into list
            imgs_path = list( data_dir.rglob('**/**/image_02/data/*.png') )
            # sort lst to confirm the order
            imgs_path.sort()
            return imgs_path

        def sample_interval(imgs_path, interval):
            from os.path import sep
            path_lst = []
            prv_dr, cur_dr = '', ''
            cnt = 0
            for path in imgs_path:
                path = str(path)
                for dr in path.split(sep):
                    if 'drive' in dr: cur_dr = dr ; break

                if cur_dr != prv_dr:
                    cnt = 0 ; prv_dr = cur_dr

                if (cnt % interval) == 0:
                    path_lst.append(path)

                cnt += 1

            return path_lst

        super().__init__()
        self.resize_size = resize_size
        self.random_flip = random_flip
        self.rnd_rng = rnd_rng
        self.subset = subset

        imgs_path = grap_path( Path(data_dir) )
        self.imgs_path = sample_interval(imgs_path, interval)

    def __pil_read(self, path, read_mode='RGB'):
        with Image.open(path) as meta_im:
            meta_im.load()
        return meta_im.convert(read_mode)

    def __ratio_resize(self, pil_im, resample_method):
        min_len = min(*pil_im.size)
        # directly resize wrt. scale : "ratio is kept"  or "input square image, H == W"
        if (self.resize_size % min_len == 0) or len(set(pil_im.size)) == 1:
            scale = self.resize_size / min_len
            resiz_pil_im = pil_im.resize(
                tuple(int(x * scale) for x in pil_im.size), resample=Image.BOX
            )

        ## two-stage resize to mimic the distortion of resize
        # 1. stage : keep ratio downsampling (fast)
        while min(*pil_im.size) >= 2 * self.resize_size:
            pil_im = pil_im.resize(
                tuple(x // 2 for x in pil_im.size), resample=Image.BOX
            )
        # 2. stage : bicubic style, carefully resize without keeping ratio
        scale = self.resize_size / min(*pil_im.size)
        resiz_pil_im = pil_im.resize(
            tuple(round(x * scale) for x in pil_im.size), resample=resample_method
        )
        return resiz_pil_im

    # confirm: crop_size is the same for all of input tensor (img, inst_im, ..., etc.)
    def __center_crop(self, pil_im, crop_size, rnd_rng=False):
        arr_im = np.array(pil_im)
        if rnd_rng:
            samp_rng = (arr_im.shape[1] - crop_size[1])
            crop_x = np.random.randint(0, samp_rng)
        else:
            crop_x = (arr_im.shape[1] - crop_size[1]) // 2

        return arr_im[:, crop_x: crop_x + crop_size[1]]

    def __getitem__(self, idx):
        # read img from pil format
        im_path = self.imgs_path[idx]
        img = self.__pil_read(im_path)

        # resize img & labels
        # dwn img with better quality : https://pillow.readthedocs.io/en/stable/handbook/concepts.html#filters-comparison-table
        img = self.__ratio_resize(img, Image.BICUBIC)

        # center-crop (hard-code 1080, 1440 currently..)
        sc = 1080 // self.resize_size
        crop_size = ( self.resize_size, 1440//sc )

        if self.rnd_rng:
            img = self.__center_crop(img, crop_size, self.rnd_rng)
        else:
            img = self.__center_crop(img, crop_size, False)

        if self.random_flip and random.random() < 0.5:
            img = img[:, ::-1].copy()

        img = img.astype(np.float32) / 127.5 - 1
        # switch to channel first format due to torch tensor..
        return {"pixel_values":np.transpose(img, [2, 0, 1]), "label":{'path':im_path} }

    def __len__(self):
        return len(self.imgs_path)

DS_DICT = {
    'cache' : {
        'cityscape' : Cityscape_cache,
        'kitti' : Kitti_cache
    },
    'ds' : {
        'cityscape' : Cityscape_ds,
        'kitti' : Kitti_ds
    }
}

# default collate function
def collate_fn(examples):
    segmap = {}
    ret = random.randint(0, 1)
    for k in examples[0]["label"].keys():
        if k != 'path':
            if isinstance(examples[0]["label"][k], list):
                segmap[k] = torch.stack([torch.from_numpy(example["label"][k][ret]) for example in examples])
            else:
                segmap[k] = torch.stack([torch.from_numpy(example["label"][k]) for example in examples])
            segmap[k] = segmap[k].to(memory_format=torch.contiguous_format).float()

    pixel_values = torch.stack([torch.from_numpy(example["pixel_values"]) for example in examples])
    pixel_values = pixel_values.to(memory_format=torch.contiguous_format).float()
    filename_lst = [ os.path.join(example['label']['path']) for example in examples ]

    return {"pixel_values": pixel_values, "segmap": segmap, 'filename': filename_lst}

# unittest..
if __name__ == "__main__":
    from easy_configer.Configer import Configer
    cfger = Configer()
    cfger.cfg_from_str('''
    [ds1]
        data_dir = '/data/joseph/kitti_ds'
        resize_size = 540
        subset_type = 'train'

    [ds2]
        data_dir = '/data1/dataset/Cityscapes'
        resize_size = 540
        subset_type = 'train'

    [ld]
        batch_size = 8
        num_workers = 0
        shuffle = True
    ''')

    import torch
    from torch.utils.data import ConcatDataset
    import torchvision.transforms as T

    kitti_ds = load_data(**cfger.ds1)
    city_ds = load_data(**cfger.ds2)
    train_dataset = ConcatDataset([kitti_ds, city_ds])

    data_ld = torch.utils.data.DataLoader(
        train_dataset,
        collate_fn=collate_fn,
        **cfger.ld
    )

    fn_lst = []
    tnsr2pil = T.ToPILImage()
    for idx, batch in enumerate(data_ld, 0):
        fn = batch['filename'][0]
        fn = os.path.basename(fn).split('.')[0]

        # Note : kitti_ds doesn't have mask..
        #clr_msk = [ clr_inst.permute(0, 3, 1, 2) / 255. for clr_inst in batch["segmap"]['clr_instance'] ][0]
        #tnsr2pil(clr_msk[0]).save(f'{fn}_clr.png')

        imgs = [((img+1)*127.5)/255. for img in batch["pixel_values"]]
        tnsr2pil(imgs[0]).save(f'tmp_im/{fn}_im.png')
        if idx >= 65:
            break  # yeah ~ take the break


#!/usr/bin/env python

import sys

def mid(s):
    if len(s) % 2 != 0:
        return(s[len(s)//2])
    else:
        return('No middle character!')

def main():
    for lines in sys.stdin:
        words = mid(lines.strip())
        print(words)
if __name__ == '__main__':
    main()


# Tarjan's strongly connected components algorithm
# Taken from https://stackoverflow.com/a/6575693 and adopted to Python3

from itertools import chain
from collections import defaultdict
from typing import *

Edge = Tuple[Any,Any]

class _Graph:
    def __init__(self, edges: List[Edge]):
        edges = list(list(x) for x in edges)
        self.edges = edges
        self.vertices = set(chain(*edges))
        self.tails = defaultdict(list)
        for head, tail in self.edges:
            self.tails[head].append(tail)

def strongly_connected_components(edges: List[Edge]):
    graph = _Graph(edges)
    counter = 0
    count = dict()
    lowlink = dict()
    stack = []
    connected_components = []

    def strong_connect(head):
        nonlocal counter
        lowlink[head] = count[head] = counter = counter + 1
        stack.append(head)

        for tail in graph.tails[head]:
            if tail not in count:
                strong_connect(tail)
                lowlink[head] = min(lowlink[head], lowlink[tail])
            elif count[tail] < count[head]:
                if tail in stack:
                    lowlink[head] = min(lowlink[head], count[tail])

        if lowlink[head] == count[head]:
            component = []
            while stack and count[stack[-1]] >= count[head]:
                component.append(stack.pop())
            connected_components.append(component)

    for v in graph.vertices:
        if v not in count:
            strong_connect(v)

    return connected_components