import copy
import cv2
import dask.array as da
import numpy as np
import xarray as xr

from pyproj import Transformer
from rasterio.transform import Affine, xy
from rasterio.crs import CRS
from rasterio import warp
from scipy.optimize import differential_evolution
from scipy.signal import convolve2d
from scipy.interpolate import interp1d

def affine_from_grid(xi, yi):
    """Retrieve the affine transformation from a gridded set of coordinates.
    This function (unlike rasterio.transform functions) can also handle rotated grids

    Parameters
    ----------
    xi: np.ndarray (2D)
        gridded x-coordinates
    yi: np.ndarray (2D)
        gridded y-coordinates

    Returns
    -------
    obj : rasterio.transform.Affine
    """

    xul, yul = xi[0, 0], yi[0, 0]
    xcol, ycol = xi[0, 1], yi[0, 1]
    xrow, yrow = xi[1, 0], yi[1, 0]
    dx_col = xcol - xul
    dy_col = ycol - yul
    dx_row = xrow - xul
    dy_row = yrow - yul
    return Affine(dx_col, dy_col, xul, dx_row, dy_row, yul)

def delayed_to_da(delayed_das, shape, dtype, coords, attrs={}, name=None, object_type=xr.DataArray):
    """Convert a list of delayed 2D arrays (assumed to be time steps of grids) into a 3D xr.DataArray with dask arrays
    with all axes.

    Parameters
    ----------
    delayed_das: xr.DataArray or list
    shape: tuple
        foreseen shape of data arrays (rows, cols)
    dtype: str or dtype
        e.g. "uint8" of data arrays
    coords: tuple of str
        dimensions of the xr.DataArray being prepared, usually ("time", "y", "x").
    attrs: dict, optional
        attributes for xr.DataArray (default: {})
    name: str, optional
        name of variable, default: None
    object_type: type
        type of object to create from lazy array (default: xr.DataArray)

    Returns
    -------
    object of object_type
        default: xr.DataArray
    """
    if isinstance(delayed_das, list):
        data_array = da.stack(
            [da.from_delayed(
                d,
                dtype=dtype,
                shape=shape
            ) for d in delayed_das],
            axis=0
        )
    else:
        data_array = da.from_delayed(
            delayed_das,
            dtype=dtype,
            shape=shape
        )
    for n, (coord, values) in enumerate(coords.items()):
        assert(len(values) == data_array.shape[n]), f"Length of {coord} axis {len(values)} is not equal to amount of data arrays {data_array.shape[n]}"
    return object_type(
        data_array,
        dims=tuple(coords.keys()),
        coords=coords,
        attrs=attrs,
        name=name
    )

def depth_integrate(depth, v, v_corr=0.85, name="q"):
    """Integrate velocities [m s-1] to depth-integrated velocity [m2 s-1] using depth information

    Parameters
    ----------
    depth : DataArray (points)
        bathymetry depths (ref. CRS)
    v : DataArray (time, points)
        effective velocity at surface [m s-1]
    v_corr : float (range: 0-1), optional
        typically close to 1, correction factor from surface to depth-average (default: 0.85)
    name: str, optional
        name of DataArray (default: "q")

    Returns
    -------
    q: DataArray (time, points)
        depth integrated velocity [m2 s-1]
    """
    # compute the depth average velocity
    q = v * v_corr * depth
    q.attrs = {
        "standard_name": "velocity_depth",
        "long_name": "velocity averaged over depth",
        "units": "m2 s-1",
    }
    # set name
    q.name = name
    return q

def distance_pts(c1, c2):
    """Compute distance between c1 and c2

    Parameters
    ----------

    c1 : tuple
        (x, y), coordinate 1
    c2: tuple
        (x, y), coordinate 2

    Returns
    -------
    dist : float
        distance between c1 and c2
    """
    x1, y1 = c1
    x2, y2 = c2
    return ((x2 - x1) ** 2 + (y2 - y1) ** 2) ** 0.5

def deserialize_attr(data_array, attr, dtype=np.array, args_parse=False):
    """Return a deserialized version of said property (assumed to be stored as a string) of DataArray.

    Parameters
    ----------
    obj: xr.DataArray
        attributes of interest
    attr: str
        name of attributes
    dtype: object type, optional
        function will try to perform type(eval(attr)), default np.array
    args_parse: boolean, optional
        if True, function will try to return type(*eval(attr)), assuming attribute contains list
        of arguments (default: False)

    Returns
    -------
    parsed_arg: type defined by user
        parsed attribute, of type defined by arg type
    """
    assert hasattr(data_array, attr), f'frames do not contain attribute with name "{attr}'
    attr_obj = getattr(data_array, attr)
    if args_parse:
        return dtype(*eval(attr_obj))
    return dtype(eval(attr_obj))

def get_axes(cols, rows, resolution):
    """Retrieve a locally spaced axes for surface velocimetry results on the basis of resolution and row and
    col distances from the original frames

    Parameters
    ----------
    cols: list
        ints, columns, sampled from the original projected frames
    rows: list
        ints, rows, sampled from the original projected frames
    resolution: float
        resolution of original frames

    Returns
    -------
    obj : np.ndarray
        x-axis with origin at the left
    obj2 : np.ndarray
        y-axis with origin on the top

    """
    spacing_x = np.diff(cols[0])[0]
    spacing_y = np.diff(rows[:, 0])[0]
    x = np.linspace(
        resolution / 2 * spacing_x,
        (len(cols[0]) - 0.5) * resolution * spacing_x,
        len(cols[0]),
    )
    y = np.flipud(
        np.linspace(
            resolution / 2 * spacing_y,
            (len(rows[:, 0]) - 0.5) * resolution * spacing_y,
            len(rows[:, 0]),
        )
    )
    return x, y

def get_xs_ys(cols, rows, transform):
    """Computes rasters of x and y coordinates, based on row and column counts and a defined transform.

    Parameters
    ----------
    cols: list of ints
        column counts
    rows: list of ints
        row counts
    transform: np.ndarray (1D)
        rasterio compatible transform parameters

    Returns
    -------
    xs : np.ndarray (MxN)
        x-coordinates
    ys : np.ndarray (MxN)
        y-coordinates
    """
    xs, ys = xy(transform, rows, cols)
    xs, ys = np.array(xs), np.array(ys)
    return xs, ys

def get_lons_lats(xs, ys, src_crs, dst_crs=CRS.from_epsg(4326)):
    """Computes raster of longitude and latitude coordinates (default) of a certain raster set of coordinates in a local
    coordinate reference system. User can supply an alternative coordinate reference system if projection other than
    WGS84 Lat Lon is needed.

    Parameters
    ----------
    xs : np.ndarray (MxN)
        x-coordinates
    ys : np.ndarray (MxN)
        y-coordinates
    src_crs : int, dict or str
        Coordinate Reference System (of source coordinates). Accepts EPSG codes (int or str) proj (str or dict) or wkt
        (str).
    dst_crs : int, dict or str, optional
        Coordinate Reference System (of target coordinates). Accepts EPSG codes (int or str) proj (str or dict) or wkt
        (str). default: CRS.from_epsg(4326) for wGS84 lat-lon

    Returns
    -------
    lons : np.ndarray (MxN)
        longitude coordinates
    lats: np.ndarray (MxN)
        latitude coordinates
    """
    lons, lats = warp.transform(src_crs, dst_crs, xs.flatten(), ys.flatten())
    lons, lats = (
        np.array(lons).reshape(xs.shape),
        np.array(lats).reshape(ys.shape),
    )
    return lons, lats

def log_profile(X, z0, k_max, s0=0., s1=0.):
    """Returns values of a log-profile function

    Parameters
    ----------
    X: tuple with np.ndarrays
        (depth [m], distance to bank [m]) arrays of equal length
    z0: float
        depth with zero velocity [m]
    k_max: float
        maximum scale factor of log-profile function [-]
    s0: float, optional
        distance from bank (defaukt: 0.) where k equals zero (and thus velocity is zero) [m]
    s1: float, optional
        distance from bank (default: 0. meaning no difference over distance) where k=k_max (k cannot be larger than
        k_max) [m]

    Returns
    -------
    velocity : np.ndarray
        V values from log-profile, equal shape as arrays inside X [m s-1]
    """
    z, s = X
    k = k_max * np.minimum(np.maximum((s-s0)/(s1-s0), 0), 1)
    v = k*np.maximum(np.log(np.maximum(z, 1e-6)/z0), 0)
    return v

def mse(pars, func, X, Y):
    """mean of sum of squares between evaluation of function with provided parameters and X input, and Y as dependent variable.

    Parameters
    ----------
    pars : list or tuple
        parameter passed as *args to func
    func: function def
        receiving X and *pars as input and returning predicted Y as result
    X: tuple with lists or array-likes
        indepent variable(s).
    Y: list or array-like
        dependent variable, predicted by func

    Returns
    -------

    Y_pred : list or array-like
        predicted Y from X and pars by func
    """
    Y_pred = func(X, *pars)
    mse = np.sum((Y_pred - Y) ** 2)
    return mse

def neighbour_stack(array, stride=1, missing=-9999.):
    """Builds a stack of arrays from a 2-D input array, constructed by permutation in space using a provided stride.

    Parameters
    ----------
    array : np.ndarray (2D)
        any values (may contain NaN)
    stride : int, optional
        stride used to determine relevant neighbours (default: 1)
    missing : float, optional
        a temporary missing value, used to be able to convolve NaNs

    Returns
    -------
    obj : np.array (3D)
        stack of 2-D arrays, with strided neighbours (length 1st dim : (stride*2+1)**2 )
    """
    array[np.isnan(array)] = missing
    array_move = []
    for vert in range(-stride, stride+1):
        for horz in range(-stride, stride+1):
            conv_arr = np.zeros((abs(vert)*2+1, abs(horz)*2+1))
            _y = int(np.floor((abs(vert)*2+1)/2)) + vert
            _x = int(np.floor((abs(horz)*2+1)/2)) + horz
            conv_arr[_y, _x] = 1
            array_move.append(convolve2d(array, conv_arr, mode="same", fillvalue=np.nan))
    array_move = np.stack(array_move)
    # replace missings by Nan
    array_move[np.isclose(array_move, missing)] = np.nan
    return array_move

def optimize_log_profile(
        z,
        v,
        dist_bank=None,
        bounds=([0.001, 0.1], [-20, 20], [0., 5], [0., 100]),
        workers=2,
        popsize=100,
        updating="deferred",
        seed=0,
        **kwargs
):

    """optimize velocity log profile relation of v=k*max(z/z0) with k a function of distance to bank and k_max
    A differential evolution optimizer is used.

    Parameters
    ----------
    z : list
        depths [m]
    v : list
        surface velocities [m s-1]
    dist_bank : list
        distances to bank [m]
    **kwargs : keyword arguments for scipy.optimize.differential_evolution

    Returns
    -------
    pars : dict
        fitted parameters of log_profile {z_0, k_max, s0 and s1}
    """
    if dist_bank is None:
        dist_bank = np.inf(len(v))
    v = np.array(v)
    z = np.array(z)
    X = (z, dist_bank)
    Y = v
    result = differential_evolution(
        wrap_mse,
        args=(log_profile, X, Y),
        bounds=bounds,
        workers=workers,
        popsize=popsize,
        updating=updating,
        seed=seed,
        **kwargs
    )
    # unravel parameters
    z0, k_max, s0, s1 = result.x
    return {"z0": z0, "k_max": k_max, "s0": s0, "s1": s1}

def rotate_u_v(u, v, theta, deg=False):
    """Rotate u and v components of vector counter clockwise by an amount of rotation.

    Parameters
    ----------
    u : float, np.ndarray or xr.DataArray
        x-direction component of vector
    v : float, np.ndarray or xr.DataArray
        y-direction component of vector
    theta : float
        amount of counter clockwise rotation in radians or degrees (dependent on deg)
    deg : boolean, optional
        if True, theta is defined in degrees, otherwise radians (default: False)

    Returns
    -------
    u_rot : float, np.ndarray or xr.DataArray
        rotated x-direction component of vector
    v_rot : float, np.ndarray or xr.DataArray
        rotated y-direction component of vector
    """
    if deg:
        # convert to radians first
        theta = np.radians(theta)
    c, s = np.cos(theta), np.sin(theta)
    r = np.array(((c, -s), (s, c)))
    # compute rotations with dot-product
    u2 = r[0, 0] * u + r[0, 1] * v
    v2 = r[1, 0] * u + r[1, 1] * v
    return u2, v2

def velocity_fill(x, y, depth, v, groupby="quantile"):
    """Fill missing surface velocities using a velocity depth profile with

    Parameters
    ----------
    x : xr.DataArray (points)
        x-coordinates of bathymetry depths (ref. CRS)
    y : xr.DataArray (points)
        y-coordinates of bathymetry depths (ref. CRS)
    depth : xr.DataArray (points)
        depths [m]
    v : xr.DataArray (time, points)
        effective velocity at surface [m s-1]
    groupby: str, optional
        dimension over which data should be grouped, default: "quantile", dimension must exist in v, typically
        "quantile" or "time"

    Returns
    -------

    v_fill: xr.DataArray (quantile or time, points)
        filled surface velocities  [m s-1]
    """
    def fit(_v):
        pars = optimize_log_profile(depth[np.isfinite(_v).values], _v[np.isfinite(_v).values], dist_bank[np.isfinite(_v).values])
        _v[np.isnan(_v).values] = log_profile((depth[np.isnan(_v).values], dist_bank[np.isnan(_v).values]), **pars)
        # enforce that velocities are zero with zero depth
        _v[depth<=0] = 0.
        return np.maximum(_v, 0)

    z_dry = depth <= 0
    dist_bank = np.array([(((x[z_dry] - _x) ** 2 + (y[z_dry] - _y) ** 2) ** 0.5).min() for _x, _y, in zip(x, y)])
    # per time slice or quantile, fill missings
    v_group = copy.deepcopy(v).groupby(groupby)
    return v_group.map(fit)

def wrap_mse(pars_iter, *args):
    return mse(pars_iter, *args)

def xy_equidistant(x, y, distance, z=None):
    """Transforms a set of ordered in space x, y (and z if provided) coordinates into x, y (and z) coordinates with equal
    1-dimensional distance between them using piece-wise linear interpolation. Extrapolation is used for the last point
    to ensure the range of points covers at least the full range of x, y coordinates.

    Parameters
    ----------
    x : np.ndarray (1D)
        set of (assumed ordered) x-coordinates
    y : np.ndarray (1D)
        set of (assumed ordered) x-coordinates
    distance : float
        user demanded distance between equidistant samples measured in cumulated 1-dimensional distance from xy
        origin (first point)
    z : np.ndarray (1D), optional
        set of (assumed ordered) z-coordinates (default: None, meaning only x, y interpolated points are returned)

    Returns
    -------
    x_sample : np.ndarray (1D)
        interpolated x-coordinates for x, y, s (distance from first point), z
    y_sample : np.ndarray (1D)
        interpolated y-coordinates
    s_sample : np.ndarray (1D)
        interpolated s-coordinates, s being piece-wise linear distance from first point
    z_sample : np.ndarray (1D), optional
        interpolated z-coordinates (only returned if z is not None):
    """
    # estimate cumulative distance between points, starting with zero
    x_diff = np.concatenate((np.array([0]), np.diff(x)))
    y_diff = np.concatenate((np.array([0]), np.diff(y)))
    s = np.cumsum((x_diff ** 2 + y_diff ** 2) ** 0.5)

    # create interpolation functions for x and y coordinates
    f_x = interp1d(s, x, fill_value="extrapolate")
    f_y = interp1d(s, y, fill_value="extrapolate")

    # make equidistant samples
    s_sample = np.arange(s.min(), np.ceil((1 + s.max() / distance) * distance), distance)

    # interpolate x and y coordinates
    x_sample = f_x(s_sample)
    y_sample = f_y(s_sample)
    if z is None:
        return x_sample, y_sample, s_sample
    else:
        f_z = interp1d(s, z, fill_value="extrapolate")
        z_sample = f_z(s_sample)
        return x_sample, y_sample, z_sample, s_sample

def xy_angle(x, y):
    """Determine angle between x, y points.

    Parameters
    ----------
    x : np.ndarray (1D)
        set of (assumed ordered) x-coordinates
    y : np.ndarray (1D)
        set of (assumed ordered) x-coordinates

    Returns
    -------
    angle : np.ndarray (1D)
        angle between the point left and right of the point under consideration. The most left and right coordinates
        are based on the first and last 2 points respectively
    """
    angles = np.zeros(len(x))
    angles[1:-1] = np.arctan2(x[2:] - x[0:-2], y[2:] - y[0:-2])
    angles[0] = np.arctan2(x[1] - x[0], y[1] - y[0])
    angles[-1] = np.arctan2(x[-1] - x[-2], y[-1] - y[-2])
    return angles

def xy_to_perspective(x, y, resolution, M, reverse_y=None):
    """
    Back transform local meters-from-top-left coordinates from frame to original perspective of camera using M
    matrix, belonging to transformation from orthographic to local.

    Parameters
    ----------
    x : np.ndarray (1D)
        axis of x-coordinates in local projection with origin top-left, to be backwards projected
    y : np.ndarray (1D)
        axis of y-coordinates in local projection with origin top-left, to be backwards projected
    resolution : float
        resolution of original projected frames coordinates of x and y
    M : np.ndarray
        2x3 transformation matrix (generated with cv2.getPerspectiveTransform)

    Returns
    -------
    xp : np.ndarray (2D)
        perspective columns with shape len(y), len(x)
    yp : np.ndarray (2D)
        perspective rows with shape len(y), len(x)
    """
    cols, rows = x / resolution - 0.5, y / resolution - 0.5
    if reverse_y is not None:
        rows = reverse_y - rows
    # make list of coordinates, compatible with cv2.perspectiveTransform
    coords = np.float32([np.array([cols.flatten(), rows.flatten()]).transpose([1, 0])])
    coords_trans = cv2.perspectiveTransform(coords, M)
    xp = coords_trans[0][:, 0].reshape(cols.shape)
    yp = coords_trans[0][:, 1].reshape(cols.shape)
    return xp, yp

def xy_transform(x, y, crs_from, crs_to):
    """transforms set of x and y coordinates from one CRS to another

    Parameters
    ----------
    x : np.ndarray
        x-coordinates in crs_from
    y : np.ndarray
        y-coordinates in crs_from
    crs_from : int, dict or str, optional
        Coordinate Reference System (source). Accepts EPSG codes (int or str) proj (str or dict) or wkt (str).
    crs_to : int, dict or str, optional
        Coordinate Reference System (destination). Accepts EPSG codes (int or str) proj (str or dict) or wkt (str).

    Returns
    -------
    x_trans : np.ndarray
        x-coordinates transformed
    y_trans : np.ndarray
        y-coordinates transformed
    """
    transform = Transformer.from_crs(crs_from, crs_to, always_xy=True)
    # transform dst coordinates to local projection
    x_trans, y_trans = transform.transform(x, y)
    if np.all(np.isinf(x_trans)):
        raise ValueError("Transformation did not give valid results, please check if the provided crs of input coordinates is correct.")
    return transform.transform(x, y)


import utilities as ut
import kd_methods
import random
import average_day_rates
import sys
import pickle

testing = False

def add_origin_stations_to_pruned_data():

    origin_lat_index = 7
    origin_lng_index = 6

    tree = kd_methods.KdTree()

    input_file_name = 'pruned_kornhauser_chicago.csv' # 'sample_subset_pruned_kornhauser.csv' #
    output_file_name = 'pruned_kornhauser_w_origins.csv' # 'pruned_kornhauser_w_origins_test.csv' #

    if testing:
        input_file_name = 'sample_subset_pruned_kornhauser.csv'
        output_file_name = 'pruned_kornhauser_w_origins_test.csv'

    input_file = open(input_file_name, 'r')
    output_file = open(output_file_name, 'w')

    header = input_file.readline() # Ignore the header
    output_file.write(header[:-1] + ",Origin Station\n")

    status_every = 500000

    print("Writing origins to: " + output_file_name)
    counter = 0
    for line in input_file:

        items = line.split(",")
        start_lat = float(items[origin_lat_index])
        start_lng = float(items[origin_lng_index])
        origin_station = tree.nearest(start_lat, start_lng)[0]

        output_file.write(line[:-1] + "," + ut.decomma(origin_station) + "\n")

        counter += 1
        if counter % status_every == 0:
            print("Processed", str(counter), "lines")

    input_file.close()
    output_file.close()
    print("\nDone\n")

def sort_by_origin_station():

    input_file_name = 'pruned_kornhauser_w_origins.csv' # 'pruned_kornhauser_w_origins_test.csv' #
    output_file_name = 'pruned_kornhauser_w_origins_sorted.csv' # 'pruned_kornhauser_w_origins_sorted_test.csv' #

    if testing:
        input_file_name = 'pruned_kornhauser_w_origins_test.csv'
        output_file_name = 'pruned_kornhauser_w_origins_sorted_test.csv'

    input_file = open(input_file_name, 'r')
    output_file = open(output_file_name, 'w')

    header = input_file.readline() # Copy the header
    output_file.write(header)

    all_trips = []

    origin_station_index = 19

    status_every = 500000

    print("Sorting trips in", input_file_name, "by origin.")
    print("Writing sorted results to: ", output_file_name)
    counter = 0
    # Read in all trips into a list of trips
    for line in input_file:

        items = line.split(",")
        all_trips.append(items)

        if counter % status_every == 0:
            print("So far added to all_trips", str(counter), "lines")
        counter += 1

    # sort the list of trips by origin station name
    print("All lines added to all_trips. Sorting now.")
    all_trips.sort(key = lambda trip: trip[origin_station_index])

    # Write the list of trips sorted by origin station to the output file
    counter = 0
    for trip in all_trips:
        output_file.write(",".join(trip))

        if counter % status_every == 0:
            print("So far wrote", str(counter), " sorted lines to output file.")
        counter += 1
    print("Wrote", str(counter), "lines to output file.")

    input_file.close()
    output_file.close()
    print("\nDone\n")

def sample_from_sorted_origins():

    input_file_name = 'pruned_kornhauser_w_origins_sorted.csv' # 'pruned_kornhauser_w_origins_sorted_test.csv' #
    output_file_name = 'sampled_kornhauser_trips.csv' # 'sampled_kornhauser_trips_test.csv' #

    if testing:
        input_file_name = 'pruned_kornhauser_w_origins_sorted_test.csv'
        output_file_name = 'sampled_kornhauser_trips_test.csv'

    # Initializes object that has getRidership(node) method that
    # returns the avg ridership at that node
    Avgs = average_day_rates.AvgRidership()

    input_file = open(input_file_name, 'r')
    output_file = open(output_file_name, 'w')

    header = input_file.readline() # Copy the header
    output_file.write(header)

    all_trips = []

    origin_station_index = 19

    status_every = 500000

    print("Sampling from: " + input_file_name)
    print("Writing sampled results to: " + output_file_name)

    # Go through each trip, adding it to the list of the current origin
    # station, and if a new origin station is reached, randomly sample
    # from the amassed last origin stations and write them to the
    # output file.
    counter = 0
    sample_counter = 0

    # Handle first line to initialize stuff
    current_station_trips = []
    first_line = input_file.readline()
    items = first_line.split(",")
    current_origin_station = items[origin_station_index].strip()
    current_station_trips.append(first_line)

    for line in input_file:

        items = line.split(",")

        if items[origin_station_index].strip() != current_origin_station:

            # Set k to the avg number of people who get on at the current_origin_station
            k = int(Avgs.getRidership(current_origin_station))

            if testing:
                k = int(0.01 * k)

            sampled_trips = []
            # print(current_origin_station, ":", k)
            if k > len(current_station_trips):
                if not testing: print("ATTENTION: STATION", current_origin_station, "HAD FEWER PRUNED KORNHAUSER DATA OCCURRENCES THAT IT'S AVERAGE RIDERSHIP VALUE")

                for i in range(k):
                    sampled_trips.append(random.choice(current_station_trips))

            else:
                sampled_trips = random.sample(current_station_trips, k)

            for f in sampled_trips:
                output_file.write(f)
            sample_counter += k

            current_origin_station = items[origin_station_index].strip()
            current_station_trips = []

        current_station_trips.append(line)

        if counter % status_every == 0:
            print("Went through", str(counter), "lines")
        counter += 1

    print("len of current_station_trips = ", len(current_station_trips))
    # print("current_station_trips = \n", current_station_trips)
    # Handle edge case of last origin station

    k = int(Avgs.getRidership(current_origin_station))
    if testing:
        k = int(0.01 * k)

    sampled_trips = []
    # print(current_origin_station, ":", k)
    if k > len(current_station_trips):
        if not testing: print("ATTENTION: STATION", current_origin_station, "HAD FEWER PRUNED KORNHAUSER DATA OCCURRENCES THAT IT'S AVERAGE RIDERSHIP VALUE")

        for i in range(k):
            sampled_trips.append(random.choice(current_station_trips))
    else:
        sampled_trips = random.sample(current_station_trips, k)

    for line in sampled_trips:
        output_file.write(line)
    sample_counter += k

    print("Wrote", str(sample_counter), "sampled trips to the output file")

    input_file.close()
    output_file.close()
    print("Done")

def add_destination_stations():

    input_file_name = 'sampled_kornhauser_trips.csv'
    output_file_name = 'sampled_kornhauser_metro_journeys.csv' # 'sampled_kornhauser_trips_test.csv' #

    if testing:
        input_file_name = 'sampled_kornhauser_trips_test.csv'
        output_file_name = 'sampled_kornhauser_metro_journeys_test.csv'

    input_file = open(input_file_name, 'r')
    output_file = open(output_file_name, 'w')

    dest_lat_index = 15
    dest_lng_index = 14

    tree = kd_methods.KdTree()

    header = input_file.readline() # Handle the header
    output_file.write(header[:-1] + ",Dest Station\n")

    status_every = 500000

    print("Writing destinations to: " + output_file_name)
    counter = 0
    for line in input_file:

        items = line.split(",")
        stop_lat = float(items[dest_lat_index])
        stop_lng = float(items[dest_lng_index])
        dest_station = tree.nearest(stop_lat, stop_lng)[0]

        output_file.write(line[:-1] + "," + ut.decomma(dest_station) + "\n")

        counter += 1
        if counter % status_every == 0:
            print("Processed", str(counter), "lines")

    input_file.close()
    output_file.close()

    print("Wrote", counter, "lines to the output file")
    print("\nDone\n")

def remove_extra_columns_from_journeys():

    input_file_name = 'sampled_kornhauser_metro_journeys.csv'
    output_file_name = 'sampled_kornhauser_metro_journeys_basic.csv' # 'sampled_kornhauser_trips_test.csv' #

    if testing:
        input_file_name = 'sampled_kornhauser_metro_journeys_test.csv'
        output_file_name = 'sampled_kornhauser_metro_journeys_test_basic.csv'

    input_file = open(input_file_name, 'r')
    output_file = open(output_file_name, 'w')

    orig_index = 19
    dest_index = 20

    tree = kd_methods.KdTree()

    header = input_file.readline() # Handle the header
    output_file.write("Origin Station,Dest Station\n")

    status_every = 100000

    print("Writing just orig/dest stations to: " + output_file_name)
    counter = 0
    for line in input_file:

        items = line.split(",")
        orig = items[orig_index]
        dest = items[dest_index]

        output_file.write(orig + "," + dest)

        counter += 1
        if counter % status_every == 0:
            print("Wrote", str(counter), "lines")

    input_file.close()
    output_file.close()

    print("Wrote", counter, "lines to the output file")
    print("\nDone\n")

# returns a list of the station names
def get_station_names():

    with open('node_names.txt') as f:
        node_names = f.readlines()
    # you may also want to remove whitespace characters like `\n` at the end of each line
    node_names = [x.strip() for x in node_names]
    return node_names

def tally_up_journeys():

    input_file_name = 'sampled_kornhauser_metro_journeys_basic.csv'
    output_file_name = 'journey_counts.pkl'

    if testing:
        input_file_name = 'sampled_kornhauser_metro_journeys_test_basic.csv'
        output_file_name = "journey_counts_test.pkl"

    output_path = ut.get_path(output_file_name)
    input_file = open(input_file_name, 'r')
    output_file = open(output_path, 'wb')

    Journey_Dict = {}
    station_names = get_station_names()

    for orig_station in station_names:
        for dest_station in station_names:
            if orig_station != dest_station:
                # initialize the counter for this journey to 0
                key = (orig_station, dest_station)
                Journey_Dict[key] = 0

    status_every = 100000

    print("Tallying up journeys from: " + input_file_name)
    print("Writing journeys dict to: " + output_path)

    header = input_file.readline() # Ignore header

    counter = 0
    for line in input_file:

        items = line.split(",")
        orig = ut.recomma(items[0].strip())
        dest = ut.recomma(items[1].strip())
        key = (orig, dest)

        if orig != dest:
            Journey_Dict[key] += 1

        counter += 1
        if counter % status_every == 0:
            print("Tallied up", str(counter), "journeys")

    pickle.dump(Journey_Dict, output_file)
    input_file.close()
    output_file.close()

    print("\nDone\n")

if __name__ == "__main__":

    # print("\nADDING ORIGIN STATIONS TO PRUNED DATA\n\n")
    # add_origin_stations_to_pruned_data()
    # print("\n\nSORTING BY ORIGIN STATION \n\n")
    # sort_by_origin_station()
    # print("\n\nSAMPLING FROM SORTED ORIGIN STATIONS \n\n")
    # sample_from_sorted_origins()
    # print("\nADDING DESTINATION STATIONS \n")
    # add_destination_stations()
    # print("\nREMOVING EXTRA COLUMNS FROM JOURNEYS\n")
    # remove_extra_columns_from_journeys()
    print("\nTALLYING UP JOURNEYS\n")
    tally_up_journeys()


"""Finding the prime number under 1000 that is written as sum of the consecutive prime numbers"""
import time

def is_prime(n):
    '''
    check if integer n is a prime, return True or False
    '''
    # 2 is the only even prime
    if n == 2:
        return True
    # integers less than 2 and even numbers other than 2 are not prime
    if n < 2:
        return False
    if not n & 1:  # even numbers
        return False
    # loop looks at odd numbers 3, 5, 7, ... to sqrt(n)
    for i in range(3, int(n**0.5)+1, 2):
        if n % i == 0:
            return False
    return True
tic = time.clock()
i = 2
somme = 0
while somme < 1000000:
    if is_prime(i):
        somme += i
        print(i, somme)
    i += 1
print(somme)
toc = time.clock()
print('time elapsed ', toc-tic)


# 30. Escreva um programa que solicite ao usuário um número e exiba se ele é primo ou não (um número é primo se é divisível apenas por 1 e por ele mesmo).

numero = int(input("Verificar numeros primos ate: "))
multiplo=0

for cont in range(2,numero):
    if (numero % cont == 0):
        print("Múltiplo de",cont)
        multiplo += 1

if(multiplo == 0):
    print("É primo")
else:
    print("Tem",multiplo," múltiplos acima de 2 e abaixo de",numero)


from src.player.base_player import BasePlayer
import random
from random import randint
import copy

class Player(BasePlayer):

    def __init__(self, name, health=None):
        if health is None:
            self.health = self.DEFAULT_HEALTH_FOR_PLAYER
        else:
            self.health = health
        self.actions = [Player.default_damage, Player.lucky_damage, Player.healing]
        self.name = name
        self.start_health = self.health

    @property
    def health(self):
        return self._health

    @health.setter
    def health(self, value):
        self._health = value

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value

    # Deal default damage to an enemy in the range of 18-25
    def default_damage(self, player):
        print("\nCall default hit")
        default_result = random.randint(self.DEFAULT_AND_HEALING_START_VALUE, self.DEFAULT_AND_HEALING_END_VALUE)
        player.health -= default_result
        if player.health < 0:
            player.health = 0

    # Deal lucky damage to an enemy in the range of 10-35
    def lucky_damage(self, player):
        print("\nCall lucky hit")
        lucky_result = random.randint(self.LUCKY_START_VALUE, self.LUCKY_END_VALUE)
        player.health -= lucky_result
        if player.health < 0:
            player.health = 0

    # Heal in the range of 18-25
    def healing(self, player):
        print("\nCall healing")
        healing_result = random.randint(self.DEFAULT_AND_HEALING_START_VALUE, self.DEFAULT_AND_HEALING_END_VALUE)
        self.health += healing_result
        if self.health > self.start_health:
            self.health = self.start_health

    # Random choice of player action
    def get_random_action(self):
        """When the health of the Computer reaches 35%, its chance of healing increases."""
        if (self.health * 100) / self.start_health <= 35 and self.name == 'Computer':
            new_actions = copy.copy(self.actions)
            new_actions.extend([Player.healing, Player.healing, Player.healing])
            return new_actions[randint(0, len(new_actions)-1)]
        """The chance remains the same"""
        return self.actions[randint(0, len(self.actions)-1)]


from . import *
from .sd_fonction import sd_fonction
from .sd_modele import sd_modele

class sd_char_cine(AsBase):
#===========================
    nomj = SDNom(fin=19)

    AFCK = AsVK8(lonmax=3)
    AFCI = AsVI()
    AFCV = Facultatif(OJBVect(type=Parmi('C', 'R', 'K')))

    def exists(self):
        # retourne "vrai" si la SD semble exister (et donc qu'elle peut etre
        # vérifiée)
        return self.AFCK.exists

    def u_veri1(self):   # retourne (CIME/CITH/CIAC, RE/CX/FT)
    #---------------------------------------------------------------
        if not self.exists():
            return
        afck = self.AFCK.get()
        l1 = afck[0].strip().split('_')
        assert len(l1) == 2, afck
        phen, tsca = l1[0], l1[1]
        assert phen in ('CIME', 'CITH', 'CIAC'), afck
        assert tsca in ('RE', 'CX', 'FT'), tsca
        return phen, tsca

    def check_AFCK(self, checker):
    #---------------------------------------------
        if not self.exists():
            return
        phen, tsca = self.u_veri1()
        afck = self.AFCK.get()
        nomo = afck[1].strip()
        sd2 = sd_modele(nomo)
        sd2.check(checker)
        if afck[2].strip() != '':
            assert (phen == 'CIME' or phen == 'CITH') and tsca == 'FT', afck

    def check_AFCI(self, checker):
    #---------------------------------------------
        if not self.exists():
            return
        phen, tsca = self.u_veri1()
        afci = self.AFCI.get()
        nbloc = afci[0]
        assert len(afci) >= 3 * nbloc + 1, afci
        for k in range(nbloc):
            nuno = afci[3 * k + 1]
            nucmp = afci[3 * k + 2]
            assert afci[3 * k + 3] == 0, (k, afci)
            assert nuno > 0, (k, afci)
            assert nucmp > 0, (k, afci)

    def check_AFCV(self, checker):
    #-------------------------------------------------
        if not self.exists():
            return
        phen, tsca = self.u_veri1()
        afci = self.AFCI.get()
        nbloc = afci[0]
        if not self.AFCV.exists:
            assert tsca == 'FT', tsca
            afck = self.AFCK.get()
            assert afck[2].strip() != '', afck
        else:
            tsca2 = self.AFCV.type.strip()
            assert self.AFCV.lonmax >= nbloc, (nbloc, self.AFCV.lonmax)

            if tsca == 'RE':
                assert tsca2 == 'R', tsca2  # champ de réels
            if tsca == 'FT':
                assert tsca2 == 'K', tsca2  # champ de fonctions
            if tsca == 'CX':
                assert tsca2 == 'C', tsca2  # champ de complexes

            # vérification des fonctions :
            if tsca == 'FT':
                afcv = self.AFCV.get()
                for fonc in afcv[:nbloc]:
                    sd2 = sd_fonction(fonc)
                    sd2.check(checker)


# Use https://www.si.umich.edu/programs/bachelor-science-
# information/bsi-admissions as a template.
# STEPS
# Create a similar HTML file but
# 1) Replace every occurrence of the word “student” with “AMAZING
# student.”
# 2) Replace the main picture with a picture of yourself.
# 3) Replace any local images with the image I provided in media.  (You
# must keep the image in a separate folder than your html code.

# Deliverables
# Make sure the new page is uploaded to your GitHub account.

import urllib.request, urllib.parse, urllib.error
from bs4 import BeautifulSoup

url = "https://www.si.umich.edu/programs/bachelor-science-information/bsi-admissions"
page = urllib.request.urlopen(url).read()
soup = BeautifulSoup(page, "html.parser")

soup1 = str(soup)
soup2 = soup1.replace("student", "AMAZING student")
soup3 = soup2.replace("https://www.youtube.com/embed/mimp_3gquc4?feature=oembed", "me.jpg")

final_soup = BeautifulSoup(soup3, "html.parser")

for p in final_soup.find_all(class_= "logo"):
	if p.img:
		p.img['src'] = "media/logo.png"
for p1 in final_soup.find_all(class_= "footer-logo"):
	if p1.img:
		p1.img['src'] = "media/logo.png"

f = open("bshw3.html", "w")
f.write (final_soup.prettify())
f.close()


from unittest import TestCase
import punctilious as pu
import punctilious.random_data as random_data

class TestDefinitionDeclaration(TestCase):

    def test_title(self):
        pu.configuration.echo_default = False
        pu.configuration.encoding = pu.encodings.plaintext
        u = pu.UniverseOfDiscourse()
        content1 = random_data.random_sentence()
        d1 = u.d.declare(natural_language=content1)
        self.assertEqual('Definition (D1)', d1.rep_title(cap=True, encoding=pu.encodings.plaintext))
        self.assertEqual('definition (D1)', d1.rep_title(cap=False, encoding=pu.encodings.plaintext))
        self.assertEqual('𝗗𝗲𝗳𝗶𝗻𝗶𝘁𝗶𝗼𝗻 (𝒟₁)', d1.rep_title(cap=True, encoding=pu.encodings.unicode))
        self.assertEqual('𝗱𝗲𝗳𝗶𝗻𝗶𝘁𝗶𝗼𝗻 (𝒟₁)', d1.rep_title(cap=False, encoding=pu.encodings.unicode))

    def test_definition_declaration(self):
        pu.configuration.echo_default = False
        pu.configuration.encoding = pu.encodings.plaintext
        u = pu.UniverseOfDiscourse()
        u_plaintext = u.nameset.rep_symbol(encoding=pu.encodings.plaintext)
        u_unicode = u.nameset.rep_symbol(encoding=pu.encodings.unicode)
        content1 = '𝒷ℴℴ𝓇ℴ𝒶 𝒷ℯℯ𝒹𝓊 𝓇𝒾ℴ𝒷𝓁ℴ 𝒷𝓎𝓊𝒹𝒾ℯ𝓀𝓊.'  # random_data.random_sentence()
        content2 = '𝓇ℯ𝒹𝓊 𝒹ℴ𝒷ℴℯ 𝒹𝒾𝓃𝓊 𝒷𝒶𝒹𝓇𝓎𝓊 𝒽𝓎𝓋𝒾𝓋ℴ𝒾 𝒷ℯ𝒷𝒾 𝓏𝓎𝒶ℊ𝓃𝓊 𝒹𝓎𝒶𝓀ℯ 𝓈𝒾𝓏ℴℴ 𝒷𝒶𝓁ℴ 𝓅ℴ𝒶𝒹ℴℯ 𝒷𝒾𝓃ℴ𝒷𝓇𝓊 𝒻𝓁ℴ𝒹𝒾ℴ 𝓁𝒾𝒽𝒶 𝓇𝓎ℯ𝓅ℯ 𝓅ℴ𝒾𝓈𝓉𝒾𝒻𝓁𝓎ℴ 𝓉ℯ𝒷𝓁ℴℯ 𝒹ℴ𝒷𝓇ℯ 𝒿ℴℯ𝒷𝒾 𝒷𝒶𝒿𝓎𝓊 𝓈𝓉𝒶𝒷𝓎𝒶𝓁𝒶 ℊ𝓃𝒶𝒷𝓇𝒾 𝓂ℴ𝒶𝒻ℴ𝓊 𝒹ℴ𝓊𝓉𝓊 𝓈𝓊𝒹𝒾 ℊ𝓃ℴ𝒷𝓊 𝓂𝓊𝒹ℯ𝒷𝒾 𝒷ℯ𝒹ℴ𝒷ℯ 𝒻𝓁𝒾ℴ𝒷𝒾 𝒹𝒶𝓏𝒶ℯ.'  # random_data.random_sentence(min_words=30)
        content3 = '𝒷𝓇𝓎ℯ𝒷𝓇𝒾 𝒷𝓎𝒶𝒷𝓇𝓎ℴ𝒷ℴ𝓊 𝒷𝓊𝒷𝒾.'  # random_data.random_sentence()
        content4 = '𝒷𝓁ℴ𝓁ℯ 𝒷𝓁ℴ𝒿ℯ 𝒽𝓎𝓊ℊ𝓃ℯℯ.'  # random_data.random_sentence()
        content5 = '𝒹𝒾𝓅𝒾𝓋𝒾 𝓈𝓉𝒾𝓂ℴ𝓋ℴ 𝒻𝓇𝒾𝒻𝓊𝒷𝓊 𝒹ℴ𝒷ℯ 𝒹𝓇𝓊𝓂𝓊𝒹ℴ𝒾𝓉𝓊.'  # random_data.random_sentence()
        d1 = u.d.declare(natural_language=content1)
        d2 = u.d.declare(natural_language=content2)
        d3 = u.d.declare(natural_language=content3, symbol='b')
        d4 = u.d.declare(natural_language=content4, symbol='c', name='the definition of test')
        d5 = u.d.declare(natural_language=content5, acronym='oaot', symbol='d', name='the other definition of test')
        self.assertEqual(
            f'𝖫𝖾𝗍 ⌜𝒟₁⌝ 𝖻𝖾 𝗍𝗁𝖾 𝑑𝑒𝑓𝑖𝑛𝑖𝑡𝑖𝑜𝑛 ⌜𝒷ℴℴ𝓇ℴ𝒶 𝒷ℯℯ𝒹𝓊 𝓇𝒾ℴ𝒷𝓁ℴ 𝒷𝓎𝓊𝒹𝒾ℯ𝓀𝓊.⌝ 𝗂𝗇 {u_unicode}.',
            d1.rep_report(encoding=pu.encodings.unicode, wrap=False))
        self.assertEqual(
            f'𝖫𝖾𝗍 ⌜𝒟₂⌝ 𝖻𝖾 𝗍𝗁𝖾 𝑑𝑒𝑓𝑖𝑛𝑖𝑡𝑖𝑜𝑛 ⌜𝓇ℯ𝒹𝓊 𝒹ℴ𝒷ℴℯ 𝒹𝒾𝓃𝓊 𝒷𝒶𝒹𝓇𝓎𝓊 𝒽𝓎𝓋𝒾𝓋ℴ𝒾 𝒷ℯ𝒷𝒾 𝓏𝓎𝒶ℊ𝓃𝓊 𝒹𝓎𝒶𝓀ℯ 𝓈𝒾𝓏ℴℴ 𝒷𝒶𝓁ℴ 𝓅ℴ𝒶𝒹ℴℯ 𝒷𝒾𝓃ℴ𝒷𝓇𝓊 𝒻𝓁ℴ𝒹𝒾ℴ 𝓁𝒾𝒽𝒶 𝓇𝓎ℯ𝓅ℯ 𝓅ℴ𝒾𝓈𝓉𝒾𝒻𝓁𝓎ℴ 𝓉ℯ𝒷𝓁ℴℯ 𝒹ℴ𝒷𝓇ℯ 𝒿ℴℯ𝒷𝒾 𝒷𝒶𝒿𝓎𝓊 𝓈𝓉𝒶𝒷𝓎𝒶𝓁𝒶 ℊ𝓃𝒶𝒷𝓇𝒾 𝓂ℴ𝒶𝒻ℴ𝓊 𝒹ℴ𝓊𝓉𝓊 𝓈𝓊𝒹𝒾 ℊ𝓃ℴ𝒷𝓊 𝓂𝓊𝒹ℯ𝒷𝒾 𝒷ℯ𝒹ℴ𝒷ℯ 𝒻𝓁𝒾ℴ𝒷𝒾 𝒹𝒶𝓏𝒶ℯ.⌝ 𝗂𝗇 {u_unicode}.',
            d2.rep_report(encoding=pu.encodings.unicode, wrap=False))
        self.assertEqual(
            f'𝖫𝖾𝗍 ⌜𝑏₁⌝ 𝖻𝖾 𝗍𝗁𝖾 𝑑𝑒𝑓𝑖𝑛𝑖𝑡𝑖𝑜𝑛 ⌜𝒷𝓇𝓎ℯ𝒷𝓇𝒾 𝒷𝓎𝒶𝒷𝓇𝓎ℴ𝒷ℴ𝓊 𝒷𝓊𝒷𝒾.⌝ 𝗂𝗇 {u_unicode}.',
            d3.rep_report(encoding=pu.encodings.unicode, wrap=False))
        self.assertEqual(
            f'𝖫𝖾𝗍 ⌜𝑐₁⌝ 𝖻𝖾 𝗍𝗁𝖾 𝑑𝑒𝑓𝑖𝑛𝑖𝑡𝑖𝑜𝑛 ⌜𝒷𝓁ℴ𝓁ℯ 𝒷𝓁ℴ𝒿ℯ 𝒽𝓎𝓊ℊ𝓃ℯℯ.⌝ 𝗂𝗇 {u_unicode}, 𝗄𝗇𝗈𝗐𝗇 𝖺𝗌 𝗍𝗁𝖾 𝖽𝖾𝖿𝗂𝗇𝗂𝗍𝗂𝗈𝗇 𝗈𝖿 𝗍𝖾𝗌𝗍.',
            d4.rep_report(encoding=pu.encodings.unicode, wrap=False))
        self.assertEqual(
            f'𝖫𝖾𝗍 ⌜𝑑₁⌝ 𝖻𝖾 𝗍𝗁𝖾 𝑑𝑒𝑓𝑖𝑛𝑖𝑡𝑖𝑜𝑛 ⌜𝒹𝒾𝓅𝒾𝓋𝒾 𝓈𝓉𝒾𝓂ℴ𝓋ℴ 𝒻𝓇𝒾𝒻𝓊𝒷𝓊 𝒹ℴ𝒷ℯ 𝒹𝓇𝓊𝓂𝓊𝒹ℴ𝒾𝓉𝓊.⌝ 𝗂𝗇 {u_unicode}, 𝗄𝗇𝗈𝗐𝗇 𝖺𝗌 𝗍𝗁𝖾 𝗈𝗍𝗁𝖾𝗋 𝖽𝖾𝖿𝗂𝗇𝗂𝗍𝗂𝗈𝗇 𝗈𝖿 𝗍𝖾𝗌𝗍, 𝗈𝗋 𝗌𝗂𝗆𝗉𝗅𝗒 𝗈𝖺𝗈𝗍.',
            d5.rep_report(encoding=pu.encodings.unicode, wrap=False))


# -*- coding: UTF-8 -*-
from django.contrib import admin
from models import CuentaBancaria, MovimientoBanco
from conciliacion.bancos.models import MovimientoLibro
from conciliacion.bancos.forms import MovimientoLibroForm
from decimal import Decimal
from django.db.models.query_utils import Q
from django.db import transaction

def marcar_conciliado(modeladmin, request, queryset):
    queryset.update(conciliado=True)

marcar_conciliado.short_description = "Marcar como conciliados"

@transaction.commit_manually
def conciliar_por_movimiento(modeladmin, request, queryset):
    for obj in queryset:
        mov = MovimientoBanco.objects.filter(
                             conciliado=False,
                            movimiento__isnull=True,
                            cuenta = obj.cuenta,
                            fecha__month=obj.fecha.month,
                            fecha__year=obj.fecha.year,
                            monto = obj.monto
                            )
        if len(mov) == 1:
            movimiento_banco = mov[0]
            movimiento_banco.movimiento=obj
            movimiento_banco.conciliado=True
            movimiento_banco.save()

            obj.conciliado = True
            obj.save()

    transaction.commit()

conciliar_por_movimiento.short_description = "Conciliar por 1 movimiento"

class CuentaBancariaAdmin (admin.ModelAdmin):
    fields = ['id','banco','numero','moneda','saldo_inicial']
    list_display = ['id','__unicode__','saldo_inicial']

class MovimientoBancoInLine (admin.TabularInline):
    model = MovimientoBanco
    exclude = ['cuenta','conciliado','descripcion','documento']
    readonly_fields = ['monto',]
    extra = 1

class MovimientoBancoAdmin (admin.ModelAdmin):
    date_hierarchy = 'fecha'
    list_display = ['fecha','movimiento','documento','cuenta','descripcion','monto','saldo','tipo']
    list_filter = ['cuenta','conciliado','tipo']
    actions = [marcar_conciliado]

class MovimientoLibroAdmin (admin.ModelAdmin):
    date_hierarchy = 'fecha'
    list_display = ['fecha','documento','cuenta','descripcion','monto','tipo']
    list_filter = ['cuenta','conciliado']
    inlines = [MovimientoBancoInLine,]
    form = MovimientoLibroForm
    actions = [marcar_conciliado,conciliar_por_movimiento]

    def get_form(self, request, obj=None, **kwargs):

        cleaned_data = request.POST
        if "documento" in cleaned_data and cleaned_data["documento"] == "" and \
            "descripcion" in cleaned_data and cleaned_data["descripcion"] == "" and \
            "cuenta" in cleaned_data and cleaned_data["cuenta"] == "" and \
            "monto" in cleaned_data and cleaned_data["monto"] == "" and \
            "movimiento_banco" in cleaned_data and cleaned_data["movimiento_banco"] != "" :
                linea =  MovimientoBanco.objects.get(pk=cleaned_data["movimiento_banco"])
                request.POST.update({
                                     "documento": linea.documento,
                                     "descripcion": linea.descripcion,
                                     "cuenta": linea.cuenta.pk,
                                     "monto": linea.monto,

                                     })

        form = super(MovimientoLibroAdmin, self).get_form(request, obj, **kwargs)
        if obj is not None:
            form.base_fields['movimiento_banco'].queryset=MovimientoBanco.objects.filter(
                                                                 conciliado=False,
                                                                movimiento__isnull=True,
                                                                cuenta = obj.cuenta,
                                                                fecha__month=obj.fecha.month,
                                                                fecha__year=obj.fecha.year,
                                                                monto = obj.monto
                                                                )
            qs = MovimientoBanco.objects.filter(
                                                    (Q(movimiento=obj) |
                                                                (Q(conciliado=False)&
                                                                Q(movimiento__isnull=True))) &
                                                                Q(cuenta = obj.cuenta) &
                                                                Q(fecha__month=obj.fecha.month) &
                                                                Q(fecha__year=obj.fecha.year) &
                                                                Q(monto__lt=15000 if obj.cuenta.moneda == "US$" else 1500) &
                                                                (
                                                                 Q(tipo__contains="DP") |
                                                                 Q(tipo__contains="DC") |
                                                                 Q(tipo__contains="TF"
                                                                     ))
                                                                )
            form.base_fields['depositos'].queryset=qs
            form.base_fields['depositos'].initial = [l.pk for l in MovimientoBanco.objects.filter(movimiento=obj)]

        else:
            form.base_fields['movimiento_banco'].queryset=MovimientoBanco.objects.filter(conciliado=False,
                                                                movimiento__isnull=True,
                                                                )
        return form

    @transaction.commit_manually
    def save_model(self, request, obj, form, change):
        try:
            super(MovimientoLibroAdmin, self).save_model(request, obj, form, change)
            if form.cleaned_data['movimiento_banco'] != None:
                movimiento_banco = form.cleaned_data['movimiento_banco']
                movimiento_banco.movimiento=obj
                movimiento_banco.conciliado=True
                movimiento_banco.save()
                obj.conciliado=True
                obj.save()

            if "depositos" in form.cleaned_data and len(form.cleaned_data['depositos']) > 0:
                depositos = form.cleaned_data['depositos']
                monto = Decimal('0.00')
                for d in depositos:
                    monto += d.monto
                    d.movimiento=obj
                    d.conciliado=True
                    d.save()

                if monto == d.monto:
                    obj.conciliado=True
                    obj.save()
        except Exception as e:
            transaction.rollback()
            raise e
        else:
            transaction.commit()

admin.site.register(CuentaBancaria,CuentaBancariaAdmin)
admin.site.register(MovimientoBanco,MovimientoBancoAdmin)
admin.site.register(MovimientoLibro,MovimientoLibroAdmin)


import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), "../../utils"))
sys.path.append('../..')
sys.path.append('..')
import matplotlib as mpl

import numpy as np
import numpy.random as random
import time
import os.path
from astropy.io import fits
import plot

import astropy.units as u
from astropy.coordinates import SkyCoord
import sunpy.coordinates
from sunpy.coordinates import frames
import track
import misc

downsample_coef = .5
num_chunks = 1

for file_date in ["2013-02-03", "2013-02-04"]:

    hdul = fits.open(f"{file_date}_hmi.M_720s.fits")

    try:
        os.remove(f"{file_date}_2.fits")
    except:
        pass

    output = fits.open(f"{file_date}_2.fits", mode="append")
    hdu = fits.ImageHDU(data=None, header=None)
    output.append(hdu)

    metadata = hdul[1].header

    t_rec = metadata['T_REC']

    date = t_rec[:10]
    date = date[:4] + "-" + date[5:7] + "-" +date[8:10]

    hrs = t_rec[11:13]
    mins = t_rec[14:16]
    secs = t_rec[17:19]

    obs_time_0 = f"{date} {hrs}:{mins}:{secs}"

    sdo_lon_0 = metadata['CRLN_OBS']
    sdo_lat_0 = metadata['CRLT_OBS']
    sdo_dist_0 = metadata['DSUN_OBS']

    observer_0 = frames.HeliographicStonyhurst(0.*u.deg, sdo_lat_0*u.deg, radius=sdo_dist_0*u.m, obstime=obs_time_0)

    nx = int(round(metadata['NAXIS1']*downsample_coef))
    ny = int(round(metadata['NAXIS2']*downsample_coef))

    xs1 = (np.arange(1, nx + 1)).astype(float)
    ys1 = (np.arange(1, ny + 1)).astype(float)
    xys = np.transpose([np.tile(xs1, ny), np.repeat(ys1, nx)])

    data = hdul[1].data
    max_val = np.nanmax(data)
    fltr = np.isnan(data)
    data[fltr] = max_val

    if downsample_coef < 1:
        data = misc.sample_image(data, downsample_coef)
    fltr = data > .9*max_val
    data[fltr] = np.nan

    fltr0 = fltr
    xys = xys[np.logical_not(fltr).flatten()]

    for i in np.arange(1, len(hdul), 10):
        metadata = hdul[i].header

        a = metadata['CROTA2']*np.pi/180

        dx = metadata['CRVAL1']
        dy = metadata['CRVAL2']
        arcsecs_per_pix_x = metadata['CDELT1']/downsample_coef
        arcsecs_per_pix_y = metadata['CDELT2']/downsample_coef
        coef_x = 1./arcsecs_per_pix_x
        coef_y = 1./arcsecs_per_pix_y
        xc = metadata['CRPIX1']*downsample_coef
        yc = metadata['CRPIX2']*downsample_coef

        t_rec = metadata['T_REC']

        date = t_rec[:10]
        date = date[:4] + "-" + date[5:7] + "-" +date[8:10]

        hrs = t_rec[11:13]
        mins = t_rec[14:16]
        secs = t_rec[17:19]

        obs_time = f"{date} {hrs}:{mins}:{secs}"

        sdo_lon = metadata['CRLN_OBS']
        sdo_lat = metadata['CRLT_OBS']
        sdo_dist = metadata['DSUN_OBS']

        ###########################################################################
        if (i - 1) % 10 == 0:
            data = hdul[i].data
            max_val = np.nanmax(data)
            fltr = np.isnan(data)
            data[fltr] = max_val

            data = misc.sample_image(data, downsample_coef)
            fltr = data > .9*max_val
            data[fltr] = np.nan
            test_plot = plot.plot(nrows=1, ncols=1, size=plot.default_size(1000, 1000))
            test_plot.colormap(data, show_colorbar=True)
            test_plot.save(f"image{date}_{i}.png")
            test_plot.close()

            metadata['NAXIS1'] = nx
            metadata['NAXIS2'] = ny

            metadata['CDELT1'] = arcsecs_per_pix_x
            metadata['CDELT2'] = arcsecs_per_pix_y
            metadata['CRPIX1'] = xc
            metadata['CRPIX2'] = yc

            hdu = fits.ImageHDU(data=data, header=metadata)

            output.append(hdu)
            output.flush()

    output.close()
    hdul.close()


"""
test_opsiservice
"""

from pathlib import Path

import pytest

from opsicli.config import OPSIService, config
from opsicli.opsiservice import jsonrpc_client  # pylint: disable=unused-import
from opsicli.opsiservice import (
	get_service_connection,
	get_service_credentials_from_backend,
)

from .utils import container_connection

@pytest.mark.skipif(not Path("/etc/opsi/backends").exists(), reason="need local backend for this test")
def test_get_service_credentials_from_backend() -> None:
	(host_id, host_key) = get_service_credentials_from_backend()
	assert host_id
	assert host_key

@pytest.mark.skipif(not Path("/etc/opsi/backends").exists(), reason="need local backend for this test")
def test_get_service_connection_local() -> None:
	local_connection = get_service_connection()
	assert local_connection
	result = local_connection.jsonrpc("backend_getInterface")
	print(result)
	assert "host_getObjects" in str(result)

@pytest.mark.skipif(not Path("/etc/opsi/backends").exists(), reason="need local backend for this test")
def test_get_service_connection_half_configured_service() -> None:
	global jsonrpc_client  # pylint: disable=global-statement
	jsonrpc_client = None
	config.services.append(OPSIService("pytest_test_service", "https://localhost:4447"))
	config.service = "pytest_test_service"
	connection = get_service_connection()
	result = connection.jsonrpc("backend_getInterface")
	print(result)
	assert "host_getObjects" in str(result)

@pytest.mark.requires_testcontainer
def test_get_service_connection() -> None:
	with container_connection():
		connection = get_service_connection()
		assert connection
		result = connection.jsonrpc("backend_getInterface")
		print(result)
		assert "host_getObjects" in str(result)


#Script to scrape metacritic and exrapolate description, metascore link, and description
from bs4 import BeautifulSoup
from datetime import datetime
from django.utils import timezone
from django.core.cache import cache
import logging, traceback, urllib2

BASE_URL='http://www.metacritic.com'
TEMPLATE_GENRE_URL=BASE_URL + '/browse/games/genre/metascore/GENRE/PLATFORM?view=condensed&page=PAGE'
GENRES=('action','adventure','fighting','first-person','flight',
        'party','platformer','puzzle','racing','real-time','role-playing',
        'simulation','sports','strategy','third-person',)
PLATFORM_MAP={'pc':'pc', 'ps4':'playstation-4', 'ps3':'playstation-3',
  'xboxone':'xbox-one','xbox360':'xbox-360','wii-u':'wii-u','3ds':'3ds',
  'ios':'ios'}
LOGGER = logging.getLogger(__name__)

def parse(link):
  response = urllib2.urlopen(link)
  soup = BeautifulSoup(response.read())
  response.close()
  return soup

#return critic,user rating tuple
def getMetacriticScore(url):
  result=cache.get(url)
  if result is None:
    soup=parse(url)
    try:
      critic=int(soup.find('span',{'itemprop':'ratingValue'}).string)
    except:
      critic='tbd'
    try:
      user=float(soup.find('div',{'class':'metascore_w user large game positive'}).string)
    except:
      user='tbd'
    result=(critic,user,getColor(critic,100),getColor(user,10))
    cache.set(url,result,60*60*24)
  return result

def getColor(score,scale):
  if isinstance(score, basestring):
    return 'LightGray'
  elif score >= scale*.6:
    return 'Green'
  elif score >= scale*.5:
    return 'Orange'
  else:
    return 'Red'

def extractString(element):
  return str(element.contents[0]).strip()

#Parse summary from soup
def parseSummary(soup):
  summary = ''
  for summaryDetailsElement in soup.findAll('ul', attrs={'class':'summary_details'}):
    for summaryElement in summaryDetailsElement.findAll('span', attrs={'class':'blurb blurb_collapsed'}):
      summary = extractString(summaryElement)
    for summaryElement in summaryDetailsElement.findAll('span', attrs={'class':'blurb blurb_expanded'}):
      summary += extractString(summaryElement)
    if summary == '':
      for spanElement in summaryDetailsElement.findAll('span', attrs={'class':'data','itemprop':'description'}):
        summary = spanElement.findNext('span').string.strip()
  if summary == '':
    for element in soup.findAll('meta', attrs={'name':'og:description'}):
      summary = element['content'].strip()
  return summary

#Return (description,published) tuple
def parseGame(url,gamename):
  soup=parse(BASE_URL+url)
  for element in soup.findAll('span', attrs={'itemprop':'datePublished'}):
    published = element.string.strip()
  summary=''
  try:
    summary=parseSummary(soup)
  except:
    LOGGER.error('Failed to parse ' + gamename)
    traceback.print_exc()
    pass
  return (summary,getTime(published))

def getTime(time):
  try:
    return timezone.get_current_timezone().localize(datetime.strptime(time, '%b %d, %Y'))
  except:
    LOGGER.error('Exception converting time ' + time)
    traceback.print_exc()
    return timezone.now()

def isValid(soup):
  return len(soup.findAll('p', attrs={'class':'no_data'})) == 0

#Get all URLS for all games
def getGameURLs(games,tags,genre,platform):
  x=0
  while True:
    url=TEMPLATE_GENRE_URL.replace('GENRE',genre).replace('PAGE',str(x)).replace('PLATFORM',platform)
    x+=1
    LOGGER.info('Getting URL: ' + url)
    try:
      soup=parse(url)
      if not isValid(soup):
        return
      for productTitleContainer in soup.findAll('div', attrs={'class':'basic_stat product_title'}):
        gameElement=productTitleContainer.findNext('a')
        gamename=gameElement.string.strip()

        #keep track of tags
        gametags=list()
        if gamename in tags:
          gametags=tags[gamename]
        gametags.append(genre)
        gametags.append(platform)
        tags[gamename]=gametags

        if not gamename in games:
          link=str(gameElement['href']).strip()
          games[gamename]=link
    except:
      traceback.print_exc()
      pass

def parseGameURL(name,url):
  try:
    summary,published=parseGame(url,name)
  except:
    LOGGER.error('Error encountered on: ' + name + ' with url: ' + url)
    summary=''
    published=timezone.now()
    pass
  return (name,BASE_URL+url,summary,published)

def scrapeGamesAndTags():
  games={}
  tags={}
  for genre in GENRES:
    for platform in PLATFORM_MAP.keys():
      getGameURLs(games,tags,genre,platform)
  return (games,tags)


''' CS5001.38359.202230 - SEC 05 - John Wilder
    Midterm - Question 2
    clancy.co@northeastern.edu (002781018)
    11 MAR 22

    The file contains a function that finds the median in a list of values.
'''

def find_median(input_list):
    '''
    find_median takes a list of numbers as input
    and finds the median value in the list. The median value
    is defined as the value for which half the items are larger
    and half the items are smaller.

    :param input_list: A list of numbers
    :return: The median value of the list
    '''

    # function variables
    size = len(input_list)

    # sorted is a built-in function that will return a sorted list
    sorted_list = sorted(input_list)
    median = 0

    # find the median for even and odd list numbers
    if size % 2 != 0:
        median = sorted_list[int(size / 2)]
    else:
        elements = sorted_list[int((size + 1) / 2)]
        elements = elements + sorted_list[int((size - 1) / 2)]
        median = 0.5 * (elements)

    return median

def main():

    # Test 1
    my_list = list(range(0, 6))
    expected = 2.5
    actual = find_median(my_list)
    if actual != expected:
        print("FAILED ")
        print("EXPECTED = ", expected)
        print("ACTUAL = ", actual)

    # Test 2
    my_list = list(range(0, 5))
    expected = 2
    actual = find_median(my_list)
    if actual != expected:
        print("FAILED")
        print("EXPECTED = ", expected)
        print("ACTUAL = ", actual)

if __name__ == "__main__":
    main()


# db 연동
import sqlite3
from datetime import datetime

# print(sqlite3.version)
# print(sqlite3.sqlite_version)

# # 남짜 생성
now = datetime.now()
# print("now", now)
now_date_time = now.strftime("%Y-%m-%d %H:%M:%S")
# print("now_date_time", now_date_time)

conn = sqlite3.connect("data/database.db", isolation_level=None)

# 커서
cursor = conn.cursor()  # <class 'sqlite3.Cursor'>
print(type(cursor))

# 테이블 생성
cursor.execute(
    "CREATE TABLE IF NOT EXISTS user(id integer primary key,username text, phon text,website text,regdate text)"
)

# insert
# cursor.execute(
#     "INSERT INTO user VALUES(1,'KIM','010-1234-1234','kim.com',?)", (now_date_time,)
# )

# cursor.execute(
#     "INSERT INTO user VALUES(?,?,?,?,?)",
#     (
#         2,
#         "hong",
#         "010-1234-4567",
#         "hong.com",
#         now_date_time,
#     ),
# )

# 여러개 삽입
user_list = (
    (3, "park", "010-4567-1234", "park.com", now_date_time),
    (4, "choi", "010-9876-1234", "choi.com", now_date_time),
    (5, "yoo", "010-3687-1234", "yoo.com", now_date_time),
)

cursor.executemany("INSERT INTO user VALUES(?,?,?,?,?)", user_list)


import pygame
from HelperFunctions import load_full_screen_image
from buttons import MouseCollider
from buttons import ButtonType, check_for_and_handle_button_click, Button

class HelpScreen:
    """Handles the visual elements of the help screen and the buttons on it. This includes creating the buttons and responding to button clicks."""

    def __init__(self, display_surface, ScreenDimensions):
        self.display_surface = display_surface
        self.ScreenDimensions = ScreenDimensions
        self.help_screen_1 = load_full_screen_image(
            ScreenDimensions,
            "Assets\\Help Screen\\help_screen_1_high_res.png",
            "Assets\\Help Screen\\help_screen_1.png",
        )
        self.help_screen_2 = load_full_screen_image(
            ScreenDimensions,
            "Assets\\Help Screen\\help_screen_2_high_res.png",
            "Assets\\Help Screen\\help_screen_2.png",
        )
        self.current_help_screen = self.help_screen_1
        self.MouseCollider = MouseCollider(
            pygame.mouse.get_pos(), self.display_surface, self.ScreenDimensions
        )

        self.help_buttons: pygame.sprite.Group() = self.create_buttons()
        self.button_pressed: (None | ButtonType) = None

    def create_buttons(self):
        buttons = pygame.sprite.Group()
        # Level 1
        buttons.add(
            Button(
                button_type=ButtonType.help_home,
                display_surface=self.display_surface,
                ScreenDimensions=self.ScreenDimensions,
            )
        )
        buttons.add(
            Button(
                button_type=ButtonType.help_next,
                display_surface=self.display_surface,
                ScreenDimensions=self.ScreenDimensions,
            )
        )
        return buttons

    def detect_button_interaction(self, help_buttons, MouseCollider, events):
        for help_button in help_buttons:
            if pygame.sprite.collide_mask(MouseCollider, help_button):
                help_button.image = help_button.image_dict["selected"]
                button_clicks = check_for_and_handle_button_click(events, help_button)
                if button_clicks != None:
                    self.button_pressed = button_clicks
            else:
                help_button.image = help_button.image_dict["unselected"]

    def update(self, events):
        self.display_surface.blit(self.current_help_screen, (0, 0))
        if (
            self.current_help_screen == self.help_screen_1
        ):  # conditional statement to only render the next button on the first help screen
            self.help_buttons.update()
        else:
            self.help_buttons.update([ButtonType.help_next])
        self.MouseCollider.update(pygame.mouse.get_pos())
        self.detect_button_interaction(self.help_buttons, self.MouseCollider, events)


from collections import deque
import sys
input = sys.stdin.readline

N, M = map(int, input().split())
graph = [[] for _ in range(N+1)]
indgree = [0] * (N+1)

for _ in range(M):
    a, b = map(int, input().split())
    graph[a].append(b)
    indgree[b] += 1

queue = deque()
for i in range(1, N+1):
    if indgree[i] == 0:
        queue.append(i)

result = []
while queue:
    idx = queue.popleft()
    result.append(idx)

    for i in graph[idx]:
        indgree[i] -= 1
        if indgree[i] == 0:
            queue.append(i)

for i in result:
    print(i, end=" ")

# import sys
# from collections import deque
# input = sys.stdin.readline

# n, m = map(int, input().split())
# L = []
# degree = [0] * 32001
# graph = [[] for _ in range(32001)]
# queue = deque()

# for i in range(m):
#     a, b = map(int, input().split())
#     L.append([a, b])

# for a, b in L:
#     degree[b] += 1
#     graph[a].append(b)

# for i in range(1, n + 1):
#     if degree[i] == 0:
#         queue.append(i)

# while queue:
#     student = queue.popleft()
#     for j in graph[student]:
#         degree[j] -= 1
#         if degree[j] == 0:
#             queue.append(j)

#     print(student, end = ' ')


"""Distribution sample submssion by year."""
from typing import Union, Tuple

import pandas as pd
from pandas.plotting import register_matplotlib_converters
import matplotlib.pyplot as plt
import seaborn as sns
from statsmodels.formula.api import ols

from ncbi_remap.plotting import NcbiPlotter, update_kwargs

register_matplotlib_converters()

PLOT_DEFAULTS = dict()
AXES_DEFAULTS = dict(ylabel="% Aligned", xlabel="Year")

def get_data(file_name: str) -> Tuple[pd.Series, pd.Series]:
    return (
        pd.read_table(file_name, usecols=["srx", "date_created", "pct_uniquely_aligned"])
        .assign(date_created=lambda x: pd.to_datetime(x.date_created))
        .assign(year=lambda x: x.date_created.dt.year)
        .set_index("srx")
    )

class Plot(NcbiPlotter):
    def __init__(
        self,
        file_name: str,
        plot_kwargs: Union[None, dict] = None,
        ax_kwargs: Union[None, dict] = None,
        ax: plt.Axes = None,
    ):
        """
        Example
        ----------
        >>> from ncbi_remap.plotting.sample_submission import Plot
        >>> plot = Plot("../../../output/paper-wf/srx_prealn.tsv")
        """
        self.update_figsize()
        self.ax = ax or self.get_ax()
        self.ax_joint = None
        self.ax_marg_x = None
        self.ax_marg_y = None

        self.plot_kwargs = update_kwargs(PLOT_DEFAULTS, plot_kwargs)
        self.axes_kwargs = update_kwargs(AXES_DEFAULTS, ax_kwargs)

        self.data = get_data(file_name)
        self.plot()
        self.tweak()

    def plot(self):
        sns.pointplot(
            x="year", y="pct_uniquely_aligned", data=self.data, ax=self.ax, **self.plot_kwargs
        )

    def tweak(self):
        self.ax.set(**self.axes_kwargs)
        plt.setp(self.ax.get_xticklabels(), rotation=45, ha="right")
        sns.despine(ax=self.ax, left=True)


import pymongo

import config

client = pymongo.MongoClient(config.MG_URI)
db = client['telegram-media-bot']
users = db.users
images = db.images
audios = db.audios

if __name__ == '__main__':
    print(db.list_collection_names())
    for image in images.find():
        print(image)


# Problem 1539
# Date completed: 2020/08/24

# 52 ms (82%)
class Solution:
    def findKthPositive(self, arr: List[int], k: int) -> int:
        if k < arr[0]: return k

        k -= (arr[0]-1)
        for i in range(len(arr)-1):
            missing = arr[i+1] - arr[i] -1
            if missing == 0: continue
            if k - missing <= 0:
                return arr[i] + k
            k -= missing

        return arr[-1] + k


import pandas as pd
import sklearn as sk
import numpy as np
import scipy as sp

from sklearn  import model_selection
from sklearn.model_selection import train_test_split
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.metrics import roc_curve, auc, f1_score,confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns
pd.set_option('display.max_columns', None) #This option displays all the columns

#Importing dataset
car_train=pd.read_csv(r"D:\Google Drive\Training\Book\0.Chapters\Chapter7 Random Forest and Boosting\Datasets\car_accidents\car_sensors.csv")

#Get an idea on number of rows and columns
print(car_train.shape)

#Print the column names
print(car_train.columns)

#Print the column types
print(car_train.info())

##Data Exploration
#Summary
all_cols_summary=car_train.describe()
print(round(all_cols_summary,2))

#Target variable
print(car_train['safe'].value_counts())

##Defining train and test data
features=car_train.columns.values[1:]
print(features)
X = car_train[features]
y = car_train['safe']

X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y ,test_size=0.2, random_state=55)

print("X_train Shape ",X_train.shape)
print("y_train Shape ", y_train.shape)
print("X_test Shape ",X_test.shape)
print("y_test Shape ", y_test.shape)

###buildng Decision tree on the training data ####
D_tree = tree.DecisionTreeClassifier(max_depth=7)
D_tree.fit(X_train,y_train)

#####Accuracy on train data ####
tree_predict1=D_tree.predict(X_train)
cm1 = confusion_matrix(y_train,tree_predict1)
accuracy_train=(cm1[0,0]+cm1[1,1])/sum(sum(cm1))
print("Decison Tree Accuracy on Train data = ", round(accuracy_train,2) )

#####Accuracy on test data ####
tree_predict2=D_tree.predict(X_test)
cm2 = confusion_matrix(y_test,tree_predict2)
accuracy_test=(cm2[0,0]+cm2[1,1])/sum(sum(cm2))
print("Decison Tree Accuracy on Test data = ", round(accuracy_test,2) )

##AUC on Train data
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, tree_predict1)
auc_train = auc(false_positive_rate, true_positive_rate)
print("Decison Tree AUC on Train data = ", round(auc_train,2) )

##AUC on Test data
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, tree_predict2)
auc_test = auc(false_positive_rate, true_positive_rate)
print("Decison Tree AUC on Test data = ", round(auc_test,2) )

################
####Building Random Forest Model
R_forest=RandomForestClassifier(n_estimators=300, max_features=4, max_depth=10)
R_forest.fit(X_train,y_train)

#####Accuracy on train data ####
forest_predict1=R_forest.predict(X_train)
cm1 = confusion_matrix(y_train,forest_predict1)
accuracy_train=(cm1[0,0]+cm1[1,1])/sum(sum(cm1))
print("Random Forest Accuracy on Train data = ", round(accuracy_train,2) )

####Accuracy on test data ####
forest_predict2=R_forest.predict(X_test)
cm2 = confusion_matrix(y_test,forest_predict2)
accuracy_test=(cm2[0,0]+cm2[1,1])/sum(sum(cm2))
print("Random Forest Accuracy on Test data = ", round(accuracy_test,2) )

##AUC on Train data
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_train, forest_predict1)
auc_train = auc(false_positive_rate, true_positive_rate)
print("Random Forest AUC on Train data =  ", round(auc_train,2) )

##AUC on Test data
false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test, forest_predict2)
auc_test= auc(false_positive_rate, true_positive_rate)
print("Random Forest AUC on Test data =  ", round(auc_test,2) )

#################################

#####GBM illustration

import pandas as pd
pets_data = pd.read_csv(r"D:\Google Drive\Training\Book\0.Chapters\Chapter7 Random Forest and Boosting\Datasets\Pet_adoption\adoption.csv")
pets_data.columns.values
pets_data.head(10)

X=pets_data[["cust_age"]]
y=h=pets_data['adopted_pet']

for i in range (1,21):

    #Model and predictions
    boost_model=GradientBoostingClassifier(n_estimators=i,learning_rate=1, max_depth=1)
    boost_model.fit(X,y)
    pets_data["itaration_result"]=boost_model.predict_proba(X)[:,1]
    boost_predict= boost_model.predict(X)

    #Graph
    fig = plt.figure()
    plt.rcParams["figure.figsize"] = (7,5)
    plt.title(['learning_rate=1', 'Iteration :', i ], fontsize=20)
    ax1 = fig.add_subplot(111)
    ax1.scatter(pets_data["cust_age"],pets_data["adopted_pet"], s=50, c='b', marker="x")
    ax1.scatter(pets_data["cust_age"],pets_data["itaration_result"], s=50, c='r', marker="o")
    ax1.set_xlabel('cust_age')
    ax1.set_ylabel('adopted_pet')

    #SSE and Accuracy
    print("SSE : ", sum((pets_data["itaration_result"] - y)**2))
    accuracy=f1_score(y, boost_predict, average='micro')
    print("Accuracy : ", accuracy)

## New learning rate learning_rate=0.1
for i in range (1,202):

    #Model and predictions
    boost_model=GradientBoostingClassifier(n_estimators=i,learning_rate=0.1, max_depth=1)
    boost_model.fit(X,y)
    pets_data["itaration_result"]=boost_model.predict_proba(X)[:,1]
    boost_predict= boost_model.predict(X)

    #Graph
    if(np.mod(i, 10) ==1):
        fig = plt.figure()
        plt.rcParams["figure.figsize"] = (7,5)
        plt.title(['learning_rate=0.1', 'Iteration :', i ], fontsize=20)
        ax1 = fig.add_subplot(111)
        ax1.scatter(pets_data["cust_age"],pets_data["adopted_pet"], s=50, c='b', marker="x")
        ax1.scatter(pets_data["cust_age"],pets_data["itaration_result"], s=50, c='r', marker="o")
        ax1.set_xlabel('cust_age')
        ax1.set_ylabel('adopted_pet')

    #SSE and Accuracy
    print("SSE : ", sum((pets_data["itaration_result"] - y)**2))
    accuracy=f1_score(y, boost_predict, average='micro')
    print("Accuracy : ", accuracy)

#################################
#####Case Study- Income Prediction from Census Data

income = pd.read_csv(r"D:\Google Drive\Training\Book\0.Chapters\Chapter7 Random Forest and Boosting\Datasets\Adult_Census_Income\Adult_Income.csv")

#Get an idea on number of rows and columns
print(income.shape)

#Print the column names
print(income.columns)

#Print the column types
print(income.info())

##Data Exploration
#Summary
all_cols_summary=income.describe()
print(round(all_cols_summary,2))

##Categorical Variables Exploration
categorical_vars=income.select_dtypes(include=['object']).columns
print(categorical_vars)

##Frequency tables for all the categorical columns
for col in categorical_vars:
    print("\n\nFrequency Table for the column ", col )
    print(income[col].value_counts())

##Data Cleaning and Feature Engineering

##workclass
income["workclass"] = income["workclass"].replace(['?','Never-worked','Without-pay'], 'Other')
print(income["workclass"] .value_counts())

##marital.status
income["marital.status"] = income["marital.status"].replace(['Never-married','Divorced','Separated','Widowed'], 'Not-married')
print(income["marital.status"] .value_counts())

##occupation
income["occupation"] = income["occupation"].replace(['?'], 'Other-service')
print(income["occupation"] .value_counts())

##Country
freq_country=income["native.country"].value_counts()
less_frequent= freq_country[freq_country <100].index
print(less_frequent)

income["native.country"]=income["native.country"].replace([less_frequent], 'Other')
income["native.country"] = income["native.country"].replace(['?'], 'Other')
print(income["native.country"].value_counts())

## Converting Gender into class-0 and Class-1
print(income["sex"].value_counts())
income['sex']=income['sex'].map({'Male': 0, 'Female': 1})

## Converting target into class-0 and Class-1
print(income["income"].value_counts())
income['income']=income['income'].map({'<=50K': 0, '>50K': 1})

###One hot encoding
one_hot_cols=['workclass','marital.status','occupation','native.country']
one_hot_data = pd.get_dummies(income[one_hot_cols])
print(one_hot_data.shape)
print(one_hot_data.columns.values)

##Final Data
print(income.shape)
income_final = pd.concat([income, one_hot_data], axis=1)
print(income_final.shape)
print(income_final.info())

##Features
one_hot_features=list(one_hot_data.columns.values)
numerical_features=['age',  'education.num', 'sex', 'capital.gain', 'capital.loss', 'hours.per.week']
all_features=one_hot_features+numerical_features
print(all_features)

##Data
X=income_final[all_features]
y=income_final['income']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print(y_test.shape)

##Model Building
gbm_model1 = GradientBoostingClassifier(learning_rate=0.01, max_depth=4,  n_estimators=100, verbose=1)
gbm_model1.fit(X_train, y_train)

##Validation on train and test data

#Train data
predictions=gbm_model1.predict(X_train)
actuals=y_train
cm = confusion_matrix(actuals,predictions)
print("Confusion Matrix on Train data\n", cm)
accuracy=(cm[0,0]+cm[1,1])/(sum(sum(cm)))
print("Train Accuracy", accuracy)

#Test data
predictions=gbm_model1.predict(X_test)
actuals=y_test
cm = confusion_matrix(actuals,predictions)
print("Confusion Matrix on Test data\n", cm)
accuracy=(cm[0,0]+cm[1,1])/(sum(sum(cm)))
print("Test Accuracy", accuracy)

##Loop for different iterations
for i in range(5,1000, 50):
    gbm_model1 = GradientBoostingClassifier(learning_rate=0.01, max_depth=4,  n_estimators=i)
    gbm_model1.fit(X_train, y_train)

    print("N_estimators=" , i)
    #Train data
    predictions=gbm_model1.predict(X_train)
    actuals=y_train
    cm = confusion_matrix(actuals,predictions)
    accuracy=(cm[0,0]+cm[1,1])/(sum(sum(cm)))
    print("Train Accuracy", accuracy)

    #Test data
    predictions=gbm_model1.predict(X_test)
    actuals=y_test
    cm = confusion_matrix(actuals,predictions)
    accuracy=(cm[0,0]+cm[1,1])/(sum(sum(cm)))
    print("Test Accuracy", accuracy)


n, m = map(int, input().split())
table = [list(map(int, input())) for _ in range(n)]
k = int(input())

ans = 0
# 모든 행에 대하여
for i in range(n):
    zero = table[i].count(0)
    # 이 행과 똑같은 0의 개수를 가진 행의 개수 세기
    cnt = 0
    # 0의 개수가 k 이하이고, 0의 개수의 짝홀이 k와 같다면 스위치를 모두 킬 수 있다.
    if zero <= k and zero % 2 == k % 2:
        # 다시 한 번 확인해서
        for j in range(n):
            # 같은 경우에만 + 1
            if table[j] == table[i]: cnt += 1
    ans = max(ans, cnt)
print(ans)

# 메모리 31256 KB, 시간 44 ms


#!/usr/bin/python

from __future__ import print_function

import binascii
from builtins import object
from builtins import str
from collections import defaultdict

from ansible.module_utils._text import to_text

try:
    from pysnmp.entity.rfc3413.oneliner import cmdgen
    has_pysnmp = True
except BaseException:
    has_pysnmp = False

if not has_pysnmp:
    print('Missing required pysnmp module (check docs)')

class DefineOid(object):
    def __init__(self, dotprefix=False):
        """__init__."""
        if dotprefix:
            dp = "."
        else:
            dp = ""

        # From SNMPv2-MIB
        self.sysDescr = dp + "1.3.6.1.2.1.1.1.0"
        self.sysObjectId = dp + "1.3.6.1.2.1.1.2.0"
        self.sysUpTime = dp + "1.3.6.1.2.1.1.3.0"
        self.sysContact = dp + "1.3.6.1.2.1.1.4.0"
        self.sysName = dp + "1.3.6.1.2.1.1.5.0"
        self.sysLocation = dp + "1.3.6.1.2.1.1.6.0"

        # From IF-MIB
        self.ifIndex = dp + "1.3.6.1.2.1.2.2.1.1"
        self.ifDescr = dp + "1.3.6.1.2.1.2.2.1.2"
        self.ifMtu = dp + "1.3.6.1.2.1.2.2.1.4"
        self.ifSpeed = dp + "1.3.6.1.2.1.2.2.1.5"
        self.ifPhysAddress = dp + "1.3.6.1.2.1.2.2.1.6"
        self.ifAdminStatus = dp + "1.3.6.1.2.1.2.2.1.7"
        self.ifOperStatus = dp + "1.3.6.1.2.1.2.2.1.8"
        self.ifAlias = dp + "1.3.6.1.2.1.31.1.1.1.18"

        # From IP-MIB
        self.ipAdEntAddr = dp + "1.3.6.1.2.1.4.20.1.1"
        self.ipAdEntIfIndex = dp + "1.3.6.1.2.1.4.20.1.2"
        self.ipAdEntNetMask = dp + "1.3.6.1.2.1.4.20.1.3"

def decode_hex(hexstring):
    if len(hexstring) < 3:
        return hexstring
    if hexstring[:2] == "0x":
        return to_text(binascii.unhexlify(hexstring[2:]))
    else:
        return hexstring

def snmp_walk(host, version, community):

    cmdGen = cmdgen.CommandGenerator()

    # Use SNMP Version 2
    # if version == "v2" or version == "v2c":
    snmp_auth = cmdgen.CommunityData(community)

    # Use p to prefix OIDs with a dot for polling
    p = DefineOid(dotprefix=True)
    # Use v without a prefix to use with return values
    v = DefineOid(dotprefix=False)

    def Tree():
        return defaultdict(Tree)

    results = Tree()
    results['error'] = False

    errorIndication, errorStatus, errorIndex, varBinds = cmdGen.getCmd(
        snmp_auth,
        cmdgen.UdpTransportTarget((host, 161)),
        cmdgen.MibVariable(p.sysDescr,),
        cmdgen.MibVariable(p.sysObjectId,),
        cmdgen.MibVariable(p.sysUpTime,),
        cmdgen.MibVariable(p.sysContact,),
        cmdgen.MibVariable(p.sysName,),
        cmdgen.MibVariable(p.sysLocation,),
        lookupMib=False
    )

    if errorIndication:
        results['error'] = True
        results['error_msg'] = str(errorIndication)

    for oid, val in varBinds:
        current_oid = oid.prettyPrint()
        current_val = val.prettyPrint()
        if current_oid == v.sysDescr:
            results['ansible_sysdescr'] = decode_hex(current_val)
        elif current_oid == v.sysObjectId:
            results['ansible_sysobjectid'] = current_val
        elif current_oid == v.sysUpTime:
            results['ansible_sysuptime'] = current_val
        elif current_oid == v.sysContact:
            results['ansible_syscontact'] = current_val
        elif current_oid == v.sysName:
            results['ansible_sysname'] = current_val
        elif current_oid == v.sysLocation:
            results['ansible_syslocation'] = current_val

    errorIndication, errorStatus, errorIndex, varTable = cmdGen.nextCmd(
        snmp_auth,
        cmdgen.UdpTransportTarget((host, 161)),
        cmdgen.MibVariable(p.ifIndex,),
        cmdgen.MibVariable(p.ifDescr,),
        cmdgen.MibVariable(p.ifMtu,),
        cmdgen.MibVariable(p.ifSpeed,),
        cmdgen.MibVariable(p.ifPhysAddress,),
        cmdgen.MibVariable(p.ifAdminStatus,),
        cmdgen.MibVariable(p.ifOperStatus,),
        cmdgen.MibVariable(p.ipAdEntAddr,),
        cmdgen.MibVariable(p.ipAdEntIfIndex,),
        cmdgen.MibVariable(p.ipAdEntNetMask,),
        cmdgen.MibVariable(p.ifAlias,),
        lookupMib=False
    )

    if errorIndication:
        results['error'] = True
        results['error_msg'] = str(errorIndication)

    return results


#!/usr/bin/env python3
import requests
import argparse
import sys
import os
import shutil

from urllib.parse import urlparse, urljoin
from bs4 import BeautifulSoup
from url_downloader import save_file
from threading import Thread

requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)

def readFromFile(filename):
    with open(filename, "r") as f:
        urls = [line.strip() for line in f.readlines()]

    return(urls)

def getHostname(url):
    return(urlparse(url).netloc)

def formDirPath(url):
    hostname = getHostname(url)
    return(f"{os.getcwd()}/{hostname}")

def checkIfDirExists(url):
    fullPath = formDirPath(url)

    if os.path.exists(fullPath):
        shutil.rmtree(fullPath, ignore_errors=True)

    os.mkdir(fullPath)

def getJSlinks(url, download):
    jsLinks = list()

    try:
        res = requests.get(url, verify=False)

    except Exception as err:
        #print(f"[!] Get JS links error - {err}")
        pass

    else:
        soup = BeautifulSoup(res.text, 'html.parser')
        scriptTags = soup.find_all('script')

        if len(scriptTags) > 0:
            print()

            for tag in scriptTags:
                src = tag.get('src')

                if src is not None:
                    jsUrl = urljoin(url, src)
                    jsLinks.append(jsUrl)
                    print(f"[+] Found {jsUrl}")

            if len(jsLinks) > 0:
                hostname = getHostname(url)
                jsLinksFile = f"jsLinks-{hostname}.txt"

                with open(jsLinksFile, 'w') as f:
                    f.writelines("%s\n" % link.strip() for link in jsLinks)

                print(f"[o] Saved results to {jsLinksFile}")

                if download:
                    downloadJSlinks(url, jsLinks)

def downloadWrapper(link, fullPath, localFilename):
    save_file(url=link, file_path=fullPath, file_name=localFilename)

def downloadJSlinks(url, jsLinks):
    checkIfDirExists(url)
    fullPath = formDirPath(url)

    threads_downloadJSlinks = list()

    print(f"[o] Initiating download on {url}")

    for jsLink in jsLinks:
        localFilename = jsLink.split('/')[-1]

        try:
            tr = Thread(target=downloadWrapper, args=(jsLink, fullPath, localFilename))
            threads_downloadJSlinks.append(tr)
            tr.start()

        except Exception as err:
            #print(f"[!] Download erro - {jsLink} - {err}")
            pass

    for thread in threads_downloadJSlinks:
        thread.join()

def massGet(urls, download):
    threads_MassGet = list()

    for url in urls:
        tr = Thread(target=getJSlinks, args=(url, download))
        threads_MassGet.append(tr)
        tr.start()

    for thread in threads_MassGet:
        thread.join()

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Get JS file(s) from website(s).")
    subParser = parser.add_subparsers(dest="command")

    singleSiteParser = subParser.add_parser("single", help="Get JS file(s) from a single website.")
    singleSiteParser.add_argument("-u", "--url", help="Url to download JS file(s) from. Format: https://example.com", required=True)
    singleSiteParser.add_argument("-d", "--download", help="Download JS file(s).", action='store_true')

    multipleSiteParser = subParser.add_parser("multiple", help="Get a list of JS file(s) from multiple websites.")
    multipleSiteParser.add_argument("-f", "--file", help="File containing multiple Url(s).", required=True)
    multipleSiteParser.add_argument("-d", "--download", help="Download JS file(s).", action='store_true')

    args = parser.parse_args()
    command = args.command

    if command == "single":
        getJSlinks(args.url, args.download)

    elif command == "multiple":
        massGet(readFromFile(args.file), args.download)

    else:
        parser.print_usage()


import cv2
import pytesseract
import numpy as np
from PIL import ImageGrab
import time

pytesseract.pytesseract.tesseract_cmd = 'E:\pythonProject\Github\Tesseract-OCR\\tesseract.exe'
img = cv2.imread('1.png')
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
#tesseract 接受RGB
#############################################
#### Detecting Characters  ######
#############################################

print(pytesseract.image_to_string(img))
hImg, wImg, _ = img.shape
print(hImg, wImg)
boxes = pytesseract.image_to_boxes(img)
for b in boxes.splitlines():
    print(b)
    b = b.split(' ')
    print(b)
    x, y, w, h = int(b[1]), int(b[2]), int(b[3]), int(b[4])
    cv2.rectangle(img, (x,hImg- y), (w,hImg- h), (50, 50, 255), 2)
    cv2.putText(img,b[0],(x,hImg- y+25),cv2.FONT_HERSHEY_SIMPLEX,1,(50,50,255),2)

cv2.imshow('img', img)
cv2.waitKey(0)

#可用于图片文字检测数字，字母

Opencv project training/01 Text Detection OCR/code 1.py


class Solution:
    def minCostClimbingStairs(self, c: List[int]) -> int:
        dp=[0 for _ in range(len(c) + 1)]
        dp[0] = c[0]
        dp[1] = c[1]
        c.append(0)
        for i in range(2, len(c)):
            dp[i] = min(dp[i-1], dp[i-2]) + c[i]

        return dp[-1]


# -*- coding: utf-8 -*-
"""
Created on Sun Jul  2 14:07:54 2017

@author: JG
"""

NUMERIC_COLUMNS = ['FTE', 'Total']
LABELS = ['Function',
 'Use',
 'Sharing',
 'Reporting',
 'Student_Type',
 'Position_Type',
 'Object_Type',
 'Pre_K',
 'Operating_Status']

#import Excel file
data=pd.ExcelFile('book_labels_2.xlsx')
bl = data.parse(2)
bl.head()

import warnings

def multilabel_sample(y, size=1000, min_count=5, seed=None):
    """ Takes a matrix of binary labels `y` and returns
        the indices for a sample of size `size` if
        `size` > 1 or `size` * len(y) if size =< 1.
        The sample is guaranteed to have > `min_count` of
        each label.
    """
    try:
        if (np.unique(y).astype(int) != np.array([0, 1])).all():
            raise ValueError()
    except (TypeError, ValueError):
        raise ValueError('multilabel_sample only works with binary indicator matrices')

    if (y.sum(axis=0) < min_count).any():
        raise ValueError('Some classes do not have enough examples. Change min_count if necessary.')

    if size <= 1:
        size = np.floor(y.shape[0] * size)

    if y.shape[1] * min_count > size:
        msg = "Size less than number of columns * min_count, returning {} items instead of {}."
        warnings.warn(msg.format(y.shape[1] * min_count, size))
        size = y.shape[1] * min_count

    rng = np.random.RandomState(seed if seed is not None else np.random.randint(1))

    if isinstance(y, pd.DataFrame):
        choices = y.index
        y = y.values
    else:
        choices = np.arange(y.shape[0])

    sample_idxs = np.array([], dtype=choices.dtype)

    # first, guarantee > min_count of each label
    for j in range(y.shape[1]):
        label_choices = choices[y[:, j] == 1]
        label_idxs_sampled = rng.choice(label_choices, size=min_count, replace=False)
        sample_idxs = np.concatenate([label_idxs_sampled, sample_idxs])

    sample_idxs = np.unique(sample_idxs)

    # now that we have at least min_count of each, we can just random sample
    sample_count = int(size - sample_idxs.shape[0])

    # get sample_count indices from remaining choices
    remaining_choices = np.setdiff1d(choices, sample_idxs)
    remaining_sampled = rng.choice(remaining_choices,
                                   size=sample_count,
                                   replace=False)

    return np.concatenate([sample_idxs, remaining_sampled])

def multilabel_train_test_split(X, Y, size, min_count=5, seed=None):
    """ Takes a features matrix `X` and a label matrix `Y` and
        returns (X_train, X_test, Y_train, Y_test) where all
        classes in Y are represented at least `min_count` times.
    """
    index = Y.index if isinstance(Y, pd.DataFrame) else np.arange(Y.shape[0])

    test_set_idxs = multilabel_sample(Y, size=size, min_count=min_count, seed=seed)
    train_set_idxs = np.setdiff1d(index, test_set_idxs)

    test_set_mask = index.isin(test_set_idxs)
    train_set_mask = ~test_set_mask

    return (X[train_set_mask], X[test_set_mask], Y[train_set_mask], Y[test_set_mask])

# Create the new DataFrame: numeric_data_only
numeric_data_only = bl[NUMERIC_COLUMNS].fillna(-1000)

# Get labels and convert to dummy variables: label_dummies
label_dummies = pd.get_dummies(bl[LABELS])

# Define combine_text_columns()
def combine_text_columns(data_frame, to_drop=NUMERIC_COLUMNS + LABELS):
    """ converts all text in each row of data_frame to single vector """

    # Drop non-text columns that are in the df
    to_drop = set(to_drop) & set(data_frame.columns.tolist())
    text_data = data_frame.drop(to_drop, axis=1)

    # Replace nans with blanks
    text_data.fillna("", inplace=True)

    # Join all text items in a row that have a space in between
    return text_data.apply (lambda x: " ".join(x), axis=1)

# Import FunctionTransformer
from sklearn.preprocessing import FunctionTransformer

# Get the dummy encoding of the labels
dummy_labels = pd.get_dummies(bl[LABELS])

# Get the columns that are features in the original df
NON_LABELS = [c for c in bl.columns if c not in LABELS]

# Split into training and test sets
X_train, X_test, y_train, y_test = multilabel_train_test_split(bl[NON_LABELS],
                                                               dummy_labels,
                                                               0.2,
                                                               seed=123)

#######################
#Deciding what's a word
#######################

# Import the CountVectorizer
from sklearn.feature_extraction.text import CountVectorizer

# Create the text vector
text_vector = combine_text_columns(X_train)

# Create the token pattern: TOKENS_ALPHANUMERIC
TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\s+)'

# Instantiate the CountVectorizer: text_features
text_features = CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC)

# Fit text_features to the text vector
text_features.fit(text_vector)

# Print the first 10 tokens
print(text_features.get_feature_names()[:10])

############################
#N-gram range in scikit-learn
############################

# Import pipeline
from sklearn.pipeline import Pipeline

# Import classifiers
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier

# Import CountVectorizer
from sklearn.feature_extraction.text import CountVectorizer

# Import other preprocessing modules
from sklearn.preprocessing import Imputer
from sklearn.feature_selection import chi2, SelectKBest

# Select 300 best features
chi_k = 300

# Import functional utilities
from sklearn.preprocessing import FunctionTransformer, MaxAbsScaler
from sklearn.pipeline import FeatureUnion

# Perform preprocessing
get_text_data = FunctionTransformer(combine_text_columns, validate=False)
get_numeric_data = FunctionTransformer(lambda x: x[NUMERIC_COLUMNS], validate=False)

# Create the token pattern: TOKENS_ALPHANUMERIC
TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\s+)'

# Instantiate pipeline: pl
pl = Pipeline([
        ('union', FeatureUnion(
            transformer_list = [
                ('numeric_features', Pipeline([
                    ('selector', get_numeric_data),
                    ('imputer', Imputer())
                ])),
                ('text_features', Pipeline([
                    ('selector', get_text_data),
                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,
                                                   ngram_range=(1, 2))),
                    ('dim_red', SelectKBest(chi2, chi_k))
                ]))
             ]
        )),
        ('scale', MaxAbsScaler()),
        ('clf', OneVsRestClassifier(LogisticRegression()))
    ])

###############################################
#Implement interaction modeling in scikit-learn
###############################################

from itertools import combinations

import numpy as np
from scipy import sparse
from sklearn.base import BaseEstimator, TransformerMixin

class SparseInteractions(BaseEstimator, TransformerMixin):
    def __init__(self, degree=2, feature_name_separator="_"):
        self.degree = degree
        self.feature_name_separator = feature_name_separator

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        if not sparse.isspmatrix_csc(X):
            X = sparse.csc_matrix(X)

        if hasattr(X, "columns"):
            self.orig_col_names = X.columns
        else:
            self.orig_col_names = np.array([str(i) for i in range(X.shape[1])])

        spi = self._create_sparse_interactions(X)
        return spi

    def get_feature_names(self):
        return self.feature_names

    def _create_sparse_interactions(self, X):
        out_mat = []
        self.feature_names = self.orig_col_names.tolist()

        for sub_degree in range(2, self.degree + 1):
            for col_ixs in combinations(range(X.shape[1]), sub_degree):
                # add name for new column
                name = self.feature_name_separator.join(self.orig_col_names[list(col_ixs)])
                self.feature_names.append(name)

                # get column multiplications value
                out = X[:, col_ixs[0]]
                for j in col_ixs[1:]:
                    out = out.multiply(X[:, j])

                out_mat.append(out)

        return sparse.hstack([X] + out_mat)

# Instantiate pipeline: pl
pl = Pipeline([
        ('union', FeatureUnion(
            transformer_list = [
                ('numeric_features', Pipeline([
                    ('selector', get_numeric_data),
                    ('imputer', Imputer())
                ])),
                ('text_features', Pipeline([
                    ('selector', get_text_data),
                    ('vectorizer', CountVectorizer(token_pattern=TOKENS_ALPHANUMERIC,
                                                   ngram_range=(1, 2))),
                    ('dim_red', SelectKBest(chi2, chi_k))
                ]))
             ]
        )),
        ('int', SparseInteractions(degree=2)),
        ('scale', MaxAbsScaler()),
        ('clf', OneVsRestClassifier(LogisticRegression()))
    ])

# Fit to the training data
pl.fit(X_train, y_train)

# Compute and print accuracy
accuracy = pl.score(X_test, y_test)
print("\nAccuracy on budget dataset: ", accuracy)

##############################################
#Implementing the hashing trick in scikit-learn
##############################################

# Import HashingVectorizer
from sklearn.feature_extraction.text import HashingVectorizer

# Get text data: text_data
text_data = combine_text_columns(X_train)

# Create the token pattern: TOKENS_ALPHANUMERIC
TOKENS_ALPHANUMERIC = '[A-Za-z0-9]+(?=\\s+)'

# Instantiate the HashingVectorizer: hashing_vec
hashing_vec = HashingVectorizer(token_pattern=TOKENS_ALPHANUMERIC)

# Fit and transform the Hashing Vectorizer
hashed_text = hashing_vec.fit_transform(text_data)

# Create DataFrame and print the head
hashed_df = pd.DataFrame(hashed_text.data)
print(hashed_df.head())

#######################
#Build the winning model
#######################

# Import the hashing vectorizer
from sklearn.feature_extraction.text import HashingVectorizer

# Instantiate the winning model pipeline: pl
pl = Pipeline([
        ('union', FeatureUnion(
            transformer_list = [
                ('numeric_features', Pipeline([
                    ('selector', get_numeric_data),
                    ('imputer', Imputer())
                ])),
                ('text_features', Pipeline([
                    ('selector', get_text_data),
                    ('vectorizer', HashingVectorizer(token_pattern=TOKENS_ALPHANUMERIC,
                                                     non_negative=True, norm=None, binary=False,
                                                     ngram_range=(1, 2))),
                    ('dim_red', SelectKBest(chi2, chi_k))
                ]))
             ]
        )),
        ('int', SparseInteractions(degree=2)),
        ('scale', MaxAbsScaler()),
        ('clf', OneVsRestClassifier(LogisticRegression()))
    ])

#course-resources-ml-with-experts-budgets/notebooks/1.0-full-model.ipynb

Machine Learning/Learning from the experts.py


# 백준 알고리즘 - 4673 셀프넘버
# https://www.acmicpc.net/problem/4673
# 시도 횟수 : 0
# 실패 : 0
# 통과 : 0

def selfNumber ():
    creator = []
    for i in range(10001):
        Temp = i
        for j in str(i):
            Temp += int(j)
        if Temp not in creator:
            creator.append(Temp)
        if i not in creator:
            print(i)

selfNumber()


# DQN训练模型测试
# 离散
import time
import gym
import numpy as np
from keras import models
env = gym.make('MountainCar-v0')
model = models.load_model('dqn_1.model')
s = env.reset()
score = 0
while True:
    env.render()
    time.sleep(0.01)
    a = np.argmax(model.predict(np.array([s]))[0])
    s, reward, done, _ = env.step(a)
    score += reward
    if done:
        print('score:', score)
        break
env.close()


import itertools, random

deck = list(itertools.product(range(1,14),
        ['Spade', 'Hearts', 'Diamond', 'Club']))

print('How many cards?')

num_cards = int(input())

random.shuffle(deck)

print("You got:")
for i in range(num_cards):
    print(deck[i][0], "of", deck[i][1])