from .. import logging as logg

import numpy as np
from scipy.sparse import issparse, csr_matrix
from scanpy.api.pp import log1p

def show_proportions(adata):
    """Fraction of spliced/unspliced/ambiguous abundances

    Arguments
    ---------
    adata: :class:`~anndata.AnnData`
        Annotated data matrix.

    Returns
    -------
    Prints the fractions of abundances.
    """
    layers_keys = [key for key in ['spliced', 'unspliced', 'ambiguous'] if key in adata.layers.keys()]
    tot_mol_cell_layers = [adata.layers[key].sum(1) for key in layers_keys]

    mean_abundances = np.round(
        [np.mean(tot_mol_cell / np.sum(tot_mol_cell_layers, 0)) for tot_mol_cell in tot_mol_cell_layers], 2)

    print('Abundance of ' + str(layers_keys) + ': ' + str(mean_abundances))

def cleanup(data, clean='layers', keep={'spliced', 'unspliced'}, copy=False):
    """Deletes attributes not needed.

    Arguments
    ---------
    data: :class:`~anndata.AnnData`
        Annotated data matrix.
    clean: `str` or list of `str` (default: `layers`)
        Which attributes to consider for freeing memory.
    keep: `str` or list of `str` (default: `['spliced', unspliced']`)
        Which attributes to keep.
    copy: `bool` (default: `False`)
        Return a copy instead of writing to adata.

    Returns
    -------
    Returns or updates `adata` with selection of attributes kept.
    """
    adata = data.copy() if copy else data

    if any(['obs' in clean, 'all' in clean]):
        for key in list(adata.obs.keys()):
            if key not in keep: del adata.obs[key]

    if any(['var' in clean, 'all' in clean]):
        for key in list(adata.var.keys()):
            if key not in keep: del adata.var[key]

    if any(['uns' in clean, 'all' in clean]):
        for key in list(adata.uns.keys()):
            if key not in keep: del adata.uns[key]

    if any(['layers' in clean, 'all' in clean]):
        for key in list(adata.layers.keys()):  # remove layers that are not needed
            if key not in keep: del adata.layers[key]

    return adata if copy else None

def set_initial_size(adata, layers={'spliced', 'unspliced'}):
    if all([layer in adata.layers.keys() for layer in layers]):
        layers = [layer for layer in layers if 'initial_size_' + layer not in adata.obs.keys()]
        total_size = 0
        for layer in layers:
            X = adata.layers[layer]
            size = X.sum(1).A1.copy() if issparse(X) else X.sum(1).copy()
            adata.obs['initial_size_' + layer] = size
            total_size += size
        if 'initial_size' not in adata.obs.keys(): adata.obs['initial_size'] = total_size

def get_initial_size(adata, layer, by_total_size):
    if layer not in {'spliced', 'unspliced'}: return None
    else: return adata.obs['initial_size'].copy() if by_total_size and 'initial_size' in adata.obs.keys() else \
        adata.obs['initial_size_' + layer].copy() if 'initial_size_' + layer in adata.obs.keys() else None

def filter(X, min_counts=None, min_cells=None, max_counts=None, max_cells=None):
    counts = X if (min_counts is not None or max_counts is not None) else X > 0
    counts = counts.sum(0).A1 if issparse(counts) else counts.sum(0)
    lb = min_counts if min_counts is not None else min_cells if min_cells is not None else -np.inf
    ub = max_counts if max_counts is not None else max_cells if max_cells is not None else np.inf
    return (lb <= counts) & (counts <= ub), counts

def filter_genes(data, min_counts=None, min_cells=None, max_counts=None, max_cells=None,
                 min_counts_u=None,  min_cells_u=None, max_counts_u=None, max_cells_u=None, copy=False):
    """Filter genes based on number of cells or counts.
    Keep genes that have at least `min_counts` counts or are expressed in at
    least `min_cells` cells or have at most `max_counts` counts or are expressed
    in at most `max_cells` cells.
    Only provide one of the optional parameters `min_counts`, `min_cells`,
    `max_counts`, `max_cells` per call.

    Parameters
    ----------
    data : :class:`~anndata.AnnData`, `np.ndarray`, `sp.spmatrix`
        The (annotated) data matrix of shape `n_obs` × `n_vars`. Rows correspond
        to cells and columns to genes.
    min_counts : `int`, optional (default: `None`)
        Minimum number of counts required for a gene to pass filtering.
    min_cells : `int`, optional (default: `None`)
        Minimum number of cells expressed required for a gene to pass filtering.
    max_counts : `int`, optional (default: `None`)
        Maximum number of counts required for a gene to pass filtering.
    max_cells : `int`, optional (default: `None`)
        Maximum number of cells expressed required for a gene to pass filtering.
    min_counts_u : `int`, optional (default: `None`)
        Minimum number of unspliced counts required for a gene to pass filtering.
    min_cells_u : `int`, optional (default: `None`)
        Minimum number of unspliced cells expressed required for a gene to pass filtering.
    max_counts_u : `int`, optional (default: `None`)
        Maximum number of unspliced counts required for a gene to pass filtering.
    max_cells_u : `int`, optional (default: `None`)
        Maximum number of unspliced cells expressed required for a gene to pass filtering.
    copy : `bool`, optional (default: `False`)
        Determines whether a copy is returned.

    Returns
    -------
    Filters the object and adds `n_counts` to `adata.var`.
    """
    adata = data.copy() if copy else data

    # set initial cell sizes before filtering
    set_initial_size(adata)

    def filter(X, min_counts=None, min_cells=None, max_counts=None, max_cells=None):
        counts = X if (min_counts is not None or max_counts is not None) else X > 0
        counts = counts.sum(0).A1 if issparse(counts) else counts.sum(0)
        lb = min_counts if min_counts is not None else min_cells if min_cells is not None else -np.inf
        ub = max_counts if max_counts is not None else max_cells if max_cells is not None else np.inf
        return (lb <= counts) & (counts <= ub), counts

    gene_subset = np.ones(adata.n_vars, dtype=bool)
    if min_counts is not None or max_counts is not None:
        gene_subset, counts = filter(adata.X, min_counts=min_counts, max_counts=max_counts)
        adata.var['n_counts'] = counts
        adata._inplace_subset_var(gene_subset)

    if min_cells is not None or max_cells is not None:
        gene_subset, cells = filter(adata.X, min_cells=min_cells, max_cells=max_cells)
        adata.var['n_cells'] = cells
        adata._inplace_subset_var(gene_subset)

    s = np.sum(~gene_subset)
    if s > 0:
        logg.info('filtered out {} genes that are detected'.format(s), end=' ')
        if min_cells is not None or min_counts is not None:
            logg.info('in less than',
                      str(min_cells) + ' cells'
                      if min_counts is None else str(min_counts) + ' counts', no_indent=True)
        if max_cells_u is not None or max_counts_u is not None:
            logg.info('in more than ',
                      str(max_cells) + ' cells'
                      if max_counts is None else str(max_counts) + ' counts', no_indent=True)

    layer = 'unspliced'
    if layer in adata.layers.keys():
        gene_subset = np.ones(adata.n_vars, dtype=bool)
        if min_counts_u is not None or max_counts is not None:
            gene_subset, _ = filter(adata.layers[layer], min_counts=min_counts_u, max_counts=max_counts_u)
            adata._inplace_subset_var(gene_subset)

        if min_cells_u is not None or max_cells_u is not None:
            gene_subset, _ = filter(adata.layers[layer], min_cells=min_cells_u, max_cells=max_cells_u)
            adata._inplace_subset_var(gene_subset)

        s = np.sum(~gene_subset)
        if s > 0:
            logg.info('filtered out {} genes that are detected'.format(s), end=' ')
            if min_cells_u is not None or min_counts_u is not None:
                logg.info('in less than',
                          str(min_cells_u) + ' cells (' + str(layer) + ')'
                          if min_counts_u is None else str(min_counts_u) + ' counts(' + str(layer) + ')', no_indent=True)
            if max_cells_u is not None or max_counts_u is not None:
                logg.info('in more than ',
                          str(max_cells_u) + ' cells(' + str(layer) + ')'
                          if max_counts_u is None else str(max_counts_u) + ' counts(' + str(layer) + ')', no_indent=True)

    return adata if copy else None

def filter_genes_dispersion(data, flavor='seurat', min_disp=None, max_disp=None, min_mean=None, max_mean=None,
                            n_bins=20, n_top_genes=None, log=True, copy=False):
    """Extract highly variable genes.
    The normalized dispersion is obtained by scaling with the mean and standard
    deviation of the dispersions for genes falling into a given bin for mean
    expression of genes. This means that for each bin of mean expression, highly
    variable genes are selected.

    Parameters
    ----------
    data : :class:`~anndata.AnnData`, `np.ndarray`, `sp.sparse`
        The (annotated) data matrix of shape `n_obs` × `n_vars`. Rows correspond
        to cells and columns to genes.
    flavor : {'seurat', 'cell_ranger', 'svr'}, optional (default: 'seurat')
        Choose the flavor for computing normalized dispersion. If choosing
        'seurat', this expects non-logarithmized data - the logarithm of mean
        and dispersion is taken internally when `log` is at its default value
        `True`. For 'cell_ranger', this is usually called for logarithmized data
        - in this case you should set `log` to `False`. In their default
        workflows, Seurat passes the cutoffs whereas Cell Ranger passes
        `n_top_genes`.
    min_mean=0.0125, max_mean=3, min_disp=0.5, max_disp=`None` : `float`, optional
        If `n_top_genes` unequals `None`, these cutoffs for the means and the
        normalized dispersions are ignored.
    n_bins : `int` (default: 20)
        Number of bins for binning the mean gene expression. Normalization is
        done with respect to each bin. If just a single gene falls into a bin,
        the normalized dispersion is artificially set to 1. You'll be informed
        about this if you set `settings.verbosity = 4`.
    n_top_genes : `int` or `None` (default: `None`)
        Number of highly-variable genes to keep.
    log : `bool`, optional (default: `True`)
        Use the logarithm of the mean to variance ratio.
    copy : `bool`, optional (default: `False`)
        If an :class:`~anndata.AnnData` is passed, determines whether a copy
        is returned.

    Returns
    -------
    If an AnnData `adata` is passed, returns or updates `adata` depending on \
    `copy`. It filters the `adata` and adds the annotations
    """
    adata = data.copy() if copy else data
    set_initial_size(adata)

    if flavor == 'svr':
        mu = adata.X.mean(0).A1 if issparse(adata.X) else adata.X.mean(0)
        sigma = np.sqrt(adata.X.multiply(adata.X).mean(0).A1 - mu ** 2) if issparse(adata.X) else adata.X.std(0)
        log_mu = np.log2(mu)
        log_cv = np.log2(sigma / mu)

        from sklearn.svm import SVR
        clf = SVR(gamma=150. / len(mu))
        clf.fit(log_mu[:, None], log_cv)
        score = log_cv - clf.predict(log_mu[:, None])
        nth_score = np.sort(score)[::-1][n_top_genes]
        adata._inplace_subset_var(score >= nth_score)
    else:
        from scanpy.api.pp import filter_genes_dispersion
        filter_genes_dispersion(adata, flavor=flavor, min_disp=min_disp, max_disp=max_disp, min_mean=min_mean,
                                max_mean=max_mean, n_bins=n_bins, n_top_genes=n_top_genes, log=log)
    return adata if copy else None

def counts_per_cell_quantile(X, max_proportion_per_cell=.05, counts_per_cell=None):
    if counts_per_cell is None:
        counts_per_cell = X.sum(1).A1 if issparse(X) else X.sum(1)
    gene_subset = np.all(X <= counts_per_cell[:, None] * max_proportion_per_cell, axis=0)
    if issparse(X): gene_subset = gene_subset.A1
    return X[:, gene_subset].sum(1).A1 if issparse(X) else X[:, gene_subset].sum(1)

def normalize_layers(data, layers={'spliced', 'unspliced'}, by_total_size=None, max_proportion_per_cell=None, copy=False):
    """Normalize by total counts to median.
    """
    adata = data.copy() if copy else data
    from scanpy.api.pp import normalize_per_cell

    def not_normalized_yet(adata, layer):
        X = adata.layers[layer]
        return np.allclose((X.data[:10] if issparse(X) else X[0]) % 1, 0, atol=1e-3)

    for layer in layers:
        if not_normalized_yet(adata, layer):
            counts_per_cell = get_initial_size(adata, layer, by_total_size)
            if max_proportion_per_cell is not None and (0 < max_proportion_per_cell < 1):
                counts_per_cell = counts_per_cell_quantile(adata.X, max_proportion_per_cell, counts_per_cell)
            adata.layers[layer] = normalize_per_cell(adata.layers[layer], None, counts_per_cell, copy=True)
    return adata if copy else None

def normalize_per_cell(data, counts_per_cell_after=None, counts_per_cell=None,
                       key_n_counts=None, max_proportion_per_cell=None, layers={'spliced', 'unspliced'}, copy=False):
    """Normalize each cell by total counts over all genes.

    Parameters
    ----------
    data : :class:`~anndata.AnnData`, `np.ndarray`, `sp.sparse`
        The (annotated) data matrix of shape `n_obs` × `n_vars`. Rows correspond
        to cells and columns to genes.
    counts_per_cell_after : `float` or `None`, optional (default: `None`)
        If `None`, after normalization, each cell has a total count equal
        to the median of the *counts_per_cell* before normalization.
    counts_per_cell : `np.array`, optional (default: `None`)
        Precomputed counts per cell.
    key_n_counts : `str`, optional (default: `'n_counts'`)
        Name of the field in `adata.obs` where the total counts per cell are
        stored.
    max_proportion_per_cell : `int` (default: `None`)
        Exclude genes counts that account for more than a specific proportion of cell size, e.g. 0.05.
    layers : `str` or `list` (default: `{'spliced', 'unspliced'}`)
        Keys for layers to be also considered for normalization.
    copy : `bool`, optional (default: `False`)
        If an :class:`~anndata.AnnData` is passed, determines whether a copy
        is returned.

    Returns
    -------
    Returns or updates `adata` with normalized version of the original `adata.X`, depending on `copy`.
    """
    adata = data.copy() if copy else data
    from scanpy.api.pp import normalize_per_cell
    if max_proportion_per_cell is not None and (0 < max_proportion_per_cell < 1):
        counts_per_cell = counts_per_cell_quantile(adata.X, max_proportion_per_cell)
    normalize_per_cell(adata, counts_per_cell_after, counts_per_cell, key_n_counts)
    normalize_layers(adata, layers, max_proportion_per_cell)
    return adata if copy else None

def filter_and_normalize(data, min_counts=None, min_counts_u=None, min_cells=None, min_cells_u=None, n_top_genes=None,
                         flavor='seurat', log=True, copy=False):
    """Filtering, normalization and log transform

    Expects non-logarithmized data. If using logarithmized data, pass `log=False`.

    Runs the following steps

    .. code:: python

        scv.pp.filter_genes(adata)
        scv.pp.normalize_per_cell(adata)
        if n_top_genes is not None:
            scv.pp.filter_genes_dispersion(adata)
        if log:
            scv.pp.log1p(adata)

    Arguments
    ---------
    data: :class:`~anndata.AnnData`
        Annotated data matrix.
    min_counts: `int` (default: `None`)
        Minimum number of counts required for a gene to pass filtering (spliced).
    min_counts_u: `int` (default: `None`)
        Minimum number of counts required for a gene to pass filtering (unspliced).
    min_cells: `int` (default: `None`)
        Minimum number of cells expressed required for a gene to pass filtering (spliced).
    min_cells_u: `int` (default: `None`)
        Minimum number of cells expressed required for a gene to pass filtering (unspliced).
    n_top_genes: `int` (default: `None`)
        Number of genes to keep.
    flavor: {'seurat', 'cell_ranger', 'svr'}, optional (default: 'seurat')
        Choose the flavor for computing normalized dispersion. If choosing 'seurat', this expects non-logarithmized data.
    log: `bool` (default: `True`)
        Take logarithm.
    copy: `bool` (default: `False`)
        Return a copy of `adata` instead of updating it.

    Returns
    -------
    Returns or updates `adata` depending on `copy`.
    """
    adata = data.copy() if copy else data
    filter_genes(adata, min_counts=min_counts, min_counts_u=min_counts_u, min_cells=min_cells, min_cells_u=min_cells_u)
    normalize_per_cell(adata)
    if n_top_genes is not None: filter_genes_dispersion(adata, n_top_genes=n_top_genes, flavor=flavor)
    if log: log1p(adata)
    return adata if copy else None

def recipe_velocity(adata, min_counts=3, min_counts_u=3, n_top_genes=None, n_pcs=30, n_neighbors=30, log=True, copy=False):
    """Runs pp.filter_and_normalize() and pp.moments()
    """
    from .moments import moments
    filter_and_normalize(adata, min_counts=min_counts, min_counts_u=min_counts_u, n_top_genes=n_top_genes, log=log)
    moments(adata, n_neighbors=n_neighbors, n_pcs=n_pcs)
    return adata if copy else None


class Solution:
    def kidsWithCandies(self, candies: List[int], extraCandies: int) -> List[bool]:
        # result = [0]*len(candies)
        # for i in range(0, len(candies)):
        #     if candies[i] + extraCandies >= max(candies):
        #         result[i] = True
        # return result
        result = []
        n = max(candies)
        for i in candies:
            result.append(i+extraCandies >= n)
        return result


import pandas as pd

from vax.utils.incremental import enrich_data, increment
from vax.utils.dates import localdate

def read(source: str) -> pd.Series:
    df = pd.read_csv(source, usecols=[
        "primera_dosis_cantidad", "segunda_dosis_cantidad", "vacuna_nombre",
    ])

    known_vaccines = set((
        "AstraZeneca ChAdOx1 S recombinante",
        "COVISHIELD ChAdOx1nCoV COVID 19",
        "Sinopharm Vacuna SARSCOV 2 inactivada",
        "Sputnik V COVID19 Instituto Gamaleya",
    ))
    assert set(df.vacuna_nombre) == known_vaccines, "New vaccine found!"

    return df.drop(columns="vacuna_nombre").sum(level=0, axis=1).sum()

def translate_index(ds: pd.Series) -> pd.Series:
    return ds.rename({
        'primera_dosis_cantidad': 'people_vaccinated',
        'segunda_dosis_cantidad': 'people_fully_vaccinated',
    })

def add_totals(ds: pd.Series) -> pd.Series:
    total_vaccinations = ds['people_vaccinated'] + ds['people_fully_vaccinated']
    return enrich_data(ds, 'total_vaccinations', total_vaccinations)

def format_date(ds: pd.Series) -> pd.Series:
    date = localdate("America/Argentina/Buenos_Aires", 8)
    return enrich_data(ds, 'date', date)

def enrich_location(ds: pd.Series) -> pd.Series:
    return enrich_data(ds, 'location', "Argentina")

def enrich_vaccine(ds: pd.Series) -> pd.Series:
    return enrich_data(ds, 'vaccine', "Oxford/AstraZeneca, Sinopharm/Beijing, Sputnik V")

def enrich_source(ds: pd.Series) -> pd.Series:
    return enrich_data(
        ds,
        'source_url',
        "http://datos.salud.gob.ar/dataset/vacunas-contra-covid-19-dosis-aplicadas-en-la-republica-argentina"
    )

def pipeline(ds: pd.Series) -> pd.Series:
    return (
        ds
        .pipe(translate_index)
        .pipe(add_totals)
        .pipe(format_date)
        .pipe(enrich_location)
        .pipe(enrich_vaccine)
        .pipe(enrich_source)
    )

def main(paths):
    source = "https://sisa.msal.gov.ar/datos/descargas/covid-19/files/Covid19VacunasAgrupadas.csv.zip"
    data = read(source).pipe(pipeline)
    increment(
        paths=paths,
        location=str(data['location']),
        total_vaccinations=int(data['total_vaccinations']),
        people_vaccinated=int(data['people_vaccinated']),
        people_fully_vaccinated=int(data['people_fully_vaccinated']),
        date=str(data['date']),
        source_url=str(data['source_url']),
        vaccine=str(data['vaccine'])
    )

if __name__ == "__main__":
    main()


import tensorflow as tf
import os

os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

def myrgression():
    """
    自实现一个线性规划的模型
    :return: None
    """
    # 1. 准备数据
    with tf.variable_scope('data'):
        x = tf.random_normal([100, 1], mean=1.75, stddev=0.5, name='x_data')
        y_true = tf.matmul(x, [[0.7]]) + 1.0
    # 2. 定义模型
    with tf.variable_scope('model'):
        weight = tf.Variable(tf.random_normal([1, 1], mean=0.0, stddev=1.0, name="w"))
        bias = tf.Variable(0.0, name="b")
    # 3. 计算误差
    with tf.variable_scope('loss'):
        y_predict = tf.matmul(x, weight) + bias
        loss = tf.reduce_mean(tf.square(y_true - y_predict))
    # 4. 梯度优化
    with tf.variable_scope('gradient'):
        train_op = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(loss)

    # 收集tensor
    tf.summary.scalar('loss', loss)
    tf.summary.scalar('bias', bias)
    tf.summary.histogram('weight', weight)
    # 合并tensor
    merged = tf.summary.merge_all()

    initial_variables = tf.global_variables_initializer()
    with tf.Session() as sess:
        sess.run(initial_variables)
        # print(sess.run(weight))
        # print(weight)
        print("初始的权重是 %s, 偏置是 %s" % (weight.eval(), bias.eval()))
        # 建立事件文件
        filewriter = tf.summary.FileWriter("./temp/summary/test/", graph=sess.graph)

        for i in range(500):
            sess.run(train_op)
            summary = sess.run(merged)
            filewriter.add_summary(summary, i)

            print("第 %s 次的权重是：%s, 偏置是：%s" % (i, weight.eval(), bias.eval()))
    return None

if __name__ == '__main__':
    myrgression()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Aug  5 16:34:31 2019

@author: lagerwer
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 28 11:40:55 2018

@author: lagerwer
"""
import odl
import numpy as np
import ddf_fdk as ddf
ddf.import_astra_GPU()
#import real_data_class as RD
import time
import pylab
import scipy.interpolate as sp
import gc

pylab.close('all')
t = time.time()
# %% Set variables
# The size of the measured objects in voxels
pix = 1024
voxels = [pix, pix, pix]

# Pick your phantom
# Options: 'Shepp-Logan', 'Defrise', 'Derenzo', 'Hollow cube', 'Cube', 'Var obj'
phantom = 'FORBILD'
#lp = '/export/scratch2/lagerwer/NNFDK_results/nTrain_optim_1024_lim_ang/'
#f_load_path = lp + 'CS_f.npy'
#g_load_path = lp + 'CS_A64_g.npy'
noise = None #['Poisson', 2 ** 8]
det_rad = 0
src_rad = 10
angles = 360
up_samp = [1.5, 2, np.exp(1), 3, np.pi]

Q_TFDK = np.zeros((np.size(up_samp), 3))
Q_FDK = np.zeros((np.size(up_samp), 3))
i = 0
for us in up_samp:
    print('starting case sc=', us)
    data_obj = ddf.phantom(voxels, phantom, angles, noise, src_rad, det_rad,
                           samp_fac=us)

    ## %% Create the circular cone beam CT class
    case = ddf.CCB_CT(data_obj)#
    ## Initialize the algorithms (FDK, SIRT)
    case.init_algo()
    case.init_DDF_FDK()
    # %%
    if i == 0:
        lam = case.TFDK.optim_param()
    case.FDK.do('Ram-Lak')
    case.TFDK.do(lam=lam)
    Q_TFDK[i, :] = case.TFDK.results.Q
    Q_FDK[i, :] = case.FDK.results.Q
    # %% Show results
    case.table()
    i += 1
    case = None
    data_obj = None
    gc.collect()

headers = ['Method', '$sc=1.5$', '$sc=2$', '$sc=e$', '$sc=3$', '$sc=\pi$']

Ql = [['Ram-Lak: MAE', *Q_FDK[:, 1]], ['Ram-Lak: SSIM', *Q_FDK[:, 2]],
      ['MR-filter: MAE', *Q_TFDK[:, 1]], ['MR-filter: SSIM', *Q_TFDK[:, 2]]]

import tabulate as tab
latex_table = tab.tabulate(Ql, headers, tablefmt='latex', floatfmt=('.s',".4e",
                                                         ".4f", ".4f"))
table = open(case.WV_path + '_table.txt', 'w')
table.write(latex_table)
table.close()
print(tab.tabulate(Ql, headers, tablefmt='latex', floatfmt=('.s',".4f",
                                                         ".4f", ".4f",
                                                         ".4f", ".4f")))


from tokflow import TokFlow

from .test_tokflow_02_data import get_example_texts

def test_matched_one_type_pattern_1():
    tokf = TokFlow([("<NL>", "\n")])

    condition = {"in_type": "full", "out_type": "full"}
    prev_len = 0
    for input_token_base in get_example_texts():
        output_sentence = tokf.put(input_token_base, condition)

        # print(f"output_sentence:{output_sentence}")

        if prev_len > len(output_sentence):
            raise ValueError("Length error")

        if "<NL>" in output_sentence:
            raise Exception("Failure Must be converted str found.")

        prev_len = len(output_sentence)

    output_sentence = tokf.flush(condition)

    assert output_sentence == f"""はい、こちらをお勧めします。

「ハチ公像」は、最も有名な観光スポットの一つとされています。

もう一つ有名な場所は、スクランブル交差点です。

また、ハチ公像は、ハチ公像の向かいにあるスクランブル交差点のランドマークでもあります。

ハチ公像は、多くの人々にとって最も有名な観光スポットの1つです。

ハチ公像は、多くの人々が訪れる場所でもあります。"""


#!/usr/bin/env python
# -*- coding:utf-8 _*-

"""
:Author :weijinlong
:Time:  :2020/1/17 10:32
:File   :worker.py
:content:

"""

import json
import os

import numpy as np
import tensorflow as tf
from tensorflow.python.framework import graph_util
from tensorflowonspark import TFNode

from tfos.base import logger
from tfos.base.gfile import ModelDir
from tfos.base.worker import Worker
from tfos.tf import TFCompile

class TFWorker(Worker):

    def __init__(self, *args, **kwargs):
        super(TFWorker, self).__init__(*args, **kwargs)
        self.model_suffix = 'pb'

    @property
    def global_step(self):
        return tf.train.get_global_step()

    @property
    def model_config_path(self):
        return os.path.join(self.save_dir, '{}.config'.format(self.name))

    def common_dict(self, epoch):
        return {
            "_task_index": self.task_index,
            "_epoch": epoch
        }

    @property
    def generate_rdd_data(self):
        while not self.tf_feed.should_stop():
            batches = self.tf_feed.next_batch(self.batch_size)
            inputs = []
            labels = []
            # if not batches:
            #     raise StopIteration()
            for row in batches:
                inputs.append(row.feature)
                labels.append(row.label)
            inputs = np.array(inputs).astype('float32')
            labels = np.array(labels).astype('float32')
            return inputs, labels

    def feed_dict(self, x, y=None):
        data = self.model.feed_dict
        data[self.model.inputs['x']] = x
        if self.model.icy and y is not None:
            data[self.model.inputs['y']] = y
        return data

    def build_model(self):
        pass

    def execute(self):
        raise NotImplementedError

class TFTrainWorker(TFWorker):
    def __init__(self, model_rdd, go_on=False, *args, **kwargs):
        super(TFTrainWorker, self).__init__(*args, **kwargs)
        self.model_config = json.loads(getattr(model_rdd.first(), TFCompile.MODEL))
        self.go_on = go_on
        self.initial_epoch = 0
        self.summary_writer = None

    def build_model(self):
        if self.task_index is None:
            raise ValueError("task_index cannot None!!!")
        with tf.device(tf.train.replica_device_setter(
                worker_device="/job:worker/task:{}".format(self.task_index), cluster=self.cluster)):
            self.model = TFCompile.from_json(self.model_config)
            self.model.valid_model()

    def restore_checkpoint(self, sess):
        saver = tf.train.Saver()
        ckpt = tf.train.get_checkpoint_state(self.checkpoint_dir)
        if ckpt and ckpt.model_checkpoint_path:
            # K.set_learning_phase(False)
            self.initial_epoch = int(ckpt.model_checkpoint_path.split('-')[-1])
            saver.restore(sess, ckpt.model_checkpoint_path)

    def save_checkpoint(self, sess, epoch, summary_str):
        if self.task_index == 0:
            if summary_str:
                self.summary_writer.add_summary(summary_str, global_step=epoch)
            saver = tf.train.Saver(max_to_keep=5)
            saver.save(sess, self.checkpoint_path, epoch)

    def save_model(self, sess):
        constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, self.model.output_node_names)
        with tf.gfile.FastGFile(self.model_path, mode='wb') as f:
            f.write(constant_graph.SerializeToString())

    def execute(self):
        result_file = os.path.join(self.result_dir, "train_result_{}.txt".format(self.task_index))
        config = tf.ConfigProto(allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        summary_op = tf.summary.merge_all()
        with tf.Session(self.server.target, config=config) as sess:
            sess.run(tf.global_variables_initializer())
            sess.run(tf.local_variables_initializer())
            self.summary_writer = tf.summary.FileWriter(self.log_dir, sess.graph)
            coord = tf.train.Coordinator()
            tf.train.start_queue_runners(coord=coord, sess=sess)

            if self.go_on:
                self.restore_checkpoint(sess)
            names, values = zip(*self.model.fetches.items())
            names = list(names)
            values = list(values)
            res, summary_str = None, None
            for epoch in range(1, self.epochs + 1):
                for _ in range(self.steps_per_epoch):
                    x, y = self.generate_rdd_data
                    if len(x) == 0:
                        break
                    if summary_op is not None:
                        *res, summary_str = sess.run(values + [summary_op], self.feed_dict(x=x, y=y))
                    else:
                        res = sess.run(values, self.feed_dict(x=x, y=y))
                result = dict((k, v) for k, v in zip(names, res) if v is not None)
                result.update(self.common_dict(epoch + self.initial_epoch))
                ModelDir.write_result(result_file, [result], True)
                self.save_checkpoint(sess, epoch + self.initial_epoch, summary_str)

            self.model.write_model(self.model_config_path, False)
            self.save_model(sess)
            self.tf_feed.terminate()

class TFEvaluateWorker(TFWorker):

    def load_model(self, sess):
        with tf.io.gfile.GFile(self.model_path, 'rb') as f:
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(f.read())
            sess.graph.as_default()
            tf.import_graph_def(graph_def, name='')  # 导入计算图

    def execute(self):
        config = tf.ConfigProto(allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        with tf.Session(self.server.target, config=config) as sess:
            self.load_model(sess)

class TFPredictWorker(TFWorker):

    def __init__(self, params: (dict, type(None)) = None, *args, **kwargs):
        super(TFPredictWorker, self).__init__(*args, **kwargs)
        self.params = params if params else {}

    def get_results(self, y_pred, y_true=None):
        results = []
        length = len(y_pred)
        if y_true is not None:
            assert length == len(y_true)

        for i in range(length):
            results.append({
                '_task_index': self.task_index,
                'y_pred': y_pred[i],
                'y_true': y_true[i]
            })
        return results

    def load_model(self, sess):
        with tf.io.gfile.GFile(self.model_path, 'rb') as f:
            graph_def = tf.GraphDef()
            graph_def.ParseFromString(f.read())
            logger.debug(graph_def)
            sess.graph.as_default()
            tf.import_graph_def(graph_def, name='')  # 导入计算图
            self.model = TFCompile.read_model(self.model_config_path)
            self.model.icy = False
            self.model.update_params(self.params)

    def execute(self):
        result_file = os.path.join(self.result_dir, "predict_result_{}.txt".format(self.task_index))
        config = tf.ConfigProto(allow_soft_placement=True)
        config.gpu_options.allow_growth = True
        with tf.Session(self.server.target, config=config) as sess:
            self.load_model(sess)
            for _ in range(self.steps_per_epoch):
                x, y = self.generate_rdd_data
                if len(x) == 0:
                    break
                predictions = sess.run(self.model.outputs['y'], self.feed_dict(x=x))
                y_pred = np.argmax(predictions, 1)
                y_true = np.argmax(y, 1) if y is not None else None
                logger.debug(predictions)
                results = self.get_results(y_pred, y_true)
                ModelDir.write_result(result_file, results, True)
            self.tf_feed.terminate()


import streamlit as st
from config import cfg, TRANSLATOR_MODELS, SP2_MODELS
from extract_text import YoutubeTextExtractor
from utils import is_from_youtube, get_youtube_id

#  ==== SIDEBAR ====

st.sidebar.title('Navigation Menu ⬇')
st.sidebar.subheader("Input")
input_video_type = st.sidebar.radio('Select input video:', ["Url", "File"])
st.sidebar.subheader("Translator")
source, dest = st.sidebar.radio('Path:', ['en-it', 'it-en']).split("-")
# trans_model = st.sidebar.radio('Select Model:', TRANSLATOR_MODELS)
# st.sidebar.subheader("Speech2Text AI model")
# s2t_model = st.sidebar.radio('Select Model', SP2_MODELS)

st.title("Video Translator")
st.text("""Web tool kit for downloading, translating, transcribing, generating video""" )

#  ==== INPUT ====

input_container = st.container()
if input_video_type == "Url":
    video_input = input_container.text_input("Insert video url here", value="https://www.youtube.com/watch?v=KG0Q05Lnm7s")
    if video_input and is_from_youtube(video_input):
        if input_container.button("Show"):
            input_container.video(video_input)
else:
    video_input = st.file_uploader("Upload Video",type=['mp4', 'avi'])

st.subheader("Transcript and Translate")
st.write("Extract and translate captions using youtube_transcript_api")
formatter = st.selectbox("Select formatter: ", ["Text", "JSON"])

col1, col2 = st.columns(2)

if input_video_type == "Url" and is_from_youtube(video_input):
    video_id = get_youtube_id(video_input)
    if st.button('Run'):
        with st.spinner('Extracting captions from given video'):
            text_extractor = YoutubeTextExtractor(formatter)
            transcript = text_extractor.get_transcript_from_id(video_id)
            if formatter == "JSON":
                col1.json(transcript)
            else:
                col1.text_area('Text Transcription', transcript)
        with st.spinner('Translating captions from given video'):
            translated_transcript = YoutubeTextExtractor.translate_from_id(video_id, source, dest)
            col2.json(translated_transcript)

#         with st.spinner('Translating Text...'):
#             result = translator.translate(text, src=f'{source}', dest=f'{dst}').text
#         st.subheader("Translated Text is ...")
#         st.write(result)

#         st.write('')
#         st.header('Generated Audio')

#         with st.spinner('Generating Audio ...'):
#             ta_tts2 = gTTS(result,lang=f'{dst}')
#             ta_tts2.save('trans2.mp3')
#         st.audio('trans2.mp3',format='audio/mp3')
#         st.balloons()

#     else:
#         st.subheader('Image not found! Please Upload an Image.')


#!/usr/bin/env python3
#
# Helper for github actions, intended to be ran in the cross-compilation
# environment
#

import argparse
from html.parser import HTMLParser
import posixpath
import subprocess
import sys
import typing
from urllib.parse import unquote_plus
from urllib.request import urlopen
from urllib.error import HTTPError

from packaging.tags import parse_tag, sys_tags
import tomllib

# https://www.mschweighauser.com/fast-url-parsing-with-python/
class LinkFinder(HTMLParser):
    @classmethod
    def extract_links(cls, content: str) -> typing.List[str]:
        parser = cls()
        parser.feed(content)
        return parser.links

    def __init__(self):
        super().__init__()
        self.links = []

    def handle_starttag(self, tag, attrs):
        if tag == "a":
            for attr in attrs:
                if "href" in attr[0]:
                    self.links.append(attr[1])
                    return

def find_wheel_url(project: str, version: str, content: str):
    project = project.replace("-", "_")
    tags = set(sys_tags())

    links = LinkFinder.extract_links(content)
    not_matched = []
    found = False

    for link in links:
        link = link.split("#")[0]
        link = posixpath.basename(link)
        if not link.endswith(".whl"):
            continue

        wproject, wversion, wtags = link[:-4].split("-", 2)
        if wproject.lower() != project.lower():
            continue

        # Add to list so we can print it at the end if nothing matches
        not_matched.append(link)

        if wversion != version:
            continue

        for wtag in parse_tag(wtags):
            if wtag in tags:
                print("Found matching wheel", link)
                return True

    if not found:
        print("Did not find matching wheels in:")
        for link in not_matched:
            print("-", link)

    return False

def get_index_url() -> str:
    """
    Retrieves the project URL
    """
    content = subprocess.check_output(
        [sys.executable, "-m", "pip", "--disable-pip-version-check", "config", "list"],
        encoding="utf-8",
    )
    for line in content.splitlines():
        s = line.split("extra-index-url=", 1)
        if len(s) == 2:
            return s[1].strip("'")

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("project")
    parser.add_argument("--config", default="packages.toml")

    group = parser.add_mutually_exclusive_group()
    group.add_argument("--url", default=None)
    group.add_argument("-f", "--file", default=None)

    args = parser.parse_args()

    with open(args.config, "rb") as fp:
        cfg = tomllib.load(fp)

    try:
        version = cfg["packages"][args.project]["version"]
    except KeyError:
        parser.error(f"{args.project} not found in {args.config}")

    if args.file:
        with open(args.file) as fp:
            content = fp.read()
    else:
        url = args.url
        if not url:
            url = get_index_url()

        if not url:
            parser.error("Index URL to parse must be specified!")

        print("Checking", url)

        # Since we're using a real index, construct the name
        if url.endswith("/"):
            url = url[:-1]

        project_url = f"{url}/{args.project}/"
        print("..", project_url)

        try:
            with urlopen(project_url) as f:
                content = f.read().decode("utf-8")
        except HTTPError as e:
            if e.code != 404:
                raise
            content = ""

    found = find_wheel_url(args.project, version, content)

    # Sets variable for use in github actions
    print(f"::set-output name=should_build::{str(not found).lower()}")


import sys
import json
import sqlite3

conn = sqlite3.connect('data')

# Table schema : create table marks("an_rank" text, "code" text, "name" text, "district" text, "course" text, "cut_off" text, "caste" text)

with open(sys.argv[1]) as f:
    rows = json.loads(f.read())
    cursor = conn.cursor()
    for row in rows:
        columns = ', '.join(map(lambda x: '_'.join(x.split()), row.keys()))
        placeholders = ", ".join('?' * len(row.values()))
        values = list(map(str, row.values()))
        actual = []
        for value in values:
            try:
                actual.append(float(value))
            except Exception as e:
                actual.append(value)
        sql = 'INSERT INTO marks ({}) VALUES ({})'.format(columns, placeholders)
        cursor.execute(sql, values)
    cursor.close()
conn.commit()
conn.close()


#!/usr/bin/env python

# bugs:
#   need to check whether rewrite is legal on English side as well

from __future__ import print_function
from six import iteritems
from grammar import *
import bigfloat
import heapq
from math import log
import bintrees

verbose = 1

LIM = 20000

class UnificationFailure(Exception):
    pass

class Agenda(object):
    def __init__(self):
        self.agenda = bintrees.FastAVLTree()
        self.set = set()

    def __len__(self):
        return len(self.agenda)

    def add(self, d, pri=1.):
        if len(self.agenda) > LIM:# and self.agenda.min_key()[0] < pri:
            #print("Added ", pri, ": ", self.agenda[-1][0], self.agenda[0][0])

            self.agenda.insert((pri,d), d)
            d_p, _ = self.agenda.pop_min()
            #print("Removed ", pri, ": ", d_p[0])
            #self.set.add(d)
            #self.set.remove(d_p)

        else:
            self.agenda.insert((pri,d), d)
            #self.set.add(d)

    def get(self):
        p, d = self.agenda.pop_max()
        #d = self.agenda.popleft()
        #self.set.remove(d)
        return d

class Subgraph(object):
    """Represents a weakly connected edge-induced subgraph of
    a weakly connected Graph/DiGraph h.
    """

    def __init__(self, h, marker=False, exts=None):
        self.h = h
        self.boundary = {}
        self.marker = marker
        self.forgotten_edges = 0

    @staticmethod
    def singleton(h, edge):
        """Creates a Subgraph with a single edge and no external nodes."""
        g = Subgraph(h)
        for node in edge:
            g.boundary[node] = {edge}
            g.marker = g.marker or h.node[node].get('marker', False)
        return g

    def copy(self):
        g = Subgraph(self.h)
        g.boundary = dict((node, edges.copy()) for (node, edges) in iteritems(self.boundary))
        g.marker = self.marker
        g.forgotten_edges = self.forgotten_edges
        return g

    def __eq__(self, other):
        return self.__handle__() == other.__handle__()
    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self.__handle__())

    def __handle__(self):
        return (
            id(self.h), self.marker, frozenset((node, frozenset(edges)) for node, edges in iteritems(self.boundary)))

    def __str__(self):
        return '{{{0}}}'.format(','.join(
            "{0}{{{1}}}".format(node, ','.join(str(edge) for edge in edges)) for node, edges in
            iteritems(self.boundary)))

    def empty(self):
        return len(self.boundary) == 0 and not self.marker

    def full(self):
        if not self.marker: return False
        for node in self.boundary:
            if len(self.boundary[node]) < self.h.degree(node):
                return False
        return True

    def size(self):
        sizer = set()
        for node in self.boundary:
            for e in self.boundary[node]:
                sizer.add(e)
        return len(sizer) + self.forgotten_edges

    def merge(self, other):
        """Merge with another subgraph.
           To ensure that the resulting subgraph is connected,
           require that the new boundary has at least one node in common with ours."""
        if self.marker and other.marker:
            if all(not self.h.node[node]['marker'] for node in set(self.boundary) & set(other.boundary)):
                raise UnificationFailure("subgraphs must be disjoint")

                #        for node, edges in iteritems(other.boundary):
                # if node in self.boundary and self.boundary[node].intersection(edges):
                # raise UnificationFailure("subgraphs must be disjoint")

        for node, edges in iteritems(other.boundary):
            res = []
            if node in self.boundary:  #
                for e in self.boundary[node]:
                    for f in edges:
                        if e == f and type(e) == type(f):
                            res.append(e)
                        if e == f and type(e) != type(f):
                            print('alarm')
                if res:
                    raise UnificationFailure("subgraphs must be disjoint")
            if node not in self.boundary:
                self.boundary[node] = set()
            self.boundary[node].update(edges)
        self.marker = self.marker or other.marker
        self.forgotten_edges += other.forgotten_edges

    @staticmethod
    def multigraph_intersection(self, e1, e2):
        res = []
        for e in e1:
            for f in e2:
                if e == f and type(e) == type(f):
                    res.append(e)
        return res

    def forget_node(self, node):
        """Forget about a node. But this is only possible if all the node's edges have been added.
           Otherwise, we lose information."""
        if len(self.boundary[node]) != self.h.degree(node):
            raise UnificationFailure("can't forget a node until all its edges are recognized")

        edges = set()
        for x,y in self.boundary.items():
            for z in y:
                edges.add(z)
        del self.boundary[node]
        nedges = set()
        for x,y in self.boundary.items():
            for z in y:
                nedges.add(z)
        forgotten = (len(edges)-len(nedges))
        self.forgotten_edges += forgotten

class Item(object):
    def __init__(self, rule, tnode, dot, map, prob):
        """
        Let H be the input graph.

        rule is a grammar rule
        tnode is a node of rule.rhs_tree
        dot is the number of edges in tnode that have been recognized

        Let R' be the subgraph of rule.rhs induced by the union of:
          - the edges to the left of the dot in tnode
          - the edges in the descendants of tnode

        map is a one-to-one mapping from the boundary nodes and edges of R' to H.

        Parser invariant: There is an extension of map that is an
        isomorphism f between R' and an induced subgraph H'.
        """
        self.rule = rule
        self.tnode = tnode
        self.dot = dot
        self.map = map
        self.prob = prob

    def __handle__(self):
        return (self.rule, self.tnode, self.dot, self.map.__handle__())

    def __eq__(self, other):
        return isinstance(other, Item) and self.__handle__() == other.__handle__()

    def __ne__(self, other):
        return not (isinstance(other, Item) and self.__handle__() == other.__handle__())

    def __hash__(self):
        return hash(self.__handle__())

    @property
    def edges(self):
        return self.rule.rhs_tree.node[self.tnode]['edges']

    @property
    def nextedge(self):
        return self.edges[self.dot]

    @property
    def nextedgelabel(self):
        return hypergraphs.edge(self.rule.rhs, self.nextedge)['label']

    def __str__(self):
        return "[{},{},{},{}]".format(self.rule.id, self.tnode, self.dot, self.map)

class Goal(object):
    def __init__(self):
        self.prob = 1.0

    def __eq__(self, other):
        return isinstance(other, Goal)

    def __ne__(self, other):
        return not isinstance(other, Goal)

    def __hash__(self):
        return hash("Goal")

    def __str__(self):
        return "[Goal]"

class Mapping(object):
    """Represents a bijection between a subgraph of R and a subgraph of H."""

    def __init__(self, r, h):
        self.r = r
        self.h = h
        self.rspan = Subgraph(r)  # R'
        self.hspan = Subgraph(h)  # H'
        self.nodemap = {}  # node of R' -> node of H'

    def copy(self):
        newmap = Mapping(self.r, self.h)
        newmap.nodemap = self.nodemap.copy()
        newmap.rspan = self.rspan.copy()
        newmap.hspan = self.hspan.copy()
        return newmap

    def __str__(self):
        s = ','.join("{0}->{1}".format(
            rnode,
            hnode) for rnode, hnode in iteritems(self.nodemap))
        return "{{{0}}}".format(s)

    def __handle__(self):
        return (self.rspan.__handle__(),
                self.hspan.__handle__(),
                frozenset(iteritems(self.nodemap)))

    def __eq__(self, other):
        return isinstance(other, Item) and self.__handle__() == other.__handle__()

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self.__handle__())

    def add(self, rnode, hnode):
        if self.r.node[rnode]['label'] != self.h.node[hnode]['label']:
            raise UnificationFailure()

        # ensure that nodemap is bijective, and
        # that we update it monotonically
        if rnode in self.nodemap:
            if self.nodemap[rnode] != hnode:
                raise UnificationFailure()
        else:
            if hnode in self.hspan.boundary:
                raise UnificationFailure()
            self.nodemap[rnode] = hnode

    # All modifications to mapping are made via the methods below

    def set_domain(self, domain):
        for rnode in list(self.nodemap):
            if rnode not in domain:
                #print ("FORGET: " + str(rnode))
                self.rspan.forget_node(rnode)
                self.hspan.forget_node(self.nodemap[rnode])
                del self.nodemap[rnode]

    def add_edge(self, redge, hedge):
        if hypergraphs.edge(self.r, redge)['label'] != hypergraphs.edge(self.h, hedge)['label']:
            raise UnificationFailure()

        for rnode, hnode in zip(redge, hedge):
            self.add(rnode, hnode)

        self.rspan.merge(Subgraph.singleton(self.r, redge))
        self.hspan.merge(Subgraph.singleton(self.h, hedge))

    def add_rewrite(self, redge, subrule, submap):
        external = external_nodes(subrule.rhs)
        if hypergraphs.edge(self.r, redge)['label'] != subrule.lhs:
            raise UnificationFailure()
        if len(redge.h) != len(external):
            raise UnificationFailure()

        for rnode, snode in zip(redge, external):
            # Special case for when snode is just a node with no edges.
            # In this case, rnode is mapped to a node but snode is not.
            # So we need to check that the node labels match,
            # but don't need to update self.nodemap.
            if subrule.rhs.degree(snode) == 0:
                if self.r.node[rnode]['label'] != subrule.rhs.node[snode]['label']:
                    raise UnificationFailure("node labels do not match")
                if rnode not in self.nodemap:
                    pass
                    # print('e')
                    # self.add(rnode, submap.nodemap[snode])
                    # assert rnode in self.nodemap
            else:
                if snode in submap.nodemap:  # and rnode not in self.nodemap:
                    self.add(rnode, submap.nodemap[snode])

        self.rspan.merge(Subgraph.singleton(self.r, redge))
        self.hspan.merge(submap.hspan)

    def merge(self, other):
        for rnode in other.nodemap:
            self.add(rnode, other.nodemap[rnode])

        self.rspan.merge(other.rspan)
        self.hspan.merge(other.hspan)

class Chart(object):
    def __init__(self):
        self.chart = networkx.Graph()
        self.tnode_index = collections.defaultdict(list)
        self.lhs_index = collections.defaultdict(list)
        self.edge_index = collections.defaultdict(list)
        self.agenda = Agenda()
        self.indent_level = 2
        self.max_size = 0
        self.bucket = {}
        self.cur_bucket = 0

    def add(self, item, ants=(), label=None, weight=1.):

        # if verbose >= 3:
        #     print(" " * self.indent_level + "add: " + str(item))

        if item in self.chart:
            if verbose >= 3:
                print(" " * self.indent_level + "  already in chart")
                if item not in self.agenda.agenda:
                    print(" " * self.indent_level + "  warning: not in agenda (inside weights will be incorrect)")
        else:
            if isinstance(item, Goal):
                self.bucket['Goal']=Agenda()
                self.bucket['Goal'].add(item)
            else:
                buck = item.map.hspan.size()
                if buck < self.cur_bucket:
                    print (buck, item, id(item.map.hspan))
                    return
                #elif buck == self.cur_bucket and buck > 10:
                #    return
                g = buck
                while g not in self.bucket and g >= 0:
                    self.bucket[g] = Agenda()
                    g -= 1
                self.bucket[buck].add(item, item.prob)
        hypergraphs.add_hyperedge(self.chart, (item,) + ants, label=label, weight=weight)

    def is_empty(self):
        if 'Goal' in self.bucket:
            return True
        sum = 0
        for x in self.bucket.keys():
            sum += len(self.bucket[x].agenda)
            if sum > 0: return False
        return True

    def get(self):
        if len(self.bucket[self.cur_bucket].agenda) == 0:
            if verbose >= 1:
                for x,y in self.bucket.items():
                    print(str(x) + ":" + str(len(y.agenda)), end=" ")
                print()
            self.cur_bucket+=1

            while self.cur_bucket not in self.bucket or len(self.bucket[self.cur_bucket].agenda) == 0:
                self.cur_bucket += 1

        item = self.bucket[self.cur_bucket].get()

        if isinstance(item, Goal):
            return item

        # We index the item only upon removing it from the agenda
        # so that a deduction always involves exactly one agenda item
        # and zero or more non-agenda items.

        # index for completed tree nodes
        if item.dot == len(item.edges):
            self.tnode_index[item.rule, item.tnode].append(item)

            # index for completed rhs's
            if item.tnode == item.rule.rhs_tree.graph['root']:
                lhs = item.rule.lhs_signature()
                self.lhs_index[lhs].append(item)

        else:
            redge = item.edges[item.dot]
            if isinstance(hypergraphs.edge(item.rule.rhs, redge)['label'], Nonterminal):
                lhs = item.rule.rhs_signature(redge)
                self.edge_index[lhs].append(item)

        return item

    def __contains__(self, item):
        return item in self.chart

    def __getitem__(self, item):
        return self.chart[item]

def viterbi(chart):
    weight = {}
    ant = {}

    # Find maximum weight
    def visit(u, clo):
        if u not in chart:
            return bigfloat.bigfloat(0.)
        if u in weight:
            return weight[u]
        w_max = None
        e_max = None
        for e in hypergraphs.edges(chart, u):
            if e.h[0] != u: continue
            w = bigfloat.bigfloat(1.)
            for v in e.h[1:]:
                #print (v)
                if v in clo :
                    continue
                clo.add(v)
                w *= visit(v, clo.copy())
            if w_max is None or w > w_max:
                w_max = w
                e_max = e
        weight[u] = w_max
        ant[u] = e_max
        return w_max

    visit(Goal(), set())

    # Reconstruct derivation
    deriv = networkx.DiGraph()

    def visit(ritem, item, clo, i=0):
        # ritem: item at the root of the rule
        # item: current item
        deriv.add_node(ritem, rule=ritem.rule.id)
        e = ant[item]
        if hypergraphs.edge(chart, e)['label'] == "Complete":
            _, aitem, pitem = e
            # print(ritem, aitem, pitem)
            if pitem in clo:
                return
            link = hypergraphs.edge(aitem.rule.rhs, aitem.nextedge)['link']
            clo.add(pitem)
            visit(ritem, aitem, clo.copy(), i+1)
            visit(pitem, pitem, clo.copy(), i+1)
            deriv.add_edge(ritem, pitem, link=link)

        else:
            for item in e.h[1:]:
                visit(ritem, item, clo.copy(), i+1)

    [_, item] = ant[Goal()]
    visit(item, item, set())
    [deriv.graph['root']] = [v for v in deriv if len(deriv.predecessors(v)) == 0]
    return deriv

class ParserError(Exception):
    pass

def parse(g, starts, h):
    # Preprocess input

    # Arbitrarily choose a marker node in each weakly connected component
    if verbose >= 3:
        for e in h.edges():
            print(e)
    if not networkx.is_weakly_connected(h):
        raise NotImplementedError("parsing of disconnected graphs not implemented")
    for comp in networkx.weakly_connected_components(h):
        marker = True
        for v in comp:
            if not isinstance(v, hypergraphs.Hyperedge):
                h.node[v]['marker'] = marker
                if verbose >= 3 and marker:
                    print("  marker node:", v)
                marker = False

    # Preprocess grammar

    # Could filter out rules that use labels not found in h

    for rule in g:
        if verbose >= 3:
            print('Rule', str(rule.id) + ':')
            print('\t' + str(rule.lhs) + '->')
            print('\tNodes:')
            for n in rule.rhs.nodes(data=True):
                print('\t\t' + str(n))
            print('\tEdges:')
            for e in rule.rhs.edges():
                print('\t\t' + str(e))
            print()
            # print("Rule {}: {} -> {}, (may be disconnected)".format(str(rule.id), rule.lhs, format_rhs(rule.rhs, show_all=True)))

        # Form tree decomposition.
        # Assume (at most) binary branching
        # For each node tnode \in rule.rhs_tree, assume a list of edges
        # such that every edge in h appears exactly once among all the lists

        t = decompose_graphlet(rule.rhs)
        for v in t.nodes():
            # We need the edges in each bag to be ordered
            t.node[v]['edges'] = list(t.node[v]['edges'])
        rule.rhs_tree = t
        if verbose >= 2:
            print("  tree decomposition:", t.edges())
            for v in t.nodes():
                print("    bag {}".format(v))
                print("      nodes:", " ".join(map(str, t.node[v]['nodes'])))
                print("      edges:", " ".join('-'.join(map(str, e)) for e in t.node[v]['edges']))

    chart = Chart()

    # Axioms
    if verbose >= 3:
        print("axioms:")
    for rule in g:
        for tnode in rule.rhs_tree.nodes():
            if len(rule.rhs_tree.successors(tnode)) == 0:
                #nds = sum(1 for x in rule.rhs.nodes(data=True) if x[1]['label'] is 'u' and 'external' not in x[1])
                item = Item(rule, tnode, 0, Mapping(rule.rhs, h), log(rule.weight))
                chart.add(item, label="Leaf")

                #   try this code below under different orderings of chart.agenda

    while not chart.is_empty():
        trigger = chart.get()
        if trigger == Goal(): continue
        tnode = trigger.tnode
        rule = trigger.rule
        # if verbose >= 2:
        #     print("trigger:", trigger, trigger.cnt, trigger.map.hspan.size())

        if trigger.dot < len(trigger.edges):
            # There are still edges left to process. Choose the next one, redge.
            redge = trigger.edges[trigger.dot]

            if isinstance(hypergraphs.edge(rule.rhs, redge)['label'], Nonterminal):
                # If redge is labeled with a nonterminal,
                # search for possible rewrites
                lhs = rule.rhs_signature(redge)
                for rewrite in chart.lhs_index[lhs]:
                    # if verbose >= 3:
                    #     print("  rewrite:", rewrite)
                    newmap = trigger.map.copy()
                    try:
                        newmap.add_rewrite(redge, rewrite.rule, rewrite.map)
                    except UnificationFailure:
                        pass
                    else:
                        newitem = Item(rule, tnode, trigger.dot + 1, newmap, trigger.prob + log(rewrite.rule.weight))
                        chart.add(newitem, ants=(trigger, rewrite),
                                  label="Complete", weight=rewrite.rule.weight)

            else:
                # If redge is labeled with a terminal,
                # search for possible matches with edges in h, hedge,
                for hedge in hypergraphs.edges(h):  # to do: more general
                    # and try to add redge->hedge to trigger.map
                    newmap = trigger.map.copy()
                    try:
                        newmap.add_edge(redge, hedge)
                    except UnificationFailure:
                        pass
                    else:
                        newitem = Item(rule, tnode, trigger.dot + 1, newmap, trigger.prob)
                        chart.add(newitem, ants=(trigger,), label="Shift")

        elif tnode != rule.rhs_tree.graph['root']:
            # No more edges in tnode left to process; move to parent
            [tparent] = rule.rhs_tree.predecessors(tnode)
            tchildren = rule.rhs_tree.successors(tparent)
            if len(tchildren) == 1:
                # only child
                newmap = trigger.map.copy()
                try:
                    newmap.set_domain(rule.rhs_tree.node[tparent]['nodes'])
                except UnificationFailure:
                    pass
                else:
                    newitem = Item(rule, tparent, 0, newmap, trigger.prob)
                    chart.add(newitem, ants=(trigger,), label="Unary")

            elif len(tchildren) == 2:
                # find sister
                tchildren.remove(tnode)
                (tsister,) = tchildren

                for sister in chart.tnode_index[rule, tsister]:
                    if verbose >= 3:
                        print("  sister:", sister)
                    newmap = trigger.map.copy()
                    try:
                        newmap.merge(sister.map)
                        newmap.set_domain(rule.rhs_tree.node[tparent]['nodes'])
                    except UnificationFailure:
                        pass
                    else:
                        newitem = Item(rule, tparent, 0, newmap, trigger.prob)
                        chart.add(newitem, ants=(trigger, sister), label="Binary")
            else:
                raise ParserError("tree decomposition not binary branching")

        else:
            # root of rule rhs: look for items to rewrite
            lhs = rule.lhs_signature()
            for rewritee in chart.edge_index[lhs]:
                # if verbose >= 3:
                #     print("  rewritee:", rewritee)
                oedge = rewritee.edges[rewritee.dot]
                newmap = rewritee.map.copy()
                try:
                    if str(trigger).startswith('[53,2,2,{_1->19,_0->28'):
                        pass
                    newmap.add_rewrite(oedge, trigger.rule,
                                       trigger.map)  # trigger: [5,2,2,{_2->5,_1->1,_0->4}] rewritee: [3,2,2,{_2->3,_1->4,_0->0}]
                except UnificationFailure:
                    pass
                else:
                    newitem = Item(rewritee.rule, rewritee.tnode, rewritee.dot + 1, newmap, trigger.prob + log(rewritee.rule.weight))
                    chart.add(newitem, ants=(rewritee, trigger),
                              label="Complete", weight=rule.weight)

            # and check if we're done
            if trigger.map.hspan.full() and len(external_nodes(rule.rhs)) == 0 and rule.lhs in starts:
                chart.add(Goal(), ants=(trigger,), weight=rule.weight, label="Goal")
                if verbose >= 1:
                    print("GOAL: ", trigger.map.hspan.size())

    return chart.chart

if __name__ == "__main__":
    import sys
    import argparse
    import amr

    parser = argparse.ArgumentParser()
    # parser.add_argument('grammars', type=str, metavar='grammar', nargs='+', help='file(s) containing grammar rules')
    parser.add_argument('-s', '--start', dest='start', type=str, default='Truth', metavar='nonterminal',
                        help='start symbol')
    args = parser.parse_args()

    frules = []
    erules = []
    start = Nonterminal(args.start)

    for f in range(1, 11):

        print()
        print('**** File #' + str(f) + '****')

        for ri, line in enumerate(open('tests/grammar' + str(f))):
            lhs, frhs, erhs, weight = line.split("|||")
            lhs = Nonterminal(lhs.strip())

            frule = Rule(lhs, parse_rhs(frhs), id=ri, weight=weight)
            frules.append(frule)

            erule = Rule(lhs, parse_rhs(erhs), id=ri)
            erules.append(erule)

        for ri, line in enumerate(open('tests/input' + str(f))):
            h = amr.parse_amr(line)

            if 1 <= verbose < 3:
                sys.stderr.write("Input graph: {}\n".format(amr.format_amr(h)))

            forest = parse(frules, [start], h)

            if forest:
                sys.stderr.write("Items: %s\n" % len(forest))
                if verbose >= 1:
                    sys.stderr.write("Output graph: ")
                print(amr.format_amr(derive(viterbi(forest), erules)))

            else:
                if verbose >= 1:
                    sys.stderr.write("No derivations\n")
                print("")


import numpy as np
import cv2

rectangle = np.zeros((300, 300), dtype="uint8")
cv2.rectangle(rectangle, (25, 25), (275, 275), 255, -1)
cv2.imshow("Rectangle", rectangle)
cv2.waitKey(0)

circle = np.zeros((300, 300), dtype="uint8")
cv2.circle(circle, (150, 150), 150, 255, -1)
cv2.imshow("Circle", circle)
cv2.waitKey(0)

# esta fazendo um AND, a intersecao das duas imagens
bitwiseAnd = cv2.bitwise_and(rectangle, circle)
cv2.imshow("AND", bitwiseAnd)
cv2.waitKey(0)

# esta fazendo um OR, a uniao das duas imagens
bitwiseOr = cv2.bitwise_or(rectangle, circle)
cv2.imshow("OR", bitwiseOr)
cv2.waitKey(0)

bitwiseXor = cv2.bitwise_xor(rectangle, circle)
cv2.imshow("XOR", bitwiseXor)
cv2.waitKey(0)

bitwiseNot = cv2.bitwise_not(circle)
cv2.imshow("NOT", bitwiseNot)
cv2.waitKey(0)

bitwiseNot = cv2.bitwise_not(rectangle)
cv2.imshow("NOT", bitwiseNot)
cv2.waitKey(0)


import pyrealsense2 as rs
import numpy as np
import cv2
import os
import shutil
import argparse

# reset data folder
if os.path.exists('data'):
    shutil.rmtree('data')
os.mkdir('data')
os.mkdir('data/color')
os.mkdir('data/depth')

# input
parser = argparse.ArgumentParser(description="Read recorded bag file and display depth stream in jet colormap.\
                                Remember to change the stream resolution, fps and format to match the recorded.")
parser.add_argument("-i", "--input", type=str, help="Path to the bag file")
args = parser.parse_args()
if not args.input:
    print("No input paramater have been given.")
    print("For help type --help")
    exit()
if os.path.splitext(args.input)[1] != ".bag":
    print("The given file is not of correct file format.")
    print("Only .bag files are accepted")
    exit()

pipeline = rs.pipeline()
config = rs.config()
rs.config.enable_device_from_file(config, args.input)
# config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30)
# config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 30)

align_to = rs.stream.color
align = rs.align(align_to)

pipeline.start(config)

i = 0
try:
    while True:
        frames = pipeline.wait_for_frames()
        timestamp = int(frames.get_timestamp() * 10000)
        print(timestamp)

        frames = align.process(frames)

        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()
        if not depth_frame or not color_frame:
            continue

        image_color = np.array(color_frame.get_data())
        image_color = cv2.cvtColor(image_color, cv2.COLOR_RGB2BGR)
        image_depth = np.array(depth_frame.get_data())

        cv2.imwrite('data/color/color{:012}.png'.format(timestamp), image_color)
        cv2.imwrite('data/depth/depth{:012}.png'.format(timestamp), image_depth)
        i += 1

finally:
    pipeline.stop()


import pandas as pd

salary = pd.read_csv("C:\\Data\\PreppinData\\PD 2021 Wk 49 Input - Input.csv",
                     parse_dates=['Date'], date_parser=lambda x: pd.to_datetime(x, format='%d/%m/%Y'))
salary['Report Year'] = salary['Date'].dt.year
salary['Min Date'] = salary['Date'].groupby(salary['Name']).transform('min').dt.strftime('%b %Y')
salary['Max Date'] = salary['Date'].groupby(salary['Name']).transform('max').dt.strftime('%b %Y')
salary['Employment Range'] = salary['Min Date']+' to '+salary['Max Date']
salary['Tenure by End of Reporting Year'] = salary.sort_values('Date').groupby(['Name']).cumcount()+1
salary['Salary Paid'] = salary['Annual Salary']/12
salary['Yearly Bonus'] = salary['Sales']*0.05

salary = salary.groupby(['Name', 'Report Year'], as_index=False).agg({
    'Employment Range':'first', 'Tenure by End of Reporting Year':'max', 'Salary Paid':'sum', 'Yearly Bonus':'sum'})
salary['Salary Paid'] = round(salary['Salary Paid'],2)
salary['Total Paid'] = round(salary['Salary Paid']+salary['Yearly Bonus'])
salary = salary[['Name', 'Employment Range', 'Report Year',
       'Tenure by End of Reporting Year', 'Salary Paid', 'Yearly Bonus', 'Total Paid']]


#### desafio aula 007 desafio 005: https://www.youtube.com/watch?v=Vw6gLypRKmY

#### iniciado no video 31:10

#### soluções dos desafio: https://youtu.be/Vw6gLypRKmY

#### desafio 005: Faça um programa que leia um número inteiro e mostre na tela o seu sucessor e seu antecessor.

n1 = int(input('Digite um número inteiro: '))
s = n1 + 1
sub = n1 - 1

print('O número que você digitou é {} e o seu sucessor é {} e o antecessor é {}'.format(n1, s, sub))


# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right
class Solution:
    def findTilt(self, root: TreeNode) -> int:
        """
        :param root: root node
        :return: personal tilt of the node
        this is a brute force appraoch
        recursively sum personal tilts of the node
        for every node use the node sum method to find the sum of left and right branches
        find the node tilt, return sum of left tilt, right tilt and node tilt
        null nodes have 0 tilts
        """
        if root == None:
            return 0
        lval = 0
        rval = 0
        personalTilt = 0
        if root.left != None:
            lval += self.nodeSum(root.left)
            personalTilt += self.findTilt(root.left)
        if root.right != None:
            rval += self.nodeSum(root.right)
            personalTilt += self.findTilt(root.right)
        personalTilt += abs(lval - rval)
        return personalTilt

    def nodeSum(self, node):
        if (node == None):
            return 0
        else:
            return node.val + self.nodeSum(node.left) + self.nodeSum(node.right)

class Solution2:
    """
    optimized solution
    create a new function that recursively calculates node sums and total tile
    when a null node is reached it returns 0 tilt and 0 sum
    for non null nodes, return their tilt+tilt sum returned and thier left sum+right sum+node val
    call the function in out main
    return 0 index of final tuple
    """

    def findTilt(self, root: TreeNode) -> int:
        def calc_tilts(root):
            if not root:
                return 0, 0
            ltilt, lsum = calc_tilts(root.left)
            rtilt, rsum = calc_tilts(root.right)
            return ltilt + rtilt + abs(lsum - rsum), lsum + rsum + root.val

        return calc_tilts(root)[0]


# pylint: disable=redefined-outer-name
import pytest
import requests
from sqlalchemy.orm import clear_mappers
from ShipRaitingslib import bootstrap, config
from ShipRaitingslib.Domain import commands
from ShipRaitingslib.adapters import notifications
from ShipRaitingslib.Services import unit_of_work
from ..random_refs import random_ShipName

@pytest.fixture
def bus(sqlite_session_factory):
    bus = bootstrap.bootstrap(
        start_orm=True,
        uow=unit_of_work.SqlAlchemyUnitOfWork(sqlite_session_factory),
        notifications=notifications.EmailNotifications(),
        publish=lambda *args: None,
    )
    yield bus
    clear_mappers()

def get_email_from_mailhog(ShipName):
    host, port = map(config.get_email_host_and_port().get,
                     ["host", "http_port"])
    all_emails = requests.get(f"http://{host}:{port}/api/v2/messages").json()
    return next(m for m in all_emails["items"] if ShipName in str(m))

def test_Duplicate_Review_email(bus):
    ShipName = random_ShipName()
    bus.handle(commands.CreateBatch("batch1", ShipName, 9, None))
    bus.handle(commands.Allocate("order1", ShipName, 10))
    email = get_email_from_mailhog(ShipName)
    assert email["Raw"]["From"] == "allocations@example.com"
    assert email["Raw"]["To"] == ["stock@made.com"]
    assert f"Duplicate Review for {ShipName}" in email["Raw"]["Data"]


import pandas as pd

class Write:
    # Write admin data to new file
    def Write_New(self, dictionary, batch):
        data = {'Header': ['Klubbnavn', 'Kontaktperson', 'Mobil', 'Epost', 'Fødselsdato']}
        data.update(dictionary)
        temp_df = pd.DataFrame.from_dict(data, orient='index')
        print(f'\n=====\nWriting completish list of admin information for {batch}...\n=====\n')
        # appended comment to make pyLint ignore false positive problem
        with pd.ExcelWriter(f'files/admin/Admin_{batch}.xlsx') as writer: # pylint: disable=abstract-class-instantiated
                temp_df.to_excel(writer, index=False, header=False)

    # Append admin data to existing file
    def Append_File(self, dictionary, batch, df_migrated, path):
        # Format data to input into existing file.
        admin_users = {}
        for key in dictionary:
            if len(dictionary[key]) == 5:
                admin_users.update({key: f'{dictionary[key][1]} / {dictionary[key][4]}'})
        # Make df to append admin info as new sheet
        data = {'Header': ['Klubbnavn', 'Kontaktperson', 'Mobil', 'Epost', 'Fødselsdato']}
        data.update(dictionary)
        temp_df = pd.DataFrame.from_dict(data, orient='index')
        # Add admin data to df
        for index in admin_users:
            df_migrated.Bruker[index] = admin_users[index]
        print(f'\n=====\nAppending Admin user data for {batch} to existing file at {path}...')
        # Write user row, and admin data to new sheet
        with pd.ExcelWriter(path, engine='openpyxl', mode='a') as writer: # pylint: disable=abstract-class-instantiated
                df_migrated.to_excel(writer, sheet_name='Tabell', index=False, header=False, startcol=7, startrow=1)
                temp_df.to_excel(writer, sheet_name=f'Admin Info {batch}', index=False, header=False)
        print('\nDone.\n=====')


import httpx
import asyncio
from random import choice, randint
from nonebot.plugin import on_command, on_keyword, on_message
from nonebot.rule import to_me
from nonebot.typing import T_State
from nonebot.adapters.cqhttp import MessageEvent
from nonebot.adapters.cqhttp.message import Message, MessageSegment
from nonebot.adapters._base import Event
from nonebot.adapters.cqhttp.bot import Bot

trans = on_command("nbnhhsh", rule=to_me(), block=True,
                   aliases={"翻译翻译", "能不能好好说话", "翻译", "你给我翻译翻译，什么叫他妈的"})

@trans.handle()
async def _(bot: Bot, event: Event, state: dict):
  args = str(event.get_message()).strip()
  if args:
    state["text"] = args
  if args == "惊喜":
    await trans.send("这还用翻译，都说了......惊喜嘛。")
    asyncio.sleep(0)
    await trans.send("我让你翻译给我听，什么叫惊喜？")
    asyncio.sleep(0)
    await trans.send("难道你听不懂什么叫惊喜？")
    asyncio.sleep(0)
    await trans.send("我就想让你翻译翻译，什么叫惊喜！")
    asyncio.sleep(0)
    await trans.send("翻译出来给我听，什么他妈的叫惊喜！什么他妈的叫他妈的惊喜！")
    asyncio.sleep(0)
    await trans.send("什么他妈的叫惊喜啊？")
    asyncio.sleep(0)
    await trans.send("惊喜就是三天之后，我出一百八十万给你们出城剿匪，接上我的腿！明白了吗？")
    asyncio.sleep(0)
    await trans.send("这就是惊喜啊。")
    asyncio.sleep(0)
    await trans.send("翻译翻译，翻译翻译！")
    asyncio.sleep(0)
    await trans.send("惊喜就是三天之后，给你一百八十万两银子出城剿匪，接上他的腿！")
    asyncio.sleep(0)
    await trans.send("哈，大哥，原来这就是他妈的惊喜啊。")
    asyncio.sleep(0)
    await trans.finish(message="懂了吗，cdd", at_sender=True)
  api = "https://lab.magiconch.com/api/nbnhhsh/guess"
  data = {"text": state["text"]}
  async with httpx.AsyncClient() as client:
    response = await client.post(api, data=data)
  response = response.json()

  if not response:
    await trans.finish("bt，wm00hdzyjh，没有对应的翻译")
  try:
    msg = "得到的翻译如下：\n"
    for i in response[0]['trans']:
      msg += i + "，"
    msg = msg.strip("，")
    await trans.finish(msg)
  except KeyError:
    await trans.finish("坏了Z世代又出新词了，没有对应的翻译")


# Project name : HackerRank: Day 11: 2D Arrays
# Link         : https://www.hackerrank.com/challenges/30-2d-arrays/problem
# Try it on    :
# Author       : Wojciech Raszka
# E-mail       : contact@gitpistachio.com
# Date created : 2020-06-29
# Description  :
# Status       : Accepted (166392174)
# Tags         : python, hourglass
# Comment      :

#!/bin/python3

import math
import os
import random
import re
import sys

def hourglass(arr, li, lj):
    return (arr[li][lj] + arr[li][lj + 1] + arr[li][lj + 2] +
                          arr[li + 1][lj + 1] +
            arr[li + 2][lj] + arr[li + 2][lj + 1] + arr[li + 2][lj + 2])

if __name__ == '__main__':
    arr = []

    for _ in range(6):
        arr.append(list(map(int, input().rstrip().split())))

    n = len(arr)
    m = len(arr[0])
    max_sum_of_hours = -7*9
    for li in range(n - 2):
        for lj in range(m - 2):  # left bottom corner og hourglass
            sum_of_hours = hourglass(arr, li, lj)
            if sum_of_hours > max_sum_of_hours:
                max_sum_of_hours = sum_of_hours

    print(max_sum_of_hours)

HackerRank/30 Days of Code/Day 11 2D Arrays/2D Arrays.py


# -*- coding: utf-8 -*-

from openpyxl_simple_service import SimpleExcelService
from docx_simple_service import SimpleDocxService

def start():
    e = SimpleExcelService()
    e.open_wb('users.xlsx')
    e.open_ws('Sheet1')
    # users.xslxから値を取得
    row_result = e.get_rows(6, 1)

    # document作成
    docx = SimpleDocxService()
    date = input('Enter the date you want to create')
    for i, row in enumerate(row_result):
        # 名前
        name = row[0]
        # 機能訓練Ⅰ
        one_long_goal = row[2]
        one_long_level = row[3]
        one_short_goal = row[4]
        one_short_level = row[5]
        # 機能訓練Ⅱ
        two_long_goal = row[6]
        two_long_level = row[7]
        two_short_goal = row[8]
        two_short_level = row[9]
        # 評価
        evaluate = row[10]
        # 表題
        docx.add_head("機能訓練報告書", 0)
        # 御中とかそんなの
        docx.open_text()
        docx.add_text('石川内科胃腸科医院デイケア　切中様御中')
        docx.close_text()
        # 日付、発信元、住所
        docx.open_text()
        docx.add_text('{}\n'.format(date))
        docx.add_text('発信者 岡本荘デイサービスセンター 下大寺\n')
        docx.add_text('〒761-8047　高松市岡本町527-1\n')
        docx.add_text('TEL 087-885-2828\n')
        docx.paragraph_alignment_right()
        docx.close_text()
        # 礼文的なやつ
        docx.open_text()
        docx.add_text(
            'いつもお世話になっております。\n' \
            '各月における評価内容や、目標の達成度合いについて報告をさせていただきます。' \
            'ご確認のほどよろしくお願いします。また、当該利用者の目標、訓練内容の継続、' \
            'ADL及びIADLの改善状況を含めた目標の見直しや、訓練内容の変更など、お手数では' \
            'ありますが、電話等で助言のほどよろしくお願いいたします。'
        )
        docx.close_text()
        # 利用者氏名
        name = name
        docx.add_head('利用者氏名', 1)
        docx.open_text()
        docx.add_text('{}'.format(name))
        docx.close_text()
        # 機能訓練Ⅰ
        docx.add_head('個別機能訓練1', 1)
        docx.open_text()
        docx.add_text('【長期目標】：{}\n'.format(one_long_goal))
        docx.add_text('　【達成度】：{}\n'.format(one_long_level))
        docx.add_text('【短期目標】：{}\n'.format(one_short_goal))
        docx.add_text('　【達成度】：{}'.format(one_short_level))
        docx.close_text()
        # 機能訓練Ⅱ
        docx.add_head('個別機能訓練2', 1)
        docx.open_text()
        docx.add_text('【長期目標】：{}\n'.format(two_long_goal))
        docx.add_text('　【達成度】：{}\n'.format(two_long_level))
        docx.add_text('【短期目標】：{}\n'.format(two_short_goal))
        docx.add_text('　【達成度】：{}'.format(two_short_level))
        docx.close_text()
        # 評価内容
        docx.add_head('評価内容', 1)
        docx.open_text()
        docx.add_text('{}'.format(evaluate))
        # 改行
        if len(row_result)-1 != i:
            docx.add_page_break()

    # 保存して終了
    docx.save("機能訓練報告書.docx")
    print("complete")

if __name__ == '__main__':
    start()


from hedra.core.hooks.types.base.hook import Hook
from hedra.core.hooks.types.base.hook_type import HookType

class HookSetupError(Exception):

    def __init__(self, hook: Hook, hook_type: HookType, message: str) -> None:

        hook_type = hook_type.name.lower()
        super().__init__(
            f'Hook Error - @{hook_type} hook {hook.shortname} from stage {hook.stage}\n{message}'
        )


#!/usr/bin/env python
# coding: utf-8

# In[1]:

get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import mean_absolute_error
import pickle

# In[1]:

import warnings
warnings.filterwarnings('ignore')

#

# Загрузим DataFrame, уберем ненужные столбцы (ID и пустой)

# In[3]:

df = pd.read_csv('data.csv')
df = df.drop('Unnamed: 32', axis=1).drop("id", axis=1)

df.head()
# malignant - "bad"
# benign - "good"

# Убедимся, что в датасете нет ячеек с NaN

# In[4]:

df.isnull().values.any()

# Посмотрим типы данных, наименования столбцов

# In[5]:

df.info()

# | №  | Название          | Перевод             | Толкование                                                                                |
# | -- | ----------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
# | 0  | diagnosis         | диагноз             | диагноз                                                                                |
# | 1  | Radius            | Радиус              | Расстояние от центра до периферии                                                                                |
# | 2  | Texture           | Размер              | Размер раковых клеток.                                                                                |
# |    |                   |                     | Однородность: Обозначает локальную изменчивость длины радиуса. Однородность это разность между длиной радиуса и средним значением длины линий вокруг него..                                                                                |
# | 3  | Perimeter         | Периметр            |  Величина основной опухоли.                                                                                |
# |    |                   |                     | Общее расстояние между точками дают периметр.                                                                                |
# | 4  | Area              | Зона                | Область раковых клеток                                                                                |
# | 5  | Smoothness        | Однородность        | Обозначает локальную изменчивость длины радиуса. Однородность это разность между длиной радиуса и средним значением длины линий вокруг него.                                                                                |
# | 6  | Compactness       | Компактность        | Это мера сравнения периметра и размера, находится как (периметр^2/размер – 1.0)                                                                                |
# | 7  | Concavity         | Вогнутость          | Глубина впадин. На меньших струнах чаще образуются меньшие впадины. Это связано с их длиной.                                                                                |
# |    |                   |                     | Количество впадин: Вогнутость измеряет частоту впадин контура, в то время как этот параметр измеряет их количество.                                                                                |
# | 8  | Concave points    | Количество впадин   | Вогнутость измеряет частоту впадин контура, в то время как этот параметр измеряет их количество.                                                                                |
# | 9  | Symmetry          | Симметрия           | Самая длинная струна берётся за основную ось. Разница длин основной оси и линии перпендикулярной ей называется симметрией                                                                                |
# | 10 | fractal dimension | Фрактальные размеры | Показатель нелинейного роста. По мере увеличения масштаба измерения точность снижается и, как следствие, уменьшается значение периметра. Эти данные, представленные в виде графика кривой с отрицательным наклонном, дают приблизительное значение фрактальных размеров. |

# **Mean** – среднее значение
# **SE(standard error)** – обычные ошибки
# **Worst** – худший результат
#

# # Немного графиков
# ## Отношение статистики по доброкачественным и злокачественым опухалям

# In[6]:

sns.set_style('darkgrid')
plt.figure(figsize=(10,5))
sns.countplot(x="diagnosis", data=df, palette='rocket');
pd.value_counts(df['diagnosis'])

# Отношение количества злокачественных(malignant) к доброкачественным(benign) оп

# ## Построение тепловой карты корреляции значений

# In[7]:

plt.figure(figsize=(25, 20))
plt.title("CORRELATION HEATMAP")
matrix = np.triu(df.corr())
sns.heatmap(df.corr(), annot=True, linewidth=.25, mask=matrix, cmap="coolwarm");

# Тепловая карта кореляций всех значений

# Распределения радиуса относительно размера. Так как Однородность: Обозначает локальную изменчивость длины радиуса. Однородность - это разность между длиной радиуса и средним значением длины линий вокруг него.

# In[8]:

sns.scatterplot(x='radius_mean', y='texture_mean', hue='diagnosis', data=df,palette='rocket')

# Распределение однородности относительно компактности, так как оба этих параметра связаны с радиусом

# In[9]:

sns.scatterplot(x='smoothness_mean', y='compactness_mean', data=df, hue='diagnosis')

# Распределение периметра относительно области поражения

# In[10]:

sns.lmplot(x='perimeter_mean', y='area_mean', data=df, hue='diagnosis')

# Распределение однородности относительно компактности

# In[11]:

sns.lmplot(x='smoothness_mean', y='compactness_mean', data=df, hue='diagnosis')

# ##  Данные можно разделить прямой и получить относительно корректный результат

# In[12]:

fig, table =  plt.subplots(1, 2, figsize=(20,6))
sns.boxplot(ax = table[0], x=df.diagnosis, y=df['area_mean'])
table[0].set_title('Area')

sns.boxplot(ax = table[1], x=df.diagnosis, y=df['perimeter_mean'])
table[1].set_title('Perimeter')

plt.show()

# In[13]:

sns.lmplot(x='area_mean', y='perimeter_mean', data=df, hue='diagnosis')

# In[14]:

bins = 20 #Number of bins is set to 20, bins are specified to divide the range of values into intervals
def histogram(features):
  plt.figure(figsize=(10,15))
  for i, feature in enumerate(features):
      plt.subplot(5, 2, i+1)  #subplot function: the number of rows are given as 5 and number of columns as 2, the value i+1 gives the subplot number, subplot numbers start with 1
      sns.distplot((df[df.diagnosis=='M'].drop("diagnosis",axis=1))[feature], bins=bins, color='red', label='Malignant');
      sns.distplot((df[df.diagnosis=='B'].drop("diagnosis",axis=1))[feature], bins=bins, color='green', label='Benign');
      plt.title(str(' Density Plot of:  ')+str(feature))
      plt.xlabel('X variable')
      plt.ylabel('Density Function')
      plt.legend(loc='upper right')
  plt.tight_layout()
  plt.show()

# In[2]:

df.columns

#

#
Mean features (Какая-нибудь недокументированная доп. возможность)

# In[16]:

mean_features = list(filter(lambda x: "mean" in x, df.columns))  #['radius_mean', 'texture_mean']
histogram(mean_features)

#

#
Error features

# In[17]:

error_features = list(filter(lambda x: "_se" in x, df.columns))  #['radius_mean', 'texture_mean']
histogram(error_features)

#

#
Worst features

# In[18]:

worst_features = list(filter(lambda x: "worst" in x, df.columns))  #['radius_mean', 'texture_mean']
histogram(worst_features)

# ## Нормальное распределение ⇒ "обрезать" данные не требуется
#

# ##### Преобразуем Malignant и Benign в числа

# In[19]:

# M - 1
# B - 0

print("B:", df.iloc[20]['diagnosis'])
print("M:", df.iloc[0]['diagnosis'])

le = LabelEncoder()
df['diagnosis'] = le.fit_transform(df['diagnosis'])

print("B:", df.iloc[20]['diagnosis'])
print("M:", df.iloc[0]['diagnosis'])

# In[20]:

ss = StandardScaler()
cols = list(df)[1:]
df[cols] = ss.fit_transform(df[cols])

# ## Разделяем выборку (датасет) на 3 части
#
#

# In[21]:

#street magic
train, test, validate = np.split(df.sample(frac=1, random_state=42),
                       [int(.6*len(df)), int(.8*len(df))])

# ##### Отделяем У от Х

# In[22]:

separate = lambda x: (x['diagnosis'], x.drop("diagnosis",axis=1))

y_train, X_train = separate(train)
y_validate, X_validate = separate(validate)
y_test, X_test = separate(test)

y_full, X_full = separate(df)

# ##### Создаем, модель NN. Подбираем для нее параметры

# In[23]:

rfc = RandomForestClassifier(random_state=0)

parameters = {
    'n_estimators': list(range(100,501,100)),
    'max_depth': list(range(2,10)),

}

cvgrid = GridSearchCV(estimator=rfc, param_grid=parameters,n_jobs=-1,verbose=3)

# In[24]:

cvgrid.fit(X_train,y_train)

# ## Выбираем модель с лучшим показателем

# In[25]:

cvgrid.best_estimator_

# ## Посмотрим процент "точных" предсказаний модели для разного кол-ва эпох

# In[26]:

rfc_model = RandomForestClassifier(max_depth=2, n_estimators=200, random_state=0)
np.random.seed(0)

N_TRAIN_SAMPLES = X_train.shape[0]
N_EPOCHS = 3 # 12 # 4
N_BATCH = 128
N_CLASSES = np.unique(y_train)

scores_train = []
scores_test = []

# EPOCH
epoch = 0
while epoch < N_EPOCHS:
    print('epoch: ', epoch+1)
    # SHUFFLING
    random_perm = np.random.permutation(X_train.shape[0])
    mini_batch_index = 0
    while True:
        # MINI-BATCH
        indices = random_perm[mini_batch_index:mini_batch_index + N_BATCH]
        rfc_model.fit(X_train.iloc[indices], y_train.iloc[indices])
        mini_batch_index += N_BATCH

        if mini_batch_index >= N_TRAIN_SAMPLES:
            break

    # SCORE TRAIN
    scores_train.append(rfc_model.score(X_train, y_train))

    # SCORE TEST
    scores_test.append(rfc_model.score(X_test, y_test))

    epoch += 1
    pass

print('Epochs:', epoch)
fig, ax = plt.subplots(2, sharex=True, sharey=True)
ax[0].plot(scores_train)
ax[0].set_title('Train')
ax[1].plot(scores_test)
ax[1].set_title('Test')
fig.suptitle("Accuracy over epochs", fontsize=14)
plt.show()

m=max(scores_test)
print('N:',scores_test.index(m)+1, m)

# In[27]:

plt.title("Train")
sns.barplot(x=[i+1 for i in range(len(scores_train))],y=scores_train)

# In[28]:

plt.title("Test")
sns.barplot(x=[i+1 for i in range(len(scores_test))],y=scores_test)

# ## Рассмотрим структурные единицы древа
#

# In[29]:

from sklearn import tree
plt.figure(figsize=(20,10))
for i in range(2):
    plt.subplot(2, 1, i+1)  #subplot function: the number of rows are given as 5 and number of columns as 2, the value i+1 gives the subplot number, subplot numbers start with 1
    tree.plot_tree(cvgrid.best_estimator_[i])
plt.tight_layout()
plt.show()

#

#
Посмотрим на точность и Среднюю Абсолютную Ошибку

# In[30]:

predicted = rfc_model.predict(X_validate)
aim=list(y_validate)
c=0
for i in range(len(aim)):
    if aim[i]==predicted[i]:
        c+=1
print(f"Accuracy:\t{round(100*c/len(aim), 2)}%\nMAE:\t\t{round(100*mean_absolute_error(y_validate, predicted), 2)}%")

#

#
Экспортируем обученную модель с помощью модуля pickle

# In[3]:

model_fin = rfc_model # final model
with open("model.pckl", 'wb') as file:
    file.write(pickle.dumps(model_fin))

validate.to_csv('validate.csv')

# In[ ]:


# -*- coding: utf-8 -*-
"""
While24. n butun soni berilgan (n > 1). n sonini Fibonachchi sonlari orasida bor – yo’qligini
aniqlovchi programma tuzilsin. Fibonachchi sonlari quyidagi qonuniyat asosida topiladi.
F1 = 1; F2 = 1; Fk = Fk-1 + Fk-2; k = 3, 4, …

Created on Wed Jun 30 06:21:48 2021

@author: Mansurjon Kamolov
"""

n=int(input('n='))
if n>1:
    a=1
    b=1
    fib=1
    while a<n:
        a,b=a + b,a
    if n==a:
        print('Bor')
    else:
        print("Mavjud emas")
else:
    print('Masala shartiga mos son kiritilmadi! n>1 bo\'lishi kerak edi')


## author: changhao
##  date:  2018-11-13
##  info: 北控项目地图供应站位置分布

from pyecharts import Map, Geo

## 先定义一下数据

# 世界地图数据
value = [95.1, 23.2, 43.3, 66.4, 88.5]
attr = ["China", "Canada", "Brazil", "Russia", "United States"]

# 直辖市
province_distribution = {'河南': 45.23, '北京': 37.56, '河北': 21, '辽宁': 12, '江西': 6, '上海': 20, '安徽': 10, '江苏': 16, '湖南': 9, '浙江': 13, '海南': 2, '广东': 22, '湖北': 8, '黑龙江': 11, '澳门': 1, '陕西': 11, '四川': 7, '内蒙古': 3, '重庆': 3, '云南': 6, '贵州': 2, '吉林': 3, '山西': 12, '山东': 11, '福建': 4, '青海': 1, '舵主科技，质量保证': 1, '天津': 1, '其他': 1}
provice=list(province_distribution.keys())
values=list(province_distribution.values())

# 城市 -- 指定省的城市 xx市
city = ['海淀区', '通州区', '房山区', '延庆区', '密云区', '平谷区', '顺义区', '昌平区', '朝阳区', '石景山区', '大兴区', '东城区', '西城区']
values2 = ['13552134043', 3.85, 6.38, 8.21, 2.53, 4.37, 9.38, 4.29, 6.1, 1.1, 2.2, 3.3, 4.4]

# # 区县 -- 具体城市内的区县  xx县
# quxian = ['夏邑县', '民权县', '梁园区', '睢阳区', '柘城县', '宁陵县']
# values3 = [3, 5, 7, 8, 2, 4]

#河南地图 数据必须是省内放入城市名
map2 = Map("北京市液化石油气公司南郊灌瓶厂供应站分布情况", '北京地区', width=1200, height=600)
map2.add('联系方式：', city, values2, visual_range=[1, 20], maptype='北京', is_visualmap=True, visual_text_color='#000')
map2.show_config()
map2.render(path="04-01北京地图.html")

'''
option = {
    // 加载 bmap 组件
    bmap: {
        // 百度地图中心经纬度
        center: [120.13066322374, 30.240018034923],
        // 百度地图缩放
        zoom: 14,
        // 是否开启拖拽缩放，可以只设置 'scale' 或者 'move'
        roam: true,
        // 百度地图的自定义样式，见 http://developer.baidu.com/map/jsdevelop-11.htm
        mapStyle: {}
    },
    series: [{
        type: 'scatter',
        // 使用百度地图坐标系
        coordinateSystem: 'bmap',
        // 数据格式跟在 geo 坐标系上一样，每一项都是 [经度，纬度，数值大小，其它维度...]
        data: [ [120, 30, 1] ]
    }]
}

// 获取百度地图实例，使用百度地图自带的控件
var bmap = chart.getModel().getComponent('bmap').getBMap();
bmap.addControl(new BMap.MapTypeControl());

'''


#coding:utf-8
'''
'''
def sort_num():
    while True:
        num_str = input("请输入多个数字,并以','进行分隔:")
        num_list = num_str.split(',')
        right = True
        for a in range(0,len(num_list)):
            if num_list[a].isdigit():
                num_list[a] = int(num_list[a])
            else:
                right = False
        if right:
            break
        else:
            print("输入有误,请重新输入")
    num_len = len(num_list)
    for num_counter in range(1,num_len):
        key = num_list[num_counter]
        j = num_counter - 1
        while j > -1 and key < num_list[j]:
            num_list[j+1] = num_list[j]
            j -= 1
        num_list[j+1] = key
    print("排序结果:%s" % num_list)

def sort_str():
    import string
    key_dict = string.ascii_letters + string.digits +string.punctuation +string.whitespace
    sort_str = input("请输入字符串:")
    sort = list()
    for i in sort_str:
        sort.append(i)
    length = len(sort)
    for list_counter in range(0, length - 1):
        flag = True
        int_sorter = 0
        while int_sorter <= length - 2:
            if int_sorter >= length - list_counter - 1:
                break
            if key_dict.find(sort[int_sorter]) > key_dict.find(sort[int_sorter + 1]):
                sort[int_sorter], sort[int_sorter + 1] = sort[int_sorter + 1], sort[int_sorter]
                flag = False
            int_sorter += 1
        if flag is True:
            break
    contect = ''
    print("排序结果:", end='')
    print(contect.join(sort))

if __name__ == "__main__":
    print("习题1：输入数字列并从小到大排序")
    sort_num()
    print("习题2:输入字符串并按a-z字典序排列")
    sort_str()


from django.urls import path

from .views import EmailView, ThankYouView

app_name = 'contact'

urlpatterns = [
    path('email/', EmailView.as_view(), name="contact_email"),
    path('thank_you/', ThankYouView.as_view(), name="thank_you"),
]


import requests,json
import pandas as pd

def hot10jqka():
    headers = {
        'Accept': 'application/json',
        'Origin': 'https://eq.10jqka.com.cn',
        'Accept-Encoding': 'gzip, deflate, br',
        'Host': 'ai.iwencai.com',
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.2 Safari/605.1.15',
        'Accept-Language': 'zh-cn',
        'Referer': 'https://eq.10jqka.com.cn/',
        'Connection': 'keep-alive',
    }

    params = (
        ('tag', '同花顺热榜_每小时热股_新版'),
        ('userid', '0'),
        ('appName', 'thsHotList'),
        ('hexin-v', 'A7SRabINadOsnv0flt5ctI-qg3Anjdh3GrFsu04VQD_CuV6ndp2oB2rBPEmd'),
    )
    response = requests.get('https://ai.iwencai.com/index/urp/getdata/basic', headers=headers, params=params)
    rj=pd.DataFrame(json.loads(response.text)['answer']['components'][1]['data']['datas'])
    return rj

if __name__ == "__main__":
    rj=hot10jqka()
    concepts=dict()
    exclude=['机构重仓','上海金改', '央视财经50', '融资融券', '深股通', '沪股通', '转融券标的', '富时罗素概念', '富时罗素概念股', 'MSCI概念', '标普道琼斯A股', '同花顺漂亮100','半年报预增']
    for k,v in rj.iterrows():
        cpts=v['所属概念'].split(';')
        for cpt in cpts :
            if cpt in exclude:
                continue
            if cpt not in concepts.keys():
                concepts[cpt]={'概念':cpt,'个股':[],'个股数':0,'热度累加':0,'平均热度':0}
            concepts[cpt]['个股'].append(v['股票名称'])
            concepts[cpt]['个股数']=len(concepts[cpt]['个股'])
            concepts[cpt]['热度累加']+=int(v['个股热度'])
            concepts[cpt]['平均热度']=int(concepts[cpt]['热度累加']/concepts[cpt]['个股数'])
    df=pd.DataFrame(data=concepts.values())
    df=df[df['个股数']>=7]
    print(df['平均热度'].mean())
    df.sort_values(by=['平均热度'],ascending=False,inplace=True)
    df.to_csv('hotConcept10jqka.csv',index=False)


from django.shortcuts import redirect, render
from .models import Shortcut

def index(request):
    """Main view for search app"""
    query = (request.POST.get('q') or request.GET.get('q', '')).strip()
    short = (request.POST.get('short') or request.GET.get('short', '')).strip()
    shortcut = None

    # Split query if no shortcut is to selected
    if query and not short:
        q = query.split(' ', 1)
        short = q[0]
        query = q[1] if len(q) >= 2 else ''

    # Find shortcut
    if short:
        try:
            shortcut = Shortcut.objects.get(short=short)
            url = shortcut.get_url(query)
            return redirect(url)
        except Shortcut.DoesNotExist:
            query = (short + ' ' + query) if query else short
            short = ''

    return render(request, 'search.html', {
        'q': query,
        'short': short,
        'shortcuts': Shortcut.objects.order_by('short')})