#!/usr/bin/env python3

def __csv_to_sqlite(db_path, table_name, csv_file):
    import csv

    buff = []
    with open(csv_file, 'r') as f:
        for row in csv.reader(f):
            buff.append(row)

    __new_table(db_path, table_name, buff)

def __new_table(db_path, table_name, buff):
    import sqlite3

    conn = sqlite3.connect(db_path)
    cur = conn.cursor()
    cur.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'")
    if cur.fetchone():
        print(f'Table {table_name} exists')
    else:
        header = buff[0]
        sample = buff[1]

        column_types = ''
        for pair in zip(header, sample):
            col_type = __detect_type(pair[1])
            if col_type == 'NULL':
                col_type = 'TEXT'
            column_types += f'{pair[0]} {col_type},'
        column_types = column_types[:-1]
        creation_sql = f'CREATE TABLE {table_name}({column_types})'

        print(creation_sql)

        conn.execute(creation_sql)
        conn.commit()

        column_values = ''
        values = ''
        for col in header:
            column_values += f'{col},'
            values += '?,'

        column_values = column_values[:-1]
        values = values[:-1]
        insertion_sql = f'INSERT INTO {table_name}({column_values}) VALUES({values})'

        print(insertion_sql)

        for row in buff[1:]:
            conn.execute(insertion_sql, [__parse(value.strip()) for value in row])
        conn.commit()

    conn.close()

def __detect_type(sample):
    import re

    if re.match('^(?!00)[-+]?\d+$', sample.strip()):
        return 'INTEGER'
    elif re.match('^(?!00)[-+]?\d+\.\d+$', sample.strip()):
        return 'REAL'
    elif not sample.strip():
        return 'NULL'
    else:
        return 'TEXT'

def __parse(raw_data):
    data_type = __detect_type(raw_data)

    if data_type == 'INTEGER':
        return int(raw_data)
    elif data_type == 'REAL':
        return float(raw_data)
    elif data_type == 'NULL':
        return None
    else:
        return raw_data

if __name__ == '__main__':
    from sys import argv, exit

    argv.pop(0)

    file = None
    table = None
    database = None

    while argv:
        opt = argv.pop(0)

        if opt == '-f':
            file = argv.pop(0)
        elif opt == '-t':
            table = argv.pop(0)
        elif opt == '-d':
            database = argv.pop(0)

    for k, v in {'file': file, 'table': table, 'database': database}.items():
        if not v:
            print(f'{k} is missing\n'
                  '-f | set the path to csv file\n'
                  '-t | set the table name\n'
                  '-d | set the path to database file\n')
            exit(1)

    __csv_to_sqlite(database, table, file)


import numpy as np
import pandas as pd
from copy import copy
from sklearn.base import BaseEstimator, ClassifierMixin

class RSAClassifier(BaseEstimator, ClassifierMixin):
    def __init__(self, subject_dir, mask, item_labels, category_labels=None):
        self.metric = 'item'
        self._estimator_type = "regressor"
        self.subject_dir = subject_dir
        self.mask = mask
        self.item_labels = item_labels
        self.category_labels = category_labels

        if category_labels is not None:
            self.metric = 'category'

    def _get_item_category(self, item_id):
        return self.category_labels[self.item_labels==item_id]

    def fit(self, X, y=None):
        # X = neural data
        # y = object_id labels

        # create an "empty" localizer, fill it with input data
        from localiza import Localiza
        localizer = Localiza(self.subject_dir, self.mask)
        localizer.imgs = X
        localizer.features  = None
        localizer.nfeatures = X.shape[1]
        localizer.nframes   = X.shape[0]
        localizer.beta_labels = {}
        #localizer.label = {}

        # fill object_id field with labels
        localizer.beta_labels['object_id'] = np.array(y)
        #localizer.label['object_id'] = np.array(y)

        # create object_id set
        y_ids = set(y)
        localizer.object_ids = y_ids

        # item-item correlations
        corr = localizer.compare_objects_betas()
        #corr = localizer.compare_objects()

        #print "using", self.metric, "metric"
        if self.metric=='item':
            # lower the better!
            corr[corr==1] = np.nan
            scores = corr.mean()

        elif self.metric=='category':
            # column categories
            categories = np.array([self._get_item_category(item)[0] for item in corr.columns])
            #print "categories=", categories

            # within/between-category separation
            scores = {}
            for cat in set(list(categories)):
                is_cat = categories==cat
                within = corr.loc[is_cat,  is_cat]
                between= corr.loc[~is_cat, is_cat]
                # remove identity
                within[within==1] = np.nan
                # higher the better!
                scores[cat] = np.square(np.exp(within.median())-np.exp(between.median()))

            scores = pd.DataFrame(scores).transpose().mean().values
            #print "shape(scores)=", scores.shape

        # update estimator
        self.scores_ = scores

        return self

    def predict(self, X, y=None):
        try:
            getattr(self, "scores_")
        except AttributeError:
            raise RuntimeError("You must train classifer before predicting data!")

        return self.scores_

    def score(self, X, y=None):
        return np.mean(self.predict(X))


class Solution:
    """
    @param array: an integer array
    @return: nothing
    """

    def pancakeSort(self, array):
        n = len(array)

        for i in range(n - 1, 0, -1):
            Max = 0
            for j in range(1, i + 1):
                if array[j] > array[Max]:
                    Max = j
            if Max != 0 and Max != i:
                FlipTool.flip(array, Max)
                FlipTool.flip(array, i)

            if Max == 0:
                FlipTool.flip(array, i)

class Solution2:
    def pancakeSort(self, arr: List[int]) -> List[int]:
        res = []
        n = len(arr)
        for i in range(n - 1, 0, -1):
            max_i = 0
            for j in range(1, i + 1):
                if arr[j] > arr[max_i]:
                    max_i = j
            if max_i != 0 and max_i != i:
                res.extend([max_i + 1, i + 1])
                arr[:max_i+1] = reversed(arr[:max_i+1])
                arr[:i+1] = reversed(arr[:i+1])
            elif max_i == 0:
                res.append(i + 1)
                arr[:i+1]= reversed(arr[:i+1])

        return res


import sys
import os
import functools

class BlocGenerator:
	def __init__(self, arguments):
		self.varName = functools.reduce(lambda a, b : a+b.capitalize(), arguments[1:], arguments[0])
		self.className = functools.reduce(lambda a, b : a+b.capitalize(), arguments, "")
		self.underName = str.join("_", arguments)
		self.folder = os.getcwd() + "/lib/"
		if not os.path.isdir(self.folder):
			os.mkdir(self.folder)

	def create_file(self, addition, subFolder, text):
		if not os.path.isdir(self.folder + subFolder):
			os.mkdir(self.folder + subFolder)
		fileName = os.path.join(self.folder + subFolder, self.underName + "_" + addition + ".dart")
		print("Created: " + fileName)
		with open(fileName, "w+") as f:
			f.write(text)

	def create_event(self):
		code = """import 'package:equatable/equatable.dart';
import 'package:meta/meta.dart';

abstract class {0}Event extends Equatable {{
  const {0}Event();

  @override
  List<Object> get props => [];

}}
			""".format(self.className)
		self.create_file("event", "application/bloc/{0}/".format(self.underName), code)

	def create_state(self):
		code = """import 'package:equatable/equatable.dart';
import 'package:meta/meta.dart';

abstract class {0}State extends Equatable {{
  const {0}State();

  @override
  List<Object> get props => [];

}}
			""".format(self.className)
		self.create_file("state", "application/bloc/{0}/".format(self.underName), code)

	def create_bloc(self):
		code = """import 'package:flutter/material.dart';
import '{0}_event.dart';
import '{0}_state.dart';
import 'package:bloc/bloc.dart';

class {1}Bloc extends Bloc<{1}Event, {1}State>{{

  {1}State get initialState => null;

  @override
  Stream<{1}State> mapEventToState({1}Event event) async* {{
  }}

}}
			""".format(self.underName, self.className)
		self.create_file("bloc", "application/bloc/{0}/".format(self.underName), code)

	def create_assembly(self):
		code = """import 'package:dioc/src/container.dart';
import 'package:flutter_bloc/flutter_bloc.dart';
import 'package:memnder/application/assembly/assembly.dart';
import 'package:memnder/application/bloc/{0}/{0}_bloc.dart';
import 'package:memnder/application/bloc/{0}/{0}_event.dart';
import 'package:memnder/application/bloc/{0}/{0}_state.dart';
import 'package:memnder/application/view/{0}/{0}_view.dart';
import 'package:memnder/application/extension/dioc//dioc_widget.dart';

class {1}Assembly extends ModuleAssembly<{1}View>{{

  Bloc<{1}Event, {1}State> _{2}Bloc;

  @override
  void assemble(Container container) {{
    container.register<Bloc<{1}Event, {1}State>>((c){{
      return {1}Bloc(
      );
    }});

    container.registerBuilder<{1}View>((context, c){{
      _{2}Bloc?.close();
      return {1}View(
        bloc: _{2}Bloc = c.create(),
      );
    }});
  }}

  @override
  void unload(Container container) {{
    _{2}Bloc.close();
  }}

}}
			""".format(self.underName, self.className, self.varName)
		self.create_file("assembly", "application/assembly/module/", code)

	def create_view(self):
		code = """import 'package:flutter/material.dart';
import 'package:flutter_bloc/flutter_bloc.dart';
import 'package:memnder/application/bloc/{0}/{0}_event.dart';
import 'package:memnder/application/bloc/{0}/{0}_state.dart';

class {1}View extends StatefulWidget{{

  final Bloc<{1}Event, {1}State> bloc;

  const {1}View(
    {{
      @required this.bloc
    }}
  );

  @override
  State<StatefulWidget> createState() => _{1}ViewState();

}}

class _{1}ViewState extends State<{1}View>{{

  @override
  Widget build(BuildContext context) {{
    return Container();
  }}

}}
			""".format(self.underName, self.className, self.varName)
		self.create_file("view", "application/view/{0}/".format(self.underName), code)

def main():
	arguments = sys.argv[1:]
	generator = BlocGenerator(arguments)
	generator.create_event()
	generator.create_state()
	generator.create_bloc()
	generator.create_assembly()
	generator.create_view()

	print("Ты сэкономил кучу времени=)))")

if __name__ == '__main__':
    main()


from frontend.models import (
    Social, Phone, Email,
)

def frontend_processor(request):
    social = Social.objects.all()[:3]
    phones = Phone.objects.all()[:2]
    email = Email.objects.all().first()
    return {
        'frontend': {
            'social': social,
            'phones': phones,
            'email': email,
        }
    }


import cv2
detector_wajah = cv2.CascadeClassifier(
    "../model/haarcascade_frontalface_default.xml")
cap = cv2.VideoCapture(0)
while True:
    image = cap.read()
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    bounding_box = detector_wajah.detectMultiScale(gray, scalefactor=1.01,
                                                   inNeighbors=5, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE)
    for (x, y, w, h) in bounding_box:
        # x,y koordinat awal box
        # w, h ukuran box lebar & tinggi
        cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
        # grayb = cv2.cytColor(box_wajah, cv2.COLOR_BGR2GRAY)
        # end = x
        # start = y
    cv2.imshow("hasil", image)
    cv2.waitKey(0)
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
# @Author: sijanonly
# @Date:   2018-03-15 12:59:25
# @Last Modified time: 2018-07-27 14:26:14

import json

from urllib import parse

import requests

from .handle_exceptions import CustomException

class SolutionByTextHTTPInterface(object):
    """
    Base class providing interface for actual requests to url.
    """

    def __init__(self, base_url=None, service_end_point=None):
        self.base_url = base_url
        self.service_end_point = service_end_point

    def get_full_path(self):
        """
        Returns full url for service
        """
        return "%s%s" % (self.base_url, self.service_end_point)

    def _post(self, params):
        headers = {'content-type': 'application/json'}
        absolute_url = self.get_full_path()
        return requests.post(absolute_url, data=json.dumps(params), headers=headers)

    def _get(self, params=None):
        absolute_url = self.get_full_path()
        return requests.get(absolute_url, params=params)

class SolutionByTextAPI(SolutionByTextHTTPInterface):
    """
    Base for SolutionByText API
    """

    __base_url = 'https://{}.solutionsbytext.com/SBT.App.SetUp/RSServices/'

    def __init__(self, security_token, org_code, stage='test'):
        if stage not in ['test', 'ui']:
            raise CustomException('Stages must be either "test" or "ui"')
        # self.api_key = api_key
        self.__org_code = org_code
        self.__stage = stage
        self.__security_token = security_token

    @property
    def stage(self):
        """
        """
        return self.__stage

    @property
    def base_url(self):
        """
        Returns API base url
        """
        return self.__base_url.format(self.stage)

    @property
    def security_token(self):
        """
        """

        return self.__security_token

    @property
    def org_code(self):
        """
        """

        return self.__org_code

    def get(self, **kwargs):
        """
        Calls BASE _get method
        """
        payload = {
            "securityToken": self.security_token,
            "orgCode": self.org_code,

        }
        payload.update(**kwargs)
        return self._get(
            params=payload
        )

    def post(self, **kwargs):
        """
        Calls BASE _post method
        """
        payload = {
            "securityToken": self.security_token,
            "orgCode": self.org_code,

        }
        payload.update(**kwargs)
        return self._post(
            params=payload
        )


import sys
import os
sys.path.append(os.path.abspath('../../stratipy_cluster'))
from stratipy import load_data, formatting_data
# GO Enrich
import pandas as pd
import goenrich
import goenrich.tools as tools
import numpy as np
import scipy.stats as st
from tqdm import trange, tqdm

def ratio(x, y):
    try:
        return x/y
    except ZeroDivisionError:
        return 0

def get_enrichment_p_count(O, interesting, min_category_size,
                           max_category_size, max_category_depth):
    query = set(interesting['GeneID'])
    options = {'min_category_size': min_category_size,
               'max_category_size': max_category_size,
               'max_category_depth': max_category_depth}
    enrichment = goenrich.enrich.analyze(O,
                                     query,
                                     'reference',
                                     **options).dropna().sort_values(by='q')

    count_bio = enrichment[
        enrichment['namespace'] == 'biological_process'].shape[0]
    count_mol = enrichment[
        enrichment['namespace'] == 'molecular_function'].shape[0]
    count_cel = enrichment[
        enrichment['namespace'] == 'cellular_component'].shape[0]

    enrichment = enrichment[enrichment['p'] < 0.05]

    p_count_bio = enrichment[
        enrichment['namespace'] == 'biological_process'].shape[0]
    p_count_mol = enrichment[
        enrichment['namespace'] == 'molecular_function'].shape[0]
    p_count_cel = enrichment[
        enrichment['namespace'] == 'cellular_component'].shape[0]

    ratio_bio = ratio(p_count_bio, count_bio)
    ratio_mol = ratio(p_count_mol, count_mol)
    ratio_cel = ratio(p_count_cel, count_cel)

    enrichment['LOD'] = (enrichment.x / enrichment.n) / \
                    (enrichment.N / enrichment.M - enrichment.N)
    enrichment['Z'] = st.norm.ppf(enrichment.p)
    enrichment['Score'] = -np.log(enrichment.p)
    df_enrich = enrichment[['x', 'n', 'namespace', 'name', 'p', 'Score', 'Z', 'LOD']]

    return (df_enrich, p_count_bio, p_count_mol, p_count_cel, ratio_bio,
            ratio_mol, ratio_cel)

def merge_p_count(data_folder, ssc_mutation_data, ssc_subgroups, gene_data,
                  ppi_data, n_components, mut_type, alpha, ngh_max,
                  n_permutations, lambd, O, min_category_size,
                  max_category_size, max_category_depth):
    p_count_bio_list = []
    p_count_mol_list = []
    p_count_cel_list = []
    ratio_bio_list = []
    ratio_mol_list = []
    ratio_cel_list = []

    for k in range(n_components):
        cluster = k+1
        file_directory = (data_folder +
                          'text/clusters_EntrezGene/{}_{}_{}_{}/k={}/'.
                          format(ssc_mutation_data, ssc_subgroups, gene_data,
                                 ppi_data, n_components))
        text_file = file_directory + (
            '{}_{}_ngh={}_permut={}_lambd={}'
            .format(mut_type, alpha, ngh_max, n_permutations, lambd))
        cluster_file = text_file + '_cluster={}.txt'.format(cluster)
        interesting = pd.read_table(cluster_file, names=['GeneID'])

        (df_enrich, p_count_bio, p_count_mol, p_count_cel,
         ratio_bio, ratio_mol, ratio_cel) = get_enrichment_p_count(
             O, interesting, min_category_size, max_category_size,
             max_category_depth)

        p_count_bio_list.append(p_count_bio)
        p_count_mol_list.append(p_count_mol)
        p_count_cel_list.append(p_count_cel)
        ratio_bio_list.append(ratio_bio)
        ratio_mol_list.append(ratio_mol)
        ratio_cel_list.append(ratio_cel)

#     print("---")
#     print("bio ", sum(p_count_bio_list), max(ratio_bio_list) )
#     print("mol ", sum(p_count_mol_list), max(ratio_mol_list) )
#     print("cel ", sum(p_count_cel_list), max(ratio_cel_list) )

    return(df_enrich,
           sum(p_count_bio_list), max(ratio_bio_list),
           sum(p_count_mol_list), max(ratio_mol_list),
           sum(p_count_cel_list), max(ratio_cel_list))

def p_df_by_subgroup(data_folder, ssc_mutation_data, ssc_subgroups, gene_data,
                     ppi_data, n_components, mut_type, alpha, ngh_max,
                     n_permutations, lambd, O, min_category_size,
                     max_category_size, max_category_depth):
    k = []
    p_bio_list = []
    p_mol_list = []
    p_cel_list = []
    r_bio_list = []
    r_mol_list = []
    r_cel_list = []

    df_enrich_list =[]

    for n_components in tqdm(range(2, 51), desc='k'):
        df_enrich, p_bio, r_bio, p_mol, r_mol, p_cel, r_cel = merge_p_count(
            data_folder, ssc_mutation_data, ssc_subgroups, gene_data, ppi_data,
            n_components, mut_type, alpha, ngh_max, n_permutations, lambd,
            O, min_category_size, max_category_size, max_category_depth)

        df_enrich['ind_group'] = ssc_subgroups
        df_enrich['k'] = n_components
        df_enrich_list.append(df_enrich)

        k.append(n_components)
        p_bio_list.append(p_bio)
        p_mol_list.append(p_mol)
        p_cel_list.append(p_cel)
        r_bio_list.append(r_bio)
        r_mol_list.append(r_mol)
        r_cel_list.append(r_cel)

    df_enrich_merged = pd.concat(df_enrich_list)

    df_pcount = pd.DataFrame({'k': k,
                              'p_bio': p_bio_list,
                              'r_bio': r_bio_list,
                              'p_mol': p_mol_list,
                              'r_mol': r_mol_list,
                              'p_cel': p_cel_list,
                              'r_cel': r_cel_list})
    return df_enrich_merged, df_pcount

def merge_2_subgroups(data_folder, ssc_mutation_data, ssc_subgroups, gene_data,
                      ppi_data, n_components, mut_type, alpha, ngh_max,
                      n_permutations, lambd, O, gene2go, min_category_size,
                      max_category_size, max_category_depth):
    ################ SSC 1 ################
    ssc_subgroups = 'SSC1'
    print(ssc_subgroups)
    mutation_profile, gene_id_patient, patient_id = (
    load_data.load_specific_SSC_mutation_profile(
        data_folder, ssc_mutation_data, ssc_subgroups, gene_data))

    gene_id_ppi, network = load_data.load_PPI_network(data_folder, ppi_data)

    network, mutation_profile, idx_ppi, idx_mut, idx_ppi_only, idx_mut_only = (
        formatting_data.classify_gene_index(
             network, mutation_profile, gene_id_ppi, gene_id_patient))

    # idx_mut : List of common genes' indexes in individuals' mutation profiles.
    reference = pd.DataFrame({'GeneID': [gene_id_patient[i] for i in idx_mut]})
    background = tools.generate_background(
        gene2go, reference, 'GO_ID', 'GeneID')
    goenrich.enrich.propagate(O, background, 'reference')

    df_enrich1, df_pcount1 = p_df_by_subgroup(
        data_folder, ssc_mutation_data, ssc_subgroups, gene_data, ppi_data,
        n_components, mut_type, alpha, ngh_max, n_permutations, lambd, O,
        min_category_size, max_category_size, max_category_depth)

    ################ SSC 2 ################
    ssc_subgroups = 'SSC2'
    print(ssc_subgroups)
    mutation_profile, gene_id_patient, patient_id = (
    load_data.load_specific_SSC_mutation_profile(
        data_folder, ssc_mutation_data, ssc_subgroups, gene_data))

    gene_id_ppi, network = load_data.load_PPI_network(data_folder, ppi_data)

    network, mutation_profile, idx_ppi, idx_mut, idx_ppi_only, idx_mut_only = (
        formatting_data.classify_gene_index(
             network, mutation_profile, gene_id_ppi, gene_id_patient))

    # idx_mut : List of common genes' indexes in individuals' mutation profiles.
    reference = pd.DataFrame({'GeneID': [gene_id_patient[i] for i in idx_mut]})
    background = tools.generate_background(
        gene2go, reference, 'GO_ID', 'GeneID')
    goenrich.enrich.propagate(O, background, 'reference')

    df_enrich2, df_pcount2 = p_df_by_subgroup(
        data_folder, ssc_mutation_data, ssc_subgroups, gene_data, ppi_data,
        n_components, mut_type, alpha, ngh_max, n_permutations, lambd, O,
        min_category_size, max_category_size, max_category_depth)

    ################ merge ################
    df_enrich = pd.concat([df_enrich1, df_enrich2], ignore_index=True)
    # change col order
    cols = list(df_enrich)
    cols.insert(0, cols.pop(cols.index('k')))
    cols.insert(0, cols.pop(cols.index('ind_group')))
    df_enrich = df_enrich.loc[:, cols]

    df_pcount = df_pcount1.merge(df_pcount2, how='inner', left_on='k',
                                 right_on='k', suffixes=[1, 2])

    directory = (
        data_folder + 'result_biostat_genes_GOslim/' + ssc_mutation_data +
        '_' + gene_data + '/')
    os.makedirs(directory, exist_ok=True)
    file_name_enrich = 'GOslim_{}_lambd={}_{}.pkl'.format(
        mut_type, lambd, ppi_data)
    file_name_pcount = 'p_count_{}_lambd={}_{}.pkl'.format(
        mut_type, lambd, ppi_data)
    # save
    df_enrich.to_pickle('{}{}'.format(directory, file_name_enrich))
    df_pcount.to_pickle('{}{}'.format(directory, file_name_pcount))

def biostat_go_enrichment(
    alpha, result_folder, mut_type, patient_data, data_folder,
    ssc_mutation_data, ssc_subgroups, gene_data, ppi_data, lambd, n_components,
    ngh_max, n_permutations):

    O = goenrich.obo.ontology(data_folder + 'goslim_generic.obo')
    gene2go = goenrich.read.gene2go(data_folder + 'gene2go.gz')
    min_category_size = 2
    # max_category_size = 10000
    max_category_size = 500
    max_category_depth = 100000

    merge_2_subgroups(
        data_folder, ssc_mutation_data, ssc_subgroups, gene_data, ppi_data,
        n_components, mut_type, alpha, ngh_max, n_permutations, lambd, O,
        gene2go, min_category_size, max_category_size, max_category_depth)


import time
import socket
import json

class Client:
    curent_time = time.time()
    def __init__(self, host, port, timeout=None):
        self.host = host
        self.port = port
        self.timeout = timeout

    def get(self, key):
        try:
            with socket.create_connection((self.host, self.port),self.timeout) as sock:
                message = 'get {}\n'.format(key)
                sock.sendall(message.encode("utf8"))
                received_data = sock.recv(1024)
                return self._parser(received_data)
        except socket.error:
            raise ClientError("error\nwrong command\n\n")

    def _parser(self, inner_message):
        """Convert inner bite string to dictionary"""
        inner = inner_message.decode("utf8")
        new_dict = dict()
        try:
            if inner == 'ok\n\n':
                return new_dict
            else:
                data_sring = inner.replace("ok\n","").replace("\n\n","").splitlines()
                data_list = []
                for line in data_sring:
                    splitted_line = line.lstrip().split()
                    data_list.append(splitted_line)
                for string in data_list:
                    val = (int(string[2]), float(string[1]),)
                    if string[0] in new_dict:
                        new_dict[string[0]].append(val)
                    else:
                        new_dict[string[0]] = [val]
                return new_dict
        except IndexError:
            raise ClientError("error\nwrong command\n\n")

    def put(self, key, value, timestamp=curent_time):
        try:
            with socket.create_connection((self.host, self.port),self.timeout) as sock:
                message = 'put {} {} {}\n'.format(key, value, int(timestamp))
                sock.sendall(message.encode("utf8"))
                received_data = sock.recv(1024)
                return received_data.decode("utf8")
        except socket.error:
            raise ClientError("error\nwrong command\n\n")

class ClientError(Exception):
    pass

# client = Client("127.0.0.1", 8888, timeout=150)
# print(client.put("palm.cpu", 0.5, timestamp=1150864247))


#!/usr/bin/env python3
import pytrellis
import tiles
import nets
import database
import heapq

# Simple automatic router
class Autorouter:
    def __init__(self, chip):
        self.chip = chip
        self.chip_size = (self.chip.get_max_row(), self.chip.get_max_col())
        self.row_bias = self.chip.info.row_bias
        self.col_bias = self.chip.info.col_bias
        self.dh_arc_cache = {}
        self.net_to_wire = {}
        self.wire_to_net = {}

    # Get arcs downhill of a wire
    # Returns list [(dest, configurable, loc), ...]
    def get_arcs_downhill(self, wire):
        if wire in self.dh_arc_cache:
            return self.dh_arc_cache[wire]
        else:
            drivers = []
            chip_size = (self.chip.get_max_row(), self.chip.get_max_col())
            try:
                npos = tiles.pos_from_name(wire, chip_size, self.row_bias, self.col_bias)
            except AssertionError:
                return []
            wname = wire.split("_", 1)[1]
            hspan = 0
            vspan = 0
            if wname.startswith("H") and wname[1:3].isdigit():
                hspan = int(wname[1:3])
            if wname.startswith("V") and wname[1:3].isdigit():
                vspan = int(wname[1:3])
            positions = {(npos[0], npos[1]), (npos[0] + vspan, npos[1]), (npos[0] - vspan, npos[1]),
                         (npos[0], npos[1] + hspan), (npos[0], npos[1] - hspan)}
            for pos in positions:
                for tile in self.chip.get_tiles_by_position(pos[0], pos[1]):
                    tinf = tile.info
                    tname = tinf.name
                    if tname.startswith("TAP"):
                        continue
                    pos = tiles.pos_from_name(tname, chip_size, self.row_bias, self.col_bias)

                    if abs(pos[0] - npos[0]) not in (vspan, 0) or abs(pos[1] - npos[1]) not in (hspan, 0):
                        continue
                    if wire.startswith("G_"):
                        twire = wire
                    else:
                        twire = nets.normalise_name(self.chip_size, tname, wire, self.chip.info.family)

                    tdb = pytrellis.get_tile_bitdata(
                        pytrellis.TileLocator(self.chip.info.family, self.chip.info.name, tinf.type))
                    downhill = tdb.get_downhill_wires(twire)
                    for sink in downhill:
                        nn = nets.canonicalise_name(self.chip_size, tname, sink.first, self.row_bias, self.col_bias)
                        if nn is not None:
                            drivers.append((nn, sink.second, tname))
            self.dh_arc_cache[wire] = drivers
            return drivers

    # Enable an Arc
    def bind_arc(self, net, uphill_wire, arc, config):
        dest_wire, configurable, tile = arc
        assert (dest_wire not in self.wire_to_net) or (self.wire_to_net[dest_wire] == net)
        self.wire_to_net[dest_wire] = net
        exists = False
        if net in self.net_to_wire:
            exists = dest_wire in self.net_to_wire[net]
            self.net_to_wire[net].add(dest_wire)
        else:
            self.net_to_wire[net] = {dest_wire}
        if configurable and not exists:
            src_wirename = nets.normalise_name(self.chip_size, tile, uphill_wire, self.chip.info.family)
            sink_wirename = nets.normalise_name(self.chip_size, tile, dest_wire, self.chip.info.family)
            config[tile].add_arc(sink_wirename, src_wirename)

    # Bind a net to a wire (used for port connections)
    def bind_net_to_port(self, net, port_wire):
        assert (port_wire not in self.wire_to_net) or (self.wire_to_net[port_wire] == net)
        self.wire_to_net[port_wire] = net
        if net in self.net_to_wire:
            self.net_to_wire[net].add(port_wire)
        else:
            self.net_to_wire[net] = {port_wire}

    # Route a net to a wire
    def route_net_to_wire(self, net, wire, config):
        print("     Routing net '{}' to wire/pin '{}'...".format(net, wire))
        chip_size = (self.chip.get_max_row(), self.chip.get_max_col())
        dest_pos = tiles.pos_from_name(wire, chip_size, self.row_bias, self.col_bias)
        def get_score(x_wire):
            pos = tiles.pos_from_name(x_wire, chip_size, self.row_bias, self.col_bias)
            score = abs(pos[0] - dest_pos[0]) + abs(pos[1] - dest_pos[1])
            x_wname = x_wire.split("_", 1)[1]
            if x_wname[1:3].isdigit() and score > 3:
                score -= int(x_wname[1:3])
            return score

        assert net in self.net_to_wire
        bfs_queue = [(get_score(x), x) for x in sorted(self.net_to_wire[net])]
        heapq.heapify(bfs_queue)

        seen_wires = set()
        routed = False
        backtrace = {}  # map wire -> (source, arc)

        while not routed and len(bfs_queue) > 0:
            score, curr_wire = heapq.heappop(bfs_queue)
            #print(curr_wire)
            arcs = self.get_arcs_downhill(curr_wire)
            for arc in arcs:
                dest, cfg, loc = arc
                if dest == wire:
                    backtrace[dest] = (curr_wire, arc)
                    routed = True
                    break
                elif dest not in seen_wires:
                    if dest in self.wire_to_net and self.wire_to_net[dest] != net:
                        continue
                    backtrace[dest] = (curr_wire, arc)
                    heapq.heappush(bfs_queue, (get_score(dest), dest))
                    seen_wires.add(dest)

        if routed:
            cursor = wire
            while cursor in backtrace:
                cursor, arc = backtrace[cursor]
                self.bind_arc(net, cursor, arc, config)
        else:
            assert False, "failed to route net {} to wire {}".format(net, wire)

def main():
    pytrellis.load_database(database.get_db_root())
    chip = pytrellis.Chip("LFE5U-45F")
    rt = Autorouter(chip)
    config = {_.info.name: pytrellis.TileConfig() for _ in chip.get_all_tiles()}
    rt.bind_net_to_port("x", "R15C10_Q0")
    rt.route_net_to_wire("x", "R15C50_A0", config)
    for tile, tcfg in sorted(config.items()):
        cfgtext = tcfg.to_string()
        if len(cfgtext.strip()) > 0:
            print(".tile {}".format(tile))
            print(cfgtext)

if __name__ == "__main__":
    main()


from bs4 import BeautifulSoup
import requests
import json
from task_1 import adventure
from task_4 import details_movie

movie_details = []
for i in adventure[:100]:
# for i in range(100):
    url=i["movie URL"]

    def details_movie(movie_url):

        page = requests.get(movie_url)

        soup = BeautifulSoup(page.text,'html.parser')

        movie_dic = {}
        name=soup.find("h1",class_="scoreboard__title").get_text()
        movie_dic.update({'name':name})

        title = soup.find_all('div',class_='meta-label subtle')

        value = soup.find_all('div',class_='meta-value')

        for i in range(len(title)):
            movie_dic[str(title[i].get_text().strip())[:-1]] = value[i].get_text().replace(" ","").strip().replace("\n","")
        movie_details.append(movie_dic)

        with open('task_5.json','w') as file:
            json.dump(movie_details,file,indent=4)
        return movie_details

    details_movie(url)


from flask import current_app as app
from utils.auth import AdminAuth
import copy
import routes.requests

# copy from users schema, but remove readonly
schema = copy.deepcopy(routes.requests.schema)
schema['candidates']['readonly'] = False
schema['publicRequestInviteId']['readonly'] = False
schema['pickedUp']['readonly'] = False
schema['isMutuallyCancelled']['readonly'] = False
schema['numberOfComments']['readonly'] = False

config = {
    'item_title': 'adminRequest',
    'public_methods': [],
    'public_item_methods': [],
    'datasource': {
        'source': 'requests'
    },
    #'extra_response_fields': ['username'],
    'item_methods': ['GET', 'PATCH', 'DELETE'],
    'pagination': False,
    'allowed_filters': ['*'],
    'resource_methods': ['GET', 'POST'],
    'schema': schema,
    'authentication': AdminAuth()
}

def init(app):
    ''' (LocalProxy) -> NoneType
    Adds this route's specific hooks to this route.
    '''

    app.on_insert_adminRequests += routes.requests.onInsert
    app.on_inserted_adminRequests += routes.requests.onInserted
    app.on_fetched_item_adminRequests += routes.requests.onFetchedItem
    app.on_fetched_resource_adminRequests += routes.requests.onFetched
    app.on_update_adminRequests += routes.requests.onUpdate
    app.on_updated_adminRequests += routes.requests.onUpdated


import os
os.remove("jupyterNotebook/filme.db") if os.path.exists("jupyterNotebook/filmes.db") else None

import sqlite3
conex = sqlite3.connect("jupyterNotebook/filmes.bd")
cur2 = conex.cursor()

def create_tabela():
    cur2.execute('CREATE TABLE IF NOT EXISTS filmes(id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, nome TEXT, diretor TEXT, estudio TEXT)')

def insert_datebase():
    cur2.execute("INSERT INTO filmes VALUES (1, 'Bond 25', 'Cary Fukunaga', 'Eon Productions, Metro-Goldwyn-Mayer')")
    conex.commit()
    cur2.close()
    conex.close()

create_tabela()
insert_datebase()

#Recuperando os dados
conex2 = sqlite3.connect("jupyterNotebook/filmes.bd")
cur3 = conex2.cursor()
select_tabela = 'select * from filmes'
cur3.execute(select_tabela)
dados_retorno = cur3.fetchall()

for i in dados_retorno:
    print(i)

cur3.close()
conex2.close()

#Alterando a func insert_datebase() para receber paramentros de entrada
def insert_datebase2(lista_filmes):

    insert_table = ("INSERT INTO filmes VALUES (?,?,?,?)")
    for i in lista_filmes:
        cur4.execute(insert_table,i)

    conex3.commit()
    cur4.close()
    conex3.close()

filmes = [(2, "A Quiet Place: Part II", "John Krasinski", "Platinum Dunes"),
        (3,"Kingsman: O Grande Jogo","Matthew Vaughn","Fox")]

conex3 = sqlite3.connect("jupyterNotebook/filmes.bd")
cur4 = conex3.cursor()

insert_datebase2(filmes)

#listar banco
def selet_datebase():

    select_tabela = 'select * from filmes'
    cur5.execute(select_tabela)
    dados_retorno = cur5.fetchall()

    for i in dados_retorno:
        print(i)

    conex4.commit()
    cur5.close()
    conex4.close()

conex4 = sqlite3.connect("jupyterNotebook/filmes.bd")
cur5 = conex4.cursor()
selet_datebase()


import base64
import json
import uuid
from io import BytesIO

import requests

import config
import queries
from database import execute_query
from message_queue import message_queue
from models import Event, RawImage, Domain
from object_storage import object_storage
from utils import get_datetime_for_db, create_image_relative_path, get_generation_args

def set_event(event: Event) -> None:
    execute_query(
        queries.SET_EVENT,
        {
            '$uuid': str(event.uuid),
            '$name': event.name,
            '$created_at': get_datetime_for_db(event.created_at)
        }
    )

def set_domain(domain: Domain) -> None:
    execute_query(
        queries.SET_DOMAIN,
        {
            '$uuid': str(domain.uuid),
            '$domain': domain.domain,
            '$created_at': get_datetime_for_db(domain.created_at)
        }
    )

def save_raw_image(raw_image: RawImage) -> str:
    event_uuid = get_current_event_uuid()
    image_relative_path = create_image_relative_path(raw_image, event_uuid)
    object_storage.upload_fileobj(
        Fileobj=BytesIO(raw_image.image_data),
        Bucket=config.OS_RAW_BUCKET_NAME,
        Key=image_relative_path
    )
    return image_relative_path

def create_generation_image_task(raw_image: RawImage, image_relative_path: str) -> None:
    generation_args = get_generation_args(raw_image, image_relative_path)
    message_queue.send_message(
        QueueUrl=config.MQ_QUERY_ENDPOINT_URL,
        MessageBody=json.dumps(generation_args)
    )

def get_raw_image_data(image_relative_path: str) -> bytes:
    response = object_storage.get_object(
        Bucket=config.OS_RAW_BUCKET_NAME,
        Key=image_relative_path
    )
    return response['Body'].read()

def generate_image(raw_image_data: bytes,
                   generation_args: dict[str, str | int | float]) -> bytes:
    domain = get_current_domain()
    raw_image_base64 = base64.b64encode(raw_image_data).decode()
    generation_args['init_images'] = [raw_image_base64]
    response = requests.post(domain, json=generation_args)
    return base64.b64decode(response.json()['images'][0])

def save_generated_image(generated_image_data: bytes, image_relative_path: str) -> None:
    object_storage.upload_fileobj(
        BytesIO(generated_image_data),
        config.OS_GENERATED_BUCKET_NAME,
        image_relative_path
    )

def get_current_event_uuid() -> uuid.UUID:
    response = execute_query(
        queries.GET_CURRENT_EVENT_UUID,
        {}
    )
    return uuid.UUID(response[0].rows[0]['uuid'].decode())

def get_current_domain() -> str:
    response = execute_query(
        queries.GET_CURRENT_DOMAIN,
        {}
    )
    domain = response[0].rows[0]['domain'].decode()
    return 'https://' + domain + config.IMAGE_GENERATION_RELATIVE_ENDPOINT


# Train stage
import pandas as pd
import xgboost as xgb
import yaml
import json
import pickle
from bayes_opt import BayesianOptimization
from sklearn.model_selection import train_test_split
import numpy as np
from sklearn.metrics import confusion_matrix, recall_score

def get_confusion_coefs(y_actual, y_pred):
    """  Returns the true positives rate & the true negatives rate for a set of y_real values and y_predicted values """
    cf = confusion_matrix(y_actual, y_pred)
    tnr = cf[0][0] / (cf[0][0]+cf[1][0])
    tpr = cf[1][1] / (cf[1][1]+cf[0][1])
    return tpr, tnr

def optimize_classifier(train, init_points, n_iter, cv, stratified, shuffle, num_boost_round, balanced,
                        scale_pos_weight) -> dict:
    """ Performs bayesian optimization over the most significant XGBoost parameters and returns the ones with
    the highest score """

    bounds = {'max_depth': (4, 10),
              'reg_alpha': (0.0, 1),
              'reg_lambda': (0.0, 1),
              'gamma': (0, 100),
              'min_child_weight': (1, 20),
              'max_delta_step': (0, 10),
              'subsample': (0.5, 1),
              'colsample_bytree': (0, 1),
              'learning_rate': (0.01, 0.3)
              }

    def get_score(max_depth, reg_alpha, reg_lambda, gamma, min_child_weight, max_delta_step,
                  subsample, colsample_bytree, learning_rate):
        params = {'booster': 'gbtree',
                  'objective': 'binary:logistic',
                  'eval_metric': 'aucpr',
                  'scale_pos_weight': scale_pos_weight,
                  'is_balanced': balanced,
                  'max_depth': int(max_depth),
                  'reg_alpha': reg_alpha,
                  'reg_lambda': reg_lambda,
                  'gamma': gamma,
                  'min_child_weight': int(min_child_weight),
                  'max_delta_step': max_delta_step,
                  'subsample': subsample,
                  'colsample_bytree': colsample_bytree,
                  'learning_rate': learning_rate,
                  'verbosity': 0,
                  'n_jobs': -1,
                  'use_label_encoder': False
                  }

        results = xgb.cv(params, train, num_boost_round=num_boost_round, nfold=cv, stratified=stratified,
                         shuffle=shuffle)
        return (results['train-aucpr-mean'].iloc[-1] * 0.4
                + results['test-aucpr-mean'].iloc[-1] * 0.6
                - abs(results['train-aucpr-mean'].iloc[-1] - results['test-aucpr-mean'].iloc[-1]) * 5)

    optimizer = BayesianOptimization(get_score, bounds)
    optimizer.probe({'max_depth': 6,
                     'reg_alpha': 0,
                     'reg_lambda': 1,
                     'gamma': 0,
                     'min_child_weight': 1,
                     'max_delta_step': 0,
                     'subsample': 1,
                     'colsample_bytree': 1,
                     'learning_rate': 0.3},
                    lazy=True)
    optimizer.maximize(init_points=init_points, n_iter=n_iter)
    best_params = {'booster': 'gbtree',
                   'objective': 'binary:logistic',
                   'is_balanced': balanced,
                   'eval_metric': 'aucpr',
                   'scale_pos_weight': scale_pos_weight,
                   'max_depth': int(optimizer.max['params']['max_depth']),
                   'reg_alpha': optimizer.max['params']['reg_alpha'],
                   'reg_lambda': optimizer.max['params']['reg_lambda'],
                   'gamma': optimizer.max['params']['gamma'],
                   'min_child_weight': int(optimizer.max['params']['min_child_weight']),
                   'max_delta_step': optimizer.max['params']['max_delta_step'],
                   'subsample': optimizer.max['params']['subsample'],
                   'colsample_bytree': optimizer.max['params']['colsample_bytree'],
                   'learning_rate': optimizer.max['params']['learning_rate'],
                   'verbosity': 0,
                   'n_jobs': -1,
                   'use_label_encoder': False
                   }
    return best_params

def main():
    params = yaml.safe_load(open('params.yaml'))['train']
    test_size = params['test_size']
    val_size = params['val_size']
    n_iter = params['n_iter']
    init_points = params['init_points']
    cv = params['cv']
    stratified = params['stratified']
    shuffle = params['shuffle']
    balanced = params['balanced']
    num_boost_round = params['num_boost_round']
    scale_pos_weight = params['scale_pos_weight']

    # Load data
    features = pd.read_pickle('data/features.pkl')
    labels = pd.read_pickle('data/labels.pkl')

    # Manage default parameters values
    if scale_pos_weight is None:
        scale_pos_weight = labels.value_counts()[0]/labels.value_counts()[1]

    # Split data in 3 sets
    x_train, x_test, y_train, y_test = train_test_split(features, labels, test_size=test_size, stratify=labels)
    x_test, x_val, y_test, y_val = train_test_split(x_test, y_test, test_size=val_size, stratify=y_test)

    train_data = xgb.DMatrix(x_train, label=y_train)
    test_data = xgb.DMatrix(x_test, label=y_test)
    val_data = xgb.DMatrix(x_val, label=y_val)

    # Look for the best parameters with bayesian optimization
    best_parameters = optimize_classifier(train_data, init_points, n_iter, cv, stratified, shuffle,
                                          num_boost_round, balanced, scale_pos_weight)

    # Train the model
    model = xgb.train(best_parameters, train_data, num_boost_round=num_boost_round,
                      evals=[(train_data, 'train'), (test_data, 'test')], verbose_eval=False)
    # Save Model
    with open('pickles/model.pkl', 'wb') as file:
        pickle.dump(model, file)

    # AUC
    train_score = model.eval(train_data)
    test_score = model.eval(test_data)
    val_score = model.eval(val_data)

    # Get test tpr, tnr & recall
    test_predictions = model.predict(test_data)
    test_predictions = np.round(test_predictions)
    test_tpr, test_tnr = get_confusion_coefs(y_test, test_predictions)
    test_recall = recall_score(y_test, test_predictions)

    # Get val tpr, tnr & recall
    val_predictions = model.predict(val_data)
    val_predictions = np.round(val_predictions)
    val_tpr, val_tnr = get_confusion_coefs(y_val, val_predictions)
    val_recall = recall_score(y_val, val_predictions)

    # Dump Metrics
    results = {
        'train-aucpr': float(train_score[train_score.find(':')+1:]),
        'test-aucpr': float(test_score[test_score.find(':')+1:]),
        'test-tpr': test_tpr,
        'test-tnr': test_tnr,
        'test-recall': test_recall,
        'val-aucpr': float(val_score[val_score.find(':')+1:]),
        'val-tpr': val_tpr,
        'val-tnr': val_tnr,
        'val-recall': val_recall
    }
    with open('pickles/metrics.json', 'w') as file:
        json.dump(results, file, indent=4)

if __name__ == '__main__':
    main()


import os
import warnings

import h5py
import numpy as np

from pynpoint.core.pypeline import Pypeline
from pynpoint.processing.darkflat import DarkCalibrationModule, FlatCalibrationModule
from pynpoint.util.tests import create_config, remove_test_data

warnings.simplefilter("always")

limit = 1e-10

class TestDarkAndFlatCalibration(object):

    def setup_class(self):

        self.test_dir = os.path.dirname(__file__) + "/"

        np.random.seed(1)

        images = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100))
        dark = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100))
        flat = np.random.normal(loc=0, scale=2e-4, size=(10, 100, 100))

        h5f = h5py.File(self.test_dir+"PynPoint_database.hdf5", "w")
        h5f.create_dataset("images", data=images)
        h5f.create_dataset("dark", data=dark)
        h5f.create_dataset("flat", data=flat)
        h5f.close()

        create_config(self.test_dir+"PynPoint_config.ini")

        self.pipeline = Pypeline(self.test_dir, self.test_dir, self.test_dir)

    def teardown_class(self):

        remove_test_data(self.test_dir)

    def test_dark_calibration(self):

        dark = DarkCalibrationModule(name_in="dark",
                                     image_in_tag="images",
                                     dark_in_tag="dark",
                                     image_out_tag="dark_cal")

        self.pipeline.add_module(dark)
        self.pipeline.run_module("dark")

        data = self.pipeline.get_data("dark")
        assert np.allclose(data[0, 10, 10], 3.528694163309295e-05, rtol=limit, atol=0.)
        assert np.allclose(np.mean(data), 7.368663496379876e-07, rtol=limit, atol=0.)
        assert data.shape == (10, 100, 100)

    def test_flat_calibration(self):

        flat = FlatCalibrationModule(name_in="flat",
                                     image_in_tag="dark_cal",
                                     flat_in_tag="flat",
                                     image_out_tag="flat_cal")

        self.pipeline.add_module(flat)
        self.pipeline.run_module("flat")

        data = self.pipeline.get_data("flat")
        assert np.allclose(data[0, 10, 10], -0.0004053528990466237, rtol=limit, atol=0.)
        assert np.allclose(np.mean(data), -4.056978234798532e-07, rtol=limit, atol=0.)
        assert data.shape == (10, 100, 100)


#!/usr/bin/env python3
"""
Purpose: Process a CSV file to combine data fields for unique key fields
Customize: Change QUOTE_CHAR, DATA_DELIMITER, LINE_TERMINATOR as required/desired
Define: KEYFIELD, DATAFIELD, SAVEFIELDS, MATCHFIELDS, SKIPFIELDS
Python: Use python or python3 below as appropriate to your system; verify that you have version 3
 $ python -V   or   python3 -V
 Python 3.x.y
Usage:
 $ python3 CSVKMD.py ./Input.csv ./Output.csv
Example:
You have a CSV file CourseStudent.csv with three columns: id,name,student
The columns are: course ID, course Name, student email address
id,name,student
57121690282,science,testuser1@domain.com
57121690282,science,testuser2@domain.com
57121690282,science,testuser3@domain.com
56941282690,english,testuser4@domain.com
56941282690,english,testuser5@domain.com
56941282690,english,testuser6@domain.com
47491913641,math,testuser7@domain.com
47491913641,math,testuser8@domain.com
47491913641,math,testuser9@domain.com

Process all courses
KEYFIELD = 'id'
DATAFIELD = 'student'
SAVEFIELDS = ['name']
MATCHFIELDS = {}
SKIPFIELDS = {}

$ python3 CSVKMD.py CourseStudent.csv CourseStudentCombined.csv
$ more CourseStudentCombined.csv
id,name,student
47491913641,math,testuser9@domain.com testuser7@domain.com testuser8@domain.com
56941282690,english,testuser5@domain.com testuser6@domain.com testuser4@domain.com
57121690282,science,testuser2@domain.com testuser1@domain.com testuser3@domain.com
$ gam csv CourseStudentCombined.csv gam courses "~id" add students users "~student"

Process only english course
KEYFIELD = 'id'
DATAFIELD = 'student'
SAVEFIELDS = ['name']
MATCHFIELDS = {'name': re.compile(r'english')}
SKIPFIELDS = {}

$ python3 CSVKMD.py CourseStudent.csv CourseStudentCombined.csv
$ more CourseStudentCombined.csv
id,name,student
56941282690,english,testuser5@domain.com testuser6@domain.com testuser4@domain.com
$ gam csv CourseStudentCombined.csv gam courses "~id" add students users "~student"

Process all courses except english
KEYFIELD = 'id'
DATAFIELD = 'student'
SAVEFIELDS = ['name']
MATCHFIELDS = {}
SKIPFIELDS = {'name': re.compile(r'english')}

$ python3 CSVKMD.py CourseStudent.csv CourseStudentCombined.csv
$ more CourseStudentCombined.csv
id,name,student
47491913641,math,testuser8@domain.com testuser7@domain.com testuser9@domain.com
57121690282,science,testuser1@domain.com testuser2@domain.com testuser3@domain.com
$ gam csv CourseStudentCombined.csv gam courses "~id" add students users "~student"
"""

import csv
import re
import sys

QUOTE_CHAR = '"' # Adjust as needed to properly read CSV files
DATA_DELIMITER = ' '# Delimiter between data field items
LINE_TERMINATOR = '\n' # On Windows, you probably want '\r\n'

ignore = re.compile(r'') # Keep pylint happy

# Key field name
# e.g., KEYFIELD = 'id'
KEYFIELD = ''
# Data field name
# e.g., DATAFIELD = 'student'
DATAFIELD = ''
# List of additional field names to include in output
# e.g., SAVEFIELDS = ['name']
SAVEFIELDS = []
# A dictionary of fields and associated Python Regular Expressions. Rows where all fields match will be processed
# e.g., MATCHFIELDS = {'name': re.compile(r'english')}
MATCHFIELDS = {}
# A dictionary of fields and associated Python Regular Expressions. Rows where with all fields don't match will be processed
# SKIPFIELDS = {'name': re.compile(r'english')}
SKIPFIELDS = {}

def fieldError(category, fieldName):
  sys.stderr.write(f'Error: {category}field "{fieldName}" not in file {sys.argv[1]} field names: {",".join(inputFieldNames)}\n')

def checkMatchSkipFields(row, matchFields, skipFields):
  for matchField, matchPattern in iter(matchFields.items()):
    if (matchField not in row) or not matchPattern.search(row[matchField]):
      return False
  for skipField, matchPattern in iter(skipFields.items()):
    if (skipField in row) and matchPattern.search(row[skipField]):
      return False
  return True

data = {}

if (len(sys.argv) > 1) and (sys.argv[1] != '-'):
  inputFile = open(sys.argv[1], 'r', encoding='utf-8')
else:
  inputFile = sys.stdin
inputCSV = csv.DictReader(inputFile, quotechar=QUOTE_CHAR)
inputFieldNames = inputCSV.fieldnames
fieldErrors = 0
if KEYFIELD not in inputFieldNames:
  fieldError('key', KEYFIELD)
  fieldErrors += 1
if DATAFIELD not in inputFieldNames:
  fieldError('data', DATAFIELD)
  fieldErrors += 1
for field in SAVEFIELDS:
  if field not in inputFieldNames:
    fieldError('save', field)
    fieldErrors += 1
for field in MATCHFIELDS:
  if field not in inputFieldNames:
    fieldError('match', field)
    fieldErrors += 1
for field in SKIPFIELDS:
  if field not in inputFieldNames:
    fieldError('skip', field)
    fieldErrors += 1
if fieldErrors:
  sys.exit(1)
outputFieldNames = []
for field in inputFieldNames:
  if field == KEYFIELD or field == DATAFIELD or field in SAVEFIELDS:
    outputFieldNames.append(field)
for irow in inputCSV:
  keyfield = irow[KEYFIELD]
  datafield = irow[DATAFIELD]
  if keyfield and datafield and checkMatchSkipFields(irow, MATCHFIELDS, SKIPFIELDS):
    data.setdefault(keyfield, {DATAFIELD: set()})
    data[keyfield][DATAFIELD].add(datafield)
    for field in SAVEFIELDS:
      data[keyfield][field] = irow[field]
if inputFile != sys.stdin:
  inputFile.close()

if (len(sys.argv) > 2) and (sys.argv[2] != '-'):
  outputFile = open(sys.argv[2], 'w', newline='')
else:
  outputFile = sys.stdout
outputCSV = csv.DictWriter(outputFile, outputFieldNames, lineterminator=LINE_TERMINATOR, quotechar=QUOTE_CHAR)
outputCSV.writeheader()

for kp, kv in sorted(data.items()):
  orow = {KEYFIELD: kp, DATAFIELD: DATA_DELIMITER.join(kv[DATAFIELD])}
  for field in SAVEFIELDS:
    orow[field] = kv[field]
  outputCSV.writerow(orow)

if outputFile != sys.stdout:
  outputFile.close()


"""
Creates self-supervised training images by pasting manipulator and occluded object on random backgrounds.

Expects the following folder structure for manipulator images:
root
|--rgb
|  |--000000.png
|  |--000001.png
|  |--...
|--pred
|  |--000000.png
|  |--000001.png
|  |--...

Expects the following folder structure for occluding objects:
root
|--<urandom-path>
|  |--0.hdf5
|  |--1.hdf5
|  |--...
|--<urandom-path>
|  |--0.hdf5
|  |--1.hdf5
|  |--...
"""

import os
import argparse
from tqdm import tqdm, trange
import numpy as np
import cv2
import glob
import random
from utils.train_utils import load_hdf5

def scale_ims(im, ma, lower_scale_ratio=0, upper_scale_ratio=0):
    # ratio is in correspondence to final image size
    h, w = ma.shape
    min_h = int(lower_scale_ratio * h)
    max_h = int(upper_scale_ratio * h)

    im_h, im_w = im.shape[:2]

    im_ratio = h / w
    new_h = np.random.randint(min_h, max_h)
    new_w = int(new_h / im_h * im_w)

    im = cv2.resize(im, dsize=(new_w, new_h), interpolation=cv2.INTER_LINEAR)
    ma = cv2.resize(ma, dsize=(new_w, new_h), interpolation=cv2.INTER_NEAREST)

    return im, ma

def calc_bbox(mask):
    h, w = mask.shape
    inds = np.where(mask > 0)

    x_min = max(inds[1].min(), 0)
    y_min = max(inds[0].min(), 0)
    x_max = min(inds[1].max(), w)
    y_max = min(inds[0].max(), h)

    return x_min, y_min, x_max - x_min, y_max - y_min

def crop_ims_by_mask(mask, ims, h=960, w=1280):
    inds = np.where(mask > 0)

    x_min = max(inds[1].min(), 0)
    y_min = max(inds[0].min(), 0)
    x_max = min(inds[1].max(), w - 1)
    y_max = min(inds[0].max(), h - 1)

    for i in range(len(ims)):
        if len(ims[i].shape) == 3:
            ims[i] = ims[i][y_min:y_max, x_min:x_max, :]
        else:
            ims[i] = ims[i][y_min:y_max, x_min:x_max]

    return ims

def rotate_ims(ims, angle=None):
    """
    Rotates an image (angle in degrees) and expands image to avoid cropping
    """
    if angle is None:
        angle = np.random.rand() * 360
    rotated_ims = []
    for im in ims:
        height, width = im.shape[:2]  # image shape has 3 dimensions
        image_center = (width/2, height/2)  # getRotationMatrix2D needs coordinates in reverse order (width, height) compared to shape

        rotation_mat = cv2.getRotationMatrix2D(image_center, angle, 1.)

        # rotation calculates the cos and sin, taking absolutes of those.
        abs_cos = abs(rotation_mat[0,0])
        abs_sin = abs(rotation_mat[0,1])

        # find the new width and height bounds
        bound_w = int(height * abs_sin + width * abs_cos)
        bound_h = int(height * abs_cos + width * abs_sin)

        # subtract old image center (bringing image back to origo) and adding the new image center coordinates
        rotation_mat[0, 2] += bound_w/2 - image_center[0]
        rotation_mat[1, 2] += bound_h/2 - image_center[1]

        # rotate image with the new bounds and translated rotation matrix
        rotated_mat = cv2.warpAffine(im, rotation_mat, (bound_w, bound_h))
        rotated_ims.append(rotated_mat)

    return rotated_ims

def load_random_manipulator():
    manipulator_id = np.random.randint(len(manipulator_ims))
    manipulator_im = cv2.imread(manipulator_ims[manipulator_id])
    manipulator_im = cv2.resize(manipulator_im, (736, 414), interpolation=cv2.INTER_LINEAR)
    manipulator_ma = cv2.imread(manipulator_mas[manipulator_id], cv2.IMREAD_GRAYSCALE)

    # small processing with erosion
    kernel = np.ones((5, 5), np.uint8)
    manipulator_ma = cv2.erode(manipulator_ma, kernel, iterations=1)

    return manipulator_im, manipulator_ma

def get_shapenet_obj(obj_id=None):
    if obj_id is None:
        while obj_id is None:
            tm = random.choice(occluding_paths)
            if len(glob.glob(tm + '/*.hdf5')) != 0:
                obj_id = tm
    hdf5s = glob.glob(obj_id + '/*.hdf5')
    path = random.choice(hdf5s)

    data = load_hdf5(path, keys=['colors', 'segmap'])
    rgb = data['colors']
    uniques, counts = np.unique(rgb, return_counts=True)
    if len(counts) < 25:
        print('RGB image contains too little color, skipping ...')
        return None, None, None

    rgb = cv2.cvtColor(data['colors'], cv2.COLOR_BGR2RGB)
    return rgb, data['segmap'], obj_id

def load_occluding_object(obj_id=None):
    obj_im, obj_ma, obj_id = get_shapenet_obj(obj_id)

    if obj_im is None:
        return None
    lsr = args.lower_scale_ratio_obj
    usr = args.upper_scale_ratio_obj

    obj_im, obj_ma = rotate_ims([obj_im, obj_ma])

    if obj_ma.max() == 0:
        print('Segmentation mask is 0, skipping ...')
        return None
    obj_im, obj_ma = crop_ims_by_mask(mask=obj_ma, ims=[obj_im, obj_ma], h=414, w=736)
    tmp_h, tmp_w = obj_ma.shape[:2]

    if obj_ma.max() == 0 or tmp_h < 25:
        print('Segmentation mask is either 0 or too small, skipping ...')
        return None
    obj_im, obj_ma = scale_ims(im=obj_im, ma=obj_ma, lower_scale_ratio=lsr, upper_scale_ratio=usr)
    kernel = np.ones((5, 5), np.uint8)
    obj_ma = cv2.erode(obj_ma, kernel, iterations=1)
    obj_im[:, :][obj_ma == 0] = [0, 0, 0]
    return obj_im, obj_ma, obj_id

def get_random_distractor_objects(num_objs=5, obj_ids=None, orig_ma=None):
    distractor_ids = []
    if obj_ids is None:
        obj_ids = [None for _ in range(num_objs)]

    fnl_overlay_im = np.zeros((414, 736, 3), dtype=np.uint8)
    fnl_overlay_ma = np.zeros((414, 736), dtype=np.uint8)

    for i in range(num_objs):
        # load data
        rets = None
        while rets is None:
            rets = load_occluding_object(obj_id=obj_ids[i])
        obj_im, obj_ma, obj_id = rets
        distractor_ids.append(obj_id)

        h, w = obj_ma.shape
        size_reduction = np.random.uniform(1., 3.)
        h = int(h / size_reduction)
        w = int(w / size_reduction)
        obj_im = cv2.resize(obj_im, (w, h), interpolation=cv2.INTER_LINEAR)
        obj_ma = cv2.resize(obj_ma, (w, h), interpolation=cv2.INTER_NEAREST)

        # crop
        bbox = calc_bbox(obj_ma)
        x1, y1, x2, y2 = bbox
        x2 += x1
        y2 += y1
        obj_im = obj_im[y1:y2, x1:x2, :]
        obj_ma = obj_ma[y1:y2, x1:x2]

        # paste with space
        im, ma = paste_with_space(obj_im, obj_ma, orig_ma)
        ma[ma != 0] = i + 3
        fnl_overlay_im[ma != 0] = im[ma != 0]
        fnl_overlay_ma[ma != 0] = ma[ma != 0]

    return fnl_overlay_im, fnl_overlay_ma, distractor_ids

def paste_with_space(im, ma, tar_ma, max_overlap=0.75):
    """
    Very simple pasting of a mask outside a bounding box with some max iou overlap.
    """

    max_h, max_w = tar_ma.shape
    bbox = calc_bbox(tar_ma)
    bbox = list(bbox)

    # pad results with overlap ratio
    h, w = ma.shape
    dummy_im = np.zeros((max_h + int(2 * h * max_overlap), max_w + int(2 * w * max_overlap), 3), dtype=np.uint8)
    dummy_ma = np.zeros((max_h + int(2 * h * max_overlap), max_w + int(2 * w * max_overlap)), dtype=np.uint8)
    bbox[0] += int(h * max_overlap)
    bbox[2] += int(h * max_overlap)
    bbox[1] += int(w * max_overlap)
    bbox[3] += int(w * max_overlap)

    # paste somewhere
    x_st, y_st = np.random.randint(0, max_w - w + int(2 * w * max_overlap) - 1), \
                 np.random.randint(0, max_h - h + int(2 * h * max_overlap))
    while not _is_invalid(bbox, (x_st, y_st, x_st + h, x_st + w), h=int(max_h + 2 * h * max_overlap),
                          w=int(max_w + 2 * w * max_overlap)):
        x_st, y_st = np.random.randint(0, max_w - w + int(2 * w * max_overlap) - 1), \
                     np.random.randint(0, max_h - h + int(2 * h * max_overlap))

    dummy_im[y_st:y_st + h, x_st:x_st + w, :] = im
    dummy_ma[y_st:y_st + h, x_st:x_st + w] = ma

    dummy_im = dummy_im[int(h * max_overlap):int(h * max_overlap) + max_h,
               int(w * max_overlap):int(w * max_overlap) + max_w, :]
    dummy_ma = dummy_ma[int(h * max_overlap):int(h * max_overlap) + max_h,
               int(w * max_overlap):int(w * max_overlap) + max_w]

    # cv2.imshow('im', im)
    # cv2.imshow('ma', ma / ma.max())
    # cv2.imshow('dummy im', dummy_im)
    # cv2.imshow('dummy ma', dummy_ma / dummy_ma.max())
    # cv2.waitKey(0)

    return dummy_im, dummy_ma

def _is_invalid(bbox1, bbox2, max_iou_overlap=0.2, h=414, w=736):
    x1, y1, xs1, ys1 = bbox1
    x2, y2, xs2, ys2 = bbox2

    arr1 = np.zeros((h, w), dtype=np.uint8)
    arr1[y1:y1 + ys1, x1:x1 + xs1] = 1

    arr2 = np.zeros_like(arr1)
    arr2[y2:y2 + ys2, x2:x2 + xs2] = 1

    sm = arr1 + arr2

    iou = (sm == 2).sum() / (sm > 0).sum()
    if iou > max_iou_overlap:
        return False

    return True

def generate_first_image():
    # load manipulator and occluding object
    manipulator_im, manipulator_ma = load_random_manipulator()

    rets = None
    while rets is None:
        rets = load_occluding_object(obj_id=None)
    obj_im, obj_ma, obj_id = rets

    # crop
    obj_im, obj_ma = crop_ims_by_mask(mask=obj_ma, ims=[obj_im, obj_ma], h=414, w=736)

    # final object image + mask
    fnl_obj_im = np.zeros_like(manipulator_im)
    fnl_obj_ma = np.zeros_like(manipulator_ma)

    # left-most hand part
    x, _, _, _ = calc_bbox(manipulator_ma)
    ys = np.where(manipulator_ma[:, x] != 0)[0]
    x += np.random.randint(0, 50)
    y = ys[int(len(ys) / 2)]

    # subtract half of obj bb to get start pos
    objh, objw = obj_ma.shape
    x -= int(objw / 2)
    x = max(0, x)
    y -= int(objh / 2)
    y = max(0, y)
    xend = x + objw
    yend = y + objh

    # cut obj mask in case it is too long
    if xend >= 736:
        obj_ma = obj_ma[:, :736-x]
        obj_im = obj_im[:, :736-x]
        xend = 736
    if yend >= 414:
        obj_ma = obj_ma[:414-y, :]
        obj_im = obj_im[:414-y, :]
        yend = 414

    # paste
    fnl_obj_im[y:yend, x:xend, :] = obj_im
    fnl_obj_ma[y:yend, x:xend] = obj_ma
    manipulator_im[fnl_obj_ma != 0] = fnl_obj_im[fnl_obj_ma != 0]
    manipulator_ma[manipulator_ma != 0] = 1
    manipulator_ma[fnl_obj_ma != 0] = 2
    #manipulator_im[manipulator_ma == 0] = [0, 0, 0]

    # crop, resize and rotate
    hand_im, hand_ma = crop_ims_by_mask(mask=manipulator_im, ims=[manipulator_im, manipulator_ma], h=414, w=736)
    hand_im, hand_ma = rotate_ims([hand_im, hand_ma])

    curr_h, curr_w = hand_ma.shape[:2]
    scl = np.random.uniform(0.5, 0.7)
    new_h = int(curr_h * scl)
    new_w = int(curr_w * scl)
    hand_ma = cv2.resize(hand_ma, (new_w, new_h), interpolation=cv2.INTER_NEAREST)
    hand_im = cv2.resize(hand_im, (new_w, new_h), interpolation=cv2.INTER_LINEAR)

    # random flip
    if np.random.rand() > 0.5:
        hand_im = cv2.flip(hand_im, 1)
        hand_ma = cv2.flip(hand_ma, 1)

    h, w = hand_ma.shape
    h2, w2 = int(h/2), int(w/2)
    if h > 414 or w > 736:
        hand_im = hand_im[:414, :736, :]
        hand_ma = hand_ma[:414, :736]
        h, w = hand_ma.shape

    x_st = np.random.randint(64, 736-64)
    y_st = np.random.randint(36, 414-36)

    fnl_train_im = np.zeros((414+h, 736+w, 3), dtype=np.uint8)
    fnl_train_ma = np.zeros((414+h, 736+w), dtype=np.uint8)

    fnl_train_im[y_st:y_st+h, x_st:x_st+w, :] = hand_im
    fnl_train_ma[y_st:y_st+h, x_st:x_st+w] = hand_ma
    fnl_train_im = fnl_train_im[h2:h2+414, w2:w2+736, :]
    fnl_train_ma = fnl_train_ma[h2:h2+414, w2:w2+736]

    idx = np.random.randint(len(background_ims))
    im = cv2.imread(background_ims[idx])
    im = cv2.resize(im, (736, 414), interpolation=cv2.INTER_LINEAR)
    background_image = im.copy()

    # paste
    im[fnl_train_ma != 0] = fnl_train_im[fnl_train_ma != 0]

    # paste distractor objects, if any
    if args.num_distractor_objects > 0:
        distractor_im, distractor_ma, distractor_ids = get_random_distractor_objects(num_objs=np.random.randint(args.num_distractor_objects + 1), obj_ids=None, orig_ma=fnl_train_ma)

        # overlay
        im[distractor_ma != 0] = distractor_im[distractor_ma != 0]
        fnl_train_ma[distractor_ma != 0] = distractor_ma[distractor_ma != 0]

        return im, fnl_train_ma, background_image, obj_id, distractor_ids

    return im, fnl_train_ma, background_image, obj_id, None

def generate_other_images(bg_im, obj_id, distractor_ids=None):

    # load robot, obj
    manipulator_im, manipulator_ma = load_random_manipulator()

    rets = None
    while rets is None:
        rets = load_occluding_object(obj_id=obj_id)
    obj_im, obj_ma, obj_id = rets

    # crop
    obj_im, obj_ma = crop_ims_by_mask(mask=obj_ma, ims=[obj_im, obj_ma], h=414, w=736)

    # final object image + mask
    fnl_obj_im = np.zeros_like(manipulator_im)
    fnl_obj_ma = np.zeros_like(manipulator_ma)

    # left-most hand part
    x, _, _, _ = calc_bbox(manipulator_ma)
    ys = np.where(manipulator_ma[:, x] != 0)[0]
    x += np.random.randint(0, 50)
    y = ys[int(len(ys) / 2)]

    # subtract half of obj bb to get start pos
    objh, objw = obj_ma.shape
    x -= int(objw / 2)
    x = max(0, x)
    y -= int(objh / 2)
    y = max(0, y)
    xend = x + objw
    yend = y + objh

    # cut obj mask in case it is too long
    if xend >= 736:
        obj_ma = obj_ma[:, :736-x]
        obj_im = obj_im[:, :736-x]
        xend = 736
    if yend >= 414:
        obj_ma = obj_ma[:414-y, :]
        obj_im = obj_im[:414-y, :]
        yend = 414

    # paste
    fnl_obj_im[y:yend, x:xend, :] = obj_im
    fnl_obj_ma[y:yend, x:xend] = obj_ma
    manipulator_im[fnl_obj_ma != 0] = fnl_obj_im[fnl_obj_ma != 0]
    manipulator_ma[manipulator_ma != 0] = 1
    manipulator_ma[fnl_obj_ma != 0] = 2

    # crop, resize and rotate
    hand_im, hand_ma = crop_ims_by_mask(mask=manipulator_ma, ims=[manipulator_im, manipulator_ma], h=414, w=736)
    hand_im, hand_ma = rotate_ims([hand_im, hand_ma])

    curr_h, curr_w = hand_ma.shape[:2]
    scl = np.random.uniform(0.5, 0.7)
    new_h = int(curr_h * scl)
    new_w = int(curr_w * scl)
    hand_ma = cv2.resize(hand_ma, (new_w, new_h), interpolation=cv2.INTER_NEAREST)
    hand_im = cv2.resize(hand_im, (new_w, new_h), interpolation=cv2.INTER_LINEAR)

    # random flip
    if np.random.rand() > 0.5:
        hand_im = cv2.flip(hand_im, 1)
        hand_ma = cv2.flip(hand_ma, 1)

    h, w = hand_ma.shape
    h2, w2 = int(h / 2), int(w / 2)
    if h > 414 or w > 736:
        hand_im = hand_im[:414, :736, :]
        hand_ma = hand_ma[:414, :736]
        h, w = hand_ma.shape

    x_st = np.random.randint(64, 736-64)
    y_st = np.random.randint(36, 414-36)

    fnl_train_im = np.zeros((414 + h, 736 + w, 3), dtype=np.uint8)
    fnl_train_ma = np.zeros((414 + h, 736 + w), dtype=np.uint8)

    fnl_train_im[y_st:y_st + h, x_st:x_st + w, :] = hand_im
    fnl_train_ma[y_st:y_st + h, x_st:x_st + w] = hand_ma
    fnl_train_im = fnl_train_im[h2:h2 + 414, w2:w2 + 736, :]
    fnl_train_ma = fnl_train_ma[h2:h2 + 414, w2:w2 + 736]

    # paste
    im = bg_im.copy()
    im[fnl_train_ma != 0] = fnl_train_im[fnl_train_ma != 0]

    # paste distractor objects, if any
    if args.num_distractor_objects > 0:
        distractor_im, distractor_ma, distractor_ids = get_random_distractor_objects(
            num_objs=len(distractor_ids), obj_ids=distractor_ids, orig_ma=fnl_train_ma)

        # overlay
        im[distractor_ma != 0] = distractor_im[distractor_ma != 0]
        fnl_train_ma[distractor_ma != 0] = distractor_ma[distractor_ma != 0]

    return im, fnl_train_ma

def check_manipulator_files(im_paths, ma_paths):
    print(f"Checking manipulator files ...")
    checked_im_paths, checked_ma_paths = [], []
    for i in trange(len(im_paths)):
        ma = cv2.imread(ma_paths[i], cv2.IMREAD_GRAYSCALE)
        kernel = np.ones((5, 5), np.uint8)
        ma = cv2.erode(ma, kernel, iterations=1)
        if ma.max() == 0:
            continue
        else:
            checked_im_paths.append(im_paths[i])
            checked_ma_paths.append(ma_paths[i])

    return checked_im_paths, checked_ma_paths

def generate(ctr):
    np.random.seed(ctr)

    # dummy loop to try 10 times
    for i in range(10):
        try:
            # generate first image
            im, ma, bg_im, obj_id, distractor_ids = generate_first_image()

            # save image, mask and gripper mask
            cv2.imwrite(os.path.join(args.output_path, f"rgb1/{str(ctr).zfill(6)}.png"), im)
            cv2.imwrite(os.path.join(args.output_path, f"gt1/{str(ctr).zfill(6)}.png"), ma)

            # generate sequence images (default just one second image)
            for j in range(args.sequence_length - 1):
                rets = generate_other_images(bg_im=bg_im, obj_id=obj_id, distractor_ids=distractor_ids)
                im, ma = rets
                cv2.imwrite(os.path.join(args.output_path, f"rgb{j+2}/{str(ctr).zfill(6)}.png"), im)
                cv2.imwrite(os.path.join(args.output_path, f"gt{j+2}/{str(ctr).zfill(6)}.png"), ma)
            break
        except Exception as e:
            print(i, e)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--output-path', type=str, required=True)
    parser.add_argument('--min-obj-size', type=int, default=1000)
    parser.add_argument('--name', type=str, default='test')
    parser.add_argument('--lower-scale-ratio-obj', type=float, default=0.5)
    parser.add_argument('--upper-scale-ratio-obj', type=float, default=1.)
    parser.add_argument('--num-ims', default=100, type=int)
    parser.add_argument('--background-root', type=str, required=True)
    parser.add_argument('--background-suffix', type=str, default='png')
    parser.add_argument('--manipulator-root', type=str, required=True)
    parser.add_argument('--occluding-root', type=str, required=True)
    parser.add_argument('--num-distractor-objects', type=int, default=0)
    parser.add_argument('--sequence-length', type=int, default=2)
    parser.add_argument('--pools', type=int, default=8)
    args = parser.parse_args()

    # load background
    background_ims = glob.glob(os.path.join(args.background_root, f'*.{args.background_suffix}'))
    assert len(background_ims) != 0, f"No background images found in {args.background_root}"
    print(f"Found {len(background_ims)} background images in {args.background_root}")

    # load manipulator images
    manipulator_ims = sorted(glob.glob(os.path.join(args.manipulator_root, 'rgb/*.png')))
    manipulator_mas = sorted(glob.glob(os.path.join(args.manipulator_root, 'pred/*.png')))
    if len(manipulator_ims) == len(manipulator_mas) + 1:
        manipulator_ims = manipulator_ims[:-1]
        print(f"Assuming that the number of rgb ims is one larger than the number of annotations, so removing last rgb image")
    assert len(manipulator_ims) != 0, f"No manipulator images found in {args.manipulator_root}"
    assert len(manipulator_ims) == len(manipulator_mas), f"Unequal length of manipulator images and masks: {len(manipulator_ims)} != {len(manipulator_mas)}"
    # check manipulator images - skip all that have 0 prediction
    manipulator_ims, manipulator_mas = check_manipulator_files(manipulator_ims, manipulator_mas)
    print(f"Found {len(manipulator_ims)} valid ims and masks in {args.manipulator_root}")

    # load occluding images
    occluding_paths = glob.glob(os.path.join(args.occluding_root, '*'))
    assert len(occluding_paths) != 0, f"No occluding paths found in {args.occluding_root}"
    print(f"Found {len(occluding_paths)} different occluding objects in {args.occluding_root}")

    # create output folder
    os.makedirs(args.output_path, exist_ok=True)
    for i in range(args.sequence_length):
        os.makedirs(os.path.join(args.output_path, f'rgb{i+1}'), exist_ok=True)
        os.makedirs(os.path.join(args.output_path, f'gt{i+1}'), exist_ok=True)

    #for i in trange(args.num_ims):
    #    generate(i)
    #exit(0)

    counters = list(range(args.num_ims))
    from multiprocessing import Pool
    print('Running with {} pools'.format(args.pools))
    pool = Pool(args.pools)
    for _ in tqdm(pool.imap(generate, counters), total=len(counters)):
        pass

    pool.close()
    pool.join()


from scipy.optimize import fsolve
import matplotlib.pyplot as plt
import numpy as np
import math
X = []

s =0

f = open(r'C:\Users\user\Desktop\волна\data\20mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close
s =0
f = open(r'C:\Users\user\Desktop\волна\data\40mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close
s =0
f = open(r'C:\Users\user\Desktop\волна\data\60mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close
s =0
f = open(r'C:\Users\user\Desktop\волна\data\80mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close
s =0
f = open(r'C:\Users\user\Desktop\волна\data\100mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close
s =0
f = open(r'C:\Users\user\Desktop\волна\data\120mm.txt')
for line in f:
    s = s + int(line)
X.append(s/2000)
f.close

Y = np.array([20,40,60,80,100,120])

a = np.polyfit(X,Y,4)
n = np.linspace(400, 3100, 2000, True)
yn = a[0]*n**4 + a[1]*n**3 + a[2]*n**2 + a[3]*n +a[4]

plt.plot(n,yn)
plt.scatter(X,Y)

#i = 0
#f = open(r'C:\Users\user\Desktop\волна\data\potop3.txt')
#for line in f:
#    p=int(line)
#    y=a[0]*p**4 + a[1]*p**3 + a[2]*p**2 + a[3]*p +a[4]
#    if (i > 1 and i < 13333):
#        plt.plot(i*3/20000,y, ',r')
#    else:
#        plt.plot(i*3/20000,y, ',b')
#
#    i = i+1
#f.close

plt.show()


import pytube
from pytube import YouTube
from aiogram import Bot, types
from aiogram.dispatcher import Dispatcher
from aiogram.utils import executor
import telebot
import time

bot = telebot.TeleBot(token="1671450952:AAHsVNOu_Fp6LrPvwAQYwxGZ-sDFzgMMxH8")

@bot.message_handler(commands=['start', 'help',])
def first_or_help_commands(message):
    bot.send_message(message.chat.id, "Привет! Отправь мне ссылку на видео, я скачаю тебе его.:) ")

@bot.message_handler(content_types=['text'])
def download_video(message):
    youtube = pytube.YouTube(message.text)
    video = youtube.streams.first()
    video.download('../Video')
    bot.send_message(message.chat.id, "Подожди мгновение я колдую..Вжух")
    time.sleep(1)
    video_location = video.title
    video_adr = open(f'../Video/{video_location}.mp4', 'rb')
    bot.send_video(message.chat.id, video_adr)

    bot.send_message(message.chat.id, "А вот и твое видео")
    bot.send_message(message.chat.id, "Отправь мне еще ссылку я тебе что нибудь наколдую...Вжух")

bot.polling()


from pytube import Channel
from time import process_time
import re
from pytube.exceptions import LiveStreamError
from pyrogram import Client
import schedule
import datetime
import os
from configs import api_id, api_hash, youtube_channel, running_interval, regex_matches, chat_id, bot_token

print('started at', datetime.datetime.now())

bot = Client('bot', api_id, api_hash, bot_token=bot_token)
bot.start()
id_trash = eval(open('id_trash.txt', 'r').read())

def update_id(vid):
    """Update channel id store if new id available"""
    id_trash.append(vid)
    open('id_trash.txt', 'w').write(f'{id_trash}')

def get_channel():
    """Recursive function to find list of videos from given channel"""
    channel = Channel(youtube_channel)
    if len(channel) > 15:
        return channel
    else:
        return get_channel()

def find_video():
    """Function to find new video from given channel and sending the video to telegram chat"""
    print(datetime.datetime.now())
    start = process_time()
    ch = get_channel()
    print(len(ch))
    for video in ch.videos[:10]:
        if video.video_id not in id_trash:
            if re.match(regex_matches, video.title):
                if video.vid_info['playabilityStatus']['status'] == 'OK':
                    print(video.title)
                    try:
                        stream = video.streams.filter(resolution='720p', progressive=True,  file_extension='mp4')[0]
                        dow = stream.download()
                        if dow:
                            print('downloaded')
                            bot.send_video(chat_id, dow, duration=video.length, width=1280, height=720)
                            update_id(video.video_id)
                            os.remove(dow)
                    except LiveStreamError:
                        print('passed')
            else:
                id_trash.append(video.video_id)

    end = process_time()
    print(f'Finished in {end - start} seconds at', datetime.datetime.now())

# Scheduler to run the program on a given interval
schedules = schedule.Scheduler()
schedules.every(running_interval).minutes.do(find_video)

while True:
    schedules.run_pending()


import asyncio
import logging

from dns_cache.server import ServerProtocol
from dns_cache.client import ClientProtocol

logging.basicConfig(level=logging.DEBUG)

def main():
    query_q = asyncio.Queue()
    result_q = asyncio.Queue()
    loop = asyncio.get_event_loop()

    listen = loop.create_datagram_endpoint(
        lambda: ServerProtocol(query_q, result_q),
        local_addr=('127.0.0.1', 53))
    s_transport, s_protocol = loop.run_until_complete(listen)

    connect = loop.create_datagram_endpoint(
        lambda: ClientProtocol(query_q, result_q),
        remote_addr=('223.5.5.5', 53))
    c_transport, c_protocol = loop.run_until_complete(connect)

    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass

    s_transport.close()
    c_transport.close()
    loop.close()

if __name__ == '__main__':
    main()


'''
Uses Flask-Assets package to monitor and compile "static" assets.
Currently requires 'sass' Ruby gem on your system.
'''

import os.path

from flask.ext.assets import Environment, Bundle
from . import app

assets = Environment(app)

# Main Site Styles
css_main = Bundle('scss/main/general.scss',
                  'scss/main/layout.scss',
                  'scss/main/tables.scss',
                  filters='scss',
                  output='css/main.css'
                 )
assets.register('css_main', css_main)

# Ability Pages
css_ability = Bundle('scss/ability.scss',
                     filters='scss',
                     output='css/ability.css'
                    )
assets.register('css_ability', css_ability)

# Character Pages
css_character = Bundle('scss/character.scss',
                        filters='scss',
                        output='css/character.css'
                       )
assets.register('css_character', css_character)

# Enemy Pages
css_enemy = Bundle('scss/enemy.scss',
                   filters='scss',
                   output='css/enemy.css'
                  )
assets.register('css_enemy', css_enemy)

# Item Pages
css_item = Bundle('item.scss',
                   filters='scss',
                   output='css/item.css'
                  )
assets.register('css_item', css_item)


class LRUCache:
    # dict랑.. doubly linked list 쓰면 될 듯.
    def __init__(self, capacity: int):
        self._capacity = capacity
        self._dt = {}
        self._head = Node(None, None)
        self._tail = Node(None, None)
        self._head.next = self._tail
        self._tail.prev = self._head

    def get(self, key: int) -> int:
        if key not in self._dt:
            return -1

        node = self._dt[key]
        self._remove(node)
        self._append(node)
        return node.val

    def put(self, key: int, value: int) -> None:
        if key in self._dt:
            self._remove(self._dt[key]) # LL에서만 갱신하기 위해...

        node = Node(key, value)
        self._append(node)
        self._dt[key] = node

        if len(self._dt) > self._capacity:
            head = self._head.next
            self._remove(head)
            self._dt.pop(head.key)

    def _remove(self, node):
        node.prev.next = node.next
        node.next.prev = node.prev

    def _append(self, node):
        prev_tail = self._tail.prev

        prev_tail.next = node
        node.next = self._tail
        node.prev = prev_tail
        self._tail.prev = node

class Node:
    def __init__(self, key, val):
        self.key = key
        self.val = val
        self.prev = None
        self.next = None

# Your LRUCache object will be instantiated and called as such:
# obj = LRUCache(capacity)
# param_1 = obj.get(key)
# obj.put(key,value)


#!/usr/bin/env python3

from firestore_client_handler import FirestoreClientHandler
import logging
import threading

logger = logging.getLogger(__name__)

class FirestoreTunnelsHandler(FirestoreClientHandler):

    KEYWORD = "tunnels"
    RESTART_DELAY_TIME_S = 0.5

    def __init__(self, api_key, project_id, refresh_token, device_id, tunnels_update_callback=lambda _: None):
        super().__init__(api_key, project_id, refresh_token)
        self.device_id = device_id
        self.users_path = None
        self.devices_path = None
        self.tunnels_update_callback = tunnels_update_callback
        self.device_subscription = None
        self.tunnels_cache = None

    def start(self):
        logger.debug("Starting Firestore Tunnels Handler")
        self.initialize_client()

    def stop(self):
        logger.debug("Stopping Firestore Tunnels Handler")
        if self.device_subscription:
            self.device_subscription.unsubscribe()
            self.device_subscription = None
        self.stop_client()

    def restart(self):
        logger.debug("Restarting Firestore Tunnels Handler")
        self.stop()
        self.start()

    def on_client_initialized(self):
        logger.info("Firestore client initialized")
        self.devices_path = f"users/{self.user_id}/devices"
        if self.device_subscription:
            return
        self.device_subscription = self.client.collection(self.devices_path).document(
            self.device_id).on_snapshot(self._on_device_update)

    def on_server_not_responding(self):
        logger.error("Firestore server not responding")
        threading.Timer(self.RESTART_DELAY_TIME_S, self.restart).start()

    def on_token_expired(self):
        logger.debug("Refreshing Firebase client token id")
        threading.Timer(self.RESTART_DELAY_TIME_S, self.restart).start()

    def get_tunnel_token(self):
        return self._get_user_field("tunnel_token")

    def get_user_email(self):
        return self._get_user_field("email")

    def get_user_id(self):
        return self.user_id

    def update_tunnel_url(self, port, url):
        self.initialize_client(notify=False)
        if not self.client:
            logger.debug("Firebase client not ready, cannot update tunnel url")
            return
        updated_field = {f"{self.KEYWORD}.{port}.url": url}
        self.client.collection(self.devices_path).document(
            self.device_id).update(updated_field)

    def _get_user_field(self, field):
        self.users_path = f"users"
        self.initialize_client(notify=False)
        if not self.client:
            logger.error("Firebase client not ready, cannot get user field")
            return
        user_info = self.client.collection(self.users_path).document(
            self.user_id).get().to_dict()
        return user_info.get(field)

    def _on_device_update(self, document_snapshot, changes, read_time):
        if len(document_snapshot) != 1:
            return
        device_info = document_snapshot[0].to_dict()

        if not self.KEYWORD in device_info:
            logger.warn(
                f"'{self.KEYWORD}' information not available, creating the new field")
            updated_field = {f"{self.KEYWORD}": {}}
            self.client.collection(self.devices_path).document(self.device_id).update(updated_field)
            return

        tunnels = device_info.get(self.KEYWORD)

        if tunnels != self.tunnels_cache:
            self.tunnels_cache = tunnels
            self.tunnels_update_callback(tunnels)


import numpy as np

from geometry_info.geometry_contour import GeometryContour

class MuR205(GeometryContour):
    # region Member
    _lead_vector_mir_to_ur5: np.array

    # Contains rotation and translation of the mir base link cs to the ur5 base link cs
    _tf_mir_to_ur5_cs: np.array
    _tf_ur5_to_mir_cs: np.array

    # endregion

    def __init__(self, mir_base_link_world_cs: np.array = np.array([0, 0]), mir_base_link_cs_rotation: float = 0.0):
        super().__init__(lead_vector_world_cs=mir_base_link_world_cs,
                         world_to_geometry_cs_rotation=mir_base_link_cs_rotation)

        # Add outer dimensions of the MiR200 base to corner list
        self.add_contour_corner_geometry_cs(np.array([0.4475, 0.291]))
        self.add_contour_corner_geometry_cs(np.array([-0.4475, 0.291]))
        self.add_contour_corner_geometry_cs(np.array([-0.4475, -0.291]))
        self.add_contour_corner_geometry_cs(np.array([0.4475, -0.291]))

        # Create contour edges of the MiR200 base platform
        self.create_contour_edges()

        # tf between mir base link and ur5 base link
        self._set_mur205_transformation()

    # region Properties
    @property
    def lead_vector_base_link_to_ur5(self):
        # TODO Docstring
        return self._lead_vector_mir_to_ur5

    @property
    def lead_vector_ur5_to_base_link(self) -> np.array:
        # TODO Docstring
        return -1 * self._lead_vector_mir_to_ur5

    @property
    def tf_ur5_to_world_cs(self) -> np.array:
        # TODO Docstring
        return self._tf_geometry_to_world_cs.dot(self._tf_ur5_to_mir_cs)

    @property
    def tf_world_to_ur5_cs(self) -> np.array:
        # TODO Docstring
        return self._tf_mir_to_ur5_cs.dot(self._tf_world_to_geometry_cs)

    # endregion

    # region private methods
    def _set_mur205_transformation(self):
        # TODO Docstring
        # Define lead vector from mir-base-link to ur5-base-link
        # These values were measured with a laser tracking system
        self._lead_vector_mir_to_ur5 = np.array([0.244311, -0.140242])

        # There is only translation between mir and ur5 base link, no rotation
        self._tf_ur5_to_mir_cs = self._create_transformation_matrix(self._lead_vector_mir_to_ur5, 0)
        self._tf_mir_to_ur5_cs = np.linalg.inv(self._tf_ur5_to_mir_cs)

    # endregion

    # region public methods
    def move_mur205_by_ur5_base_link(self, ur5_lead_vector_world_cs: np.array, mur205_rotation_world_cs: float = 0.0):
        tf_ur5_to_world_cs: np.array = self._create_transformation_matrix(ur5_lead_vector_world_cs,
                                                                          mur205_rotation_world_cs)
        new_tf_mir_to_world_cs: np.array = tf_ur5_to_world_cs.dot(self._tf_mir_to_ur5_cs)
        new_tf_world_to_mir_cs: np.array = np.linalg.inv(new_tf_mir_to_world_cs)

        self._lead_vector_world_cs = self._get_translation_from_tf(new_tf_mir_to_world_cs)
        self._geometry_cs_rotation = self._get_rotation_from_tf(new_tf_mir_to_world_cs)
        self._tf_geometry_to_world_cs = new_tf_mir_to_world_cs
        self._tf_world_to_geometry_cs = new_tf_world_to_mir_cs

    def rotate_relative_around_ur5_base_cs(self, relative_rotation_radian: float):
        # TODO Docstring
        ur5_lead_vector_world_cs: np.array = self._get_translation_from_tf(self.tf_ur5_to_world_cs)

        new_tf_ur5_to_world_cs: np.array = self._create_transformation_matrix(
            ur5_lead_vector_world_cs,
            self._geometry_cs_rotation + relative_rotation_radian)

        new_tf_mir_to_world_cs: np.array = new_tf_ur5_to_world_cs.dot(self._tf_mir_to_ur5_cs)

        self._lead_vector_world_cs = self._get_translation_from_tf(new_tf_mir_to_world_cs)
        self._geometry_cs_rotation = self._get_rotation_from_tf(new_tf_mir_to_world_cs)
        self._tf_geometry_to_world_cs = new_tf_mir_to_world_cs
        self._tf_world_to_geometry_cs = np.linalg.inv(new_tf_mir_to_world_cs)

    def rotate_absolute_around_ur5_base_cs(self, absolute_rotation_radian: float):
        # TODO Docstring
        ur5_lead_vector_world_cs: np.array = self._get_translation_from_tf(self.tf_ur5_to_world_cs)

        new_tf_ur5_to_world_cs: np.array = self._create_transformation_matrix(ur5_lead_vector_world_cs,
                                                                              absolute_rotation_radian)

        new_tf_mir_to_world_cs: np.array = new_tf_ur5_to_world_cs.dot(self._tf_mir_to_ur5_cs)

        self._lead_vector_world_cs = self._get_translation_from_tf(new_tf_mir_to_world_cs)
        self._geometry_cs_rotation = self._get_rotation_from_tf(new_tf_mir_to_world_cs)
        self._tf_geometry_to_world_cs = new_tf_mir_to_world_cs
        self._tf_world_to_geometry_cs = np.linalg.inv(new_tf_mir_to_world_cs)
    # endregion