import collections
import itertools
import random

from model.clusters import AbstractClusterRules
from utils import stats_utils, bdsa_utils

def aggregate_clusters(edges: list, eid2source2element: dict, score:int, cluster_rules: AbstractClusterRules,
                                  compare_previous_limit=None):
    """
    Aggregate former cluster (eid2source2element) with new edges
    :param edges:
    :param eid2source2element:
    :param cluster_rules:
    :param compare_previous_limit:
    :return:
    """
    for source2elements in eid2source2element.values():
        for source_pair in itertools.combinations(source2elements.keys(), 2):
            for element_pair in itertools.product(source2elements[source_pair[0]], source2elements[source_pair[1]]):
                edges.append(WeightedEdge(element_pair[0], element_pair[1], score))
    partition_using_agglomerative(edges, eid2source2element, cluster_rules, compare_previous_limit)

def partition_using_agglomerative(edges: list, eid2source2element: dict, cluster_rules: AbstractClusterRules,
                                  compare_previous_limit=None):
    """
    Cluster graph in this way:
    - Initially each node is a cluster
    - Foreach edge (sorted by decreasing weight):
     - If nodes for this edge are in different clusters, if a specific precondition for merging is met (tipically there
        should be elements from same source)
     - Stop at MIN_EDGE_WEIGHT
    Inspired (but different) by paper 'Clustering large probabilistic graph'
    :param edges: list of edges with nodes and weight. Each node must have an attribute 'source'.
    :param eid2source2element: the dictionary that represents the output
    :param cluster_rules: class that says if elements can be merged, and build cluster objects.
    :param: compare_previous_limit: Compute detailed similarity (costly) if only if gross similarity is bigger \
    than this. If None, never compute detailed similarity. Cf stats_utils.compare_clusters
    :return:
    """

    edges_sorted = sorted(edges, key=lambda e: (e.weight, e.node1, e.node2), reverse=True)
    node2cid = {}
    cid2cluster = {}
    cluster_counter = 0
    for edge in edges_sorted:
        node1 = edge.node1
        node2 = edge.node2
        c1 = node2cid[node1] if node1 in node2cid else None
        c2 = node2cid[node2] if node2 in node2cid else None
        if c1 is None and c2 is None:
            if cluster_rules.is_pair_mergeable(node1, node2):
                node2cid[node1] = cluster_counter
                node2cid[node2] = cluster_counter
                cid2cluster[cluster_counter] = cluster_rules.build_cluster(node1, node2)
                cluster_counter += 1
        elif c2 is None:
            added = cid2cluster[c1].add_element_if_allowed(node2)
            if added:
                node2cid[node2] = c1
        elif c1 is None:
            added = cid2cluster[c2].add_element_if_allowed(node1)
            if added:
                node2cid[node1] = c2
        # If 2 nodes already belong to the same cluster, there is nothing to do.
        elif c1 != c2:
            cmin = min(c1, c2)
            cmax = max(c1, c2)
            added = cid2cluster[cmin].merge_cluster(cid2cluster[cmax])
            if added:
                for node in cid2cluster[cmax].nodes():
                    node2cid[node] = cmin
                del cid2cluster[cmax]
    if compare_previous_limit:
        old_clusters_copy = {eid: frozenset(element for elements in source2element.values() for element in elements) for eid, source2element in eid2source2element.items()}
        eid2source2element.clear()
        new_cluster_copy = {cid: frozenset(cluster.nodes()) for cid, cluster in cid2cluster.items()}
        res = compare_clusters(old_clusters_copy, new_cluster_copy, compare_previous_limit)
        del old_clusters_copy
        del new_cluster_copy
    else:
        eid2source2element.clear()
        res = 0
    for eid, cluster in cid2cluster.items():
        for node in cluster.nodes():
            eid2source2element[eid][node.source].add(node)
    return res

class WeightedEdge:
    """
    Weighted edge, 1st and 2nd
    """
    def __init__(self, node_a, node_b, weight:float):
        self.weight = weight
        self.node1 = min(node_a, node_b)
        self.node2 = max(node_a, node_b)

    def __repr__(self):
        return "%s --- %s [%f]" % (self.node1, self.node2, self.weight)

    def __eq__(self, other):
        return self.node1 == other.node1 and self.node2 == other.node2

    def __hash__(self):
        return hash(self.node1) + hash(self.node2)

def nodes_flattener(source2elements):
    """
    Flatten nodes grouped in source2elements
    >>> nodes_flattener({10: {7,3,8}, 20: {6,4,2}})
    [2, 3, 4, 6, 7, 8]
    """

    return sorted(element for elements in source2elements.values() for element in elements)

def build_all_pairs_generic(clusters, nodes_retriever=nodes_flattener):
    """
    Build all pairs of nodes in same cluster.
    :param: clusters: list of clusters
    :param: nodes_retriever: method that retrieve nodes from a cluster
    >>> list(build_all_pairs_generic([[4,1,3,2], set([7,9,8])], lambda cluster:cluster))
    [(1, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 4), (7, 8), (7, 9), (8, 9)]
    >>> list(build_all_pairs_generic([{30:{100,200}},{10: {7,3,8}, 20: {6,4,2}}]))
    [(100, 200), (2, 3), (2, 4), (2, 6), (2, 7), (2, 8), (3, 4), (3, 6), (3, 7), (3, 8), (4, 6), (4, 7), (4, 8), (6, 7), (6, 8), (7, 8)]

    """
    for cluster in clusters:
        all_nodes_in_cluster = sorted(nodes_retriever(cluster))
        for pair in itertools.combinations(all_nodes_in_cluster, 2):
            yield pair

def add_edges_bipartite(edges, source2sas1, source2sas2, weight:int):
    """
    Add edges between all pairs from sas1 and sas2
    :param edges:
    :param sas1:
    :param sas2:
    :param weight:
    :return:
    """
    for source1, source2 in itertools.product(source2sas1.keys(), source2sas2.keys()):
        if source1 != source2:
            for sa1, sa2 in itertools.product(source2sas1[source1], source2sas2[source2]):
                edges.append(WeightedEdge(sa1, sa2, weight))

def add_group_edges(edges, sas_groups, weight: int):
    """
    Add edges between all elements in sas_groups (ie full graph)
    :param edges:
    :param sas_groups:
    :param weight:
    :param exclude_rule:
    :param group_all if True, group everything in a single clique, otherwise make separate cliques per group
    :return:
    """
    for sas in sas_groups:
        for sa_pair in itertools.combinations(sas, 2):
            edges.append(WeightedEdge(sa_pair[0], sa_pair[1], weight))

ClusterDifference = collections.namedtuple('ClusterDifference',
                                           'original_pairs added_pairs deleted_pairs sample_added sample_deleted')

def _build_flat_clusters(eid2source2elements:dict):
    """
    Build a list of clusters, represented as sorted tuple of elements
    >>> eid2source2elements = {1: {'s1':  {10,20},'s2':{30}}, 2: {'s1': {10}}, 3: {'s4': {50,60,70,80}}}
    >>> _build_flat_clusters(eid2source2elements)
    {(50, 60, 70, 80), (10,), (10, 20, 30)}
    """
    res = set()
    for source2elements in eid2source2elements.values():
        res.add(tuple(sorted(element for elements in source2elements.values() for element in elements)))
    return res

def cluster_differences(eid2source2element_old: dict, eid2source2element_new: dict,
                        sample_function=lambda x: x, overlapping_cluster=False) -> ClusterDifference:
    """
    Compute difference between 2 clustering. In particular, detect new and deleleted pairs
    :param overlapping_cluster: True if cluster are overlapping. Notice that alignment cluster are not overlapping per-se:
    there may be different virtual atts from an original att that pertain to different cluster, but technically they are
    different.

    >>> eid2source2element1 = {1: {'s1': {10,20,30}, 's2':  {40,50}}, \
                               2: {'s1': {60, 70}}, \
                               3: {'s3': {80}}, \
                               4: {'s5': {100,110}}}
    >>> eid2source2element2 = {9: {'s1': {10,20}, 's2': {40,50}}, \
                               5: {'s1': {60, 30, 70}}, \
                               6: {'s3': {80, 90}}, \
                               4: {'s5': {100,110}}}
    >>> cluster_differences(eid2source2element1, eid2source2element2)
    ClusterDifference(new_pairs=3, deleted_pairs=4, new_pairs_percent=0.25, deleted_pairs_percent=0.3333333333333333)
    """
    clusters_old = _build_flat_clusters(eid2source2element_old)
    clusters_new = _build_flat_clusters(eid2source2element_new)
    total_pairs_old = sum(bdsa_utils.nb_pairs_in_cluster(len(cluster)) for cluster in clusters_old)

    # Now we need to compute pairs of elements in same cluster, and check if some pairs are new and/or some is old.
    # If clusters are non-overlapping, we can throw away identical clusters, as they share exactly the same pairs.
    clusters_only_old = clusters_old if overlapping_cluster else clusters_old - clusters_new
    clusters_only_new = clusters_new if overlapping_cluster else clusters_new - clusters_old
    old_pairs = set().union(*(set(itertools.combinations(cluster, 2)) for cluster in clusters_only_old))
    new_pairs = set().union(*(set(itertools.combinations(cluster, 2)) for cluster in clusters_only_new))
    added_pairs = new_pairs - old_pairs
    deleted_pairs = old_pairs - new_pairs
    sample_added = stats_utils.safe_sample(added_pairs, 300)
    sample_deleted = stats_utils.safe_sample(deleted_pairs, 300)

    return ClusterDifference \
        (total_pairs_old, len(added_pairs), len(deleted_pairs), sample_added, sample_deleted)

def compare_clusters(cl1:dict, cl2:dict, min_similarity_to_continue=0.95):
    """
    Compare 2 clustering of same data. Typically used to decide when to stop iteration
    If ratio of element in identical clusters are less than min_similarity_to_continue, then stop and return this ratio.
        - For instance, [12, 345, 67, 8] and [1, 2345, 67, 8] have a ratio of 3/8 (678 over 1,2,3,4,5,6,7,8)
    This method is used to check whether 2 clusters are enough similar, so if with this quick analysis they are quite different,
    then we should stop in order to avoid losing too much time.
    If ratio is > min_similarity_to_continue, then continues:
    - build pair of "correspondent clusters", from cl1 and cl2, i.e.:
        - compute jaccard containment similarity between all cluster pairs that share at least an element
        - take the pair with biggest JS --> label as 'correspondent' clusters
        - continue iteratively last point (unless a cluster is already in a pair)
    - compute score as #elements in cl1 that are in their correspondent cluster in cl2 / total elements

    :param cl1: dict of FROZEN set
    :param cl2: same
    :param min_similarity_to_continue: see description
    :return:
    >>> cl1 = {100:frozenset([1,2,3]), 200:frozenset([4,5,6]), 300:frozenset([7,8,9]), 400:frozenset([15,16])}
    >>> cl2 = {500:frozenset([1,2]), 600:frozenset([3,4,5,6]), 700:frozenset([7,8,9,10]), 800:frozenset([15,16])}
    >>> res = compare_clusters(cl1, cl2, 0.05)
    >>> round(res, 3)
    0.833
    >>> cl1
    {100: frozenset({1, 2, 3}), 200: frozenset({4, 5, 6}), 300: frozenset({8, 9, 7}), 400: frozenset({16, 15})}
    >>> cl2
    {500: frozenset({1, 2}), 600: frozenset({3, 4, 5, 6}), 700: frozenset({8, 9, 10, 7}), 800: frozenset({16, 15})}
    """
    len1 = sum(len(group) for group in cl1.values())
    len2 = sum(len(group) for group in cl2.values())
    # Join clusters
    intersect = set(cl1.values()) & set(cl2.values())
    len_intersec = sum(len(group) for group in intersect)
    similarity_rough = len_intersec / max(len1, len2) if len1 != 0 and len2 != 0 else 1
    if similarity_rough < min_similarity_to_continue:
        return similarity_rough

    # Now compare the remaining
    cl1_filtered = {key: values for key, values in cl1.items() if values not in intersect}
    cl2_filtered = {key: values for key, values in cl2.items() if values not in intersect}
    cl2_inverted = collections.defaultdict(set)
    for cid, elements in cl2_filtered.items():
        for element in elements:
            cl2_inverted[element].add(cid)
    element_pairs = []
    for cid1, elements in cl1_filtered.items():
        counter_cid2 = collections.Counter()
        for element in elements:
            counter_cid2.update(cl2_inverted[element])
        for cid2, occs in counter_cid2.items():
            # Jaccard similarity
            element_pairs.append((cid1, cid2, occs / max(len(elements), len(cl2_filtered[cid2])), occs))
    counted_c1 = set()
    counted_c2 = set()
    total_correct = len_intersec
    for pair in sorted(element_pairs, key=lambda pair: pair[2], reverse=True):
        if pair[0] not in counted_c1 and pair[1] not in counted_c2:
            total_correct += pair[3] # TODO fix
            counted_c1.add(pair[0])
            counted_c2.add(pair[1])
    return total_correct / max(len1, len2) if len1 != 0 and len2 != 0 else 1


class Solution:
    def putMarbles(self, weights: List[int], k: int) -> int:
        seen = []
        for i in range(1, len(weights)):
            seen.append(weights[i]+weights[i-1])
        seen = sorted(seen)
        print(seen)
        if k == 1:
            return 0
        else:
            biggest = sum(seen[-(k-1):])
            smallest = sum(seen[:(k-1)])
            print(biggest, smallest)
            return biggest - smallest


import numpy as np
import SimpleITK as sitk
from skimage import transform
import os
import tre3d
import random
from scipy import ndimage
import math

mode = 'E1'

BASE_64_DIR = '/mnt/hd3/new3dDataset_64/'
TRAIN_64 = BASE_64_DIR + 'train/'
TEST_64 = BASE_64_DIR + 'test/'
PATH = '/home/puwenbo/puwenbo/Dataset/new3dDataset/test/'

if mode == 'E1':
    TEST_64 = TEST_64 + 'E1/'
    OUTPUT = '/home/puwenbo/puwenbo/Result/E1/'

if mode == 'E2':
    TEST_64 = TEST_64 + 'E2/'
    OUTPUT = '/home/puwenbo/puwenbo/Result/E2/'

p = sitk.ParameterMap()
p['FixedInternalImagePixelType'] = ['float']
p['MovingInternalImagePixelType'] = ['float']
p['FixedImageDimension'] = ['3']
p['MovingImageDimension'] = ['3']
p['UseDirectionCosines'] = ['true']
p['Registration'] = ['MultiResolutionRegistration']
p['Interpolator'] = [ "BSplineInterpolator"]
p['ResampleInterpolator'] = ["FinalBSplineInterpolator"]
p['Resampler'] = ["DefaultResampler"]
p['FixedImagePyramid'] = ["FixedRecursiveImagePyramid"]
p['MovingImagePyramid'] = ["MovingRecursiveImagePyramid"]
p['Optimizer'] = ["AdaptiveStochasticGradientDescent"]
p['Transform'] =  ["EulerTransform"]
p['Metric'] = ["AdvancedMattesMutualInformation"]
p['AutomaticScalesEstimation'] = [ "true"]
p['AutomaticTransformInitialization'] = ["true"]
p['HowToCombineTransforms'] = ["Compose"]
p['NumberOfHistogramBins']=['32']
p['ErodeMask'] = ['false']
p['NumberOfResolutions'] = ['4']
p['MaximumNumberOfIterations'] = ['250']
p['NumberOfSpatialSamples'] = ['2048']
p['NewSamplesEveryIteration'] = ['true']
p['ImageSampler'] = ['Random']
p['BSplineInterpolationOrder'] = ['1']
p['FinalBSplineInterpolationOrder'] = ['3']
p['DefaultPixelValue'] = ['0']

def get64():
    if not os.path.exists(PATH):
        os.mkdir(PATH)
    if not os.path.exists(PATH):
        os.mkdir(PATH)
    for dir in os.listdir('/mnt/hd3/new3dDataset/test'):
        if not os.path.exists(PATH+dir):
            os.mkdir(PATH+dir)
        for CT in os.listdir('/mnt/hd3/new3dDataset/test/'+dir+'/CT'):
            imageCT = sitk.ReadImage('/mnt/hd3/new3dDataset/test/'+dir+'/CT/'+CT)
            imageData = sitk.GetArrayFromImage(imageCT)
            res = transform.resize(imageData, (64,64,64))
            if not os.path.exists(PATH+dir+'/CT/'):
                os.mkdir(PATH+dir+'/CT/')
            sitk.WriteImage(sitk.GetImageFromArray(res), PATH+dir+'/CT/'+CT)
        for MR in os.listdir('/mnt/hd3/new3dDataset/test/'+dir+'/MR'):
            imageMR = sitk.ReadImage('/mnt/hd3/new3dDataset/test/' + dir + '/MR/' + MR)
            imageData = sitk.GetArrayFromImage(imageMR)
            res = transform.resize(imageData, (64, 64, 64))
            if not os.path.exists(PATH + dir + '/MR/'):
                os.mkdir(PATH + dir + '/MR/')
            sitk.WriteImage(sitk.GetImageFromArray(res), PATH + dir + '/MR/' + MR)

def getMovingIMG():
    for patient in os.listdir(TEST_64 + 'ground'):
        os.mkdir(OUTPUT + patient) if not os.path.exists(OUTPUT + patient) else print(patient)
        image = sitk.ReadImage(TEST_64+'ground/'+patient)
        imageData = sitk.GetArrayFromImage(image)
        T = getRandomT()
        # print(T)
        res = ndimage.affine_transform(imageData, tre3d.get_transform_matrix(T))
        sitk.WriteImage(sitk.GetImageFromArray(normalization(res)), OUTPUT + patient + '/moving.nii')
        np.savetxt(OUTPUT + patient + '/T', T)

def getRandomT():
    shift_z = random.randint(-10, 10)
    shift_y = random.randint(-10, 10)
    shift_x = random.randint(-10, 10)
    rotation_z = random.randint(-20, 20)
    # rotation_y = random.randint(-45, 45)
    # rotation_z = random.randint(-45, 45)
    # rotation_x = 0
    rotation_y = 0
    rotation_x = 0
    T = np.array([shift_z, shift_y, shift_x, rotation_z, rotation_y, rotation_x, 1])
    return T

def normalization(new_img):
    # ary = sitk.GetArrayFromImage(img)
    # new_img = np.copy(img)
    new_img[new_img < 0] = 0
    new_img = (new_img - np.min(new_img)) / (np.max(new_img) - np.min(new_img))
    return new_img

def elastix():
    for patient in os.listdir(TEST_64 + '/fixed'):
        try:
            elastixImageFilter = sitk.ElastixImageFilter()
            fix = sitk.ReadImage(TEST_64 + '/fixed/' + patient)
            fix_data = sitk.GetArrayFromImage(fix)
            fix_norm = sitk.GetImageFromArray(normalization(fix_data))
            elastixImageFilter.SetFixedImage(fix_norm)
            elastixImageFilter.SetMovingImage(sitk.ReadImage(OUTPUT + patient + '/moving.nii'))
            elastixImageFilter.SetParameterMap(p)
            if not os.path.isdir(OUTPUT+patient):
                os.mkdir(OUTPUT+patient)
            elastixImageFilter.SetOutputDirectory(OUTPUT+patient)
            elastixImageFilter.Execute()
            res = normalization(sitk.GetArrayFromImage(elastixImageFilter.GetResultImage()))
            sitk.WriteImage(sitk.GetImageFromArray(res), OUTPUT+patient+'/output.nii')
        except Exception:
            print(patient)

def getTrans(patient):
    file = open(OUTPUT + patient + '/TransformParameters.0.txt')
    TransformParameters = []
    for line in file:
        if line.startswith('(TransformParameters'):
            matrix = line.split(')')[0]
            matrix = matrix.split(' ')
            matrix = matrix[-6:]
            for i in range(6):
                if i > 2:
                    TransformParameters.append(math.degrees(float(matrix[5 - i])))
                else:
                    TransformParameters.append(float(matrix[5 - i]))
            TransformParameters.append(1)
            # print(TransformParameters)
            break
    file.close()
    return  TransformParameters

def get_t(patient):
    T = []
    patient = patient.split('.')[0]
    path = TEST_64
    with open(path+'ground2movingparameters.txt', 'r') as file:
        for line in file.readlines():
            parts = line.split('\t')
            name = parts[0]
            if patient == name:
                t = parts[1].strip('\n').strip('[').strip(']')
                t = t.split(' ')
                for i in t:
                    if i != '':
                        T.append(float(i))
                break
    return T

def getTRE(patient):
    TransformParameters = getTrans(patient)
    # T = get_t(patient)
    T = np.loadtxt(OUTPUT + patient + '/T')
    print(T)
    trans = np.array(TransformParameters, dtype=np.float32)
    # print(math.degrees(TransformParameters[3]))
    print(TransformParameters)
    res = tre3d.cal_tre_3d([64, 64, 64], T, trans)
    return res
#
# def getDice(patient):
#


"""
consts
======

Constants used by the app
"""

# Generating waveforms
# ====================

# The frequency of the samples in the waveform
SAMPLE_RATE = 24_000

# Printing waveforms
# ==================

# How tall to make the waveform
WAVE_HEIGHT = 10
# What to print where there is a wave
WAVE_CHAR = '█'
# What to print where there isn't a wave
SPACE_CHAR = ' '

# Type aliases
# ============

# Waveform
Waveform = list[float]


data = input()
dkt = {}
while not len(data) < 2:
    key, value = data.split('-')
    dkt[key] = value
    data = input()
for i in range(int(data)):
    name = input()
    if name in dkt:
        print(f"{name} -> {dkt[name]}")
    else:
        print(f"Contact {name} does not exist.")

SoftUni/Python fundamentals/Dictionaries - Exercise/Phonebook.py


# A neural network a day keeps stupid away
import numpy as np

np.random.seed(1)

def relu(x):
    return (x > 0) * x

def relu2deriv(x):
    return x > 0

inputs = np.array([[1, 0, 1],
                   [0, 1, 0],
                   [0, 0, 1],
                   [1, 1, 1]])

outputs = np.array([[1, 1, 0, 0]]).T

input_size = inputs.shape[1]
hidden_size = 16
output_size = outputs.shape[1]

weights_0_1 = 2 * np.random.random((input_size, hidden_size)) - 1
weights_1_2 = 2 * np.random.random((hidden_size, output_size)) - 1

epochs = 100
learning_rate = 0.02

for x in range(epochs):
    error = 0
    for i in range(len(inputs)):
        layer_0 = inputs[i:i+1]
        layer_1 = relu(np.dot(layer_0, weights_0_1))
        layer_2 = np.dot(layer_1, weights_1_2)

        target_pred = outputs[i:i+1]
        error += np.sum((layer_2 - target_pred) ** 2)

        layer_2_delta = layer_2 - target_pred
        layer_1_delta = np.dot(layer_2_delta, weights_1_2.T) * relu2deriv(layer_1)

        weights_1_2_delta = np.dot(layer_1.T, layer_2_delta)
        weights_0_1_delta = np.dot(layer_0.T, layer_1_delta)

        weights_1_2 -= weights_1_2_delta * learning_rate
        weights_0_1 -= weights_0_1_delta * learning_rate

    if x % 9 == 0:
        print("Error: {}".format(error))


import os
fname = input("Enter file name: ")
fh = open(fname)
lst = list()
for line in fh:
    list=line.split()
    for word in list:
        if word not in [c.lower() for c in lst]:
            lst.append(word)
lst.sort()
for word in lst:
    print(word)


import os
import sys
import tensorflow as tf
from argparse import Namespace
from collections import OrderedDict
from unsup_vvs.neural_fit.brainscore_mask.bs_fit_utils import color_normalize

from unsup_vvs.network_training.models.model_blocks import NoramlNetfromConv
import unsup_vvs.network_training.cmd_parser as cmd_parser
from unsup_vvs.network_training.models.config_parser import get_network_cfg
from unsup_vvs.network_training.models.model_builder import ModelBuilder
from unsup_vvs.network_training.models.rp_col_utils import rgb_to_lab
from unsup_vvs.network_training.models.mean_teacher_utils import \
        ema_variable_scope, name_variable_scope
from unsup_vvs.network_training.models.instance_task.model.instance_model \
        import resnet_embedding

MEAN_RGB = [0.485, 0.456, 0.406]

def get_simclr_ending_points(inputs):
    from unsup_vvs.network_training.models.simclr import resnet
    from absl import flags
    FLAGS = flags.FLAGS
    FLAGS(['none'])
    with tf.variable_scope('base_model'):
        model = resnet.resnet_v1(
                resnet_depth=18,
                width_multiplier=1,
                cifar_stem=False,
                drop_final_pool=True)
        output = model(
                tf.cast(inputs['images'], tf.float32) / 255,
                is_training=False)
    ending_points = resnet.ENDING_POINTS
    return ending_points

def get_network_cfg_from_setting(setting_name=None, **kwargs):
    train_args = Namespace()

    # Required fields for ModelBuilder
    train_args.seed = None
    train_args.weight_decay = 0.0
    train_args.global_weight_decay = 0.0
    train_args.enable_bn_weight_decay = False
    train_args.ignorebname_new = 1
    train_args.add_batchname = None
    train_args.tpu_flag = 1
    train_args.combine_tpu_flag = 0
    train_args.tpu_task = None
    train_args.init_stddev = .01
    train_args.init_type = 'xavier'
    train_args.sm_bn_trainable = 0
    train_args.sm_resnetv2 = 0
    train_args.sm_resnetv2_1 = 0

    for key, value in kwargs.items():
        setattr(train_args, key, value)

    if setting_name is not None:
        train_args.load_setting_func = setting_name
        train_args = cmd_parser.load_setting(train_args)
        if not hasattr(train_args, 'network_func_kwargs'):
            train_args.network_func_kwargs = None
    else:
        assert (hasattr(train_args, 'network_func') \
                or hasattr(train_args, 'pathconfig')), \
                'Must provide network cfg'
        if not hasattr(train_args, 'network_func'):
            train_args.network_func = None

    network_cfg = get_network_cfg(train_args)
    return network_cfg, train_args

def build_vm_model_from_args(args, post_input_image, module_name=['encode'], train=False):
    model_builder = ModelBuilder(args, {})

    # Preparations before building one module
    model_builder.dataset_prefix = 'imagenet'
    model_builder.reuse_dict = {}
    model_builder.init_model_block_class()
    model_builder.train = train
    model_builder.all_out_dict = OrderedDict()
    model_builder.outputs_dataset = OrderedDict()
    model_builder.save_layer_middle_output = True

    for each_module_name in module_name:
        model_builder.build_one_module(each_module_name, post_input_image)
    return model_builder.all_out_dict

def get_network_outputs(
        inputs,
        prep_type,
        model_type,
        setting_name=None,
        module_name=['encode'],
        inst_resnet_size=18,
        train=False,
        **cfg_kwargs):
    input_image = inputs['images']
    if prep_type == 'only_mean':
        input_image = tf.cast(input_image, tf.float32) / 255
        offset = tf.constant(MEAN_RGB, shape=[1, 1, 1, 3])
        post_input_image = input_image - offset
    elif prep_type == 'mean_std':
        # Divided by 255 is done inside the function
        post_input_image = color_normalize(input_image)
    elif prep_type == 'color_prep':
        input_image = tf.cast(input_image, tf.float32) / 255
        post_input_image = rgb_to_lab(input_image)
        post_input_image = post_input_image[:,:,:,:1] - 50
    elif prep_type == 'no_prep':
        post_input_image = tf.cast(input_image, tf.float32) / 255
    else:
        raise NotImplementedError('Preprocessing type not supported!')

    if model_type == 'vm_model':
        network_cfg, args = get_network_cfg_from_setting(setting_name, **cfg_kwargs)
        all_outs = build_vm_model_from_args(args, post_input_image, module_name, train=train)
    elif model_type == 'mt_vm_model':
        network_cfg, args = get_network_cfg_from_setting(setting_name, **cfg_kwargs)
        with name_variable_scope("primary", "primary", reuse=tf.AUTO_REUSE) \
                as (name_scope, var_scope):
            all_outs = build_vm_model_from_args(
                    args, post_input_image, module_name, train=train)
        with ema_variable_scope("ema", var_scope, reuse=tf.AUTO_REUSE):
            ema_out_dict = build_vm_model_from_args(
                    args, post_input_image, module_name, train=train)
            # Combine two out dicts, adding ema_ as prefix to keys
            for each_key, each_value in ema_out_dict.items():
                new_key = "ema_%s" % each_key
                assert not new_key in all_outs, \
                        "New key %s already exists" % new_key
                all_outs[new_key] = each_value
    elif model_type.startswith('inst_model'):
        # inst_model:get_all_layers_args
        if ':' not in model_type:
            get_all_layers = 'all_spatial'
        else:
            get_all_layers = model_type.split(':')[1]
        all_outs = resnet_embedding(
                inputs['images'],
                get_all_layers=get_all_layers,
                skip_final_dense=True,
                resnet_size=inst_resnet_size)
    elif model_type.startswith('vd_inst_model'):
        # model_type should be vd_inst_model:actual_model_type
        sys.path.append(os.path.expanduser('~/video_unsup/'))
        import tf_model.model.instance_model as vd_inst_model
        _model_type = model_type.split(':')[1]
        input_images = inputs['images']
        if _model_type == '3dresnet':
            curr_shape = input_images.get_shape().as_list()[1]
            input_images = tf.image.resize_images(
                    input_images, [curr_shape // 2, curr_shape // 2])
            input_images = tf.tile(
                    tf.expand_dims(input_images, axis=1),
                    [1, 16, 1, 1, 1])
        elif _model_type == '3dresnet_full':
            input_images = tf.tile(
                    tf.expand_dims(input_images, axis=1),
                    [1, 16, 1, 1, 1])
            _model_type = '3dresnet'
        elif _model_type == 'vanilla3d_single':
            input_images = tf.tile(
                    tf.expand_dims(input_images, axis=1),
                    [1, 16, 1, 1, 1])
        all_outs = vd_inst_model.resnet_embedding(
                input_images,
                get_all_layers='all_spatial',
                skip_final_dense=True,
                model_type=_model_type)
    elif model_type.startswith('vd_prednet'):
        # model_type should be vd_prednet:actual_model_type
        _model_type = model_type.split(':')[1]
        images = inputs['images']
        if _model_type == 'prednet_l9':
            curr_shape = images.get_shape().as_list()[1]
            if not curr_shape % 32 == 0:
                new_shape = curr_shape // 32 * 32
                images = tf.image.resize_images(
                        images, [new_shape, new_shape])
        import unsup_vvs.network_training.models.prednet_builder as prednet_builder
        all_outs = OrderedDict(prednet_builder.build_all_outs(images, _model_type))
    elif model_type == 'simclr_model':
        ending_points = get_simclr_ending_points(inputs)
        all_outs = OrderedDict()
        for _idx, _rep in enumerate(ending_points):
            all_outs['encode_%i' % (_idx+1)] = _rep
    elif model_type == 'simclr_model_mid':
        ending_points = get_simclr_ending_points(inputs)
        output = ending_points[-1]
        output = tf.reshape(output, [output.shape[0], -1])
        m = NoramlNetfromConv(seed=0)
        with tf.variable_scope('category_trans'):
            with tf.variable_scope('mid'):
                output = m.fc(
                        out_shape=1000,
                        init='xavier',
                        weight_decay=1e-4,
                        activation='relu',
                        bias=0.1,
                        dropout=None,
                        in_layer=output,
                        )
        all_outs = {'category_trans_1': output}
    else:
        raise NotImplementedError('Model type not supported!')
    all_outs['model_inputs'] = post_input_image
    return all_outs, {}

def test_vm_model():
    setting_name = 'part10_cate_res18_fix'
    input_image = tf.zeros([1, 224, 224, 3], dtype=tf.float32)
    network_outputs = get_network_outputs(
            {'images': input_image},
            prep_type='mean_std',
            model_type='vm_model',
            setting_name=setting_name)
    print(network_outputs)

def test_inst_model():
    input_image = tf.zeros([1, 224, 224, 3], dtype=tf.float32)
    network_outputs = get_network_outputs(
            {'images': input_image},
            prep_type='no_prep',
            model_type='inst_model',
            )
    print(network_outputs)

def test_video_model():
    input_image = tf.zeros([1, 224, 224, 3], dtype=tf.float32)
    network_outputs = get_network_outputs(
            {'images': input_image},
            prep_type='no_prep',
            model_type='vd_inst_model:3dresnet',
            )
    print(network_outputs)

def test_simclr_model():
    input_image = tf.zeros([1, 224, 224, 3], dtype=tf.float32)
    network_outputs = get_network_outputs(
            {'images': input_image},
            prep_type='no_prep',
            model_type='simclr_model',
            )
    print(network_outputs)

if __name__ == '__main__':
    #test_vm_model()
    #test_inst_model()
    #test_video_model()
    test_simclr_model()


import argparse
import datetime
import os
import shutil
import sys
import time

def migrate_to_goose(directory):

  new_migration = True
  first_iteration = True
  out_dir = os.path.join(directory, "output")
  datafix_dir = os.path.join(out_dir, "datafix")
  template_file_dir = os.path.join(directory, "template")
  template_file_path = os.path.join(template_file_dir, "create_goose_db_version_table_template.sql")
  current_filename = ""

  if not os.path.isdir(out_dir):
    os.system("mkdir %s" % out_dir)
  else:
    shutil.rmtree(out_dir)
    os.system("mkdir %s" % out_dir)

  # create dir for datafix file and create empty file
  if not os.path.isdir(datafix_dir):
    os.system("mkdir %s" % datafix_dir)
  else:
    shutil.rmtree(datafix_dir)
    os.system("mkdir %s" % datafix_dir)

  datafix_file_name = "migrate_to_goose_datafix.sql"
  datafix_file_path = os.path.join(datafix_dir, datafix_file_name)

  shutil.copyfile(template_file_path, datafix_file_path)

  up_migration_contents = ""
  down_migration_contents = ""

  for filename in sorted(os.listdir(directory)):
    f = os.path.join(directory, filename)

    if os.path.isfile(f):
      file_full_path = (os.path.join(os.path.dirname(os.path.abspath(__file__)), f))
      sliced_filename = filename[5:-4].replace(".down", "").replace(".up", "")

    # check if it is the same file we are processing
    if (sliced_filename != current_filename) and (not first_iteration):
      file_timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
      new_file_name_with_ext = file_timestamp + "_" + current_filename + ".sql"
      new_file_dir = os.path.join(out_dir, new_file_name_with_ext)

      with open(new_file_dir, 'w') as new_file:
        print("creating new migration file %s" % new_file_dir)
        # up migration
        new_file.write("-- +goose Up\n")
        new_file.write("-- +goose StatementBegin\n")
        new_file.write(up_migration_contents)
        new_file.write("\n-- +goose StatementEnd\n")
        new_file.write("\n")

        # down migration
        new_file.write("-- +goose Down\n")
        new_file.write("-- +goose StatementBegin\n")
        new_file.write(down_migration_contents)
        new_file.write("\n-- +goose StatementEnd\n")

        time.sleep(5)

      with open(datafix_file_path, 'a') as datafix_file:
        datafix_file.write("INSERT INTO goose_db_version (version_id, is_applied, tstamp) VALUES (%s, 1, NOW());\n" % file_timestamp)

      new_migration = True

    if ('.sql' in filename) and ('.down.' in filename):
      with open (file_full_path, "r") as down_mig_file:
        file_contents = down_mig_file.readlines()
        raw_file_contents = " ".join(map(str, file_contents))
        down_migration_contents = raw_file_contents

    elif ('.sql' in filename) and ('.up.' in filename):
      with open (file_full_path, "r") as up_mig_file:
        file_contents = up_mig_file.readlines()
        raw_file_contents = " ".join(map(str, file_contents))
        up_migration_contents = raw_file_contents

    else:
      # check if there is any migration file pending to be written

      print ("not a sql file - skipping migration of %s" % f)

    if first_iteration:
      first_iteration = False

    current_filename = sliced_filename

if __name__== "__main__":

  # read input args
  parser = argparse.ArgumentParser()
  parser.add_argument("-p", "--path", help = "folder path where the migrations are stored")
  args = parser.parse_args()

  if args.path:
    print("INFO: converting files under %s to goose format" % args.path)
    migrate_to_goose(args.path)
  else:
    print("ERROR: no input path supplied. aborting run")
    sys.exit()


import os
import shutil
import subprocess

from . import product
from .. import shell
from .. import targets

class TensorFlowSwiftAPIs(product.Product):
    @classmethod
    def product_source_name(cls):
        return "tensorflow-swift-apis"

    @classmethod
    def is_build_script_impl_product(cls):
        return False

    def should_build(self, host_target):
        return self.args.build_tensorflow_swift_apis

    def build(self, host_target):
        toolchain_path = targets.toolchain_path(self.args.install_destdir,
                                                self.args.install_prefix)
        swiftc = os.path.join(toolchain_path, 'usr', 'bin', 'swiftc')

        tensorflow_source_dir = os.path.join(self.source_dir, '..',
                                             'tensorflow')
        tensorflow_source_dir = os.path.realpath(tensorflow_source_dir)

        if host_target.startswith('macosx'):
            lib_name = 'libtensorflow.dylib'
        elif host_target.startswith('linux'):
            lib_name = 'libtensorflow.so'
        else:
            raise RuntimeError("Unknown host target %s" % host_target)

        # FIXME: this is a workaround for CMake <3.16 which does not correctly
        # generate the build rules if you are not in the build directory.  As a
        # result, we need to create the build tree before we can use it and
        # change into it.
        #
        # NOTE: unfortunately, we do not know if the build is using Python
        # 2.7 or Python 3.2+.  In the latter, the `exist_ok` named parameter
        # would alleviate some of this issue.
        try:
            os.makedirs(self.build_dir)
        except OSError:
            pass

        with shell.pushd(self.build_dir):

            shell.call([
                self.toolchain.cmake,
                '-G', 'Ninja',
                '-D', 'BUILD_SHARED_LIBS=YES',
                '-D', 'CMAKE_INSTALL_PREFIX={}/usr'.format(
                    self.args.install_destdir),
                '-D', 'CMAKE_MAKE_PROGRAM={}'.format(self.toolchain.ninja),
                '-D', 'CMAKE_Swift_COMPILER={}'.format(swiftc),
                # SWIFT_ENABLE_TENSORFLOW
                '-D', 'USE_BUNDLED_CTENSORFLOW=YES',
                # SWIFT_ENABLE_TENSORFLOW END
                '-D', 'TensorFlow_INCLUDE_DIR={}'.format(tensorflow_source_dir),
                '-D', 'TensorFlow_LIBRARY={}'.format(
                    os.path.join(tensorflow_source_dir, 'bazel-bin', 'tensorflow',
                                 lib_name)),
                '-D', 'CMAKE_Swift_FLAGS={}'.format('-L{}'.format(
                    os.path.join(tensorflow_source_dir, 'bazel-bin', 'tensorflow'))
                ),
                '-B', self.build_dir,
                '-S', self.source_dir,
            ])
            shell.call([
                self.toolchain.cmake,
                '--build', self.build_dir,
            ])

    def should_test(self, host_target):
        return False

    def test(self, host_target):
        pass

    def should_install(self, host_target):
        return self.args.build_tensorflow_swift_apis

    def install(self, host_target):
        shell.call([
            self.toolchain.cmake,
            '--build', self.build_dir,
            '--target', 'install',
        ])

# SWIFT_ENABLE_TENSORFLOW
def _get_tensorflow_library(host):
    if host.startswith('macosx'):
        return ('libtensorflow.2.1.0.dylib', 'libtensorflow.dylib')

    if host.startswith('linux'):
        return ('libtensorflow.so.2.1.0', 'libtensorflow.so')

    raise RuntimeError('unknown host target {}'.format(host))

def _silenced(op):
    def inner(*args, **kwargs):
        try:
            return op(*args, **kwargs)
        except OSError:
            pass
    return inner

def _symlink(dest, src):
    _silenced(os.unlink)(src)
    os.symlink(dest, src)

class TensorFlow(product.Product):
    @classmethod
    def product_source_name(cls):
        """product_source_name() -> str
        The name of the source code directory of this product.
        """
        return "tensorflow"

    @classmethod
    def is_build_script_impl_product(cls):
        return False

    def should_build(self, host_target):
        return self.args.build_tensorflow_swift_apis

    def build(self, host_target):
        with shell.pushd(self.source_dir):
            # Run the TensorFlow configure script: `yes "" | ./configure`.
            # NOTE: consider rewriting `subprocess` API usages using `shell`
            # APIs.
            yes_process = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE)
            subprocess.check_call([os.path.join(self.source_dir, 'configure')],
                                  stdin=yes_process.stdout)
            yes_process.terminate()

            # Build TensorFlow via bazel.
            shell.call([
                self.toolchain.bazel,
                "build",
                "-c", "opt",
                "--define", "framework_shared_object=false",
                "//tensorflow:tensorflow",
            ])

        # bazel builds libraries with version suffixes, e.g.
        # "libtensorflow.{dylib,so}.x.y.z".
        # Create a symlink to the standard unsuffixed library name:
        # "libtensorflow.{dylib,so}".
        (suffixed_lib_name, unsuffixed_lib_name) = \
            _get_tensorflow_library(host_target)

        # NOTE: ignore the race condition here ....
        _symlink(os.path.join(self.source_dir, 'bazel-bin', 'tensorflow',
                              suffixed_lib_name),
                 os.path.join(self.source_dir, 'bazel-bin', 'tensorflow',
                              unsuffixed_lib_name))

    def should_test(self, host_target):
        return False

    def test(self, host_target):
        pass

    def should_install(self, host_target):
        return self.args.build_tensorflow_swift_apis

    def install(self, host_target):
        (suffixed_lib_name, unsuffixed_lib_name) = \
            _get_tensorflow_library(host_target)

        subdir = None
        if host_target.startswith('macsox'):
            subdir = 'macosx'
        if host_target.startswith('linux'):
            subdir = 'linux'

        if not subdir:
            raise RuntimeError('unknown host target {}'.format(host_target))

        _silenced(os.unlink)(os.path.join(self.install_toolchain_path(),
                                          'usr', 'lib', 'swift', subdir,
                                          suffixed_lib_name))
        _silenced(os.makedirs)(os.path.join(self.install_toolchain_path(),
                                            'usr', 'lib', 'swift', subdir))
        shutil.copy(os.path.join(self.source_dir, 'bazel-bin', 'tensorflow',
                                 suffixed_lib_name),
                    os.path.join(self.install_toolchain_path(),
                                 'usr', 'lib', 'swift',
                                 subdir, suffixed_lib_name))
        # Add write permissions to `libtensorflow.{dylib,so}`.  This is required
        # for symbol stripping and code signing.
        os.chmod(os.path.join(self.install_toolchain_path(),
                              'usr', 'lib', 'swift', subdir, suffixed_lib_name),
                 0o755)

        if host_target.startswith('linux'):
            versions = (
                    'libtensorflow.so.2.1.0',
                    'libtensorflow.so.2.1',
                    'libtensorflow.so.2',
                    'libtensorflow.so',
            )

            for (index, value) in enumerate(versions[:-1]):
                _symlink(value,
                         os.path.join(self.install_toolchain_path(),
                                      'usr', 'lib', 'swift', subdir,
                                       versions[index + 1]))
        else:
            _symlink(suffixed_lib_name,
                     os.path.join(self.install_toolchain_path(),
                                  'usr', 'lib', 'swift', subdir,
                                  unsuffixed_lib_name))

        _silenced(shutil.rmtree)(os.path.join(self.install_toolchain_path(),
                                              'usr', 'lib', 'swift',
                                              'tensorflow'))
        _silenced(os.makedirs)(os.path.join(self.install_toolchain_path(),
                                            'usr', 'lib', 'swift', 'tensorflow',
                                            'c', 'eager'))
        for header in (
                'c_api.h',
                'c_api_experimental.h',
                'tf_attrtype.h',
                'tf_datatype.h',
                'tf_status.h',
                'tf_tensor.h',
                'eager/c_api.h',
        ):
            shutil.copy(os.path.join(self.source_dir, 'tensorflow', 'c', header),
                        os.path.join(self.install_toolchain_path(),
                                     'usr', 'lib', 'swift', 'tensorflow', 'c',
                                     header))

        for name in (
                'CTensorFlow.h',
                'module.modulemap',
        ):
            shutil.copy(os.path.join(self.source_dir, '..',
                                     'tensorflow-swift-apis', 'Sources',
                                     'CTensorFlow', name),
                        os.path.join(self.install_toolchain_path(),
                                     'usr', 'lib', 'swift', 'tensorflow', name))

# SWIFT_ENABLE_TENSORFLOW END


"""GitHub module

This module enables you to manage list the languages used by the 100
trending public repositories on GitHub.
"""

import requests
from operator import itemgetter

from backend.models import get_data_json, set_data_json

# URL of list all public repositories on GitHub
URL_REPOS = "https://api.github.com/repositories"

# Lists languages for the specified repository
def languages(repos) -> list:
    """This function returns a lists of languages for the
    specified repository.

    """
    rep = requests.get(repos['url'] + '/languages')
    if rep.status_code != 200:
        return None
    return list(rep.json())

# List of  100 trending public repositories on GitHub in json format
def json_repos() -> list:
    """This funtion returns a list of  100 trending public repository
    on GitHub in json format.

    """
    list_repos = requests.get(URL_REPOS)

    if list_repos.status_code != 200:
        return None

    return list_repos.json()

# List of  100 trending public repos on GitHub and languages used
def repositories() -> list:
    """This funtion returns a list of 100 trending Github public
    repositories and the  languages they use.

    Return the `list` of `dict`.

    """
    # List of  repositories on GitHub in json format
    json = json_repos()

    # If GitHub-API rate limit exceeded
    if not json:
        # used data stored in `data.json`
        return get_data_json()

    list_repos = []
    for repos in json:
        dict = {}
        dict['repository'] = {
            'html_url': repos['html_url'],
            'url': repos['url']
        }
        # If GitHub-API rate limit exceeded
        lang = languages(repos)
        if isinstance(lang, list):
            dict['languages'] = lang
        else:
            return get_data_json()
        list_repos.append(dict)

    # Store the list of repositories in `data.json`
    if list_repos != get_data_json():
        set_data_json(list_repos)

    return list_repos

# This function returns a repository specified by `id`
def repository(id :int) -> dict:
    """This function returns a repository specified by `id`.

    Parameters :
        id :int - the repository number, is integer between 0 and 99.
    """
    # List of  repositories on GitHub in json format
    json = json_repos()

    # if GitHub-API rate limit exceeded
    if not json:
        # used data stored in `data.json`
        list_repos = get_data_json()
        try:
            return list_repos[id]
        except:
            return None

    # repository specified by `id`
    try:
        # 'id' is integer between 0 and 99
        repos = json[id]
    except:
        return None

    dict = {}
    # repository info
    dict['repository'] = {
        'html_url': repos['html_url'],
        'url': repos['url']
    }
    # Lists languages for `repos`
    dict['languages'] = languages(repos)

    return dict

# Number of repositories using a language
def number_repos(language=None):
    """This function returns the number of repositories using a language.

    """
    # List of repositories and the languages they use
    list_repos = repositories()

    if not list_repos:
        return None

    # for a given language
    if language:
        # `language` is string
        if not isinstance(language, str):
            return None

        nb_repos = 0
        # browse the list of repositories
        for repos in list_repos:
            # Test if `language` is in list of languages used by `repos`
            if language.lower() in map(str.lower, repos['languages']):
                nb_repos += 1

        # if the `language` not found
        if not nb_repos:
            return None

        return {
            "language": language,
            "nb_repos": nb_repos
        }

    # for all languages
    list_number_repos = {}
    # browse the list of repositories
    for repos in list_repos:
        # browse the list of languages used by `repos`
        for language in repos['languages']:
            try:
                list_number_repos[language] += 1
            except:
                list_number_repos[language] = 1

    # sort results by order descending
    list_number_repos = sort(list_number_repos)

    return [
        {'language': language, 'nb_repos': nb_repos}
        for language, nb_repos in list_number_repos.items()
    ]

# List of repositories using a language
def list_repos(language=None):
    """This function returns the list of repositories using a language.

    """
    # List of repositories and the languages they use
    list_repos = repositories()

    if not list_repos:
        return None

    # for a given language
    if language:
        # `language` is string
        if not isinstance(language, str):
            return None

        list_repos_lang = []
        # browse the list of repositories
        for repos in list_repos:
            # Test if `language` is in list of languages used by `repos`
            if language.lower() in map(str.lower, repos['languages']):
                list_repos_lang.append(repos['repository'])

        # if the `language` not found
        if not list_repos_lang:
            return None

        return {
            "language": language,
            "repositories": list_repos_lang
        }

    # for all languages
    list_lang_list_repos = {}
    # browse the list of repositories
    for repos in list_repos:
        # browse the list of languages used by `repos`
        for language in repos['languages']:
            try:
                list_lang_list_repos[language].append(repos['repository'])
            except:
                list_lang_list_repos[language] = [repos['repository']]

    # sort results by order descending
    list_lang_list_repos = sort(
        list_lang_list_repos,
        lambda tuple: len(tuple[1])
    )

    return [
        {'language': language, 'repositories': list_repos}
        for language, list_repos in list_lang_list_repos.items()
    ]

# Framework popularity over the 100 repos
def popularity(language=None, plot=False):
    """This function returns Framework popularity over the 100 repos
    """
    # List of repositories and the languages they use
    list_repos = repositories()

    if not list_repos:
        return None

    # for a given language
    if language:
        # `language` is string
        if not isinstance(language, str):
            return None

        nb_repos, sum_number_repos = 0, 0
        # browse the list of repositories
        for repos in list_repos:
            # sum (number of `repos` using a language) for all language
            sum_number_repos += len(repos['languages'])

            # Test if `language` is in list of languages used by `repos`
            if language.lower() in map(str.lower, repos['languages']):
                nb_repos += 1

        # if the `language` not found
        if not nb_repos:
            return None

        rate = nb_repos / sum_number_repos * 100

        if plot:
            return (
                [language, 'Others languages'],
                [
                    {'name': language, 'y': rate},
                    {'name': 'Others languages', 'y': 100 - rate}
                ]
            )

        return {
            "language": language,
            "popularity": rate
        }

    # for all languages
    list_nb_repos = {}
    # browse the list of repositories
    for repos in list_repos:
        # browse the list of languages used by `repos`
        for language in repos['languages']:
            try:
                list_nb_repos[language] += 1
            except:
                list_nb_repos[language] = 1

    # sum (number of `repos` using a language) for all language
    sum_number_repos = sum(list_nb_repos.values())

    if plot:
        return (
            list(list_nb_repos.keys()),
            [{'name': k, 'y': list_nb_repos[k]} for k in list_nb_repos]
        )

    # sort results by order descending
    list_nb_repos = sort(list_nb_repos)

    return [{
            'language': language,
            'popularity': nb_repos / sum_number_repos * 100
        }
        for language, nb_repos in list_nb_repos.items()
    ]

# Sort descending a dictionary by value
def sort(d :dict, key=itemgetter(1)) -> dict:
    """ Sort descending a dictionary by value """
    return dict(
        sorted(d.items(), key=key, reverse=True)
    )


import pickle
import gymnasium as gym
import torch

from torch import nn
from torch.optim import Adam
from torch.distributions import Categorical
from utils import *
from gymnasium.utils.save_video import save_video

# Class for training an RL agent within an environment
class PGTrainer:
    def __init__(self, params):
        self.params = params
        self.env = gym.make(self.params['env_name'])
        self.agent = Agent(env=self.env, params=self.params)
        self.actor_policy = PGPolicy(input_size=self.env.observation_space.shape[0], output_size=self.env.action_space.n, hidden_dim=self.params['hidden_dim']).to(get_device())
        self.optimizer = Adam(params = self.actor_policy.parameters(), lr=self.params['lr'])

    def run_training_loop(self):
        list_ro_reward = list()

        for ro_idx in range(self.params['n_rollout']):
            trajectory = self.agent.collect_trajectory(policy=self.actor_policy)
            #print(trajectory)
            loss = self.estimate_loss_function(trajectory)
            self.update_policy(loss)
            # DONE TODO: Calculate avg reward for this rollout
            # HINT: Add all the rewards from each trajectory. There should be "ntr" trajectories within a single rollout.
            avg_ro_reward = sum(sum(traj_rewards) for traj_rewards in trajectory['reward']) / self.params['n_trajectory_per_rollout']
            print(f'End of rollout {ro_idx}: Average trajectory reward is {avg_ro_reward: 0.2f}')
            # Append average rollout reward into a list
            list_ro_reward.append(avg_ro_reward)
        # Save avg-rewards as pickle files
        pkl_file_name = self.params['exp_name'] + '.pkl'
        with open(pkl_file_name, 'wb') as f:
            pickle.dump(list_ro_reward, f)
        # Save a video of the trained agent playing
        self.generate_video()
        # Close environment
        self.env.close()

    def estimate_loss_function(self, trajectory):
        loss = list()
            # DONE TODO: Compute loss function
            # HINT 1: You should implement equation of policy gradient, reward to go and reward discounting here. Which will be used based on the flags set from the main function
            # Get trajectory action log-prob
            # Calculate discounted and reward-to-go
            # Calculate loss
            # HINT 2: Get trajectory action log-prob
            # HINT 3: Calculate Loss function and append to the list
        gamma = 0.99
        for t_idx in range(self.params['n_trajectory_per_rollout']):
            log_probs = trajectory['log_prob'][t_idx]
            rewards = trajectory['reward'][t_idx]
            if self.params['reward_to_go']:
                reward_sum = 0
                for i in range(len(rewards) - 1, -1, -1):
                    reward_sum = rewards[i] + gamma * reward_sum
                    loss.append(-log_probs[i] * reward_sum)
            else:
                reward_sum = sum(rewards)
                if self.params['reward_discount']:
                    for i in range(len(rewards)):
                        loss.append(-log_probs[i] * (reward_sum * gamma ** i))
                else:
                    for i in range(len(rewards)):
                        loss.append(-log_probs[i] * reward_sum)
        print("Discounted reward to go:", apply_discount(rewards))
        print("Reward-to-go:", apply_reward_to_go(rewards))
        print("Return-Reward: ", apply_return(rewards))

        loss = torch.stack(loss).mean()
        return loss

    def update_policy(self, loss):
        loss.backward()
        self.optimizer.step()
        self.optimizer.zero_grad()

    def generate_video(self, max_frame=1000):
        self.env = gym.make(self.params['env_name'], render_mode='rgb_array_list')
        obs, _ = self.env.reset()
        for _ in range(max_frame):
            action_idx, log_prob = self.actor_policy(torch.tensor(obs, dtype=torch.float32, device=get_device()))
            obs, reward, terminated, truncated, info = self.env.step(self.agent.action_space[action_idx.item()])
            if terminated or truncated:
                break
        save_video(frames=self.env.render(), video_folder=self.params['env_name'][:-3], fps=self.env.metadata['render_fps'], step_starting_index=0, episode_index=0)

# CLass for policy-net
class PGPolicy(nn.Module):
    def __init__(self, input_size, output_size, hidden_dim):
        super(PGPolicy, self).__init__()
        # TODO: Define the policy net
        # HINT: You can use nn.Sequential to set up a 2 layer feedforward neural network.
        self.policy_net = nn.Sequential(
            nn.Linear(input_size,hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim,output_size),
            nn.Softmax(dim = -1)
        )

    def forward(self, obs):
        # TODO: Forward pass of policy net
        # HINT: (use Categorical from torch.distributions to draw samples and log-prob from model output)
        logits = self.policy_net(obs)
        dist = Categorical(logits=logits)
        action_index = dist.sample()
        log_prob = dist.log_prob(action_index)
        return action_index, log_prob

# Class for agent
class Agent:
    def __init__(self, env, params=None):
        self.env = env
        self.params = params
        self.action_space = [action for action in range(self.env.action_space.n)]

    def collect_trajectory(self, policy):
        obs, _ = self.env.reset(seed=self.params['rng_seed'])
        rollout_buffer = list()
        for _ in range(self.params['n_trajectory_per_rollout']):
            trajectory_buffer = {'log_prob': list(), 'reward': list()}
            while True:
                # TODO: Get action from the policy (forward pass of policy net)
                action_idx, log_prob = policy(torch.tensor(obs, dtype=torch.float32, device=get_device()))
                # TODO: Step environment (use self.env.step() function)
                obs, reward, terminated, truncated, info = self.env.step(self.action_space[action_idx.item()])
                # Save log-prob and reward into the buffer
                trajectory_buffer['log_prob'].append(log_prob)
                trajectory_buffer['reward'].append(reward)
                # Check for termination criteria
                if terminated or truncated:
                    obs, _ = self.env.reset()
                    rollout_buffer.append(trajectory_buffer)
                    break
        rollout_buffer = self.serialize_trajectory(rollout_buffer)
        return rollout_buffer

    # Converts a list-of-dictionary into dictionary-of-list
    @staticmethod
    def serialize_trajectory(rollout_buffer):
        serialized_buffer = {'log_prob': list(), 'reward': list()}
        for trajectory_buffer in rollout_buffer:
            serialized_buffer['log_prob'].append(torch.stack(trajectory_buffer['log_prob']))
            serialized_buffer['reward'].append(trajectory_buffer['reward'])
        return serialized_buffer


import numpy as np
from scipy.spatial.distance import minkowski
from .base import ClusteringMethod

class KMeans(ClusteringMethod):

  def __init__(self, epochs, n_clusters, distance_method=minkowski):
    self.epochs     = epochs
    self.n_clusters = n_clusters
    self.distance_method = distance_method
    # Extra
    self.reuse = False

  def fit(self, X):
    self.init_centroids(X)
    grupos = None
    for e in range(self.epochs):
        if (e%20 == 0):
          print(f">> epoch: {e}")
        grupos = self.get_groups(X)
        new_c = self.new_centroids(X, grupos)
        #diff = np.array(self.centroids) - np.array(new_c)
        self.centroids = new_c
    return grupos

  def get_groups(self, X):
    groups = []
    for x in X:
        dist = np.array( [self.distance_method(c, x) for c in self.centroids])
        g    = np.argmin( dist )
        groups.append(g)
    groups = np.array(groups)
    return groups

  def new_centroids(self, X, grupos):
    new_c = []
    for c in range(self.n_clusters):
        elements = X[grupos == c]
        if len(elements) > 0:
          new_c.append(np.mean(elements, axis = 0))
        else:
          if not self.reuse:
            print("REUSE")
            self.reuse=True
          new_c.append(self.centroids[c])
    new_c = np.array(new_c)

    return new_c

  def init_centroids(self, X):
    # Select centoids
    self.centroids = X[np.random.choice(len(X), self.n_clusters, replace=False)]


# -*- coding: utf-8 -*-

from keybert import KeyBERT
from konlpy.tag import Mecab

def tokenizer(sentence):
    mecab = Mecab(dicpath=r"C:\mecab\mecab-ko-dic")
    pos_lst = ["NNG", "NNP"]
    filtered_lst = [word for word, pos in mecab.pos(sentence) if pos in pos_lst]

    return filtered_lst

def keyword(doc):
    kw_model = KeyBERT()

    keywords = kw_model.extract_keywords(docs=doc)

    candidates_lst = [word for word, _ in keywords]
    candidates_lst = " ".join(candidates_lst)

    fin_keywords = tokenizer(candidates_lst)

    return fin_keywords

if __name__ == "__main__":
    example1 = "강릉여행. 온 가족이 함께 강릉으로 여행을 갔다. 다같이 가서 회도 먹고 물놀이도 하고 너무 즐거웠다"
    example2 = "추석에 온 가족이 모여 찍은 가족사진. 오랜만에 모여 윳놀이도 하고, 고스톱도 치고 재미있었다."
    example3 = "진통이 정말 심했어요. 새벽에 아버지한테 제가 전화해서 너무 아프다고 울었어요. 그래서 그날 가족들 다같이 병원으로 와서 진통하는 걸 봤었지요. 그때 아머지가 계속 손잡고 옆을 지켜주셨어요. 둘째 낳고 아버지가 보양식이며 배냇저고리며 이것저것 엄청 챙겨주셨던 기억이 나요. 낳자마자 아버지가 손수 이름 지어주겠다고 사흘밤낮동안 고민하셨던 것도 눈에 선해요"

    model = KeyBERT()
    test = model.extract_keywords(docs=[example1, example2, example3])
    print(test)


#!/usr/bin/python3
# -*- coding: utf-8 -*-
from PyQt5 import QtCore, QtGui, QtWidgets,uic
from PyQt5.QtWidgets import QApplication
import sys,time,os
import serial
import serial.tools.list_ports

TITLE = 'simple Serial Tool'

qtCreatorFile = "ui/main.ui" # Enter file here.
Ui_MainWindow, QtBaseClass = uic.loadUiType(qtCreatorFile)

qtCreatorFile = "ui/message.ui" # Enter file here.
Ui_Dialog, QtBaseClass = uic.loadUiType(qtCreatorFile)

def center_on_screen(self,QtWidgets):
    resolution = QtWidgets.QDesktopWidget().screenGeometry()
    self.move((resolution.width() / 2) - (self.frameSize().width() / 2),(resolution.height() / 2) - (self.frameSize().height() / 2))

class DataCaptureThread(QtCore.QThread):
    data_ok = QtCore.pyqtSignal(object)
    def __init__(self,ser):
        self.ser = ser
        QtCore.QThread.__init__(self)
    def run(self):
        try:
            num = self.ser.inWaiting()
        except:
            print('none data')
        else:
            if num>0:
                data = self.ser.read(num)
                text = str(data, encoding = "utf8")
                self.data_ok.emit(text)
        # QtCore.QThread.sleep(1)
        time.sleep(0.1)
        self.run()

class GetCOMsThread(QtCore.QThread):
    data = QtCore.pyqtSignal(object)
    last = None
    def __init__(self):
        QtCore.QThread.__init__(self)
    def run(self):
        ports = serial.tools.list_ports.comports()
        if ports != self.last:
            self.data.emit(ports)
            self.last = ports
        # QtCore.QThread.sleep(1)
        time.sleep(1)
        self.run()

class Message(QtWidgets.QMainWindow, Ui_Dialog):
    def __init__(self, parent=None):
        super(Message, self).__init__(parent)
        self.setupUi(self)
        center_on_screen(self,QtWidgets)
        self.setFixedSize(self.size())

class App(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self, parent=None):
        super(App, self).__init__(parent)
        self.setupUi(self)
        center_on_screen(self,QtWidgets)
        self.btn_open.clicked.connect(self.btn_open_Clicked)
        self.btn_send.clicked.connect(self.btn_send_Clicked)
        self.btn_clear.clicked.connect(self.btn_clear_Clicked)
        self.btn_about.clicked.connect(self.btn_about_Clicked)
        self.getCOMs()
        self.setFixedSize(self.size())
        # self.setWindowFlags(QtCore.Qt.WindowCloseButtonHint | QtCore.Qt.WindowMinimizeButtonHint)
        # self.setWindowIcon(QtGui.QIcon('icon.jpeg'))
        scriptDir = os.path.dirname(os.path.realpath(__file__))
        self.setWindowIcon(QtGui.QIcon(scriptDir + os.path.sep + 'icon.jpeg'))

    def btn_open_Clicked(self):

        if self.btn_open.text() == 'close':
            self.btn_open.setText('open')
            self.setWindowTitle("{} ({})".format(TITLE,"closed"))
            self.ser=None
            self.btn_send.setEnabled(False)
        else:
            com = self.comboBox_COMs.currentData(self.comboBox_COMs.currentIndex())
            baud = self.comboBox_baud.currentData(self.comboBox_baud.currentIndex())
            try:
                self.ser=serial.Serial(com,baud,timeout=5000)
            except Exception as err:
                print(err)
                self.dialog = Message(self)
                self.dialog.textBrowser.setText("")
                self.dialog.textBrowser.append("
error
")
                self.dialog.textBrowser.append(str(err))
                self.dialog.show()
            else:
                print('ok')
                self.btn_open.setText('close')
                self.setWindowTitle("{} ({})".format(TITLE,"opened"))
                self.dataCollectionThread = DataCaptureThread(self.ser)
                self.dataCollectionThread.data_ok.connect(self.on_data_ready)
                self.dataCollectionThread.start()
                self.btn_send.setEnabled(True)
    def on_data_ready(self, data):
        # self.textBrowser.append("{}\n".format(data))
        self.textBrowser.append(data)

    def btn_send_Clicked(self):
        try:
            self.ser.write(("{}\r\n".format(self.lineEdit.text())).encode('utf8'))
        except Exception as err:
            self.dialog = Message(self)
            self.dialog.textBrowser.setText("")
            self.dialog.textBrowser.append("
error
")
            self.dialog.textBrowser.setOpenLinks(True)
            self.dialog.textBrowser.setOpenExternalLinks(True)
            self.dialog.textBrowser.append(str(err))
            self.dialog.show()
        else:
            self.lineEdit.setText("")

    def btn_clear_Clicked(self):
        self.textBrowser.setText("")

    def btn_about_Clicked(self):
        self.dialog = Message(self)
        self.dialog.textBrowser.setText("")
        self.dialog.textBrowser.append("
about
")
        link = "https://github.com/geek2pm/simpleSerialTool"
        self.dialog.textBrowser.append(link)
        self.dialog.textBrowser.setOpenExternalLinks(True)
        self.dialog.show()

    def getCOMs(self):
        self.getCOMsThread = GetCOMsThread()
        self.getCOMsThread.data.connect(self.on_data_coms)
        self.getCOMsThread.start()

    def on_data_coms(self, ports):
        self.btn_send.setEnabled(False)
        if len(ports)>0:
            self.btn_open.setEnabled(True)
            self.btn_clear.setEnabled(True)
            self.comboBox_COMs.clear()
            for port, desc, hwid in sorted(ports):
                print("{}: {} [{}]".format(port, desc, hwid))
                self.comboBox_COMs.addItem(port)
            self.btn_open.setText('open')
            self.setWindowTitle("{} ({})".format(TITLE,"ready to open"))
        else:
            self.btn_open.setEnabled(False)

            self.btn_clear.setEnabled(False)
            self.btn_open.setText('open')
            self.setWindowTitle("{} ({})".format(TITLE,"no device found"))
def main():
    app = QApplication(sys.argv)
    form = App()
    form.show()
    sys.exit(app.exec_())

if __name__ == '__main__':
    main()


from llama_index import Prompt

from student import Student

class SolveMamanPrompt(Prompt):
    def __init__(self):
        super().__init__(
            template=(
                "Your name is Yonatan Bebchuk. \n"
                "You are a student of philosophy at the Open University in Israel. \n"
                "You are currently taking a course called \"Morality without Laws\" or מוסר ללא כללים in Hebrew. \n"
                "The assignments are called \"Maman\" or ממ\"ן in Hebrew. \n"
                "There were 4 assignments during the course. \n"
                "The first assignment is ממ\"ן 11. \n"
                "The second assignment is ממ\"ן 12. \n"
                "The third assignment is ממ\"ן 13. \n"
                "The fourth assignment is ממ\"ן 14. \n"
                "----------------------------------------\n"
                "Your final exam is to write a 7-10 page review of \"It Ain't Necessarily So\" by Nomy Arpaly. \n"
                "----------------------------------------\n"
                "Given this information and the information in the context, please answer the question: {question}\n"
            )
        )

if __name__ == '__main__':
    student = Student()
    prompt = SolveMamanPrompt().format(question="What is the question in assignment 12 and what was your response?")
    response = student.solve(prompt)
    print(response)


import argparse
import tensorflow as tf

import models_tf as models
import utils

def optimistic_restore(session, save_file):
    reader = tf.train.NewCheckpointReader(save_file)
    saved_shapes = reader.get_variable_to_shape_map()
    var_names = sorted(
        [(var.name, var.name.split(':')[0]) for var in tf.global_variables() if var.name.split(':')[0] in saved_shapes])
    restore_vars = []

    with tf.variable_scope('', reuse=True):

        for var_name, saved_var_name in var_names:
            curr_var = tf.get_variable(saved_var_name)
            var_shape = curr_var.get_shape().as_list()

            if var_shape == saved_shapes[saved_var_name]:
                restore_vars.append(curr_var)

    saver = tf.train.Saver(restore_vars)
    saver.restore(session, save_file)

def inference(parameters, verbose=True):
    tf.set_random_seed(7)

    with tf.Graph().as_default():
        with tf.device('/' + parameters['device_type']):
            # initialize input holders
            if parameters["model_type"] == 'cnn':
                x_l_cc = tf.placeholder(tf.float32,
                                        shape=[None, parameters['input_size'][0], parameters['input_size'][1], 1])
                x_r_cc = tf.placeholder(tf.float32,
                                        shape=[None, parameters['input_size'][0], parameters['input_size'][1], 1])
                x_l_mlo = tf.placeholder(tf.float32,
                                         shape=[None, parameters['input_size'][0], parameters['input_size'][1], 1])
                x_r_mlo = tf.placeholder(tf.float32,
                                         shape=[None, parameters['input_size'][0], parameters['input_size'][1], 1])
                x = (x_l_cc, x_r_cc, x_l_mlo, x_r_mlo)
                model_class = models.BaselineBreastModel
            elif parameters["model_type"] == 'histogram':
                x = tf.placeholder(tf.float32, shape=[None, parameters['bins_histogram'] * 4])
                model_class = models.BaselineHistogramModel
            else:
                raise RuntimeError(parameters["model_type"])

            # holders for dropout and Gaussian noise
            nodropout_probability = tf.placeholder(tf.float32, shape=())
            gaussian_noise_std = tf.placeholder(tf.float32, shape=())

            # construct models
            model = model_class(parameters, x, nodropout_probability, gaussian_noise_std)
            y_prediction_density = model.y_prediction_density

        # allocate computation resources
        if parameters['device_type'] == 'gpu':
            session_config = tf.ConfigProto()
            session_config.gpu_options.visible_device_list = str(parameters['gpu_number'])
        elif parameters['device_type'] == 'cpu':
            session_config = tf.ConfigProto(device_count={'GPU': 0})
        else:
            raise RuntimeError(parameters['device_type'])

        with tf.Session(config=session_config) as session:
            session.run(tf.global_variables_initializer())

            # loads the pre-trained parameters if it's provided
            optimistic_restore(session, parameters['model_path'])

            # load input images
            datum_l_cc = utils.load_images(parameters['image_path'], 'L-CC')
            datum_r_cc = utils.load_images(parameters['image_path'], 'R-CC')
            datum_l_mlo = utils.load_images(parameters['image_path'], 'L-MLO')
            datum_r_mlo = utils.load_images(parameters['image_path'], 'R-MLO')

            # populate feed_dict for TF session
            # No dropout and no gaussian noise in inference
            feed_dict_by_model = {nodropout_probability: 1.0, gaussian_noise_std: 0.0}
            if parameters["model_type"] == 'cnn':
                feed_dict_by_model[x_l_cc] = datum_l_cc
                feed_dict_by_model[x_r_cc] = datum_r_cc
                feed_dict_by_model[x_l_mlo] = datum_l_mlo
                feed_dict_by_model[x_r_mlo] = datum_r_mlo
            elif parameters["model_type"] == 'histogram':
                feed_dict_by_model[x] = utils.histogram_features_generator(
                    [datum_l_cc, datum_r_cc, datum_l_mlo, datum_r_mlo],
                    parameters,
                )

            # run the session for a prediction
            prediction_density = session.run(y_prediction_density, feed_dict=feed_dict_by_model)

            if verbose:
                # nicely prints out the predictions
                print('Density prediction:\n' +
                      '\tAlmost entirely fatty (0):\t\t\t' + str(prediction_density[0, 0]) + '\n' +
                      '\tScattered areas of fibroglandular density (1):\t' + str(prediction_density[0, 1]) + '\n' +
                      '\tHeterogeneously dense (2):\t\t\t' + str(prediction_density[0, 2]) + '\n' +
                      '\tExtremely dense (3):\t\t\t\t' + str(prediction_density[0, 3]) + '\n')

            return prediction_density[0]

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description='Run Inference')
    parser.add_argument('model_type')
    parser.add_argument('--bins-histogram', default=50)
    parser.add_argument('--model-path', default=None)
    parser.add_argument('--device-type', default="cpu")
    parser.add_argument('--gpu-number', default=0)
    parser.add_argument('--image-path', default="images/")
    args = parser.parse_args()

    parameters_ = {
        "model_type": args.model_type,
        "bins_histogram": args.bins_histogram,
        "model_path": args.model_path,
        "device_type": args.device_type,
        "image_path": args.image_path,
        "gpu_number": args.gpu_number,
        "input_size": (2600, 2000),
    }

    if parameters_["model_path"] is None:
        if args.model_type == "histogram":
            parameters_["model_path"] = "saved_models/BreastDensity_BaselineHistogramModel/model.ckpt"
        elif args.model_type == "cnn":
            parameters_["model_path"] = "saved_models/BreastDensity_BaselineBreastModel/model.ckpt"
        else:
            raise RuntimeError(parameters_['model_class'])

    inference(parameters_)

"""
python density_model_tf.py histogram
python density_model_tf.py cnn
"""