"""

"""
# Email: Kun.bj@outlook.com
import collections
import copy
import json
import os
import pickle
import time
import traceback
from collections import Counter
from pprint import pprint

import numpy as np
from sklearn.preprocessing import StandardScaler

from fkm import vis
from fkm.cluster import centralized_minibatch_kmeans
from fkm.utils.utils_func import dump, obtain_true_centroids
from fkm.utils.utils_stats import evaluate2
from fkm.utils.utils_func import timer
# These options determine the way floating point numbers, arrays and
# other NumPy objects are displayed.
# np.set_printoptions(precision=3, suppress=True)
from fkm.vis.visualize import plot_2gaussian, plot_3gaussian

np.set_printoptions(precision=3, suppress=True, formatter={'float': '{:.3f}'.format}, edgeitems=120, linewidth=100000)

def save_history2txt(seed_history, out_file='.txt'):
	"""
		with open(seed_file + '.txt', 'w') as file:
			file.write(json.dumps(seed_history))  # not working
	Returns
	-------

	"""

	def format(data):
		res = ''
		if type(data) == dict:
			for k, v in data.items():
				res += f'{k}: ' + format(v) + '\n'
		elif type(data) == list:
			res += f'{data} \n'
		else:
			res += f'{data} \n'

		return res

	with open(out_file, 'w') as f:
		f.write('***Save data with pprint\n')
		pprint(seed_history, stream=f, sort_dicts=False)  # 'sort_dicts=False' works when python version >= 3.8

		f.write('\n\n***Save data with recursion')
		res = format(seed_history)
		f.write(res)

# def normalize2(raw_x, raw_y, raw_true_centroids, splits, params, is_federated=False):
# 	"""
# 	Only for diagonal covariance matrix:
# 		even for federated kmeans, we still can get global mean and std from each client data.
# 		Based on that, for centralized and federated kmeans, we can use the same standscaler.
#
# 	Parameters
# 	----------
# 	raw_x
# 	raw_y
# 	raw_true_centroids
# 	splits
# 	params
# 	is_federated
#
# 	Returns
# 	-------
#
# 	"""
# 	is_show = params['is_show']
# 	normalize_method = params['normalize_method']
# 	# do normalization
# 	if normalize_method == 'std':
# 		# collects all clients' data together
# 		x = copy.deepcopy(raw_x)
# 		y = copy.deepcopy(raw_y)
# 		new_true_centroids = copy.deepcopy(raw_true_centroids)
# 		for spl in splits:  # train and test
# 			x[spl] = np.concatenate(x[spl], axis=0)
# 		global_stdscaler = StandardScaler()
# 		global_stdscaler.fit(x['train'])
# 		for spl in splits:  # train and test
# 			new_true_centroids[spl] = global_stdscaler.transform(raw_true_centroids[spl])
#
# 		if is_federated == False:  # for centralized normalization
# 			new_x = copy.deepcopy(raw_x)
# 			new_y = copy.deepcopy(raw_y)
# 			for i_, _ in enumerate(new_x['train']):
# 				new_x['train'][i_] = global_stdscaler.transform(new_x['train'][i_])
# 				new_x['test'][i_] = global_stdscaler.transform(new_x['test'][i_])
# 			params['stdscaler'] = global_stdscaler
#
# 			if is_show:
# 				if '3GAUSSIANS' in params['p0']:  # for plotting
# 					plot_3gaussian(new_x['train'][0], new_y['train'], new_x['train'][1], new_y['train'][1],
# 					               new_x['train'][2], new_y['train'][2], params, title='std')
# 		else:  # federated kmeans
# 			# for each client, we can get mean and std and then use them to get global_std
# 			new_x = copy.deepcopy(raw_x)
# 			new_y = copy.deepcopy(raw_y)
# 			stds = [[]] * len(new_x['train'])  # get number of clients
# 			N = 0
# 			for i_, _ in enumerate(new_x['train']):  # for each client
# 				data = new_x['train'][i_]
# 				n = len(data)
# 				stds[i_] = (n, np.mean(data, axis=0), np.std(data, axis=0))
# 				N += n
#
# 			dim = new_x['train'][0].shape[1]
# 			global_mean = [[]] * dim
# 			global_std = [[]] * dim
# 			# the following method only works for diagonal covariance matrix
# 			for i_ in range(dim):
# 				# compute global mean and std given each client's mean and std
# 				# global_mean = 1/N * (\sum client1  + \sum client2 + \sum client3)
# 				# 			  = 1/N * (n1 * mu1 + n2 * mu2 + n3 * mu3)
# 				global_mean[i_] = sum([n[i_] * mu[i_] for n, mu, s in stds]) / N
#
# 				# global_var = E(x-mu)**2 = (1/N * (\sum x**2)) - mu**2
# 				# 					   = (1/N * (\sum client1**2 + \sum client2**2+ \sum client3**2)) - global_mean **2
# 				#                      = (1/N * (n1 * var1 + n2 * var2 + n3 * var3)) - global_mean**2
# 				#                      = (1/N * (n1 * std1**2 + n2 * std2** + n3 * std3**2)) - global_mean**2
# 				global_std[i_] = (sum([n[i_] * s[i_] ** 2 for n, mu, s in stds]) / N - global_mean[i_] ** 2) ** (1 / 2)
#
# 			# for each client, then normalize its data use the global mean and std
# 			global_stdscaler2 = StandardScaler()
# 			global_stdscaler2.mean_ = global_mean
# 			global_stdscaler2.scale_ = global_std
#
# 			for i_, _ in enumerate(new_x['train']):
# 				new_x['train'][i_] = global_stdscaler2.transform(new_x['train'][i_])
# 				new_x['test'][i_] = global_stdscaler2.transform(new_x['test'][i_])
# 			params['stdscaler'] = global_stdscaler2
#
# 			if is_show:
# 				if '3GAUSSIANS' in params['p0']:  # for plotting
# 					plot_3gaussian(new_x['train'][0], new_y['train'], new_x['train'][1], new_y['train'][1],
# 					               new_x['train'][2], new_y['train'][2], params, title='std')
#
# 	else:
# 		new_x, new_y, new_true_centroids = raw_x, raw_y, raw_true_centroids
#
# 	return new_x, new_y, new_true_centroids

def normalize(raw_x, raw_y, raw_true_centroids, splits, params):
	"""
	For federated kmeans, we still can get global mean and std from each client data.
		Based on that, for centralized and federated kmeans, we can use the same standscaler.

	Parameters
	----------
	raw_x
	raw_y
	raw_true_centroids
	splits
	params

	Returns
	-------

	"""
	try:
		is_show = params['is_show']
	except Exception as e:
		is_show = params['IS_SHOW']
	try:
		normalize_method = params['normalize_method']
	except Exception as e:
		normalize_method = params['NORMALIZE_METHOD']

	try:
		algorithm_name = params['p0']
	except Exception as e:
		algorithm_name = params['ALGORITHM']['name']
	# do normalization
	if normalize_method == 'std':
		# collects all clients' data together and get global stdscaler
		x = copy.deepcopy(raw_x)
		y = copy.deepcopy(raw_y)
		new_true_centroids = copy.deepcopy(raw_true_centroids)
		for spl in splits:  # train and test
			x[spl] = np.concatenate(x[spl], axis=0)

		global_stdscaler = StandardScaler()  # we can get the same global_stdscaler using each client mean and std.
		global_stdscaler.fit(x['train'])
		# print(global_stdscaler.mean_, global_stdscaler.scale_)

		for spl in splits:  # train and test
			new_true_centroids[spl] = global_stdscaler.transform(new_true_centroids[spl])
			# print(new_true_centroids[spl])

		# normalize data
		new_x = copy.deepcopy(raw_x)
		new_y = copy.deepcopy(raw_y)
		for spl in splits:
			for i_ in range(len(new_x[spl])):
				new_x[spl][i_] = global_stdscaler.transform(new_x[spl][i_])  # normalize data first
		# for j_ in set(new_y[spl][i_]):
		# 	new_true_centroids[spl][j_] = np.mean(new_x[spl][i_], axis=0)  # get new centroids

		# if is_show:
		# 	if '2GAUSSIANS' in algorithm_name:  # for plotting
		# 		plot_2gaussian(new_x['train'][0], new_y['train'], new_x['train'][1], new_y['train'][1],
		# 		               params, title='std')
		# 	elif '3GAUSSIANS' in algorithm_name:  # for plotting
		# 		plot_3gaussian(new_x['train'][0], new_y['train'], new_x['train'][1], new_y['train'][1],
		# 		               new_x['train'][2], new_y['train'][2], params, title='std')
		# 	elif '4GAUSSIANS' in algorithm_name:  # for plotting
		# 		plot_2gaussian(new_x['train'][0], new_y['train'], new_x['train'][1], new_y['train'][1],
		# 		               params, title='std')
	elif normalize_method == 'min_max':
		raise NotImplementedError
	else:
		# new_x, new_y, new_true_centroids = raw_x, raw_y, raw_true_centroids
		# # collects all clients' data together and get global stdscaler
		# x = copy.deepcopy(raw_x)
		# y = copy.deepcopy(raw_y)
		# for spl in splits:  # train and test
		# 	x[spl] = np.concatenate(x[spl], axis=0)
		#
		# global_stdscaler = StandardScaler()  # we can get the same global_stdscaler using each client mean and std.
		# global_stdscaler.fit(x['train'])
		return raw_x, raw_y, raw_true_centroids, None

	return new_x, new_y, new_true_centroids, global_stdscaler

class NumpyEncoder(json.JSONEncoder):
	def default(self, obj):
		if isinstance(obj, np.ndarray):
			return obj.tolist()
		return json.JSONEncoder.default(self, obj)

@timer
def run_model(args):
	"""

		Parameters
		----------
		params
		KMeansFederated
		verbose:
			0 < verbose <= 5: info
			5 < verbose <= 10: debug

		Returns
		-------

		"""
	np.random.seed(args['SEED'])  # set the global seed for numpy
	VERBOSE = args['VERBOSE']
	SEPERTOR = args['SEPERTOR']
	SPLITS = args['SPLITS']
	dataset_name = args['DATASET']['name']
	N_CLUSTERS = args['N_CLUSTERS']
	N_CLIENTS = args['N_CLIENTS']
	# args['DATASET']['detail'] = f'{SEPERTOR}'.join(args['DATASET']['detail'], f'M_{N_CLIENTS}|K_{N_CLUSTERS}')
	# dataset_detail = args['DATASET']['detail']
	# algorithm_py_name = args['ALGORITHM']['py_name']
	# initial_method = args['ALGORITHM']['initial_method']
	server_init_method = args['ALGORITHM']['server_init_method']
	client_init_method = args['ALGORITHM']['client_init_method']

	N_REPEATS = args['N_REPEATS']
	TOLERANCE = args['TOLERANCE']
	NORMALIZE_METHOD = args['NORMALIZE_METHOD']
	# algorithm_detail = args['ALGORITHM']['detail']
	# OUT_DIR = args['OUT_DIR']
	# # if os.path.exists(OUT_DIR):
	# # 	shutil.rmtree(OUT_DIR)
	history_file = os.path.join(args['OUT_DIR'], 'history.dat')
	args['history_file'] = history_file
	# if os.path.exists(history_file):
	# # here could be some issue for multi-tasks, please double-check before calling this function.
	# 	return history_file

	# settings
	history = {'args': args}  # Store all results and needed data

	# data is fixed, however, the algorithm will have different initialization centroids with different seeds.
	with open(args['data_file'], 'rb') as f:
		raw_x, raw_y = pickle.load(f)
	print(f'data_file: ', args['data_file'])
	for split in SPLITS:
		X_ = raw_x[split]
		y_ = raw_y[split]
		print(len(X_), N_CLIENTS)
		for i_c in range(N_CLIENTS):
			print(f'{split}:{i_c}-th client raw data info, where mean: {np.mean(X_[i_c], axis=0)}, '
			      f'std: {np.std(X_[i_c], axis=0)}, and y: {collections.Counter(y_[i_c])}')

	if VERBOSE >= 1:
		# print raw_x and raw_y distribution
		for split in SPLITS:
			print(f'{split} set:')
			clients_x, clients_y = raw_x[split], raw_y[split]
			if VERBOSE >= 5:
				# print each client distribution
				for c_i, (c_x, c_y) in enumerate(zip(clients_x, clients_y)):
					print(f'\tClient_{c_i}, n_datapoints: {len(c_y)}, '
					      f'cluster_size: {sorted(Counter(c_y).items(), key=lambda kv: kv[0], reverse=False)}')

			y_tmp = []
			for vs in clients_y:
				y_tmp.extend(vs)
			print(f'n_{split}_clients: {len(clients_x)}, n_datapoints: {sum(len(vs) for vs in clients_y)}, '
			      f'cluster_size: {sorted(Counter(y_tmp).items(), key=lambda kv: kv[0], reverse=False)}')

	# obtain the true centroids given raw_x and raw_y
	raw_true_centroids = obtain_true_centroids(raw_x, raw_y, SPLITS, args)
	if VERBOSE >= 3:
		# print true centroids
		for split in SPLITS:
			true_c = raw_true_centroids[split]
			print(f'{split}_true_centroids:')
			print(true_c)

	# if algorithm_name == 'FEMNIST':
	# 	save_image2disk((raw_x, raw_y), out_dir_i, params)

	# do normalization in global for kmeans and federated kmeans
	raw_x, raw_y, raw_true_centroids, global_stdscaler = normalize(raw_x, raw_y, raw_true_centroids, SPLITS, args)
	args['global_stdscaler'] = global_stdscaler
	history['raw_true_centroids'] = raw_true_centroids
	print(f'after normalization, true_centroids:\n{raw_true_centroids} \nwhen normalize_method = {NORMALIZE_METHOD}')
	# history = {'x': raw_x, 'y': raw_y, 'results': []}
	# SEEDS = [10 * v ** 2 for v in range(1, N_REPEATS + 1, 1)]
	SEEDS = [42]    # we fix the model seed; however, the data seed is different.
	history['SEEDS'] = SEEDS

	from fkm.cluster import centralized_kmeans, federated_server_init_first, federated_client_init_first, \
		federated_greedy_kmeans
	KMEANS2PY = {
		'centralized_kmeans': centralized_kmeans.KMeans,
		'federated_server_init_first': federated_server_init_first.KMeansFederated,
		'federated_client_init_first': federated_client_init_first.KMeansFederated,
		'federated_greedy_kmeans': federated_greedy_kmeans.KMeansFederated,
		'centralized_minibatch_kmeans': centralized_minibatch_kmeans.KMeans,
	}
	for idx_seed, seed in enumerate(SEEDS):  # repetitions:  to obtain average and std score.

		# if VERBOSE >= 2:
		print(f'\n***{idx_seed}th repeat with seed: {seed}:')
		X = copy.deepcopy(raw_x)
		Y = copy.deepcopy(raw_y)
		true_centroids = copy.deepcopy(raw_true_centroids)

		if not args['IS_FEDERATED']:
			# collects all clients' data together
			for spl in SPLITS:
				X[spl] = np.concatenate(X[spl], axis=0)
				Y[spl] = np.concatenate(Y[spl], axis=0)
				print(spl, X[spl].shape, Y[spl].shape)

		t1 = time.time()
		# for Centralized Kmeans, we use server_init_centroids as init_centroids.
		kmeans = KMEANS2PY[args['ALGORITHM']['py_name']](
			n_clusters=N_CLUSTERS,
			# batch_size=BATCH_SIZE,
			sample_fraction=1.0,
			epochs_per_round=args['CLIENT_EPOCHS'],
			max_iter=args['ROUNDS'],
			server_init_method=server_init_method,
			client_init_method=client_init_method,
			true_centroids=true_centroids,
			random_state=seed,
			learning_rate=0,
			adaptive_lr=0,
			epoch_lr=0,
			momentum=0,
			reassign_min=0,
			reassign_after=0,
			verbose=VERBOSE,
			tol=TOLERANCE,
			params=args
		)

		if VERBOSE > 5:
			# print all kmeans's variables.
			pprint(vars(kmeans))

		# During the training, we also evaluate the model on the test set at each iteration
		kmeans.fit(X, Y, SPLITS, record_at=None)

		# After training, we obtain the final scores on the train/test set.
		scores = evaluate2(
			kmeans=kmeans,
			x=X, y=Y,
			splits=SPLITS,
			federated=args['IS_FEDERATED'],
			verbose=VERBOSE,
		)
		# # To save the disk storage, we only save the first repeat results.
		# if params['p0'] == 'FEMNIST' and s_i == 0:
		# 	try:
		# 		predict_n_saveimg(kmeans, x, y, SPLITS, SEED,
		# 		                  federated=params['is_federated'], verbose=VERBOSE,
		# 		                  out_dir=os.path.join(out_dir_i, f'SEED_{SEED}'),
		# 		                  params=params, is_show=is_show)
		# 	except Exception as e:
		# 		print(f'Error: {e}')
		# # traceback.print_exc()

		t2 = time.time()
		print(f'{idx_seed}th repeat with seed {seed} takes {(t2 - t1):.4f}s')

		# for each seed, we will save the results.
		history[seed] = {'initial_centroids': kmeans.initial_centroids,
		                 'true_centroids': kmeans.true_centroids,
		                 'final_centroids': kmeans.centroids,
		                 'training_iterations': kmeans.training_iterations,
		                 'history': kmeans.history,
		                 'scores': scores, 'duration': t2 - t1}
		if dataset_name != 'FEMNIST':
			try:
				# # save the current 'history' to disk before plotting.
				seed_file = os.path.join(args['OUT_DIR'], f'SEED_{seed}', f'~history.dat')
				dump(history[seed], out_file=seed_file)
				save_history2txt(history[seed], out_file=seed_file + '.txt')
			except Exception as e:
				print(f'save_history2txt() fails when SEED={seed}, Error: {e}')
		if VERBOSE >= 2:
			pprint(f'seed:{seed}, '
			       f'scores:{scores}')
	try:
		results_avg = vis.visualize.stats_single(history)
	except Exception as e:
		traceback.print_exc()
		results_avg = {}
	# save the current 'history' to disk before plotting.
	history['results_avg'] = results_avg
	dump(history, out_file=history_file)

	try:
		vis.visualize.plot_single(history)
	except Exception as e:
		traceback.print_exc()
	# out_file = os.path.join( args['OUT_DIR'], f'varied_clients-Server_{server_init_method}-Client_{client_init_method}')
	# print(out_file)
	# dump(stats, out_file=out_file + '.dat')

	# # dump(histories, out_file=out_file + '-histories.dat')
	# with open(history_file + '-histories.txt', 'w') as file:
	# 	file.write(json.dumps(history, cls=NumpyEncoder))  # use `json.loads` to do the reverse

	return history_file


import os
import tempfile
from typing import List
import numpy as np
import tensorflow as tf
import tensorflow_datasets as tfds

import data  # local file import from experimental.shoshin

def _make_temp_dir() -> str:
  return tempfile.mkdtemp(dir=os.environ.get('TEST_TMPDIR'))

def _make_serialized_image(size: int) -> bytes:
  image = np.random.randint(0, 255, size=(size, size, 3), dtype=np.uint8)
  return tf.io.encode_png(image).numpy()

def _make_example(
    example_id: str,
    longitude: float,
    latitude: float,
    encoded_coordinates: str,
    label: float,
    string_label: float,
    patch_size: int,
    large_patch_size: int,
) -> tf.train.Example:
  example = tf.train.Example()
  example.features.feature['example_id'].bytes_list.value.append(
      example_id.encode()
  )
  example.features.feature['coordinates'].float_list.value.extend(
      (longitude, latitude)
  )
  example.features.feature['encoded_coordinates'].bytes_list.value.append(
      encoded_coordinates.encode()
  )
  example.features.feature['label'].float_list.value.append(label)
  example.features.feature['string_label'].bytes_list.value.append(
      string_label.encode()
  )
  example.features.feature['pre_image_png'].bytes_list.value.append(
      _make_serialized_image(patch_size)
  )
  example.features.feature['post_image_png'].bytes_list.value.append(
      _make_serialized_image(patch_size)
  )
  example.features.feature['pre_image_png_large'].bytes_list.value.append(
      _make_serialized_image(large_patch_size)
  )
  example.features.feature['post_image_png_large'].bytes_list.value.append(
      _make_serialized_image(large_patch_size)
  )
  return example

def _write_tfrecord(examples: List[tf.train.Example], path: str) -> None:
  with tf.io.TFRecordWriter(path) as file_writer:
    for example in examples:
      file_writer.write(example.SerializeToString())

def _create_test_data():
  examples_dir = _make_temp_dir()
  labeled_train_path = os.path.join(
      examples_dir, 'train_labeled_examples.tfrecord')
  labeled_test_path = os.path.join(
      examples_dir, 'test_labeled_examples.tfrecord')
  unlabeled_path = os.path.join(
      examples_dir, 'unlabeled_examples.tfrecord')

  _write_tfrecord([
      _make_example('1st', 0, 0, 'A0', 0, 'no_damage', 64, 256),
      _make_example('2nd', 0, 1, 'A1', 0, 'no_damage', 64, 256),
      _make_example('3rd', 0, 2, 'A2', 1, 'major_damage', 64, 256),
  ], labeled_train_path)

  _write_tfrecord([
      _make_example('4th', 1, 0, 'B0', 0, 'no_damage', 64, 256),
  ], labeled_test_path)

  _write_tfrecord([
      _make_example('5th', 2, 0, 'C0', -1, 'bad_example', 64, 256),
      _make_example('6th', 2, 1, 'C1', -1, 'bad_example', 64, 256),
      _make_example('7th', 2, 2, 'C2', -1, 'bad_example', 64, 256),
      _make_example('8th', 2, 3, 'C3', -1, 'bad_example', 64, 256),
  ], unlabeled_path)

  return labeled_train_path, labeled_test_path, unlabeled_path

class SkaiDatasetTest(tfds.testing.DatasetBuilderTestCase):
  """Tests for Skai dataset."""

  DATASET_CLASS = data.SkaiDataset
  SPLITS = {
      'labeled_train': 3,
      'labeled_test': 1,
      'unlabeled': 4
  }
  EXAMPLE_DIR = _make_temp_dir()
  BUILDER_CONFIG_NAMES_TO_TEST = ['test_config']
  SKIP_TF1_GRAPH_MODE = True

  @classmethod
  def setUpClass(cls):
    super().setUpClass()

    labeled_train_path, labeled_test_path, unlabeled_path = _create_test_data()

    cls.DATASET_CLASS.BUILDER_CONFIGS = [
        data.SkaiDatasetConfig(
            name='test_config',
            labeled_train_pattern=labeled_train_path,
            labeled_test_pattern=labeled_test_path,
            unlabeled_pattern=unlabeled_path)
    ]

if __name__ == '__main__':
  tfds.testing.test_main()


"""
Executor class.
"""

from __future__ import unicode_literals

import yaml
import subprocess

from voluptuous import Schema
from contextlib import closing
from functools import partial
from six import PY2
from locale import getpreferredencoding

class BaseExecutor(object):
    """
    A generic executor class.
    """
    options_schema = Schema({})

    def __init__(self, options=None):
        self.options = self.options_schema(options or {})

    @property
    def full_name(self):
        """
        Get the full name of the executor.

        :returns: The full dotted-name representation of the current instance
            class, including its definition module.
        """
        return '{module}.{name}'.format(
            module=self.__module__,
            name=self.__class__.__name__,
        )

    def execute(self, environment, commands, display):
        """
        Execute the specified commands.

        :param environment: The environment variables dictionary.
        :param commands: A list of commands to execute.
        :param display: The display to use for command output and report.
        :returns: True if the execution went fine, False otherwise.
        """
        display.set_context(commands=commands)

        for index, command in enumerate(commands):
            with display.command(
                    index,
                    command,
            ) as result:
                result.returncode = self.execute_one(
                    environment=environment,
                    command=command,
                    output=partial(display.command_output, index),
                )

                if result.returncode is None:
                    raise RuntimeError(
                        "No returncode specified for command execution "
                        "({})".format(
                            command,
                        ),
                    )
                elif result.returncode != 0:
                    return False

        return True

def executor_representer(dumper, executor):
    if executor.options:
        return dumper.represent_mapping(
            'tag:yaml.org,2002:map',
            {
                'name': executor.full_name,
                'options': executor.options,
            },
        )
    else:
        return dumper.represent_scalar(
            'tag:yaml.org,2002:str',
            executor.full_name,
        )

yaml.add_multi_representer(BaseExecutor, executor_representer)

class ShellExecutor(BaseExecutor):
    """
    An executor that execute commands through the system shell.
    """

    def execute_one(self, environment, command, output):
        # Python 2 subprocess doesn't deal well with unicode commands.
        command = (
            command.encode(getpreferredencoding())
            if PY2
            else command
        )

        process = subprocess.Popen(
            command,
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            shell=True,
            env=environment,
        )

        with closing(process.stdout):
            while True:
                data = process.stdout.read(4096)

                if data:
                    output(data)
                else:
                    break

        process.wait()
        return process.returncode


#!/usr/bin/env python3
"""
hw2main.py

UNSW COMP9444 Neural Networks and Deep Learning

DO NOT MODIFY THIS FILE
"""

import torch
from torchtext import data

from config import device
import student

def main():
    print("Using device: {}"
          "\n".format(str(device)))

    # Load the training dataset, and create a dataloader to generate a batch.
    textField = data.Field(lower=True, include_lengths=True, batch_first=True,
                           tokenize=student.tokenise,
                           preprocessing=student.preprocessing,
                           postprocessing=student.postprocessing,
                           stop_words=student.stopWords)
    labelField = data.Field(sequential=False, use_vocab=False, is_target=True)

    dataset = data.TabularDataset('train.json', 'json',
                                 {'reviewText': ('reviewText', textField),
                                  'rating': ('rating', labelField),
                                  'businessCategory': ('businessCategory', labelField)})

    textField.build_vocab(dataset, vectors=student.wordVectors)

    # Allow training on the entire dataset, or split it for training and validation.
    if student.trainValSplit == 1:
        trainLoader = data.BucketIterator(dataset, shuffle=True,
                                          batch_size=student.batchSize,
                                          sort_key=lambda x: len(x.reviewText),
                                          sort_within_batch=True)
    else:
        train, validate = dataset.split(split_ratio=student.trainValSplit)

        trainLoader, valLoader = data.BucketIterator.splits((train, validate),
                                                            shuffle=True,
                                                            batch_size=student.batchSize,
                                                            sort_key=lambda x: len(x.reviewText),
                                                            sort_within_batch=True)

    # Get model and optimiser from student.
    net = student.net.to(device)
    lossFunc = student.lossFunc
    optimiser = student.optimiser

    # Train.
    for epoch in range(student.epochs):
        runningLoss = 0

        for i, batch in enumerate(trainLoader):
            # Get a batch and potentially send it to GPU memory.
            inputs = textField.vocab.vectors[batch.reviewText[0]].to(device)
            length = batch.reviewText[1].to(device)
            rating = batch.rating.to(device)
            businessCategory = batch.businessCategory.to(device)

            # PyTorch calculates gradients by accumulating contributions to them
            # (useful for RNNs).  Hence we must manually set them to zero before
            # calculating them.
            optimiser.zero_grad()

            # Forward pass through the network.
            ratingOutput, categoryOutput = net(inputs, length)
            loss = lossFunc(ratingOutput, categoryOutput, rating, businessCategory)

            # Calculate gradients.
            loss.backward()

            # Minimise the loss according to the gradient.
            optimiser.step()

            runningLoss += loss.item()

            if i % 32 == 31:
                print("Epoch: %2d, Batch: %4d, Loss: %.3f"
                      % (epoch + 1, i + 1, runningLoss / 32))
                runningLoss = 0

    # Save model.
    torch.save(net.state_dict(), 'savedModel.pth')
    print("\n"
          "Model saved to savedModel.pth")

    # Test on validation data if it exists.
    if student.trainValSplit != 1:
        net.eval()

        correctRatingOnlySum = 0
        correctCategoryOnlySum = 0
        bothCorrectSum = 0
        with torch.no_grad():
            for batch in valLoader:
                # Get a batch and potentially send it to GPU memory.
                inputs = textField.vocab.vectors[batch.reviewText[0]].to(device)
                length = batch.reviewText[1].to(device)
                rating = batch.rating.to(device)
                businessCategory = batch.businessCategory.to(device)

                # Convert network output to integer values.
                ratingOutputs, categoryOutputs = student.convertNetOutput(*net(inputs, length))

                # Calculate performance
                correctRating = rating == ratingOutputs.flatten()
                correctCategory = businessCategory == categoryOutputs.flatten()

                correctRatingOnlySum += torch.sum(correctRating & ~correctCategory).item()
                correctCategoryOnlySum += torch.sum(correctCategory & ~correctRating).item()
                bothCorrectSum += torch.sum(correctRating & correctCategory).item()

        correctRatingOnlyPercent = correctRatingOnlySum / len(validate)
        correctCategoryOnlyPercent = correctCategoryOnlySum / len(validate)
        bothCorrectPercent = bothCorrectSum / len(validate)
        neitherCorrectPer = 1 - correctRatingOnlyPercent \
                              - correctCategoryOnlyPercent \
                              - bothCorrectPercent

        score = 100 * (bothCorrectPercent
                       + 0.5 * correctCategoryOnlyPercent
                       + 0.1 * correctRatingOnlyPercent)

        print("\n"
              "Rating incorrect, business category incorrect: {:.2%}\n"
              "Rating correct, business category incorrect: {:.2%}\n"
              "Rating incorrect, business category correct: {:.2%}\n"
              "Rating correct, business category correct: {:.2%}\n"
              "\n"
              "Weighted score: {:.2f}".format(neitherCorrectPer,
                                              correctRatingOnlyPercent,
                                              correctCategoryOnlyPercent,
                                              bothCorrectPercent, score))

if __name__ == '__main__':
    main()


from django.conf.urls import url, include
from rest_framework.urlpatterns import format_suffix_patterns
from .views import *

urlpatterns = {
    url(r'^$', default, name="default"),
    url(r'^profile/$', ProfileCreateView, name="profile"),
    url(r'^profile/(?P<pk>[0-9]+)/$', ProfileDetailsView, name="profile_details"),
    url(r'^session/$', SessionCreateView, name="session"),
    url(r'^session/(?P<pk>[0-9]+)/$', SessionDetailsView, name="session_details"),
    url(r'^question/$', QuestionCreateView, name="question"),
    url(r'^question/(?P<pk>[0-9]+)/$', QuestionDetailsView, name="question_details"),
    url(r'^response/$', UserResponseCreateView, name="response"),
    url(r'^response/(?P<pk>[0-9]+)/$', UserResponseDetailsView, name="response_details"),
    url(r'^savesession/$', SaveSession, name="save_session"),
    url(r'^saveresponse/(?P<filename>[^/]+)/$', SaveResponse, name="save_response"),
    url(r'^getresponse/$', GetResponse, name="get_response"),
}

urlpatterns = format_suffix_patterns(urlpatterns)


from ..config import Vector, ParameterName, ColorConfig, ZOrderConfig
from ..config import CompositeFigure, PathStep, PathOperation, PathShape, Rectangle, Ellipse, ellipse_arc_obj

class CulturedCellConfig(object):
    common_z_order = ZOrderConfig.default_patch_z_order
    z_order_increment = ZOrderConfig.z_order_increment
    dish_outline_z_order = common_z_order + 2 * z_order_increment
    media_z_order = common_z_order
    cell_z_order = common_z_order - z_order_increment
    cell_nucleus_z_order = cell_z_order + z_order_increment

    media_color = ColorConfig.orange
    media_side_alpha = ColorConfig.lower_alpha_value
    media_top_alpha = ColorConfig.alpha_for_bar_plot
    common_edge_width = 9.5
    infusion_rect_edge_width = common_edge_width - 1
    cell_alpha = ColorConfig.higher_alpha
    cell_nucleus_alpha = 1

    dish_outline_config = {
        ParameterName.edge_width: common_edge_width,
        ParameterName.edge_color: ColorConfig.black_color,
        ParameterName.face_color: None,
        ParameterName.z_order: dish_outline_z_order
    }
    media_config = {
        ParameterName.edge_width: None,
        ParameterName.z_order: media_z_order
    }
    media_top_config = {
        **media_config,
        ParameterName.face_color: media_color.add_transparency(media_top_alpha)
    }
    media_side_config = {
        **media_config,
        ParameterName.face_color: media_color.add_transparency(media_side_alpha)
    }
    cell_config = {
        ParameterName.edge_width: None,
        ParameterName.z_order: cell_z_order,
        ParameterName.face_color: media_color.add_transparency(cell_alpha)
    }
    cell_nucleus_config = {
        ParameterName.edge_width: None,
        ParameterName.z_order: cell_nucleus_z_order,
        ParameterName.face_color: media_color.add_transparency(cell_nucleus_alpha)
    }

class CulturedCell(CompositeFigure):
    total_width = 1
    height_to_width_ratio = 0.4

    def __init__(self, scale=1, bottom_left_offset=None, base_z_order=0, z_order_increment=1, **kwargs):
        total_width = self.total_width
        total_height = self.height_to_width_ratio * total_width
        ellipse_height_to_width_ratio = self.height_to_width_ratio / 2
        center_x = total_width / 2
        ellipse_width = total_width
        ellipse_height = ellipse_height_to_width_ratio * ellipse_width
        dish_height = (self.height_to_width_ratio - ellipse_height_to_width_ratio) * total_width

        bottom_ellipse_center_y = ellipse_height / 2
        top_ellipse_center_y = bottom_ellipse_center_y + dish_height
        media_height = dish_height * 0.6
        media_top_ellipse_center_y = bottom_ellipse_center_y + media_height

        cell_width = 0.15 * total_width
        cell_height = 0.17 * total_height
        cell_nucleus_width = 0.3 * cell_width
        cell_nucleus_height = 0.3 * cell_height

        cell_location_list = [
            Vector(center_x - 0.1 * total_width, bottom_ellipse_center_y - 0.12 * media_height),
            Vector(center_x + 0.254 * total_width, bottom_ellipse_center_y + 0.463 * media_height),
            Vector(center_x + 0.1 * total_width, bottom_ellipse_center_y + 0.32 * media_height),
            Vector(center_x - 0.34 * total_width, bottom_ellipse_center_y + 0.31 * media_height),
            Vector(center_x - 0.2386 * total_width, bottom_ellipse_center_y - 0.083 * media_height),
            Vector(center_x + 0.05 * total_width, bottom_ellipse_center_y - 0.33 * media_height),
            Vector(center_x + 0.325 * total_width, bottom_ellipse_center_y - 0.062 * media_height),
        ]
        cell_nucleus_relative_location = Vector(0, 0.1)

        cell_obj_list = []
        for cell_location in cell_location_list:
            cell_obj = Ellipse(**{
                ParameterName.center: cell_location,
                ParameterName.width: cell_width,
                ParameterName.height: cell_height,
                ParameterName.name: 'cell',
                **CulturedCellConfig.cell_config
            })
            cell_nucleus_location = cell_location + cell_nucleus_relative_location * Vector(cell_width, cell_height)
            cell_nucleus_obj = Ellipse(**{
                ParameterName.center: cell_nucleus_location,
                ParameterName.width: cell_nucleus_width,
                ParameterName.height: cell_nucleus_height,
                ParameterName.name: 'cell_nucleus',
                **CulturedCellConfig.cell_nucleus_config
            })
            cell_obj_list.extend([cell_obj, cell_nucleus_obj])

        bottom_half_ellipse = ellipse_arc_obj.generator(
            Vector(center_x, bottom_ellipse_center_y),
            -180, 0, ellipse_width / 2, ellipse_height / 2
        )
        left_vertical_path = [
            PathStep(PathOperation.moveto, Vector(center_x - ellipse_width / 2, top_ellipse_center_y)),
            PathStep(PathOperation.lineto, Vector(center_x - ellipse_width / 2, bottom_ellipse_center_y))
        ]
        right_vertical_path = [
            # PathStep(PathOperation.moveto, Vector(center_x + ellipse_width / 2, bottom_ellipse_center_y)),
            PathStep(PathOperation.lineto, Vector(center_x + ellipse_width / 2, top_ellipse_center_y))
        ]
        media_left_vertical_path = [
            PathStep(PathOperation.moveto, Vector(center_x - ellipse_width / 2, media_top_ellipse_center_y)),
            PathStep(PathOperation.lineto, Vector(center_x - ellipse_width / 2, bottom_ellipse_center_y))
        ]
        media_right_vertical_path = [
            # PathStep(PathOperation.moveto, Vector(center_x + ellipse_width / 2, bottom_ellipse_center_y)),
            PathStep(PathOperation.lineto, Vector(center_x + ellipse_width / 2, media_top_ellipse_center_y))
        ]
        media_top_half_ellipse = ellipse_arc_obj.generator(
            Vector(center_x, media_top_ellipse_center_y),
            0, -180, ellipse_width / 2, ellipse_height / 2
        )

        dish_outline_obj_list = [
            Ellipse(**{
                ParameterName.center: Vector(center_x, top_ellipse_center_y),
                ParameterName.width: ellipse_width,
                ParameterName.height: ellipse_height,
                ParameterName.name: 'top_outline_ellipse',
                **CulturedCellConfig.dish_outline_config
            }),
            PathShape(**{
                ParameterName.path_step_list: [*left_vertical_path, *bottom_half_ellipse, *right_vertical_path],
                ParameterName.closed: False,
                ParameterName.name: 'bottom_outline',
                **CulturedCellConfig.dish_outline_config
            })
        ]

        media_obj_list = [
            Ellipse(**{
                ParameterName.center: Vector(center_x, media_top_ellipse_center_y),
                ParameterName.width: ellipse_width,
                ParameterName.height: ellipse_height,
                ParameterName.name: 'media_top',
                **CulturedCellConfig.media_top_config,
            }),
            PathShape(**{
                ParameterName.path_step_list: [
                    *media_left_vertical_path, *bottom_half_ellipse, *media_right_vertical_path,
                    *media_top_half_ellipse],
                ParameterName.name: 'media_side',
                **CulturedCellConfig.media_side_config,
            })
        ]

        cultured_cell_dict = {
            ParameterName.dish_outline: {
                dish_outline_obj.name: dish_outline_obj for dish_outline_obj in dish_outline_obj_list},
            ParameterName.media: {
                media_obj.name: media_obj for media_obj in media_obj_list},
            ParameterName.cell: {
                cell_obj.name: cell_obj for cell_obj in cell_obj_list}
        }
        super().__init__(
            cultured_cell_dict, bottom_left=Vector(0, 0), size=Vector(total_width, total_height),
            scale=scale, bottom_left_offset=bottom_left_offset, base_z_order=base_z_order,
            z_order_increment=z_order_increment)


"""Added_TokenBlackList_table

Revision ID: c2d76eeeeb15
Revises: 703db21eb105
Create Date: 2021-11-20 00:13:10.270320

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'c2d76eeeeb15'
down_revision = '703db21eb105'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('token_black_list',
    sa.Column('id', sa.INTEGER(), nullable=False),
    sa.Column('jti', sa.String(length=36), nullable=False),
    sa.Column('blacklisted_at', sa.DateTime(), nullable=False),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('token_black_list')
    # ### end Alembic commands ###


from atexit import register
from unicodedata import category
from django import template
from ..models import Category

register = template.Library()

# @register.simple_tag
# def title(data="وبلاگ جنگویی"):
#     return data

# @register.inclusion_tag("pages/partials/category_navbar.html")
# def category_navbar():
#     return {
#         "category": Category.objects.filter(status=True)
#     }

@register.inclusion_tag("registration/partials/link.html")
def link(request, link_name, content, classes_a, classes_i):
    return {
        "request": request,
        "link_name": link_name,
        "link": f"account:{link_name}",
        "content": content,
        "classes_a": classes_a,
        "classes_i": classes_i,
    }


"""
autoencoder.py
Autoencoder-style image generation model.
"""
import glob
import os

import tensorflow as tf
from matplotlib import pyplot as plt
from tqdm import tqdm

import models
from utils import gauss_kernel

class AutoEncoderGAN():

    def __init__(self,
                 batch_size,
                 z_dim,
                 enc_model,
                 dec_model,
                 disc_model,
                 loss_weights,
                 max_filters,
                 conv_init='he_normal',
                 verbose=True,
                 **kwargs):
        self.batch_size = batch_size
        self.z_dim = z_dim
        self.enc_model = enc_model
        self.dec_model = dec_model
        self.disc_model = disc_model
        self.loss_weights = loss_weights
        self.max_filters = max_filters
        self.conv_init = conv_init
        self.verbose = verbose
        self.best_gen_loss = None
        self.best_disc_loss = None
        self.gen_loss_curve = []
        self.disc_loss_curve = []

        if 'blur_kernel' in kwargs.keys() and kwargs['blur_kernel']:
            self.gauss_kernel = gauss_kernel(kwargs['blur_kernel'])

    def init_models(self, in_size, out_size, disc_size, lr, b1, b2):
        """Define encoder, decoder, and discriminator models if using plus optimizers."""
        encoder_model = getattr(models, self.enc_model)
        decoder_model = getattr(models, self.dec_model)

        self.encoder = encoder_model(in_size, self.z_dim, max_filters=self.max_filters, conv_init=self.conv_init)
        self.decoder = decoder_model(out_size, self.z_dim, max_filters=self.max_filters, conv_init=self.conv_init)
        if self.loss_weights['discriminator'] > 0 and self.disc_model:
            discriminator_model = getattr(models, self.disc_model)
            self.discriminator = discriminator_model(disc_size, max_filters=self.max_filters, conv_init=self.conv_init)

        if self.verbose:
            print(self.encoder.summary())
            print(self.decoder.summary())
            if self.loss_weights['discriminator'] > 0:
                print(self.discriminator.summary())

        self.gen_optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, beta1=b1, beta2=b2)
        if self.loss_weights['discriminator'] > 0:
            self.disc_optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr*4, beta1=b1, beta2=b2)

    def train_step(self, img_in, true_img, out_size, disc_size):
        """Make one update step."""
        with tf.GradientTape() as gen_tape, tf.GradientTape() as disc_tape:
            gen_z = self.encoder(img_in, training=True)
            gen_img = self.decoder(gen_z, training=True)

            self.tot_gen_loss = 0
            if self.loss_weights['discriminator'] > 0:
                if disc_size < out_size:
                    true_img_disc = tf.random_crop(true_img, [self.batch_size, disc_size, disc_size, 3])
                    gen_img_disc = tf.random_crop(gen_img, [self.batch_size, disc_size, disc_size, 3])
                else:
                    true_img_disc = true_img
                    gen_img_disc = gen_img

                real_disc = self.discriminator(true_img_disc, training=True)
                generated_disc = self.discriminator(gen_img_disc, training=True)

                gen_loss = self.generator_loss(generated_disc) * self.loss_weights['discriminator']
                disc_loss = self.discriminator_loss(true_img_disc, gen_img_disc, real_disc, generated_disc) * \
                            self.loss_weights['discriminator']
                self.tot_disc_loss = disc_loss
                self.tot_gen_loss += gen_loss
                self.disc_loss_curve.append(self.tot_disc_loss)

            l2_loss = self.l2_loss(gen_img, true_img) * self.loss_weights['l2_gen']
            self.tot_gen_loss += l2_loss
        self.gen_loss_curve.append(self.tot_gen_loss)

        gradients_gen = gen_tape.gradient(
            self.tot_gen_loss, self.encoder.trainable_variables + self.decoder.trainable_variables)
        self.gen_optimizer.apply_gradients(
            zip(gradients_gen, self.encoder.trainable_variables + self.decoder.trainable_variables))
        self.best_gen_loss = min(self.best_gen_loss, self.tot_gen_loss) if self.best_gen_loss else self.tot_gen_loss

        if self.loss_weights['discriminator'] > 0:
            gradients_disc = disc_tape.gradient(self.tot_disc_loss, self.discriminator.trainable_variables)
            self.disc_optimizer.apply_gradients(zip(gradients_disc, self.discriminator.trainable_variables))
            self.best_disc_loss = min(self.best_disc_loss, self.tot_disc_loss) if self.best_disc_loss else self.tot_disc_loss

            return (gen_loss, disc_loss, l2_loss)
        else:
            return (l2_loss)

    def train(self, train_data, in_size, out_size, disc_size, iterations, lr, save_dir, b1, b2, save_int, **kwargs):
        """Train over specified number of iterations per settings."""
        if not os.path.isdir(save_dir): os.mkdir(save_dir)
        self.init_models(in_size, out_size, disc_size, lr, b1, b2)
        gen_steps, disc_steps = self.load_saved_models(save_dir)

        source_imgs = None
        progress = tqdm(train_data.take(iterations))
        for iteration, true_img in enumerate(progress):
            if kwargs['blur_kernel']:
                proc_img = tf.nn.depthwise_conv2d(true_img, self.gauss_kernel, strides=[1, 1, 1, 1], padding="SAME")
            else:
                proc_img = true_img
            if in_size < out_size:
                proc_img = tf.image.resize(proc_img, (in_size, in_size), align_corners=True)
            if source_imgs is None:
                source_imgs = proc_img
                self.save_img(source_imgs, save_dir, name='imgs_in_')
            losses = self.train_step(img_in=proc_img, true_img=true_img, out_size=out_size, disc_size=disc_size)
            if self.loss_weights['discriminator'] > 0:
                progress.set_postfix(best_gen_loss=self.best_gen_loss.numpy(), best_disc_loss=self.best_disc_loss.numpy(),
                                     gen_loss=losses[0].numpy(), disc_loss=losses[1].numpy(), l2_loss=losses[2].numpy())
            else:
                progress.set_postfix(l2_loss=losses.numpy())

            if iteration % save_int == 0 or iteration == iterations - 1:
                self.generate(source_imgs, iteration + 1 + gen_steps, save_dir)
                self.save_learning_curve(save_dir)
                self.save_models(save_dir, iteration, gen_steps, disc_steps)

    def generator_loss(self, generated_disc):
        """Gen loss from WGAN-GP loss: https://arxiv.org/pdf/1704.00028.pdf"""
        # Negative so that gradient descent maximizes critic score received by generated output
        return -tf.reduce_mean(generated_disc)

    def discriminator_loss(self, real_imgs, generated_imgs, real_disc, generated_disc, gp_lambda=10, epsilon=0.001):
        """Disc loss from WGAN-GP loss: https://arxiv.org/pdf/1704.00028.pdf"""
        # Difference between critic scores received by generated output vs real image
        # Lower values mean that the real image samples are receiving higher scores, therefore
        # gradient descent maximizes discriminator accuracy
        out_size = real_imgs.get_shape().as_list()
        d_cost = tf.reduce_mean(generated_disc) - tf.reduce_mean(real_disc)
        alpha = tf.random.uniform(
            shape=[self.batch_size, out_size[2], out_size[2], 3],
            minval=0.,
            maxval=1.
        )
        diff = generated_imgs - real_imgs
        interpolates = real_imgs + (alpha * diff)
        with tf.GradientTape() as tape:
            tape.watch(interpolates)
            interpolates_disc = self.discriminator([interpolates], training=False)
        # Gradient of critic score wrt interpolated imgs
        gradients = tape.gradient(interpolates_disc, [interpolates])[0]
        # Euclidean norm of gradient for each sample
        norm = tf.sqrt(tf.reduce_sum(tf.square(gradients), axis=[1, 2, 3]))
        # Gradient norm penalty is the average distance from 1
        gradient_penalty = tf.reduce_mean((norm - 1.) ** 2) * gp_lambda
        epsilon_penalty = tf.reduce_mean(real_disc) * epsilon

        return d_cost + gradient_penalty + epsilon_penalty

    def l2_loss(self, gen_img, true_img):
        """L2 loss."""
        if self.loss_weights['l2_loss_size'] != gen_img.get_shape().as_list()[1]:
            gen_img = tf.image.resize(gen_img,
                                      (self.loss_weights['l2_loss_size'], self.loss_weights['l2_loss_size']),
                                      align_corners=True)
        if self.loss_weights['l2_loss_size'] != true_img.get_shape().as_list()[1]:
            true_img = tf.image.resize(true_img,
                                       (self.loss_weights['l2_loss_size'], self.loss_weights['l2_loss_size']),
                                       align_corners=True)

        return tf.reduce_mean(tf.squared_difference(true_img, gen_img))

    def generate(self, source_imgs, epoch, save_dir):
        """Inference pass to produce and save image."""
        generated_imgs = self.decoder(self.encoder(source_imgs, training=False), training=False)

        self.save_img(generated_imgs, save_dir, name=str(generated_imgs[0].shape[-2]) + '_' + str(epoch) + '_')

    def save_img(self, img_tensor, save_dir, name):
        """Save out image as JPG."""
        img = tf.cast(255 * (img_tensor + 1)/2, tf.uint8)
        for i, ind_img in enumerate(img):
            encoded = tf.image.encode_jpeg(ind_img)
            tf.write_file(os.path.join(save_dir, name + str(i) + '.jpg'), encoded)

    def save_learning_curve(self, save_dir):
        """Save out losses as plot."""
        plt.plot(range(len(self.gen_loss_curve)), self.gen_loss_curve, color='g', linewidth='1')
        plt.xlabel("Iterations")
        plt.ylabel("Generator Loss")
        plt.savefig(os.path.join(save_dir, 'gen_loss.jpg'), bbox_inches='tight')
        plt.clf()

        if self.loss_weights['discriminator'] > 0:
            plt.plot(range(len(self.disc_loss_curve)), self.disc_loss_curve, color='g', linewidth='1')
            plt.xlabel("Iterations")
            plt.ylabel("Discriminator Loss")
            plt.savefig(os.path.join(save_dir, 'disc_loss.jpg'), bbox_inches='tight')
            plt.clf()

    def load_saved_models(self, save_dir):
        """Load models from checkpoints."""
        gen_checkpoints = glob.glob(os.path.join(save_dir, 'encoder-*.h5'))
        max_gen, max_disc = 0, 0
        if len(gen_checkpoints) > 0:
            all_steps = [int(os.path.basename(path).split('.')[0].split('-')[1]) for path in gen_checkpoints]
            max_gen = max(all_steps)
            self.encoder.load_weights(os.path.join(save_dir, 'encoder-' + str(max_gen) + '.h5'), by_name=True)
            self.decoder.load_weights(os.path.join(save_dir, 'decoder-' + str(max_gen) + '.h5'), by_name=True)
        if self.disc_model:
            disc_checkpoints = glob.glob(os.path.join(save_dir, 'disc-*.h5'))
            if len(disc_checkpoints) > 0:
                all_steps = [int(os.path.basename(path).split('.')[0].split('-')[1]) for path in disc_checkpoints]
                max_disc = max(all_steps)
                self.discriminator.load_weights(os.path.join(save_dir, 'disc-' + str(max_disc) + '.h5'), by_name=True)
        return max_gen, max_disc

    def save_models(self, save_dir, current_steps, gen_steps, disc_steps):
        """Save models as checkpoints."""
        self.encoder.save(os.path.join(save_dir, 'encoder-' + str(gen_steps + current_steps) + '.h5'))
        self.decoder.save(os.path.join(save_dir, 'decoder-' + str(gen_steps + current_steps) + '.h5'))
        if self.disc_model:
            self.discriminator.save(os.path.join(save_dir, 'disc-' + str(disc_steps + current_steps) + '.h5'))


import os

from os.path import (
    abspath,
    dirname,
    isfile,
    join as join_path,
)

from six.moves.configparser import ConfigParser

from pymud.utilities import ConsoleLogger

rel_path = abspath(join_path(dirname(__file__), 'pymud.conf'))
etc_path = join_path('/etc/pymud', 'pymud.conf')

CONFIG = ConfigParser()

if isfile(rel_path):
    CONFIG.read(rel_path)
elif isfile(etc_path):
    CONFIG.read(etc_path)
else:
    CONFIG.add_section('general')
    CONFIG.set(
        'general',
        'host',
        os.getenv('HOST', '0.0.0.0')
    )
    CONFIG.set(
        'general',
        'port',
        os.getenv('PORT', '7001')
    )

LOGGER = ConsoleLogger()


from keras.datasets import mnist
from keras.utils import np_utils

import numpy as np
import sys
import tensorflow as tf

seed = 0
np.random.seed(seed)
tf.set_random_seed(seed)

(X_train, y_class_train), (X_test, y_class_test) = mnist.load_data()

print("Num of images in Train set: %d" % (X_train.shape[0]))
print("Num of images in Test set: %d" % (X_test.shape[0]))

import matplotlib.pyplot as plt

plt.imshow(X_train[0], cmap='Greys')
plt.show()

for x in X_train[0]:
    for i in x:
        sys.stdout.write('%d\t' % i)
    sys.stdout.write('\n')

X_train = X_train.reshape(X_train.shape[0], 784)
X_train = X_train.astype('float64')
X_train = X_train / 255

X_test = X_test.reshape(X_test.shape[0], 784).astype('float64') / 255

print('class: %d ' % (y_class_train[0]))

y_train = np_utils.to_categorical(y_class_train, 10)
y_test = np_utils.to_categorical(y_class_test, 10)

print(y_train[0])


import numpy as np
from nnlib.propagation import propagation
from nnlib.sl import load_parameters
from nnlib.dataset import load_dataset, flatten, normalization
import matplotlib.pyplot as plt

train_set_x_orig, train_set_y, test_set_x_orig, test_set_y, label = load_dataset(
    "datasets/train.h5", "datasets/test.h5"
)
print(test_set_x_orig.shape)
train_set_x_flatten = flatten(train_set_x_orig)
train_set_x = normalization(train_set_x_flatten)
test_set_x_flatten = flatten(test_set_x_orig)
test_set_x = normalization(test_set_x_flatten)

L, w, b = load_parameters("parameters.h5")

a_train = propagation(train_set_x, train_set_y, w, b, returnCost=False)
train_set_y_hat = np.round(a_train[L])
m_train = train_set_y.shape[1]
index_train = np.arange(0, m_train)
train_accuracy = len(index_train[train_set_y[0] == train_set_y_hat[0]]) / m_train
print("Train set accuracy: {:.2%}".format(train_accuracy))

a_test = propagation(test_set_x, test_set_y, w, b, returnCost=False)
test_set_y_hat = np.round(a_test[L])
m_test = test_set_y.shape[1]
index_test = np.arange(0, m_test)
test_accuracy = len(index_test[test_set_y[0] == test_set_y_hat[0]]) / m_test
print("Test set accuracy: {:.2%}".format(test_accuracy))


from operator import mul
from functools import reduce
def combinations_count(n, r):
    r = min(r, n - r)
    numer = reduce(mul, range(n, n - r, -1), 1)
    denom = reduce(mul, range(1, r + 1), 1)
    return numer // denom

N, L = map(int, input().split())
l = N%L
cnt = 0
for l in range(N//L+1):
    x = N-L*l
    cnt += combinations_count(x+l, x)
print(cnt%(10**9+7))


from fastapi import Response, HTTPException
from odmantic import Field
from pydantic import BaseModel
from pydantic.main import ModelMetaclass
from starlette.background import BackgroundTask
import typing
from dojo.shared.supported_services import SupportedServices

if typing.TYPE_CHECKING:
    from dojo.shared.error_messages import ErrorMessage

class ModelResponse(Response):
    """Directly return a Pydantic model as a JSON Response."""

    media_type = "application/json"

    def __init__(
        self,
        content: typing.Any,
        status_code: int = 200,
        headers: typing.Optional[dict] = None,
        media_type: typing.Optional[str] = None,
        background: typing.Optional[BackgroundTask] = None,
    ) -> None:
        super().__init__(content, status_code, headers, media_type, background)

    def render(self, content: typing.Any) -> bytes:
        if not isinstance(content, BaseModel):
            raise Exception("Content must be a pydantic model!")

        return content.json(
            encoder={
                "ensure_ascii": False,
                "allow_nan": False,
                "indent": None,
                "separators": (",", ":"),
            }
        ).encode("utf-8")

class PredefinedModelResponse(Response):
    """Directly intiaize a pydantic model with defaults and return as a JSON Response."""

    media_type = "application/json"

    def __init__(
        self,
        content: typing.Any,
        status_code: int = 200,
        headers: typing.Optional[dict] = None,
        media_type: typing.Optional[str] = None,
        background: typing.Optional[BackgroundTask] = None,
    ) -> None:
        super().__init__(content, status_code, headers, media_type, background)

    def render(self, content: typing.Any) -> bytes:
        if not isinstance(content, ModelMetaclass):
            raise Exception("Content must be a ModelMetaclass!")

        return (
            content()
            .json(
                encoder={
                    "ensure_ascii": False,
                    "allow_nan": False,
                    "indent": None,
                    "separators": (",", ":"),
                }
            )
            .encode("utf-8")
        )

class ActionCompleted(BaseModel):
    service: SupportedServices
    action: str

class BackgroundProcessResponse(BaseModel):
    uid: str
    process_name: str

class APIErrorRaw(Exception):
    def __init__(self, error_code: str, error_message: str, status_code: int = 400):
        self.error_code = error_code
        self.error_message = error_message
        self.status_code = status_code

class PreinitErrorMessage(Exception):
    """Use this error for models that need to be initialized before they can be used."""

    def __init__(self, model: typing.Any, status_code: int = 400):
        self.model = model().json(
            encoder={
                "ensure_ascii": False,
                "allow_nan": False,
                "indent": None,
                "separators": (",", ":"),
            }
        ).encode("utf-8")
        self.status_code = status_code

class ModelErrorMessage(Exception):
    """Use this error for models that have already been initalized """
    def __init__(self, model: typing.Any, status_code: int = 400):
        self.model = model.json(
            encoder={
                "ensure_ascii": False,
                "allow_nan": False,
                "indent": None,
                "separators": (",", ":"),
            }
        ).encode("utf-8")
        self.status_code = status_code


import bpy
import os
import random
from os.path import isfile

output_fname_tpl = os.getenv('FRAME_FNAME','frame%06d')

def frame_done(n):
  return isfile(output_fname_tpl % n + ".png")

def frames_todo():
  all_frames = range(bpy.context.scene.frame_start, bpy.context.scene.frame_end+1)
  return [n for n in all_frames if not frame_done(n)]

def render(frame):
  bpy.context.scene.frame_set(frame)
  bpy.context.scene.render.filepath = output_fname_tpl % (frame)
  bpy.ops.render.render(write_still=True)

bpy.context.user_preferences.addons['cycles'].preferences.compute_device_type = 'CUDA'
bpy.context.scene.cycles.device = 'GPU'

random.seed()
frames = frames_todo()
while len(frames):
  start = random.randrange(0,len(frames))
  queue = frames[start:]+frames[0:start-1]
  for n in queue:
    if frame_done(n):
      break
    render(n)
  frames = frames_todo()


import logparser
import numpy as np
from matplotlib import pyplot as plt

access_log = open("access.log","r")

dt_counter = {}

for line in access_log:

  logDict = logparser.parser(line)

  dat = logDict['time'][:11]

  if dat not in dt_counter:

    dt_counter[dat] = 1

  else:

    dt_counter[dat] = dt_counter[dat] + 1

def hits(x):

  return x[-1]

sort = sorted(dt_counter.items(),key=hits,reverse=True)[:15]

date = []

hits = []

for item in sort:

  date.append(item[0])
  hits.append(item[1])

access_log.close()

plt.figure(figsize=(12,7))
plt.xlabel("Date -->")
plt.ylabel("Hits -->")
plt.title("Hits per Date")

plt.xticks(rotation=90)
x = date
y = hits

plt.plot(x, y, color = 'red')
plt.yticks(np.arange(min(y)+12, 8000+250, 250))
plt.show()


import os
import gym
import time
import argparse
import numpy as np
import matplotlib.pyplot as plt
from EA_components_OhGreat.EA import EA
from EA_components_OhGreat.Recombination import *
from EA_components_OhGreat.Mutation import *
from EA_components_OhGreat.Selection import *
from src.Evaluation import *
from src.Network import *
from src.utilities import *

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-exp_name', action='store',
                        dest='exp_name', type=str,
                        default='test_experiment',
                        help="Defines the name of the experiment.")
    parser.add_argument('-b', action='store',
                        dest='budget', type=int,
                        default=10000,
                        help="Defines the total amount of evaluations.")
    parser.add_argument('-min', action='store_true',
                        dest='minimize',
                        help="Use this flag if the problem is minimization.")
    parser.add_argument('-r', action='store',
                        dest='recombination', type=str,
                        default=None,
                        help="Defines the recombination strategy.")
    parser.add_argument('-m', action='store',
                        dest='mutation', type=str,
                        default='IndividualSigma',
                        help="Defines the mutation strategy.")
    parser.add_argument('-s', action='store',
                        dest='selection', type=str,
                        default='PlusSelection',
                        help="Defines the selection strategy.")
    parser.add_argument('-ps', action='store',
                        dest='parents_size', type=int,
                        default=2,
                        help="Defines the number of parents per generation.")
    parser.add_argument('-os', action='store',
                        dest='offspring_size', type=int,
                        default=4,
                        help="Defines the number of offspring per generation.")
    parser.add_argument('-mul', action='store',
                        dest='one_fifth_mul', type=float,
                        default=0.9,
                        help="Defines the multiplier for the one fifth success rule.")
    parser.add_argument('-pat', action='store',
                        dest='patience', type=int,
                        default=None,
                        help="Defines the wait time before resetting sigmas.")
    parser.add_argument('-exp_reps', action='store',
                        dest='exp_reps', type=int,
                        default=5,
                        help="Defines the number of experiments to average results.")
    parser.add_argument('-train_reps', action='store',
                        dest='train_reps', type=int,
                        default=10,
                        help="Defines the number of evaluation repetitions to use during training.")
    parser.add_argument('-eval_reps', action='store',
                        type=int, default=100,
                        help="Defines the number of evaluation repetitions to run after \
                                'training' our candidate individuals.")
    parser.add_argument('-model', action='store',
                        dest='model', type=int,
                        default=0,
                        help="Defines the model architecture to use.")
    parser.add_argument('-env', action='store', type=str,
                        dest='env', default='CartPole-v1')
    parser.add_argument('-render_train', action='store_true',
                        help='use this flag to render the training process \
                                of the individuals')
    parser.add_argument('-render_eval', action='store_true',
                        help='use this flag to render the evaluation process \
                                after training our individuals')
    parser.add_argument('-virtual_display', action='store_true',
                        help='needed for headless servers when using render')
    parser.add_argument('-plot_name', action='store', type=str,
                        default=None)
    parser.add_argument('-env_threshold', action='store',
                        type=float, default=500,
                        help="Optimum value to set as horizontal line in plot")
    parser.add_argument('-v', action='store',
                        dest='verbose', type=int, default=0,
                        help="Defines the intensity of debug prints.")
    args = parser.parse_args()
    if args.verbose > 0:
        print(args)

    # used to train on headless servers
    if args.virtual_display:
        os.environ["SDL_VIDEODRIVER"] = "dummy"

    # create gym environment
    env = gym.make(args.env)
    if env is None:
        exit("Please select an environment")
    print()

    # environment specific parameters
    n_observations = np.sum([dim for dim in env.observation_space.shape])
    if env.action_space.__class__.__name__ == "Discrete":
        n_actions = env.action_space.n
    elif env.action_space.__class__.__name__ == "Box":
        n_actions = sum(env.action_space._shape)
    else:
        exit(f"{env.action_space.__class__.__name__} action space not yet implemented")

    # create an instance of the model
    if args.model == 0:
        model = NN_regression_0(n_observations, 4, 4, n_actions).to("cpu")
    elif args.model == 1:
        model = NN_regression_1(n_observations, 4, 4, n_actions).to("cpu")
    elif args.model == 2:
        model = NN_regression_2(n_observations, 4, 4, n_actions).to("cpu")
    elif args.model == 3:
        model = NN_regression_3(n_observations, 4, 8, n_actions).to("cpu")
    else:
        exit("Choose a valid model")

    # define es individual size
    individual_size = model.total_params

    print(f"Model architecture: {args.model}\nEnvironment: {env.unwrapped.spec.id}\nNumber of observations: {n_observations}\nNumber of actions: {n_actions}\nIndividual size: {individual_size}")
    print()

    # EA specific parameters
    minimize = args.minimize
    budget = args.budget
    patience = args.patience
    parents_size = args.parents_size
    offspring_size = args.offspring_size
    # Recombination specific controls
    if args.recombination != None:
        recombination = globals()[args.recombination]()
    elif args.recombination == "GlobalIntermediary" and args.offspring_size > 1:
        print("GlobalIntermediary recombination cannot be used with more than one offspring.")
        print("Please use a valid configuration")
        exit()
    else: recombination = None
    # Mutation specific controls
    if args.mutation == "IndividualOneFifth":
        mutation = globals()[args.mutation](args.one_fifth_mul)
    else:
        mutation = globals()[args.mutation]()
    selection=globals()[args.selection]()
    evaluation = RewardMaximizationNN(env, model, args.train_reps,
                                        args.render_train)

    # loop through experiment
    best_results = []
    data_for_plots = []
    for i in range(args.exp_reps):
        # define new ea istance
        ea = EA(minimize=minimize, budget=budget, patience=patience,
        parents_size=parents_size, offspring_size=offspring_size,
        individual_size=individual_size, recombination=recombination,
        mutation=mutation, selection=selection, evaluation=evaluation,
        verbose=args.verbose)

        # run the ea
        start_time = time.time()
        best_ind, all_best_evals = ea.run()
        end_time = time.time()

        # keep track of results
        best_results.append([best_ind, max(all_best_evals)])
        data_for_plots.append(all_best_evals)
        print(f"Rep: {i+1} | average for {args.train_reps} evals: {max(all_best_evals)} | time: {np.round(end_time-start_time, 2)}")

    # save plot if name has been defined
    if args.plot_name != None:
        save_plot(args.plot_name, args.env_threshold, np.array(data_for_plots))

    # loop through final evalutation process for our best results
    eval_results = []
    for res in best_results:
        model.update_weights(res[0])
        curr_eval = eval(env, model,
                        args.eval_reps,
                        render=args.render_eval)
        eval_results.append(curr_eval)
    print("Evaluation results",np.round(eval_results, 2))

    # if saving is enabled
    if args.exp_name != None:
    # initialize directory and save results
        if not os.path.exists('model_weights'):
            os.makedirs('model_weights')
        # get the best individual
        best_ind_idx = np.argmax(eval_results)
        best_ind = best_results[best_ind_idx][0]
        # update model weights to match best inidividual
        model.update_weights(best_ind)
        # Save best individual as model weights
        torch.save(model.state_dict(), 'model_weights/'+args.exp_name)
        # Also save it as a numpy array
        np.save('model_weights/'+args.exp_name+'.npy', best_ind)

if __name__ == "__main__":
    main()


"""H1st IoT Maintenance Operations: URL configs."""

from django.conf.urls import include, url

from rest_framework.routers import DefaultRouter

from aito.iot_mgmt.maint_ops.views import (
    EquipmentInstanceDailyRiskScoreViewSet,
    EquipmentProblemTypeViewSet,
    EquipmentInstanceAlarmPeriodViewSet,
    EquipmentInstanceProblemDiagnosisViewSet,
    AlertDiagnosisStatusViewSet,
    EquipmentInstanceAlertPeriodViewSet,
)

ROUTER = DefaultRouter()

ROUTER.register(
    'equipment-instance-daily-risk-scores',
    EquipmentInstanceDailyRiskScoreViewSet)

ROUTER.register(
    'equipment-problem-types',
    EquipmentProblemTypeViewSet)

ROUTER.register(
    'equipment-instance-alarm-periods',
    EquipmentInstanceAlarmPeriodViewSet)

ROUTER.register(
    'equipment-instance-problem-diagnoses',
    EquipmentInstanceProblemDiagnosisViewSet)

ROUTER.register(
    'alert-diagnosis-statuses',
    AlertDiagnosisStatusViewSet)

ROUTER.register(
    'equipment-instance-alert-periods',
    EquipmentInstanceAlertPeriodViewSet)

URL_PATTERNS = [
    # API URLs
    url('iot/api/maint-ops/', include(ROUTER.urls)),
]


import pygame

pygame.init()

default_display = 320, 240
max_display = pygame.display.Info().current_w, pygame.display.Info().current_h
scale = 1
fullscreen = False

draw_surface = pygame.Surface(default_display)
scale_surface = pygame.Surface(default_display)
game_display = pygame.display.set_mode(default_display)
clock = pygame.time.Clock()

# Functions
def set_display():
    global game_display, scale_surface, draw_surface
    scale_surface = pygame.Surface(get_resolution())
    if fullscreen:
        draw_surface = pygame.Surface(max_display)
        gameDisplay = pygame.display.set_mode(max_display, pygame.FULLSCREEN)
    else:
        draw_surface = pygame.Surface(default_display)
        gameDisplay = pygame.display.set_mode(default_display)
    return

def get_resolution():
    if fullscreen:
        return max_display[0] * scale, max_display[1] * scale
    else:
        return default_display[0] * scale, default_display[1] * scale

# MainLoop
while True:

    # EventHandle
    for event in pygame.event.get():
        # Quit
        if event.type == pygame.QUIT:
            pygame.quit()
            quit()

        # Buttons
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_z:
                scale += 1
                if scale == 4:
                    scale = 1
                set_display()
            if event.key == pygame.K_x:
                fullscreen = not fullscreen
                set_display()
            if event.key == pygame.K_c:
                pass

    # Draw
    draw_surface.fill((255, 255, 255))
    pygame.draw.rect(draw_surface, (0, 0, 0), (50, 50, 50, 50))

    pygame.transform.scale(draw_surface, get_resolution(), scale_surface)

    game_display.blit(scale_surface, (0, 0))

    pygame.display.update()
    clock.tick(60)


# importing required modules

import tkinter as tk
from tkinter import filedialog
import xlsxwriter

from init_row_data import init_row_data
from compute_B import compute_B
from compute_A import compute_A

# console interface

print("Power bills data extractor in txt\nver.: 1.0, 10/03/2022\nAuthor: Fellipe Filgueira")
input("Press ENTER to begin...")
print("\nProcessing...")

# load files paths

root = tk.Tk()
root.withdraw()
root.call('wm', 'attributes', '.', '-topmost', True)
filePath = filedialog.askopenfilename(multiple=False) # True to load multiple files

# creating .xlsx object

findPath = filePath[len(filePath)::-1]
findPath = findPath[findPath.find("/"):]
findPath = findPath[len(findPath)::-1]
workbook = xlsxwriter.Workbook(findPath + 'data.xlsx')
worksheet = workbook.add_worksheet('main')
data_dict = init_row_data()
data_headers = list(data_dict.keys())
for itr in range(len(data_headers)): worksheet.write(0, itr, data_headers[itr])

# reading txt file

file = open(filePath, 'r', encoding='utf-8')
file_items = file.readlines()

# set lists

text = [[]]
data = []

# set loop variables

count = 0
j = 0
row = 1

# convert str into list based in break points

for item in file_items:
    if 'Federal' in item:
        text[count].append(item)
        if item != file_items[-1]:
            text.append(list())
            count += 1
    if item[0:2] == '11' or item[0:2] == '12'or item[0:2] == '13' or item[0:2] == '14' and 'Federal' not in item:
        text[count].append(item)
    else:
        pass

# algorithm loop

while j < len(text):
    if 'B3' in text[j][1]:
        data = init_row_data()
        data = compute_B(j, text, data)
        for i in range(len(list(data))): worksheet.write(row, i, data[list(data)[i]])
        row += 1
    if 'A4' in text[j][1] or 'A3' in text[j][1]:
        data = init_row_data()
        data = compute_A(j, text, data)
        for i in range(len(list(data))): worksheet.write(row, i, data[list(data)[i]])
        row += 1
    if text[j][1][81:83] not in ['B3', 'A3', 'A4'] :
        print(text[j][1][26:36] + " bill was not correctly identified")
    else:
        pass
    j += 1

# close files

workbook.close()
file.close()

# console interface

print("\ndata.xlsx file has been successfully saved!")
input("Press ENTER to finish...")


# 백준 4485. 녹색 옷 입은 애가 젤다지?
# 시간 제한 1초 / 메모리 제한 256MB
from collections import deque

dx = [-1, 1, 0, 0]
dy = [0, 0, -1, 1]

def bfs():
    global dist
    queue = deque()
    queue.append((0, 0))
    dist[0][0] = graph[0][0]

    while queue:
        x, y = queue.popleft()

        for d in range(4):
            nx = x + dx[d]
            ny = y + dy[d]

            if not(0 <= nx < n and 0 <= ny < n):
                continue

            cost = graph[nx][ny]

            if dist[nx][ny] > dist[x][y] + cost:
                dist[nx][ny] = dist[x][y] + cost
                queue.append((nx, ny))

index = 1
while True:
    n = int(input())

    if n == 0:
        break

    graph = [list(map(int, input().split())) for _ in range(n)]

    dist = [[1e9 for _ in range(n)] for _ in range(n)]
    bfs()
    print("Problem {}: {}".format(index, dist[n-1][n-1]))

    index += 1

백준/Gold/4485. 녹색 옷 입은 애가 젤다지？/녹색 옷 입은 애가 젤다지？.py

녹색 옷 입은 애가 젤다지？.py


from helpers.agent.multi_agent import MultiAgent
from helpers.env.env_utils import Init_Env
from itertools import count
import numpy as np
import torch
import glob
import time
import os
import gc
import sys
LEAVE_PRINT_EVERY_N_SECS = 300

class Env_Agent_Mix:
    # initializerrrrrrr
    def __init__(self, filename:str, training_info:dict, policy_info:dict,
                value_info:dict, buffer_info:dict, seed:int, optim_update:int,
                checkpoints:int, storage_loc:str):
        # create environment
        self.env_class = Init_Env(os.path.join(filename))

        # initialize environment
        self.env, self.brain_name = self.env_class.make_env_func(seed)

        # get state_size, number of agents, number of actions
        self.state_size, self.num_agents, \
        self.num_actions = self.env_class.reset(self.env, self.brain_name,
                                                train_mode=False, flag=True)

        # environment bounds
        self.low_bounds = np.array([-1] * self.num_actions)
        self.high_bounds = np.array([1] * self.num_actions)

        # seed
        self.seed = seed
        self.optim_update = optim_update

        # general training information
        self.update_steps = training_info["update_every_steps"]
        self.warm_up_batch = training_info["n_warmup_batches"]
        self.weight_mix_ratio = training_info["weight_mix_ratio"]

        # initialize multi agents
        self.multi_agent_init(policy_info, value_info, buffer_info)

        # storage information
        self.checkpoints = checkpoints
        self.storage_loc = storage_loc

    # multiple agents
    def multi_agent_init(self, policy_info:dict, value_info:dict, buffer_info:dict):
        general_info = {"weight_mix_ratio": self.weight_mix_ratio,
                        "bounds": (self.low_bounds, self.high_bounds),
                        "state_size": self.state_size,
                        "num_agents": self.num_agents,
                        "num_actions": self.num_actions}

        # creating the agents
        self.multi_agents = MultiAgent(general_info, policy_info, value_info, buffer_info, self.seed)

    # interaction step
    def env_agent_inter_step(self, states:np.ndarray, min_samples:int):
        actions = []
        # take random walks to initialize the and warm up the network
        if self.multi_agents.buffer.storage_length() < min_samples:
            for (agent, state) in zip(self.multi_agents.agents, states):
                # append to
                actions.append(agent.policy_model.select_random_action(state))
        else:
            for (agent, state) in zip(self.multi_agents.agents, states):
                # append to
                actions.append(agent.policy_model.select_action(state))

        # take step and obtain rewards
        parameters = self.env_class.step(self.env, self.brain_name, np.array(actions))

        dones_a = []
        ## prep parameters to be stored in the array
        for idx in range(self.num_agents):
            is_truncated = parameters["max_reached"][idx]
            dones_a.append(parameters["dones"][idx] and not is_truncated)

        # store in buffer array -> (states, action, reward, next_state, done)
        self.multi_agents.buffer.store(states, np.array(actions),
                                       parameters["rewards"],
                                       parameters["next_states"],
                                       parameters["dones"])

        # tracking parameters
        self.episode_timestep[-1] += 1
        self.episode_exploration[-1] += np.array([agent.policy_model.exploration_ratio\
                                                for agent in self.multi_agents.agents]).mean()

        return parameters["next_states"], parameters["rewards"], parameters["dones"]

    # interaction step
    def env_agent_inter_step_eval(self, states:np.ndarray):
        actions = []
        for (agent, state) in zip(self.multi_agents.agents, states):
            # append to
            actions.append(agent.policy_model.select_greedy_action(state))

        # take step and obtain rewards
        parameters = self.env_class.step(self.env, self.brain_name, np.array(actions))

        return parameters["next_states"], parameters["rewards"], parameters["dones"]

    # evaluation mode
    def evaluate(self, n_episodes=1, stage="train"):
        rs = []
        rs_per_agent = []
        for episode in range(n_episodes):
            states = self.env_class.reset(self.env, self.brain_name,
                                                train_mode=False, flag=False)
            dones = False
            reward_tracker = np.zeros(self.num_agents)

            for idx in count():
                # combines actions for each agent
                states, rewards, dones = self.env_agent_inter_step_eval(states)
                # store in array
                reward_tracker += np.array(rewards)

                if True in dones:
                    break
            # append rewards
            if stage.lower() =="train":
                rs.append(reward_tracker.mean())
            else:
                rs_per_agent.append(reward_tracker)
                rs.append(reward_tracker.mean())

        # return mean and standard deviation
        if stage.lower() =="train":
            return np.mean(rs), np.std(rs)
        else:
            return rs, np.std(rs), np.array(rs_per_agent)

    def train(self, env_info:dict):
        # intialize constants
        goal_mean_100_reward = env_info["goal_mean_100_rewards"]
        max_minutes  = env_info["max_minutes"]
        max_episodes = env_info["max_episodes"]
        gamma        = env_info["gamma"]

        # initialize tracking parameters
        self.evaluation_scores = []
        self.episode_timestep  = []
        self.episode_seconds   = []
        self.episode_reward    = []
        self.episode_exploration = []
        self.reward_tracker = np.zeros((max_episodes, self.num_agents))

        # loop and tracking parameters
        training_start, last_debug_time = time.time(), float("-inf")
        training_time = 0
        total_steps = 0

        # result for storage
        results = np.empty((env_info["max_episodes"] , 7))
        fin_episode = 0
        # storage training loop
        for episode in range(1, max_episodes + 1):
            episode_start = time.time()

            # refresh environment
            states = self.env_class.reset(self.env, self.brain_name,
                                          train_mode=True, flag=False)

            self.episode_timestep.append(0.0)
            self.episode_exploration.append(0.0)

            # warmup samples
            min_samples = self.warm_up_batch * self.multi_agents.batch_size

            for step in count():
                # get the next state, and terminal conditions
                states, rewards, dones = self.env_agent_inter_step(states, min_samples)
                self.reward_tracker[episode-1] += np.array(rewards)

                if self.multi_agents.buffer.storage_length() > min_samples:

                    for _ in range(self.optim_update):
                        # extract from memory
                        idx_batch, memory_batch, ISweights = self.multi_agents.buffer.sample()
                        # optimize agents
                        self.multi_agents.optimize(idx_batch, memory_batch, ISweights, gamma)

                        # update network
                        if np.sum(self.episode_timestep) % self.update_steps == 0:
                                self.multi_agents.update_networks_multi()

                if True in dones:
                    self.episode_reward.append(np.max(self.reward_tracker[episode-1]))
                    gc.collect()
                    break

            # stat tracking
            episode_elapsed = time.time() - episode_start
            training_time += episode_elapsed
            self.episode_seconds.append(episode_elapsed)

            # evaluation
            evaluation_score, _ = self.evaluate()
            for idx, agent in enumerate(self.multi_agents.agents):
                self.save_checkpoint(episode - 1, agent.policy_model, idx)

            total_steps = int(np.sum(self.episode_timestep))
            self.evaluation_scores.append(evaluation_score)

           # mean and std calculations
            mean_10_reward = np.mean(self.episode_reward[-10:])
            std_10_reward = np.std(self.episode_reward[-10:])
            mean_100_reward = np.mean(self.episode_reward[-100:])

            min_100_reward = np.min(self.episode_reward[-100:])
            max_100_reward = np.max(self.episode_reward[-100:])
            std_100_reward = np.std(self.episode_reward[-100:])

            mean_100_eval_score = np.mean(self.evaluation_scores[-100:])
            std_100_eval_score = np.std(self.evaluation_scores[-100:])
            min_100_eval_score = np.min(self.evaluation_scores[-100:])
            max_100_eval_score = np.max(self.evaluation_scores[-100:])

            lst_100_exp_rat = np.array(self.episode_exploration[-100:]) / np.array(self.episode_timestep[-100:])
            mean_100_exp_rat = np.mean(lst_100_exp_rat)
            std_100_exp_rat = np.std(lst_100_exp_rat)

            wallclock_elapsed = time.time() - training_start
            results[episode - 1] = total_steps, mean_100_reward,\
                                   min_100_reward, max_100_reward, \
                                   mean_100_eval_score, min_100_eval_score, \
                                   max_100_eval_score

            reached_debug_time = time.time() - last_debug_time >= LEAVE_PRINT_EVERY_N_SECS
            # termination criteria check
            reached_max_minutes = wallclock_elapsed >= max_minutes * 60
            reached_max_episodes = episode >= max_episodes
            reached_goal_mean_reward = mean_100_eval_score >= goal_mean_100_reward
            training_over = reached_max_minutes or reached_max_episodes or reached_goal_mean_reward

            #print debug message
            self.debug_message(episode, total_steps, mean_10_reward, std_10_reward,
                               mean_100_reward, std_100_reward, mean_100_exp_rat,
                               std_100_exp_rat, mean_100_eval_score, std_100_eval_score,
                               training_start)

            if training_over:
                if reached_max_minutes: print(u'--> reached_max_minutes \u2715')
                if reached_max_episodes: print(u'--> reached_max_episodes \u2715')
                if reached_goal_mean_reward: print(u'--> reached_goal_mean_reward \u2713')
                fin_episode = episode
                break

        # re-evaluate for 100 steps
        final_eval_score, score_std, agent_scores = self.evaluate(n_episodes=100, stage="fin")
        wallclock_time = time.time() - training_start

        # print final message post evalualuation
        self.final_message(np.mean(final_eval_score), score_std, training_time, wallclock_time)

        # clean up the checkpoints
        self.get_cleaned_checkpoints()

        self.env.close() ; del self.env

        return np.array(results), np.array(final_eval_score), np.array(agent_scores),\
               training_time, wallclock_time, fin_episode

    def final_message(self, final_eval_score, score_std, training_time, wallclock_time):
        print('Final evaluation score {:.2f}\u00B1{:.2f} in {:.2f}s training time,'
               ' {:.2f}s wall-clock time.\n'.format(final_eval_score, score_std,
                                                   training_time, wallclock_time))

    def debug_message(self, episode, total_steps, mean_10_reward, std_10_reward,
                      mean_100_reward, std_100_reward, mean_100_exp_rat,
                      std_100_exp_rat, mean_100_eval_score, std_100_eval_score,
                      training_start):
        # message string
        elapsed_str = time.strftime("%H:%M:%S", time.gmtime(time.time() - training_start))
        debug_message = 'el {}, ep {:04}, ts {:07}, '
        debug_message += 'ar_10 ts {:05.1f} \u00B1 {:05.1f}, '
        debug_message += 'ar_100 ts {:05.1f} \u00B1 {:05.1f}, '
        debug_message += 'ex 100 {:02.1f} \u00B1 {:02.1f}, '
        debug_message += 'ev {:05.1f} \u00B1 {:05.1f}'
        debug_message = debug_message.format(elapsed_str, episode - 1, total_steps,
                                             mean_10_reward, std_10_reward,
                                             mean_100_reward, std_100_reward,
                                             mean_100_exp_rat, std_100_exp_rat,
                                             mean_100_eval_score, std_100_eval_score)
        print(debug_message)

    def save_checkpoint(self, episode_idx, model, rank):
        root_dir = os.path.join(self.storage_loc, "checkpoints",
                                "{}".format(self.seed),
                                "model_{}".format(rank))

        # check if directory exists and create if not
        if not os.path.isdir(root_dir):
            os.makedirs(root_dir)

        # save model
        torch.save(model.state_dict(),
                   os.path.join(root_dir, 'ep_{}.tar'.format(episode_idx)))

    def get_cleaned_checkpoints(self):
        for rank in range(self.num_agents):
            checkpoint_paths = {}
            paths = glob.glob(os.path.join(self.storage_loc, "checkpoints",
                                           "{}".format(self.seed),
                                           "model_{}".format(rank),
                                           "*.tar"))
            paths_dic = {int(path.split('.')[0].split("_")[-1]): path for path in paths}
            last_ep = max(paths_dic.keys())

            checkpoint_idxs = np.linspace(1, last_ep + 1, self.checkpoints, endpoint=True, dtype=np.int) - 1

            for idx, path in paths_dic.items():
                if idx in checkpoint_idxs:
                    checkpoint_paths[idx] = path
                else:
                    os.unlink(path)


# -*- coding: utf-8 -*-
"""
Created on Sat Jul 16 12:27:12 2022

@author: Delaeyram
"""

import streamlit as st
import cv2
st.markdown("Built by Eyram Dela")
run = st.checkbox("Run")
FRAME_WINDOW = st.image([])
video = cv2.VideoCapture(0)
net = cv2.dnn.readNet("dnn_model/yolov4-tiny.weights","dnn_model/yolov4-tiny.cfg")
model = cv2.dnn_DetectionModel(net)
model.setInputParams(size=(320,320),scale=1/255)
classes = []
with open("dnn_model/classes.txt","r") as file_object:
    for class_name in file_object.readlines():
        class_name  = class_name.strip()
        classes.append(class_name)
print(classes)

while run:
    sucess,frame =video.read()
    frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
    (class_ids,scores,bboxes) = model.detect(frame)
    for class_id,score,bbox in zip(class_ids,scores,bboxes):
        (x,y,w,h) = bbox
        class_name = classes[class_id]

        cv2.putText(frame,class_name,(x,y-10),cv2.FONT_HERSHEY_PLAIN,2,(100,0,50),2)
        cv2.rectangle(frame,(x,y),(x+w,y+h),(100,0,50),2)
    FRAME_WINDOW.image(frame)


from sys import maxsize as maxint

class Solution:
    def removeInvalidParentheses(self, s):
        n = len(s)
        min_dels = maxint
        min_sols = set()

        def rec(i, cnt, dels):
            nonlocal min_dels, min_sols
            if i == n:
                if cnt == 0:
                    sol = "".join([s[i] for i in range(n) if i not in dels])
                    # print((1, i, cnt, dels, sol, min_dels, min_sols))
                    if len(dels) == min_dels:
                        min_sols.add(sol)
                    elif len(dels) < min_dels:
                        min_dels = len(dels)
                        min_sols.clear()
                        min_sols.add(sol)
                        # print((2, i, cnt, dels, sol, min_dels, min_sols))

            elif s[i] in ('(', ')'):
                dels.add(i)
                rec(i + 1, cnt, dels)
                dels.remove(i)
                if s[i] == '(':
                    rec(i + 1, cnt + 1, dels)
                elif cnt > 0:
                    rec(i + 1, cnt - 1, dels)
            else:
                rec(i + 1, cnt, dels)

        rec(0, 0, set())
        return list(min_sols)


import sys
input = sys.stdin.readline

N = int(input())
stacks = []
for _ in range(N):
	stacks.append(int(input()))
high = 0
count = 0
for _ in range(N):
	newbar = stacks.pop()
	if newbar > high:
		count += 1
		high = newbar

print(count)


"""
Calculate rates with various fitting functions
"""

import numpy as np

def calc_sse_and_mae(calc_ks, fit_ks):
    """ (1) get the sum of square error (SSE) useful when determining
            which double plog routine will be used to initialize
            the nonlinear solver
        (2) also get the mean absolute error (MAE), which is written
            to the plog file
        Only need to assess error if there are 2 or more rate constants
    """

    abs_err = []
    if len(calc_ks) > 2:
        for calc_k, fit_k in zip(calc_ks, fit_ks):
            abs_err.append(np.abs((calc_k - fit_k) / calc_k))
        abs_err = np.array(abs_err, dtype=np.float64)
        mean_abs_err = np.mean(abs_err) * 100.0
        max_abs_err = np.max(abs_err) * 100.0
    else:
        mean_abs_err = 0.0
        max_abs_err = 0.0

    return mean_abs_err, max_abs_err

def assess_pressure_dependence(tk_dct, assess_pdep_temps,
                               tolerance=20.0, plow=None, phigh=None):
    """ Assess how much the rate constants change from
        a low-pressure to high-pressure regime

        tk_dct[pressure] = [temps, k(T, P)s]
        we assume the temps and pressures give all positive, defined rates
    """
    # Get list of the sorted pressures, ignoring the high-pressure limit rates
    pressures = [pressure for pressure in tk_dct
                 if pressure != 'high']
    pressures.sort()

    # Set the low- and high-pressure if not specified by user
    if plow is None:
        plow = min(pressures)
    if phigh is None:
        phigh = max(pressures)

    # Check % difference for k(T, P) vals
    is_pressure_dependent = False
    if plow in tk_dct and phigh in tk_dct:

        for temp_compare in assess_pdep_temps:
            # For the low- and high-P, find the idx for the temp in temp_compare
            temps_low = tk_dct[plow][0]
            temps_high = tk_dct[phigh][0]
            temp_low_match = np.where(np.isclose(temps_low, temp_compare))[0]
            temp_high_match = np.where(np.isclose(temps_high, temp_compare))[0]
            if temp_low_match.size > 0 and temp_high_match.size > 0:
                temp_low_idx = temp_low_match[0]
                temp_high_idx = temp_high_match[0]
                # Grab the k(T, P) vale for the approprite temp and pressure
                ktp_low = tk_dct[plow][1][temp_low_idx]
                ktp_high = tk_dct[phigh][1][temp_high_idx]
                # Calculate the % difference and see if above threshold
                ktp_dif = (abs(ktp_low - ktp_high) / ktp_low) * 100.0
                if ktp_dif > tolerance:
                    is_pressure_dependent = True

    return is_pressure_dependent

if __name__ == '__main__':
    PAIRS = [
        [800, 27103.4],
        [900, 193348],
        [1000, 955781],
        [1100, 3.60E+06],
        [1200, 1.10E+07],
        [1300, 2.85E+07],
        [1400, 6.51E+07],
        [1500, 1.34E+08],
        [1600, 2.52E+08],
        [1700, 4.43E+08],
        [1800, 7.34E+08],
        [1900, 1.16E+09],
        [2000, 1.74E+09],
        [2100, 2.53E+09],
        [2200, 3.56E+09],
        [2300, 4.86E+09],
        [2400, 6.48E+09],
        [2500, 8.45E+09],
        [2600, 1.08E+10],
        [2700, 1.36E+10],
        [2800, 1.68E+10],
        [2900, 2.06E+10],
        [3000, 2.48E+10]
    ]

    KTP_DCT = {
    #     1:  [[pair[0] for pair in PAIRS], [pair[1] for pair in PAIRS]],
    #     10:  [[pair[0] for pair in PAIRS], [pair[1] for pair in PAIRS]],
    #      10:  [[pair[0] for pair in PAIRS2], [pair[1] for pair in PAIRS2]],
          10:  np.array([[pair[0] for pair in PAIRS], [pair[1] for pair in PAIRS]]),
    #     100:  [[pair[0] for pair in PAIRS], [pair[1] for pair in PAIRS]],
    #     'high':  numpy.array([[pair[0] for pair in PAIRS], [pair[1] for pair in PAIRS]]),
    }

    assess_pressure_dependence(KTP_DCT, [1000.0],
                               tolerance=20.0, plow=None, phigh=None)


def hackerrank(s):
    counter = 0
    hack = 'hackerrank'
    for i in range(len(hack)):
        for j in range(len(s)):
            if s[j] == hack[i]:
                s = s[j + 1:]
                counter += 1
                break
    if counter == 10:
        print('YES')
    else:
        print('NO')

hackerrank('rhackerank')


import setuptools

# with open("README.md", "r") as fh:
# 	long_description = fh.read()

long_description = 'https://github.com/suckmybigdick/flask-wechat-utils'

setuptools.setup(
	name = "flask-wechat-utils",
	version="0.1.16",
	auth="Huang Xu Hui",
	author_email="13250270761@163.com",
	description="flask-wechat-tuils for wechat-app-user's login/register/auth, and message_template",
	long_description=long_description,
	long_description_content_type="text/markdown",
	url="https://github.com/suckmybigdick/flask-wechat-utils",
	packages=setuptools.find_packages(),
	classifiers=[
		"Programming Language :: Python :: 2",
		"License :: OSI Approved :: MIT License",
		"Operating System :: OS Independent",
	],
	install_requires=[
		"Flask==0.10.1",
		"requests==2.9.1",
		"cryptography",
		"itsdangerous==0.24",
		"Werkzeug==0.14.1",
		"flask_restplus==0.12.1",
		"flask_mongoengine==0.9.5",
	],
)