#coding = utf-8
#选择信息增益最大的10维特征
import numpy as np
from InfoGain import choose_best_feature

data_path = "C:\\Users\\TJM\\OneDrive\\graduated\\研①\\人工智能算法与实践\\homework\\分享\\Test\\1-kddcup.data_10_percent_corrected"
test_path = "C:\\Users\\TJM\\OneDrive\\graduated\\研①\\人工智能算法与实践\\homework\\分享\\Test\\3-corrected.txt"
feature1 = []
feature2 = []
feature3 = []
label = []
def read_data(path):
    data = []
    with open(path) as f:
        for l in f.readlines():
            l = l.strip('.\n').split(',')
            if l[1] not in feature1:
                feature1.append(l[1])
            if l[2] not in feature2:
                feature2.append(l[2])
            if l[3] not in feature3:
                feature3.append(l[3])
            if l[41] not in label:
                label.append(l[41])
            data.append(l)
    return data

def split_data(data):
    data = np.array(data)
    features = data[:,0:41]
    labels = data[:,41]
    return features,labels

def quantify(feature,Label):
    feature1 = ['udp', 'tcp', 'icmp']
    feature2 = ['private', 'domain_u', 'http', 'smtp', 'ftp_data', 'ftp', 'eco_i', 'other', 'auth', 'ecr_i', 'IRC', 'X11', 'finger', 'time', 'domain', 'telnet', 'pop_3', 'ldap', 'login', 'name', 'ntp_u', 'http_443', 'sunrpc', 'printer', 'systat', 'tim_i', 'netstat', 'remote_job', 'link', 'urp_i', 'sql_net', 'bgp', 'pop_2', 'tftp_u', 'uucp', 'imap4', 'pm_dump', 'nnsp', 'courier', 'daytime', 'iso_tsap', 'echo', 'discard', 'ssh', 'whois', 'mtp', 'gopher', 'rje', 'ctf', 'supdup', 'hostnames', 'csnet_ns', 'uucp_path', 'nntp', 'netbios_ns', 'netbios_dgm', 'netbios_ssn', 'vmnet', 'Z39_50', 'exec', 'shell', 'efs', 'klogin', 'kshell', 'icmp']
    feature3 = ['SF', 'RSTR', 'S1', 'REJ', 'S3', 'RSTO', 'S0', 'S2', 'RSTOS0', 'SH', 'OTH']
    for j in range(len(feature)):
        if feature[j][2] not in feature1:
            feature2.append(feature[j][2])
        feature[j][1] = feature1.index(feature[j][1])+1
        feature[j][2] = feature2.index(feature[j][2])+1
        feature[j][3] = feature3.index(feature[j][3])+1
        if Label[j]=="normal":
            Label[j] = 1
        #elif Label[j] =="smurf":
            #Label[j] = 2
        else:
            Label[j] = 0
   # print(feature[99999])
    feature = feature.astype(float)
    Label = Label.astype(float)
    return feature,Label

if __name__ == "__main__":
    trainset = 1
    if trainset:#
        d = read_data(data_path)
        #print(d[0])
        print("lable种类：",label)
        features,labels = split_data(d)
        print(feature1)
        print(feature2)
        print(feature3)
        print(len(label))
        q_feature ,q_label= quantify(features,labels)
        new_data = np.array(np.column_stack((q_feature,q_label)))
        #np.savetxt("./dataset/train_data_41_bin.txt", new_data,fmt = '%f',delimiter=',')

        #np.savetxt("./dataset/train_data_41.txt",new_data)#保存10维训练数据     保存格式有点问题
        h= len(new_data)
        '''best_fearture_index,best_10_index = choose_best_feature(q_feature,q_label)#计算信息增益，取最大的前十个特征
        print("最佳特征：",best_fearture_index)
        print("最佳前10特征：",best_10_index)'''
        best_10_index = [5, 23, 3, 24, 36, 2, 33, 35, 34, 30]#计算得出的最佳特征维度
        best_5_index = [5, 23, 3, 24, 36]#取前5个特征来分类进行对比
        #new_data1 = np.zeros((h,11))
        #new_data1 = np.zeros((h, 6))
        new_data1 = np.zeros((h, 11))
        print(new_data.shape)
        print(new_data1.shape)
        best_10_index.append(42)
        best_5_index.append(42)
        '''for i in best_5_index:
            j=i-1
            new_data1[:,best_5_index.index(i)] = new_data[:,j]'''

        for i in best_10_index:
            j=i-1
            new_data1[:,best_10_index.index(i)] = new_data[:,j]
        np.savetxt("./dataset/train_data_10_bin.txt",new_data1,fmt='%f',delimiter=',')#保存10维训练数据'''
        #np.savetxt("./dataset/train_data_4.txt", new_data1, fmt='%f', delimiter=',')
    else:
        d = read_data(test_path)
        # print(d[0])
        features, labels = split_data(d)
        print(feature1)
        print(feature2)
        print(feature3)
        print(len(label))
        q_feature, q_label = quantify(features, labels)
        new_data = np.column_stack((q_feature, q_label))
        h = len(new_data)
        #np.savetxt("./dataset/test_data_41_bin.txt", new_data,fmt = "%f",delimiter=',')#保存41维的二分类测试机
        #best_fearture_index, best_10_index = choose_best_feature(q_feature, q_label)
        best_10_index = [5, 23, 3, 24, 36, 2, 33, 35, 34, 30]
        #print("最佳特征：", best_fearture_index)
        #print("最佳前10特征：", best_10_index)
        #new_data1 = np.zeros((h, 11))
        #np.savetxt("./dataset/test_data_41.txt", new_data)#保存41维测试数据
        best_5_index = [5, 23, 3, 24, 36]  # 取前5个特征来分类进行对比
        #new_data1 = np.zeros((h, 6))
        new_data1 = np.zeros((h, 11))
        print(new_data.shape)
        print(new_data1.shape)
        best_10_index.append(42)
        #best_5_index.append(42)
        '''for i in best_5_index:
            j = i - 1
            new_data1[:, best_5_index.index(i)] = new_data[:, j]'''
        #np.savetxt("./dataset/test_data_4.txt", new_data1, fmt='%f', delimiter=',')
        for i in best_10_index:
            j = i - 1
            new_data1[:, best_10_index.index(i)] = new_data[:, j]
        np.savetxt("./dataset/test_data_10_bin.txt", new_data1, fmt='%f', delimiter=',')#保存10维测试数据


# /usr/bin/python3.6
# -*- coding:utf-8 -*-
# 超时

class Solution(object):
    def minKBitFlips(self, A, K):
        """
        :type A: List[int]
        :type K: int
        :rtype: int
        """
        try:
            old_index = A.index(0)
        except:
            return 0
        length = len(A)
        ret = 0
        while old_index <= length - K:
            A[old_index:old_index+K] = [1-x for x in A[old_index:old_index+K]]
            ret += 1
            try:
                old_index = A.index(0)
            except:
                return ret
        # print(A)
        for i in range(length):
            if A[i] == 0:
                return -1
        return ret

def main():
    s = Solution()
    # print(s.minKBitFlips([0,1,0],1))
    print(s.minKBitFlips([1,1,0],2))
    print(s.minKBitFlips(A = [0,0,0,1,0,1,1,0], K = 3))

if __name__ == "__main__":
    main()


import datetime
import math

from sqlalchemy import desc, asc

from app.main import db
from app.main.model.unit import Unit
from app.main.service.language_helper import LanguageHelper

def save_unit(data, args):
    errors = {}
    language_data = LanguageHelper(args)

    # Check unique field is null or not
    if data['name'] == "":
        errors['name'] = [language_data.get_message(
            'unit.save.no_unit_name_message')]
    if len(errors) > 0:
        response_object = {
            'status': 'FAILED',
            'message': language_data.get_message('unit.save.failed_message'),
            'errors': errors
        }
        return response_object, 200
    else:
        unit = Unit.query.filter_by(
            name=data['name']).first()

        if unit:
            errors['name'] = [language_data.get_message(
                'unit.save.existed_unit_name_message')]
            response_object = {
                'status': 'FAILED',
                'message': language_data.get_message('unit.save.failed_message'),
                'errors': errors
            }
            return response_object, 200
        else:
            new_unit = Unit(
                name=data['name'],
                description=data['description'],
                created_on=datetime.datetime.utcnow(),
                updated_on=datetime.datetime.utcnow()
            )
            save_changes(new_unit)

            output = {}
            output['name'] = new_unit.name
            output['description'] = new_unit.description
            output['created_on'] = str(new_unit.created_on)
            output['updated_on'] = str(new_unit.updated_on)

            response_object = {
                'status': 'SUCCESS',
                'message': language_data.get_message('unit.save.success_message'),
                'data': output
            }
            return response_object, 201

def update_unit(id, data, args):
    unit = Unit.query.filter_by(id=id).first()
    is_updated = False
    errors = {}
    language_data = LanguageHelper(args)

    # Check if ID is valid or not
    if not unit:
        errors['id'] = ["Unit ID does not exist!"]
        response_object = {
            'status': 'FAILED',
            'message': language_data.get_message('unit.update.failed_message'),
            'errors': errors
        }
        return response_object, 200
    else:
        # Check null
        if data['name'] == "":
            errors['name'] = [language_data.get_message(
                'unit.update.no_unit_message')]

        if (len(errors) > 0):
            response_object = {
                'status': 'FAILED',
                'message': language_data.get_message('unit.update.failed_message'),
                'errors': errors
            }
            return response_object, 200
        else:
            if data['name'] != unit.name:
                # Check if unit name is existed or not
                updated_unit = Unit.query.filter_by(name=data['name']).first()
                if updated_unit:
                    errors['name'] = [language_data.get_message(
                        'unit.update.existed_unit_name_message')]
                    response_object = {
                        'status': 'FAILED',
                        'message': language_data.get_message('unit.update.failed_message'),
                        'errors': errors
                    }
                    return response_object, 200
                else:
                    is_updated = True
                    unit.name = data['name']

            if data['description'] != unit.description:
                is_updated = True
                unit.description = data['description']

            if is_updated is True:
                unit.updated_on = datetime.datetime.utcnow()
                db.session.commit()

            unit_data = {}
            unit_data['id'] = str(unit.id)
            unit_data['name'] = unit.name
            unit_data['description'] = unit.description
            unit_data['created_on'] = str(unit.created_on)
            unit_data['updated_on'] = str(unit.updated_on)

            respone_object = {
                'status': 'SUCCESS',
                'message': language_data.get_message('unit.update.success_message'),
                'data': unit_data
            }
            return respone_object, 200

def get_all_units(args):
    all_unit = Unit.query.all()
    output = []
    languages_data = LanguageHelper(args)

    for unit in all_unit:
        unit_data = {}
        unit_data['id'] = unit.id
        unit_data['name'] = unit.name
        unit_data['description'] = unit.description
        unit_data['created_on'] = str(unit.created_on)
        unit_data['updated_on'] = str(unit.updated_on)

        output.append(unit_data)

    data = {}
    data['units'] = output

    respone_object = {
        'status': 'SUCCESS',
        'message':  languages_data.get_message('unit.get_all.success_message'),
        'data': data
    }
    return respone_object, 200

def get_unit(id, args):
    unit = Unit.query.filter_by(id=id).first()
    languages_data = LanguageHelper(args)

    if not unit:
        respone_object = {
            'status': 'ERROR',
            'message': languages_data.get_message('unit.get.no_unit_message')
        }
        return respone_object, 200

    unit_data = {}
    unit_data['id'] = unit.id
    unit_data['name'] = unit.name
    unit_data['description'] = unit.description
    unit_data['created_on'] = str(unit.created_on)
    unit_data['updated_on'] = str(unit.updated_on)

    respone_object = {
        'status': 'SUCCESS',
        'message':  languages_data.get_message('unit.delete.success_message'),
        'data': unit_data
    }
    return respone_object, 200

def delete_unit(id, args):
    errors = {}
    unit = Unit.query.filter_by(id=id).first()
    languages_data = LanguageHelper(args)

    if not unit:
        respone_object = {
            'status': 'FAILED',
            'message': languages_data.get_message('unit.delete.no_unit_message'),
            'errors': errors
        }
        return respone_object, 200
    else:
        db.session.delete(unit)
        db.session.commit()

        response_object = {
            'status': 'SUCCESS',
            'message': languages_data.get_message('unit.delete.success_message')
        }
        return response_object, 200

def save_changes(data):
    db.session.add(data)
    db.session.commit()

def get_all_units_with_pagination(args):
    # Query Params
    page_size = 10
    current_page = 1
    next_page = False
    key_word = None
    sort_field = None
    sort_order = -1

    # Check query param value
    if "page_size" in args:
        page_size = int(args['page_size'])
    if "current_page" in args:
        current_page = int(args['current_page'])
    if "key_word" in args:
        key_word = args['key_word']
    if "sort_field" in args:
        sort_field = args['sort_field']
    if "sort_order" in args:
        sort_order = int(args['sort_order'])

    # Get language data
    languages_data = LanguageHelper(args)

    # Sort by order value
    if sort_field is None or sort_order is None:
        '''Default order by the lasted created_on value'''
        units = Unit.query.order_by(Unit.created_on.desc())
    else:
        if sort_order == -1:
            units = Unit.query.order_by(desc(sort_field))
        else:
            units = Unit.query.order_by(asc(sort_field))

    units_on_page = units.limit(page_size).offset(
        (current_page - 1) * page_size)
    total_pages = math.ceil(units.count() / page_size)

    if math.ceil(units.count() - page_size*current_page > 0):
        next_page = True
    else:
        next_page = False

    output = []

    for unit in units_on_page:
        # Sort by keyword
        if (key_word is not None):
            if (key_word in unit.name.lower()) or (
                    key_word in unit.description.lower()):
                unit_data = {}
                unit_data['id'] = unit.id
                unit_data['name'] = unit.name
                unit_data['description'] = unit.description
                unit_data['created_on'] = str(unit.created_on)
                unit_data['updated_on'] = str(unit.updated_on)

                output.append(unit_data)
        else:
            unit_data = {}
            unit_data['id'] = unit.id
            unit_data['name'] = unit.name
            unit_data['description'] = unit.description
            unit_data['created_on'] = str(unit.created_on)
            unit_data['updated_on'] = str(unit.updated_on)

            output.append(unit_data)

    data = {}
    data['units'] = output
    data['total_pages'] = total_pages
    data['current_page'] = current_page
    data['has_next_page'] = next_page

    response_object = {
        'status': 'SUCCESS',
        'message':  languages_data.get_message('unit.get_all_with_pagination.success_message'),
        'data': data
    }
    return response_object, 200


import torch
import torch.nn as nn

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.fc1 = nn.Linear(784, 256, bias = False)
        self.bn1 = nn.BatchNorm1d(256)
        self.relu1 = nn.ReLU()

        self.fc2 = nn.Linear(256, 128)
        self.bn2 = nn.BatchNorm1d(128)
        self.relu2 = nn.ReLU()

        self.fc3 = nn.Linear(128, 10)

    def forward(self, x):
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = self.bn1(x)
        x = self.relu1(x)

        x = self.fc2(x)
        x = self.bn2(x)
        x = self.relu2(x)

        x = self.fc3(x)
        return x


class Solution:
    def maxArea(self, height: list[int]) -> int:
        left = 0
        right = len(height) - 1
        biggest_area = 0

        while left < right:
            left_bar = height[left]
            right_bar = height[right]
            current_area = min(left_bar, right_bar) * (right - left)
            biggest_area = max(biggest_area, current_area)

            if left_bar < right_bar:
                left += 1
            else:
                right -= 1

        return biggest_area


# %%
import logging
import os.path
import shutil
import sys
from typing import Optional

import matplotlib.pyplot as plt
import pandas as pd
import pytorch_lightning as pl
import torch
import torch.nn.functional as F
import torchaudio
from icecream import ic
from torch.utils.data import Dataset, DataLoader
import matplotlib.pyplot as plt
from io import StringIO

try:
    from hubert.clustering.filter_dataframe import clean_data_parczech
except ModuleNotFoundError:
    from filter_dataframe import clean_data_parczech

class ParCzechDataset(Dataset):
    def __init__(self, df_path, resample_rate=16000, df_filters=None, sep='\t', sort=True, train_flag=True, iloc=True, *args, **kwargs):
        super(ParCzechDataset, self).__init__()
        self.df = pd.read_csv(df_path, sep=sep)
        self.filter_df(df_filters)

        if train_flag:
            self.df = self.df[(self.df.type == 'train') | (self.df.type == 'other')]

        if sort:
            self.df = self.df.sort_values(by=['duration__segments'], ascending=False).copy().reset_index(drop=True)

        self.new_sr = resample_rate
        self.resample_transform = None
        # this configures the __getitem__ method, when self.iloc is true index in __getitem__ is interpreted as integer location in dataframe
        # when self.iloc is False index in __getitem__ is interpreted as an element in self.df.index
        self.iloc = iloc

    def index_df(self, i, column_name=None):
        if self.iloc:
            row = self.df.iloc[i]
        else:
            row = self.df.loc[i]

        if column_name is not None:
            return row[column_name]
        return row

    def get_mp3_name(self, i):
        row = self.index_df(i)
        try:
            # need to remove prefix  'sentences_'
            mp3_name = row.mp3_name.split('_')[-1]
        except:
            ic(row)
            raise ValueError(f'can not find row by index {i}')
        return mp3_name

    def extract_path(self, i):
        row = self.index_df(i)
        mp3_name = self.get_mp3_name(i)
        return os.path.join(row.segment_path, mp3_name)

    def get_gold_transcript(self, path):
        with open(f'{path}.prt', 'r') as f:
            return f.read().rstrip()

    def get_asr_transcript(self, path):
        with open(f'{path}.asr', 'r') as f:
            return f.read().rstrip()

    def _safe_read_df(self, path, names, header, sep, dtypes, na_values):
        if not os.path.isfile(path):
            print(f'{path} does not exist')
        replace_dict = {
            '"': "__double_quotes__",
        }
        with open(path, 'r') as f:
            src = ''.join(f.readlines())

        for k, v in replace_dict.items():
            src = src.replace(k, v)

        df = pd.read_csv(StringIO(src), names=names, header=header, sep=sep, dtype=dtypes, na_values=na_values)

        return df

    def get_recognized_df(self, path, i):
        # will extract recognized based on word ids
        header = ['word', 'word_id', 'start_time', 'end_time', 'XXX', 'avg_char_duration', 'speaker']

        try:
            words_df = self._safe_read_df(
                f'{path}.words',
                names=header,
                header=None,
                sep='\t',
                # replace_col=['word'],
                dtypes=None,
                na_values=None,
            )
        except Exception as e:
            ic(path)
            ic(e)
            raise ValueError(f'Can not read file {path}')

        word_ids = words_df['word_id'].values.tolist()

        # read aligned file
        path_aligned = f"/lnet/express/work/people/stankov/alignment/results/full/words-aligned/jan/words_{self.get_mp3_name(i)}.tsv"
        # normalization is done by the length of the "true_word"
        header_aligned = ['true_w', 'trans_w', 'joined', 'id', 'recognized', 'dist', 'dist_norm', 'start', 'end', 'time_len_ms', 'time_len_norm']
        dtypes = dict(
            true_w=str,
            trans_w=str,
            joined=bool,
            id=str,
            recognized=bool
        )
        for name in header_aligned:
            if name not in dtypes:
                dtypes[name] = float

        aligned_df = self._safe_read_df(
            path_aligned,
            header_aligned,
            sep='\t',
            dtypes=dtypes,
            na_values='-',
            header=0
        )
        aligned_df.trans_w = aligned_df.trans_w.fillna('-')
        aligned_df = aligned_df[aligned_df['id'].isin(word_ids)]
        return aligned_df

    def get_recognized_transcript(self, path, i):
        aligned_df = self.get_recognized_df(path, i)
        # from miliseconds to seconds
        start_time = aligned_df['start'].min() / 1000
        end_time = aligned_df['end'].max() / 1000
        mp3_name = self.get_mp3_name(i)

        path_transcribed = f'/lnet/express/work/people/stankov/alignment/results/full/scrapping/jan/{mp3_name}/{mp3_name}.tsv'
        if not os.path.isfile(path_transcribed):
            # print(f'{mp3_name} is not in scraping')
            path_transcribed = f'/lnet/express/work/people/stankov/alignment/results/full/time-extracted/jan/{mp3_name}.tsv'

        header_transcribed = ['start', 'end', 'recognized', 'true_word', 'cnt', 'dist']
        transcribed_df = pd.read_csv(path_transcribed, names=header_transcribed, header=None, sep='\t')
        # ic(start_time, end_time, transcribed_df.head())
        transcript = transcribed_df[(transcribed_df.start >= start_time) & (transcribed_df.end <= end_time)].recognized.values.tolist()
        return ' '.join(transcript)

    def resample(self, sr, wav):
        if self.resample_transform is None:
            self.resample_transform = torchaudio.transforms.Resample(orig_freq=sr, new_freq=self.new_sr)
        return self.resample_transform(wav)

    def get_wav(self, path):
        wav, sr = torchaudio.load(f'{path}.wav', normalize=True)
        # stereo to mono if needed
        if wav.size(0) == 2:
            wav = torch.mean(wav, dim=0).unsqueeze(0)
        return self.resample(sr, wav)

    def duration_hours(self, filters=None):
        if filters is not None:
            df = clean_data_parczech(self.df, filters)
        else:
            df = self.df
        return df.duration__segments.sum() / 3600

    def plot_stat(self, col_name, filters=None):
        if filters is not None:
            df = clean_data_parczech(self.df, filters)
        else:
            df = self.df
        # plt.boxplot(dataset.df.avg_char_duration__segments, vert=False)

        print(df.sort_values(by=[col_name])[col_name])
        plt.plot(range(len(df)), df.sort_values(by=[col_name])[col_name])
        plt.title(f'{col_name} sorted, {self.duration_hours(filters):.2f}h')
        plt.xlabel('segments')
        plt.ylabel(col_name)
        plt.show()

    def filter_df(self, filters, reset_index=False):
        if filters is None:
            return

        self.df = clean_data_parczech(self.df, filters)
        if reset_index:
            self.df.reset_index(drop=True, inplace=True)

    def get_columns(self):
        return self.df.columns.values

    def __getitem__(self, i):
        path = self.extract_path(i)
        return dict(
            gold_transcript=self.get_gold_transcript(path),
            asr_transcript=self.get_asr_transcript(path),
            wav=self.get_wav(path),
            path=os.path.dirname(path)
        )

    def __len__(self):
        return len(self.df)

def clean_data(df, params):
    # thresholds were selected based on the plot
    df = df[(df.type == 'train') | (df.type == 'other')]
    df = df[df.recognized_sound_coverage__segments > params['recognized_sound_coverage__segments_lb']]
    df = df[df.recognized_sound_coverage__segments < params['recognized_sound_coverage__segments_ub']]
    # removed 404.5 hours
    # use only long enough segments
    ic(df.duration__segments.sum() / 3600)
    if 'duration__segments_lb' in params:
        df = df[df.duration__segments > params['duration__segments_lb']]

    if 'duration__segments_ub' in params:
        df = df[df.duration__segments < params['duration__segments_ub']]
        ic(df.duration__segments.sum() / 3600)
    return df

class CommonVoiceDataset(Dataset):
    def __init__(self, base_dir, type, resample_rate=16000):
        self.data_path = os.path.join(base_dir, 'clips')
        self.df = pd.read_csv(os.path.join(base_dir, f'{type}.tsv'), sep='\t')
        self.resample_rate = resample_rate
        self.resample_transform = None

    def resample(self, waveform, sr):
        if self.resample_transform is None:
            self.resample_transform = torchaudio.transforms.Resample(sr, self.resample_rate)
        return self.resample_transform(waveform)

    def __getitem__(self, i):
        if torch.is_tensor(i):
            i = i.item()
        waveform, sample_rate = torchaudio.load(os.path.join(self.data_path, self.df.path[i]))
        return dict(
            wav=self.resample(waveform, sample_rate),
            path=self.df.path[i]
        )

    def __len__(self):
        return len(self.df)

class MFCCExtractorPL(pl.LightningModule):
    def __init__(self, n_mffcs, n_mels, f_max, resample_rate, output_dir, n_fft=400):
        super(MFCCExtractorPL, self).__init__()
        self.output_dir = output_dir
        self.n_fft = n_fft
        self.sr = resample_rate

        self.MFCC_transform = torchaudio.transforms.MFCC(
            resample_rate,
            n_mfcc=n_mffcs,
            melkwargs=dict(
                n_mels=n_mels,
                n_fft=n_fft,  # default
                hop_length=n_fft // 2,  # default
                f_max=f_max,
            )
        )
        self.delta_transform = torchaudio.transforms.ComputeDeltas()

    def prepare_data(self):
        if os.path.exists(self.output_dir):
            shutil.rmtree(self.output_dir)
        os.makedirs(self.output_dir)

    def forward(self, batch):
        wavs, _, lens = batch

        mfccs_batch = self.MFCC_transform(wavs)
        deltas_batch = self.delta_transform(mfccs_batch)
        deltas2_batch = self.delta_transform(deltas_batch)
        # all shapes [batch_size, 1, 13, max_n_frames]

        # stacking features
        output = torch.cat([mfccs_batch, deltas_batch, deltas2_batch], dim=2).squeeze().permute(0, 2, 1)
        # [batch_size, max_n_frames, 13 * 3]
        n_frames = torch.tensor([compute_frames(l, self.sr) for l in lens], device=self.device)
        return output, n_frames

def compute_frames(wave_len, sample_rate):
    ms_int = int(wave_len / sample_rate * 1000)
    # these "random" operations mimic how hubert.feature extractor counts frames in the audio
    new_ms = (ms_int - (ms_int % 5) - 1) // 20
    return new_ms

class SaveResultsCB(pl.Callback):
    def __init__(self, target_path, n_fft, buffer_size, df_type, total_batches, resample_rate=16000, frame_length=20):
        self.df_type = df_type
        self.output_dir = target_path
        self.n_fft = n_fft
        # number of frames to store at one csv
        self.buffer_size = buffer_size
        self.frame_length = frame_length

        self.dataframes = []
        self.current_buffer = 0
        # count how many df written to disk
        self.cnt = 0
        self.resample_rate = resample_rate
        self.total_duration_sec = 0
        self.loggers = {}
        self.total_batches = total_batches

    def extract_name(self, path):
        if self.df_type == 'common_voice':
            return path
        elif self.df_type == 'parczech':
            return '/'.join(path.split('/')[-2:])
        else:
            raise NotImplementedError(f'{self.df_type} is not supported')

    def write_df(self, trainer):
        output_path = os.path.join(self.output_dir, f'{trainer.global_rank:02}-{self.cnt:04}.csv')
        result = pd.concat(self.dataframes).reset_index()
        result['path'] = result['path'] + '/' + result['index'].astype(str)
        result.drop('index', axis=1).to_csv(output_path, index=False)

        self.current_buffer = 0
        self.dataframes = []
        self.cnt += 1

    def on_predict_batch_end(self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx):
        _, paths, wave_lens = batch
        self.total_duration_sec += sum(w_len / self.resample_rate for w_len in wave_lens)

        mfcc_features, frames_cnt = outputs[0].cpu().numpy(), outputs[1].cpu().numpy()
        for n_frames, features, path in zip(frames_cnt, mfcc_features, paths):
            self.current_buffer += n_frames

            # select only useful frames without padding
            features = features[:n_frames]

            features_df = pd.DataFrame(data=features)
            features_df['path'] = self.extract_name(path)
            self.dataframes.append(features_df)

            if self.current_buffer >= self.buffer_size:
                self.write_df(trainer)

        if batch_idx % 50 == 0:
            logger = self.loggers[pl_module.global_rank]
            logger.debug(f'gpu={pl_module.global_rank:2} batches processed {batch_idx:4}/{self.total_batches} ... {batch_idx / self.total_batches:.4f}')

    def setup(self, trainer: "pl.Trainer", pl_module: "pl.LightningModule", stage: Optional[str] = None) -> None:
        # setup loggers for each gpu
        # logging.basicConfig(filename=logging_file, filemode='a', level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%H:%M:%S %d.%m.%Y')
        handler = logging.FileHandler(f'gpu-{pl_module.global_rank}.log')
        formatter = logging.Formatter(fmt='%(asctime)s - %(message)s', datefmt='%H:%M:%S %d.%m.%Y')
        handler.setFormatter(formatter)

        logger = logging.getLogger(f'{pl_module.global_rank}')
        logger.setLevel(logging.DEBUG)
        logger.addHandler(handler)
        self.loggers[pl_module.global_rank] = logger

    def on_predict_epoch_end(self, trainer, pl_module, outputs):
        if self.dataframes != []:
            self.write_df(trainer)

        logger = self.loggers[pl_module.global_rank]
        total_duration_hours = int(self.total_duration_sec // 3600)
        remaining_seconds = int(self.total_duration_sec % 3600)
        total_duration_mins = int(remaining_seconds // 60)
        total_duration_secs = int(remaining_seconds % 60)
        logger.debug(f'gpu={pl_module.global_rank:2} finished, {total_duration_hours:3}:{total_duration_mins:2}:{total_duration_secs:.3f} or'
                     f' {self.total_duration_sec:.3f} seconds')

def collate_fn(batch):
    M = max([x['wav'].size(-1) for x in batch])
    wavs = []
    paths = []
    for x in batch:
        padded = F.pad(x['wav'], (0, M - x['wav'].size(-1)))
        wavs.append(padded)
        paths.append(x['path'])

    # save lengths of waveforms, will be used to cut the padding from spectrogram
    lengths = [x['wav'].size(-1) for x in batch]
    return torch.stack(wavs, dim=0), paths, lengths

def plot_spectrogram(spec, title=None, ylabel='freq_bin', aspect='auto', xmax=None, lim=None):
    fig, axs = plt.subplots(1, 1)
    axs.set_title(title or 'Spectrogram (db)')
    axs.set_ylabel(ylabel)
    axs.set_xlabel('frame')
    im = axs.imshow(spec, origin='lower', aspect=aspect)
    if xmax:
        axs.set_xlim((0, xmax))
    fig.colorbar(im, ax=axs)
    if lim is not None:
        plt.axvline(x=lim, color='red')
    plt.show(block=False)

# %%
if __name__ == '__main__':
    # %%
    # logging.basicConfig(filename=logging_file, filemode='a', level=logging.DEBUG, format='%(asctime)s - %(message)s', datefmt='%H:%M:%S %d.%m.%Y')
    params = dict(
        resample_rate=16000,
        batch_size=70,
        n_mffcs=13,
        n_mels=40,
        n_fft=640,
        buffer_size=130000,
        df_type='parczech',
        frame_length_ms=20,
        data_type='validated'
    )
    parczech_clean_params = dict(
        recognized_sound_coverage__segments_lb=0.45,
        recognized_sound_coverage__segments_ub=0.93,
        duration__segments_lb=0.5,
    )

    if 'lnet' in os.getcwd():
        df_path = '/lnet/express/work/people/stankov/alignment/Thesis/clean_with_path_large.csv'
        # df = pd.read_csv(df_path, sep='\t')
        # directory where mfccs will be stored
        output_dir = '/lnet/express/work/people/stankov/alignment/mfcc'
        dataset = ParCzechDataset(df_path, resample_rate=params['resample_rate'], clean_params=parczech_clean_params)
    else:
        # under base dir there are tsv file and clips/ folder
        base_dir = '/root/common_voice_data/cv-corpus-7.0-2021-07-21/cs'
        # directory where mfccs will be stored
        output_dir = os.path.join(base_dir, 'mffcs')
        dataset = CommonVoiceDataset(base_dir, params['data_type'], params['resample_rate'])

    # %%
    dataloader = DataLoader(dataset, batch_size=params['batch_size'], shuffle=False, collate_fn=collate_fn, num_workers=os.cpu_count() // 4, pin_memory=True)
    extractor = MFCCExtractorPL(n_mffcs=params['n_mffcs'], n_mels=params['n_mels'], n_fft=params['n_fft'], f_max=params['resample_rate'] // 2,
                                output_dir=output_dir, resample_rate=params['resample_rate'])

    cb = SaveResultsCB(output_dir, params['n_fft'], buffer_size=params['buffer_size'], df_type=params['df_type'], frame_length=params['frame_length_ms'],
                       total_batches=len(dataloader))

    trainer = pl.Trainer(gpus=-1, strategy='ddp', num_sanity_val_steps=0, callbacks=cb, deterministic=True, progress_bar_refresh_rate=0)
    # trainer = pl.Trainer(gpus=1, num_sanity_val_steps=0, callbacks=cb, precision=16, deterministic=True, limit_predict_batches=10)
    trainer.predict(extractor, dataloader)

    ic('done')


"""
Helper function to safely convert an array to a new data type.
"""
from __future__ import absolute_import, print_function, division

import numpy as np

import theano

__docformat__ = "restructuredtext en"

def _asarray(a, dtype, order=None):
    """Convert the input to a Numpy array.

    This function is almost identical to ``numpy.asarray``, but it should be
    used instead of its numpy counterpart when a data type is provided in
    order to perform type conversion if required.
    The reason is that ``numpy.asarray`` may not actually update the array's
    data type to the user-provided type. For more information see ticket
    http://projects.scipy.org/numpy/ticket/870.

    In that case, we check that both dtype have the same string
    description (byte order, basic type, and number of bytes), and
    return a view with the desired dtype.

    This function's name starts with a '_' to indicate that it is meant to be
    used internally. It is imported so as to be available directly through
        theano._asarray
    """
    if str(dtype) == 'floatX':
        dtype = theano.config.floatX
    dtype = np.dtype(dtype)  # Convert into dtype object.
    rval = np.asarray(a, dtype=dtype, order=order)
    # Note that dtype comparison must be done by comparing their `num`
    # attribute. One cannot assume that two identical data types are pointers
    # towards the same object (e.g. under Windows this appears not to be the
    # case).
    if rval.dtype.num != dtype.num:
        # Type mismatch between the data type we asked for, and the one
        # returned by numpy.asarray.
        # If both types have the same string description (byte order, basic
        # type, and number of bytes), then it is safe to return a view.
        if (dtype.str == rval.dtype.str):
            # Silent fix.
            return rval.view(dtype=dtype)
        else:
            # Unexpected mismatch: better know what is going on!
            raise TypeError(
                'numpy.array did not return the data type we '
                'asked for (%s %s #%s), instead it returned type '
                '%s %s #%s: function '
                'theano._asarray may need to be modified to handle this '
                'data type.' %
                (dtype, dtype.str, dtype.num, rval.dtype, rval.str,
                 rval.dtype.num))
    else:
        return rval


import psutil
import time
import sys

# Nav : gzserver, move_base, amcl, robo state pub, rosout, mapsrv
# ObjTrack : gzserver, subscribr, objdetector, objtracker, controller
# Nav2D : stage, navigator, operator, mapper, rviz, joy, controller
cpu_util = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
mem_util = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]

print(sys.argv)

# t = int(sys.argv[1])
# freq = int(sys.argv[2])

# tim = int(sys.argv[3])
# r = int(sys.argv[5])

sleep_time = 0.4
# n_o = tim/sleep_time

count = 0

def is_illixr_proc(proc) -> bool:
    try:
        exe = proc.exe()
    except psutil.AccessDenied:
        exe = ""
    return "main.opt.exe" in exe

def is_running():
    return any(map(is_illixr_proc, psutil.process_iter()))

def get_cpu_mem_nav2d():
    for proc in filter(is_illixr_proc, psutil.process_iter()):
        cpu_util[0] += proc.cpu_percent()

ts_arr = []

while not is_running():
    time.sleep(0.01)

print("Detected process launch")

while is_running():
    get_cpu_mem_nav2d()
    count += 1
    # print once every 10s i.e. 25*0.4s.
    if (count % 25 == 15):
        cpu = [x/count for x in cpu_util]
        mem = [x/count for x in mem_util]
        cms = "###Count: " + str(count) + "Avg CPU: " + str(cpu) + ", Mem: " + str(mem)
        print(cms)
        ts_arr.append(cms)
    time.sleep(sleep_time)

print("ADDED all observations", count)
cpu_txt = ""
mem_txt = ""

for i in range(len(cpu_util)):
    cpu_util[i] /= count
    cpu_txt += str(cpu_util[i]) + ", "
    mem_util[i] /= count
    mem_txt += str(mem_util[i]) + ", "

# fname = "%s_cpu_mem.txt"% (sys.argv[4])
f = sys.stdout
# f = open(fname, "w")

for i in sys.argv:
    f.write(i + ", ")

for j in ts_arr:
    f.write(j)
    f.write("\n")

f.write(str(count) + ", ")
f.write(cpu_txt)
f.write(mem_txt)
f.write("\n")
print(sys.argv)
print(cpu_util)
print(mem_util)
'''
if sys.argv[8] == 'yes':
    with open('cpu_time_series_%s_%s%s.txt'% (sys.argv[3], sys.argv[4], sys.argv[9]), 'a') as fw:
        print "Writing to file for ", sys.argv[3], sys.argv[4], sys.argv[9]
        for i in move_base_cpu_arr:
            fw.write(str(i) + ', ')
        fw.write('\n')
        for i in amcl_cpu_arr:
            fw.write(str(i) + ', ')
        fw.write('\n')
'''


import argparse
import os
import torch
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
import albumentations
from albumentations.pytorch import ToTensorV2
from PIL import Image
import numpy as np

torch.backends.cudnn.benchmark = True

# Dicriminator model definition

class Discriminator(nn.Module):
    def __init__(self, in_channels=3) -> None:
        super().__init__()
        self.convlayers = nn.Sequential(
            nn.Conv2d(in_channels=in_channels*2, out_channels=64,
                      kernel_size=4, stride=2, padding=1, padding_mode="reflect",),
            nn.LeakyReLU(0.2),

            nn.Conv2d(in_channels=64, out_channels=128, kernel_size=4,
                      stride=2, bias=False, padding_mode="reflect",),
            nn.BatchNorm2d(128),
            nn.LeakyReLU(0.2),

            nn.Conv2d(in_channels=128, out_channels=256, kernel_size=4,
                      stride=2, bias=False, padding_mode="reflect",),
            nn.BatchNorm2d(256),
            nn.LeakyReLU(0.2),

            nn.Conv2d(in_channels=256, out_channels=512, kernel_size=4,
                      stride=1, bias=False, padding_mode="reflect",),
            nn.BatchNorm2d(512),
            nn.LeakyReLU(0.2),

            nn.Conv2d(in_channels=512, out_channels=1, kernel_size=4,
                      stride=1, padding=1, padding_mode="reflect",),
            nn.LeakyReLU(0.2),
        )

    def forward(self, x, y) -> None:
        out = torch.cat([x, y], dim=1)
        out = self.convlayers(out)
        return out

# generator class definition

class Generator(nn.Module):
    # U-NET encoder section
    def encoder(self, in_channels, out_channel, is_relu=False, need_batch_norm=True):
        x = nn.Sequential(
            nn.Conv2d(in_channels, out_channel, 4, 2, 1,
                      bias=False, padding_mode="reflect"),
            nn.BatchNorm2d(out_channel) if need_batch_norm else None,
            nn.ReLU() if is_relu else nn.LeakyReLU(),
        )
        return x

    # # U-NET decoder section
    def decoder(self, in_channels, out_channel, is_relu=False, need_batch_norm=True, need_dropout=True, ):
        return nn.Sequential(
            nn.ConvTranspose2d(in_channels, out_channel, 4, 2, 1, bias=False),
            nn.BatchNorm2d(out_channel),
            nn.ReLU() if is_relu else nn.LeakyReLU(),
        ) if not need_dropout else nn.Sequential(
            nn.ConvTranspose2d(in_channels, out_channel, 4, 2, 1, bias=False),
            nn.BatchNorm2d(out_channel),
            nn.ReLU() if is_relu else nn.LeakyReLU(),
            nn.Dropout(0.5),
        )

    def __init__(self, in_channels=3, features=64):
        super().__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels, features, kernel_size=4,
                      stride=2, padding=1, padding_mode="reflect"),
            nn.LeakyReLU(0.2),
        )

        self.layer2 = self.encoder(
            in_channels=features, out_channel=features * 2)
        self.layer3 = self.encoder(features * 2, features * 4)
        self.layer4 = self.encoder(features * 4, features * 8)
        self.layer5 = self.encoder(features * 8, features * 8)
        self.layer6 = self.encoder(features * 8, features * 8)
        self.layer7 = self.encoder(features * 8, features * 8)
        # self.latent = self.encoder(
        #     features * 8, features * 8, need_batch_norm=False)
        self.latent = nn.Sequential(
            nn.Conv2d(features * 8, features * 8, kernel_size=4,
                      stride=2, padding=1),
            nn.ReLU(),
        )

        self.layer8 = self.decoder(features * 8, features * 8, is_relu=True)
        self.layer9 = self.decoder(
            features * 8 * 2, features * 8, is_relu=True)
        self.layer10 = self.decoder(
            features * 8 * 2, features * 8, is_relu=True)
        self.layer11 = self.decoder(
            features * 8 * 2, features * 8, is_relu=True, need_dropout=False)
        self.layer12 = self.decoder(
            features * 8 * 2, features * 4, is_relu=True, need_dropout=False)
        self.layer13 = self.decoder(
            features * 4 * 2, features * 2, is_relu=True, need_dropout=False)
        self.layer14 = self.decoder(
            features * 2 * 2, features, is_relu=True, need_dropout=False)

        self.layer15 = nn.Sequential(
            nn.ConvTranspose2d(features*2, in_channels, 4, 2, 1, bias=False),
            nn.Tanh(),
        )

    def forward(self, x):
        layer1 = self.layer1(x)
        layer2 = self.layer2(layer1)
        layer3 = self.layer3(layer2)
        layer4 = self.layer4(layer3)
        layer5 = self.layer5(layer4)
        layer6 = self.layer6(layer5)
        layer7 = self.layer7(layer6)

        latent = self.latent(layer7)
        layer8 = self.layer8(latent)
        layer9 = self.layer9(torch.cat([layer8, layer7], 1))
        layer10 = self.layer10(torch.cat([layer9, layer6], 1))
        layer11 = self.layer11(torch.cat([layer10, layer5], 1))
        layer12 = self.layer12(torch.cat([layer11, layer4], 1))
        layer13 = self.layer13(torch.cat([layer12, layer3], 1))
        layer14 = self.layer14(torch.cat([layer13, layer2], 1))

        return self.layer15(torch.cat([layer14, layer1], 1))

# global class for constants and hyperparameters

class config:
    DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu"
    TRAIN_DIR = "data/daynight/train"
    VAL_DIR = "data/daynight/val"
    LEARNING_RATE = 0.0002
    BATCH_SIZE = 16
    NUM_WORKERS = 2
    LAMBDA = 100
    NUM_EPOCHS = 50
    LOAD_MODEL = False
    SAVE_MODEL = True
    FLIP_TRAIN = False
    CHECKPOINT_DISC = "disc.pth.tar"
    CHECKPOINT_GEN = "gen.pth.tar"
    MODEL_DEFAULT = 'maps'
    MODEL_ANIME = 'anime'
    MODEL_DAYNIGHT = 'daynight'
    MODE = 'train'

class DataTransformation:
    resize = albumentations.Compose(
        [albumentations.Resize(width=256, height=256), ], additional_targets={"image0": "image"},
    )
    transform = albumentations.Compose(
        [
            albumentations.HorizontalFlip(p=0.5),
            albumentations.ColorJitter(p=0.2),
            albumentations.Normalize(mean=[0.5, 0.5, 0.5], std=[
                0.5, 0.5, 0.5], max_pixel_value=255.0,),
            ToTensorV2(),
        ]
    )
    tranform_mask = albumentations.Compose(
        [
            albumentations.Normalize(mean=[0.5, 0.5, 0.5], std=[
                0.5, 0.5, 0.5], max_pixel_value=255.0,),
            ToTensorV2(),
        ]
    )

def save_checkpoint(model, optimizer, filename="my_checkpoint.pth.tar"):
    print("=> Saving checkpoint")
    torch.save({
        "state_dict": model.state_dict(),
        "optimizer": optimizer.state_dict(),
    }, filename)

def load_checkpoint(checkpoint_file, model, optimizer, lr):
    print("=> Loading checkpoint")
    checkpoint = torch.load(checkpoint_file, map_location=config.DEVICE)
    model.load_state_dict(checkpoint["state_dict"])
    optimizer.load_state_dict(checkpoint["optimizer"])
    for param_group in optimizer.param_groups:
        param_group["lr"] = lr

'''
  This class extends the pytorch Dataset class
'''

class SplitData(Dataset):
    def __init__(self, root_dir) -> None:
        self.root_dir = root_dir
        self.list_files = os.listdir(self.root_dir)

    def __len__(self) -> None:
        return len(self.list_files)

    def __getitem__(self, index) -> None:
        img_file = self.list_files[index]
        img_path = os.path.join(self.root_dir, img_file)
        image = np.array(Image.open(img_path))
        # get the image shape
        image_dim = int(image.shape[1]/2)
        # print('image shape: ', image_dim)
        flip = config.FLIP_TRAIN
        # print('flip: ', flip)
        if flip:
            target_image = image[:, :image_dim, :]
            input_image = image[:, image_dim:, :]
        else:
            input_image = image[:, :image_dim, :]
            target_image = image[:, image_dim:, :]

        augmentations = DataTransformation.resize(
            image=input_image, image0=target_image)
        input_image = augmentations["image"]
        target_image = augmentations["image0"]

        input_image = DataTransformation.transform(image=input_image)["image"]
        target_image = DataTransformation.tranform_mask(image=target_image)[
            "image"]

        return input_image, target_image

def get_l1_loss(weights) -> torch.Tensor:
    return torch.abs(weights).sum()

def get_l2_loss(weights) -> torch.Tensor:
    return torch.square(weights).sum()

def train_fn(
    disc, gen, loader, opt_disc, opt_gen, l1_loss, bce, gen_scaler, disc_scaler,
) -> None:
    loop = tqdm(loader, leave=True)

    for idx, (x, y) in enumerate(loop):
        x = x.to(config.DEVICE)
        y = y.to(config.DEVICE)

        # Train Discriminator
        with torch.cuda.amp.autocast():
            y_fake = gen(x)
            Disc_real = disc(x, y)
            Disc_real_loss = bce(Disc_real, torch.ones_like(Disc_real))
            Disc_fake = disc(x, y_fake.detach())
            Disc_fake_loss = bce(Disc_fake, torch.zeros_like(Disc_fake))
            Disc_loss = (Disc_real_loss + Disc_fake_loss) / 2

        disc.zero_grad()
        disc_scaler.scale(Disc_loss).backward()
        disc_scaler.step(opt_disc)
        disc_scaler.update()

        # Train generator
        with torch.cuda.amp.autocast():
            Disc_fake = disc(x, y_fake)
            Gen_fake_loss = bce(Disc_fake, torch.ones_like(Disc_fake))
            l1 = l1_loss(y_fake, y) * config.LAMBDA
            params = []
            for param in disc.parameters():
                params.append(param.view(-1))
            # l1 = config.LAMBDA * get_l1_loss(torch.cat(params))
            l2 = config.LAMBDA * get_l2_loss(torch.cat(params))
            Gen_loss = Gen_fake_loss + l1 + l2

        opt_gen.zero_grad()
        gen_scaler.scale(Gen_loss).backward()
        gen_scaler.step(opt_gen)
        gen_scaler.update()

        if idx % 10 == 0:
            loop.set_postfix(
                Disc_real=torch.sigmoid(Disc_real).mean().item(),
                Disc_fake=torch.sigmoid(Disc_fake).mean().item(),
            )

# helper functions
def _getTrainDirectoryPath(modelname):
    return 'data/'+modelname+'/train' if modelname != None or modelname != '' else 'data/maps/train'

def _getValDirectoryPath(modelname):
    return 'data/'+modelname+'/val' if modelname != None or modelname != '' else 'data/maps/val'

def _getDiscCheckpointPath(modelname):
    return modelname+'_'+config.CHECKPOINT_DISC if modelname != None or modelname != '' else 'maps_'+config.CHECKPOINT_DISC

def _getGenCheckpointPath(modelname):
    return modelname+'_'+config.CHECKPOINT_GEN if modelname != None or modelname != '' else 'maps_'+config.CHECKPOINT_GEN

def main(args) -> None:
    # get data from the command line arguments
    config.LOAD_MODEL = True if str(args.loadmodel).lower() == 'true' else False
    config.FLIP_TRAIN = True if str(args.flip).lower() == 'true' else False
    config.NUM_EPOCHS = int(
        args.epochs) if args.epochs != None else config.NUM_EPOCHS
    config.MODE = args.mode if args.mode != None else config.MODE

    disc = Discriminator(in_channels=3).to(config.DEVICE)
    gen = Generator(in_channels=3, features=64).to(config.DEVICE)
    opt_disc = optim.Adam(
        disc.parameters(), lr=config.LEARNING_RATE, betas=(0.5, 0.999),)
    opt_gen = optim.Adam(
        gen.parameters(), lr=config.LEARNING_RATE, betas=(0.5, 0.999))
    BCE = nn.BCEWithLogitsLoss()
    L1_LOSS = nn.L1Loss()

    print('saved gen checkpoint path: ', _getGenCheckpointPath(args.modelname))
    print('saved disc checkpoint path: ',
          _getDiscCheckpointPath(args.modelname))
    print('Load model value: ', config.LOAD_MODEL, type(config.LOAD_MODEL))
    if config.LOAD_MODEL:
        load_checkpoint(
            _getGenCheckpointPath(
                args.modelname), gen, opt_gen, config.LEARNING_RATE,
        )
        load_checkpoint(
            _getDiscCheckpointPath(
                args.modelname), disc, opt_disc, config.LEARNING_RATE,
        )

    train_dataset = SplitData(root_dir=_getTrainDirectoryPath(args.modelname))
    train_loader = DataLoader(
        train_dataset,
        batch_size=config.BATCH_SIZE,
        shuffle=True,
        num_workers=config.NUM_WORKERS,
    )
    gen_scaler = torch.cuda.amp.GradScaler()
    disc_scaler = torch.cuda.amp.GradScaler()
    val_dataset = SplitData(root_dir=_getValDirectoryPath(args.modelname))
    val_loader = DataLoader(val_dataset, batch_size=1, shuffle=False)
    val_itr = iter(val_loader)

    for epoch in range(1, config.NUM_EPOCHS+1):
        if(config.MODE == 'train'):
            print('Epoch: {}/{}'.format(epoch, config.NUM_EPOCHS))
            train_fn(
                disc, gen, train_loader, opt_disc, opt_gen, L1_LOSS, BCE, gen_scaler, disc_scaler,
            )

            if config.SAVE_MODEL and epoch % 5 == 0:
                save_checkpoint(
                    gen, opt_gen, filename=_getGenCheckpointPath(args.modelname))
                save_checkpoint(
                    disc, opt_disc, filename=_getDiscCheckpointPath(args.modelname))
        try:
            x, y = next(val_itr)
            # get_test_samples(gen, x, y, epoch, folder="evaluation")
            x, y = x.to(config.DEVICE), y.to(config.DEVICE)
            folder = "evaluation"
            gen.eval()
            with torch.no_grad():
                y_fake = gen(x)
                y_fake = y_fake * 0.5 + 0.5
                save_image(y_fake, folder + f"/y_gen_{epoch}.png")
                save_image(x * 0.5 + 0.5, folder + f"/input_{epoch}.png")
                save_image(y * 0.5 + 0.5, folder + f"/label_{epoch}.png")
            gen.train()
        except:
            pass

if __name__ == "__main__":
    # setting up the argument parser to parse the command line arguments
    argparser = argparse.ArgumentParser()
    argparser.add_argument("--flip", default='false',
                           help="learn the left side of the image")
    argparser.add_argument(
        "--modelname", default=config.MODEL_DEFAULT, help="which model to load")
    argparser.add_argument("--mode", default='test',
                           help='start in train or test mode')
    argparser.add_argument("--epochs", default=50,
                           help="number of epochs to train")
    argparser.add_argument("--loadmodel", default='false', help='load model or not')
    args = argparser.parse_args()

    # printing the passed args to debug
    print(args)

    # run the main function with all the passed command line arguments
    main(args)


from ..config import np, Vector, DataName, MetaboliteConfig, ParameterName, LegendConfig

from ..metabolic_network_contents.metabolite import Metabolite
from ..metabolic_network_contents.reaction import Reaction

metabolite_width = MetaboliteConfig.width

class NormalLegendConfig(object):
    metabolite_content_dict = {
        'G6P': Metabolite('G6P'),
        'LAC': Metabolite('LAC').set_mid_data_state(True),
        'MAL': Metabolite('MAL').set_mid_data_state(True).set_mixed_mid_data_state(True),
        'GLU': Metabolite('GLU').set_biomass_flux_state(True),
        'GLY': Metabolite('GLY').set_input_state(True),
        'GLC': Metabolite('GLC').set_input_state(True).set_c13_labeling_state(True),
    }
    reaction_content_dict = {
        'fluxes': (Reaction('unidirectional'), Reaction('bidirectional', reversible=True)),
        'boundary_flux': Reaction('boundary_flux').set_boundary_flux(True),
    }
    text_content_dict = {
        'G6P': 'Normal metabolites',
        'LAC': 'Metabolites with mass spec data',
        'MAL': 'Metabolites with mixed mass spec\ndata (mitochondria and cytosol)',
        'GLU': 'Metabolites with biomass flux',
        'GLY': 'Input or output metabolites\nwith fixed MID',
        'GLC': 'Input metabolites with $\mathregular{^{13}}$C\nlabelled',
        'fluxes': 'Normal fluxes (unidirectional\nor reversible)',
        'boundary_flux': 'Boundary fluxes with fixed value',
    }

class SmallerSizeLegendConfig(object):
    metabolite_content_dict = {
        'GLN': Metabolite('GLN').set_input_state(True),
        'OAC': Metabolite('OAC'),
        'MAL': Metabolite('MAL').set_data_sensitivity_state(DataName.raw_model_raw_data),
        '3PG': Metabolite('3PG').set_data_sensitivity_state(DataName.medium_data),
        'GLC': Metabolite('GLC').set_data_sensitivity_state(DataName.few_data),
    }
    reaction_content_dict = {}
    text_content_dict = {
        'GLN': 'Input or output metabolites\nwith fixed MID',
        'OAC': 'With MID data in all data set',
        'MAL': 'With MID data in all + experimental\ndata set',
        '3PG': 'With MID data in all + experimental\n+ medium data set',
        'GLC': 'With MID data in all + experimental\n+ medium + small data set',
    }

class RemovePathwayLegendConfig(object):
    metabolite_content_dict = {
        'LAC': Metabolite('LAC').set_mid_data_state(True),
        'R5P': Metabolite('R5P').set_data_sensitivity_state(DataName.data_without_ppp),
        'MAL': Metabolite('MAL').set_data_sensitivity_state(DataName.data_without_tca),
        'GLU': Metabolite('GLU').set_data_sensitivity_state(DataName.data_without_aa),
        # 'CIT': Metabolite('CIT').set_data_sensitivity_state(DataName.medium_data_without_combination),
    }
    reaction_content_dict = {}
    text_content_dict = {
        'LAC': 'Experimental data set',
        'R5P': 'Removed MID data of PPP metabolites',
        'MAL': 'Removed MID data of TCA metabolites',
        'GLU': 'Removed MID data of AA metabolites',
        # 'CIT': 'Added compartmental MID',
    }

class ConstantFluxLegendConfig(object):
    metabolite_content_dict = {}
    reaction_content_dict = {
        'fluxes': (Reaction('unidirectional'), Reaction('bidirectional', reversible=True)),
        'boundary_fluxes': Reaction('boundary_flux').set_boundary_flux(True),
    }
    text_content_dict = {
        'fluxes': 'Normal fluxes (unidirectional\nor reversible)',
        'boundary_fluxes': 'Preset fixed boundary fluxes',
    }

def legend_layout_generator(mode=ParameterName.normal):
    if mode == ParameterName.normal or mode == ParameterName.horizontal:
        legend_config = NormalLegendConfig
    elif mode == DataName.smaller_data_size:
        legend_config = SmallerSizeLegendConfig
    elif mode == DataName.data_without_pathway:
        legend_config = RemovePathwayLegendConfig
    elif mode == DataName.different_constant_flux:
        legend_config = ConstantFluxLegendConfig
    else:
        raise ValueError()
    metabolite_content_dict, reaction_content_dict, text_content_dict = \
        legend_config.metabolite_content_dict, legend_config.reaction_content_dict, legend_config.text_content_dict

    total_item_list = []
    for metabolite_key, metabolite_content in metabolite_content_dict.items():
        text_content = text_content_dict[metabolite_key]
        total_item_list.append((ParameterName.metabolite, metabolite_key, metabolite_content, text_content))
    for reaction_key, reaction_content in reaction_content_dict.items():
        text_content = text_content_dict[reaction_key]
        total_item_list.append((ParameterName.reaction, reaction_key, reaction_content, text_content))

    total_item_num = len(total_item_list)
    each_row_height = LegendConfig.legend_each_row_height
    if mode == ParameterName.horizontal:
        total_row_num = 2
        total_col_num = np.ceil(total_item_num / 2)
        total_width = LegendConfig.legend_horizontal_width
    else:
        total_row_num = total_item_num
        total_col_num = 1
        total_width = LegendConfig.legend_width
    layout_index_list = [
        (item_index % total_row_num, item_index // total_row_num) for item_index in range(total_item_num)]
    total_height = total_row_num * each_row_height
    each_col_width = total_width / total_col_num

    # patch_center_x_axis = 0.15 * total_width
    # text_left_x_axis = 0.3 * total_width
    # text_width = total_width - text_left_x_axis
    multiple_reaction_up_down_distance = 0.005
    flux_width = metabolite_width
    base_patch_center_x_axis = 0.15 * each_col_width
    base_text_left_x_axis = 0.3 * each_col_width
    text_width = each_col_width - base_text_left_x_axis

    patch_raw_obj_dict = {}
    text_param_dict = {}
    for (row_index, col_index), (item_type, item_key, item_content, text_content) \
            in zip(layout_index_list, total_item_list):
        patch_center_x_axis = col_index * each_col_width + base_patch_center_x_axis
        text_left_x_axis = col_index * each_col_width + base_text_left_x_axis
        flux_left_x_value, flux_right_x_value = (
            patch_center_x_axis - flux_width / 2, patch_center_x_axis + flux_width / 2)
        irreversible_flux_right_x_value = 0.03 * flux_left_x_value + 0.97 * flux_right_x_value

        current_row_center_y_value = (total_row_num - row_index - 0.5) * each_row_height
        text_param_dict[item_key] = {
            ParameterName.center: Vector(text_left_x_axis + text_width / 2, current_row_center_y_value),
            ParameterName.string: text_content,
            ParameterName.width: text_width,
            ParameterName.height: each_row_height,
        }
        if item_type == ParameterName.metabolite:
            item_content.set_center(Vector(patch_center_x_axis, current_row_center_y_value))
            patch_raw_obj_dict[item_key] = item_content
        elif item_type == ParameterName.reaction:
            if isinstance(item_content, tuple):
                reaction_num = len(item_content)
                reaction_subrow_height = (each_row_height - 2 * multiple_reaction_up_down_distance) / reaction_num
                for reaction_subindex, reaction_obj in enumerate(item_content):
                    current_subrow_y_value = (
                            current_row_center_y_value + each_row_height / 2 - multiple_reaction_up_down_distance -
                            (reaction_subindex + 0.5) * reaction_subrow_height)
                    if reaction_obj.reversible:
                        current_flux_right_x_value = flux_right_x_value
                    else:
                        current_flux_right_x_value = irreversible_flux_right_x_value
                    reaction_obj.extend_reaction_start_end_list([
                        (
                            ParameterName.normal,
                            Vector(current_flux_right_x_value, current_subrow_y_value),
                            Vector(flux_left_x_value, current_subrow_y_value),
                            {}
                        )
                    ])
                    patch_raw_obj_dict[reaction_obj.reaction_name] = reaction_obj
            elif isinstance(item_content, Reaction):
                if item_content.reversible:
                    current_flux_right_x_value = flux_right_x_value
                else:
                    current_flux_right_x_value = irreversible_flux_right_x_value
                item_content.extend_reaction_start_end_list([
                    (
                        ParameterName.normal,
                        Vector(current_flux_right_x_value, current_row_center_y_value),
                        Vector(flux_left_x_value, current_row_center_y_value),
                        {}
                    )
                ])
                patch_raw_obj_dict[item_key] = item_content
            else:
                raise ValueError()
        row_index += 1
    # row_index = 0
    # for reaction_key, reaction_content in reaction_content_dict.items():
    #     current_row_center_y_value = (total_row_num - row_index - 0.5) * each_row_height
    #     text_content = text_content_dict[reaction_key]
    #     text_param_dict[reaction_key] = {
    #         ParameterName.center: Vector(text_left_x_axis + text_width / 2, current_row_center_y_value),
    #         ParameterName.string: text_content,
    #         ParameterName.width: text_width,
    #         ParameterName.height: each_row_height,
    #     }
    #     if isinstance(reaction_content, tuple):
    #         reaction_num = len(reaction_content)
    #         reaction_subrow_height = (each_row_height - 2 * multiple_reaction_up_down_distance) / reaction_num
    #         for reaction_subindex, reaction_obj in enumerate(reaction_content):
    #             current_subrow_y_value = (
    #                     current_row_center_y_value + each_row_height / 2 - multiple_reaction_up_down_distance -
    #                     (reaction_subindex + 0.5) * reaction_subrow_height)
    #             if reaction_obj.reversible:
    #                 current_flux_right_x_value = flux_right_x_value
    #             else:
    #                 current_flux_right_x_value = irreversible_flux_right_x_value
    #             reaction_obj.extend_reaction_start_end_list([
    #                 (
    #                     ParameterName.normal,
    #                     Vector(current_flux_right_x_value, current_subrow_y_value),
    #                     Vector(flux_left_x_value, current_subrow_y_value),
    #                     {}
    #                 )
    #             ])
    #             patch_raw_obj_dict[reaction_obj.reaction_name] = reaction_obj
    #     elif isinstance(reaction_content, Reaction):
    #         if reaction_content.reversible:
    #             current_flux_right_x_value = flux_right_x_value
    #         else:
    #             current_flux_right_x_value = irreversible_flux_right_x_value
    #         reaction_content.extend_reaction_start_end_list([
    #             (
    #                 ParameterName.normal,
    #                 Vector(current_flux_right_x_value, current_row_center_y_value),
    #                 Vector(flux_left_x_value, current_row_center_y_value),
    #                 {}
    #             )
    #         ])
    #         patch_raw_obj_dict[reaction_key] = reaction_content
    #     else:
    #         raise ValueError()
    #     row_index += 1
    return patch_raw_obj_dict, text_param_dict, total_width, total_height


import sys
import time
from threading import Thread

class ProgressThread(Thread):
    def __init__(self):
        super(ProgressThread, self).__init__()
        self.is_stop = False
        self.cursor_index = 0
        self.cursor_str = '|/-\\'
        self.now = None
        self.info = ""

    def set_progress_info(self, info):
        self.info = info

    def get_progress_text(self):
        cursor = self.cursor_str[self.cursor_index]
        self.cursor_index = self.cursor_index + 1
        if self.cursor_index == len(self.cursor_str):
            self.cursor_index = 0
        time_second = str(int(time.time() - self.now))
        progress_text = self.info + " " + time_second + "s " + cursor
        return progress_text

    def stop_progress(self):
        self.is_stop = True
        time.sleep(0.6)

    def run(self):
        self.now = time.time()
        while True:
            if not self.is_stop:
                progress_text = self.get_progress_text()
                sys.stdout.write(progress_text)
                sys.stdout.flush()
                time.sleep(0.4)
                sys.stdout.write('\r')
            else:
                return

class Progress:

    def __init__(self):
        self.current_thread = None
        pass

    def start_progress(self, progress_info):
        if self.current_thread is not None:
            self.current_thread.stop_progress()
        self.current_thread = ProgressThread()
        self.current_thread.daemon = True
        self.current_thread.set_progress_info(progress_info)
        self.current_thread.start()

    def stop_progress(self):
        if self.current_thread is not None:
            self.current_thread.stop_progress()
        self.current_thread = None

    def show_progress(self, info):
        self.current_thread.set_progress_info(info)

if __name__ == '__main__':
    progress = Progress()
    progress.start_progress("开始上传文件")
    for i in range(10):
        time.sleep(0.5)
        progress.show_progress("文件上传中")
    progress.stop_progress()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# filename: const.py
# modified: 2019-03-30
"""
常数表
"""

__all__ = [

    "PROJECT_DIR",
    "PACKAGE_DIR",
    "CACHE_DIR",
    "CONFIG_DIR",
    "STATIC_DIR",
    "LOG_DIR",
    "INPUT_DIR",
    "OUTPUT_DIR",
    "OUTPUT_SRC_DIR",

    "STYLE_CSS",

    "CLIENT_DEFAULT_TIMEOUT",
    "CLIENT_USER_AGENT",

    "TIETUKU_TOKEN",
    "TIETUKU_AID",
    "TIETUKU_CACHE_EXPIRED",
    "TIETUKU_LINKS_CACHE_JSON",

    "ELIMAGE_LINKS_CACHE_JSON",

    "SMMS_LINKS_CACHE_JSON",

    ]

import os

def __mkdir(path):
    if not os.path.exists(path):
        os.mkdir(path)

__Base_dir = os.path.dirname(__file__)
__absP = lambda *path: os.path.abspath(os.path.join(__Base_dir, *path))

PROJECT_DIR    = __absP("../../")
PACKAGE_DIR    = __absP("../")
CACHE_DIR      = __absP("../cache/")
CONFIG_DIR     = __absP("../config/")
STATIC_DIR     = __absP("../static/")
LOG_DIR        = __absP("../../logs/")
INPUT_DIR      = __absP("../../input/")
OUTPUT_DIR     = __absP("../../output/")
OUTPUT_SRC_DIR = __absP("../../output/src/")

STYLE_CSS      = __absP(CONFIG_DIR, "style.css")

__mkdir(LOG_DIR)
__mkdir(CACHE_DIR)
__mkdir(INPUT_DIR)
__mkdir(OUTPUT_DIR)
__mkdir(OUTPUT_SRC_DIR)

CLIENT_DEFAULT_TIMEOUT   = 15
CLIENT_USER_AGENT        = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.122 Safari/537.36"

TIETUKU_TOKEN            = "305b1ec69ae2dfd54076a3f648931b9ac51a414b:EkuHTpQkLlK07Ocf69_VxR3anu8=:eyJkZWFkbGluZSI6MTUzMzI4ODMyNCwiYWN0aW9uIjoiZ2V0IiwidWlkIjoiNjU2NTU0IiwiYWlkIjoiMTQ3ODU1NSIsImZyb20iOiJmaWxlIn0="
TIETUKU_AID              = 1478555
TIETUKU_CACHE_EXPIRED    = 12*60*60  # 12 h
TIETUKU_LINKS_CACHE_JSON = "tietuku.links.json"

ELIMAGE_LINKS_CACHE_JSON = "elimage.links.json"

SMMS_LINKS_CACHE_JSON    = "sm.ms.links.json"


import socket
import hashlib
"""
data, addr = cli_socket.recvfrom(4096)
print("Server Says")
print(str(data))
cli_socket.close()
"""
cli_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
msg = "Hello"
cli_socket.sendto(msg.encode("utf-8"), ('127.0.0.1', 12345))
data,addr = cli_socket.recvfrom(1024)
numero_conexiones = data.decode('utf-8')
data,addr = cli_socket.recvfrom(1024)
# Se captura el nombre del file desde el servidor
file_name = data
titulo = file_name.decode("utf-8")
f = open("ArchivosRecibidos/"+titulo, 'wb')
# recepción del hash
data,addr = cli_socket.recvfrom(1024)
hexadecimal_hash = data.decode()

def hasheame_esta():
    """"This function returns the SHA-1 hash
       of the file passed into it"""

    # make a hash object
    h = hashlib.sha1()

    # open file for reading in binary mode
    with open("files/"+titulo, 'rb') as file:
        # loop till the end of the file
        chunk = 0
        while chunk != b'':
            # read only 1024 bytes at a time
            chunk = file.read(1024)
            h.update(chunk)

    # return the hex representation of digest
    return h.hexdigest()

print("Nicole y brayan: el número de sockets que deben crear en esta iteración es " + numero_conexiones)
print(file_name)
print("se recibe el archivo con un hash esperado de " + hexadecimal_hash)
print(numero_conexiones)

data,addr = cli_socket.recvfrom(1024)
corte = titulo + "kill"
termine = corte.encode('utf-8')
print("se recibe el archivo" + titulo)
while data != termine:
    f.write(data)
    data,addr = cli_socket.recvfrom(1024)
print("archivo recibido")
f.close()


import os
import pathlib
import argparse
import uuid
import logging
import subprocess

from nuvoloso.dependencies.install_packages import InstallPackages
from nuvoloso.dependencies.kops_cluster import KopsCluster
from nuvoloso.dependencies.kubectl_helper import KubectlHelper
from nuvoloso.api.nuvo_management import NuvoManagement

DEFAULT_AMI = '099720109477/ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-20190212.1'
DEFAULT_CENTRALD_CLUSTER_NAME = "nuvotestfiomulti.k8s.local"
NUVO_CLUSTER_NAME = "./nuvo_cluster_name.txt"

class CreateAppCluster:
    def __init__(self, args):
        self.args = args
        if not args.create_only:
            self.nuvo_mgmt = NuvoManagement(args)
        self.kubectl_helper = KubectlHelper(args)

    def install_dependencies(self):
        InstallPackages.apt_get_update()
        InstallPackages.install_kops()
        InstallPackages.install_kubectl()
        InstallPackages.install_awscli()
        InstallPackages.configure_aws(self.args)
        InstallPackages.generate_sshkeypair()
        return

    def set_protection_domains(self, csp_domain_id, nuvo_cluster_name):
        accounts = self.nuvo_mgmt.get_all_accounts()
        admin_account = self.args.account_name
        for account in accounts:
            # Creating spa to authorize accounts, so it can set protection domains
            self.nuvo_mgmt.do_service_plan_allocation(nuvo_cluster_name, account['name'])
            self.nuvo_mgmt.switch_accounts(account['name'])
            protection_domain_id = self.nuvo_mgmt.create_protection_domain()
            self.nuvo_mgmt.set_protection_domain(protection_domain_id, csp_domain_id)
            self.nuvo_mgmt.switch_accounts(admin_account)

    def create_application_cluster(self):
        self.install_dependencies()
        KopsCluster.create_kops_app_cluster(self.args)
        if self.args.create_only:
            return

        try:
            csp_domain_id = self.nuvo_mgmt.create_csp_domain()
            nuvo_cluster_name = self.nuvo_mgmt.deploy_clusterd(csp_domain_id)
            logging.info("Nuvo cluster created : %s", nuvo_cluster_name)
            self.set_protection_domains(csp_domain_id, nuvo_cluster_name)
            snapshot_catalog_pd = self.nuvo_mgmt.create_protection_domain()
            self.nuvo_mgmt.set_snapshot_catalog_policy(snapshot_catalog_pd, csp_domain_id)
        except subprocess.CalledProcessError as err:
            if err.output: logging.info(err.output)
            raise
        return nuvo_cluster_name

def main():
    """main"""

    default_cluster_name = DEFAULT_CENTRALD_CLUSTER_NAME
    default_aws_access_key = default_aws_secret_access_key = default_kops_state_store = None

    if os.environ.get('KOPS_CLUSTER_NAME'):
        default_cluster_name = os.environ.get('KOPS_CLUSTER_NAME')

    if os.environ.get('KOPS_STATE_STORE') is None:
        default_kops_state_store = os.environ.get('KOPS_STATE_STORE')

    if os.environ.get('AWS_ACCESS_KEY_ID'):
        default_aws_access_key = os.environ.get('AWS_ACCESS_KEY_ID')

    if os.environ.get('AWS_SECRET_ACCESS_KEY') is None:
        default_aws_secret_access_key = os.environ.get('AWS_SECRET_ACCESS_KEY')

    parser = argparse.ArgumentParser(description="Deploys a kops cluster with "
                                     "Nuvo data plane and runs fio against all volumes")
    parser.add_argument(
        '--kops_cluster_name', help='name of kops cluster for Nuvo Data Plane[default: ' +
        default_cluster_name + ']',
        default=default_cluster_name)
    parser.add_argument(
        '--nodes', help='number of nodes in the cluster [default=3]', type=int, default=3,
        choices=range(1, 101))
    parser.add_argument(
        '--kops_state_store', help='state store for cluster',
        default=default_kops_state_store)
    parser.add_argument(
        '--aws_access_key', help='aws AccessKey', default=default_aws_access_key)
    parser.add_argument(
        '--aws_secret_access_key', help='aws SecretAccessKey',
        default=default_aws_secret_access_key)
    parser.add_argument(
        '--region', help='aws region', default=None)
    parser.add_argument(
        '--k8s_master_zone', help='aws zone for master node',
        default=None)
    parser.add_argument(
        '--k8s_nodes_zone', help='aws zone for other nodes ',
        default=None)
    parser.add_argument(
        '--master_size', help='ec2 instance type for master node ', default=None)
    parser.add_argument(
        '--node_size', help='ec2 instance type for other nodes ', default=None)
    parser.add_argument(
        '--nuvo_kontroller_hostname', help='Hostname of https svc of Nuvo Kontroller')
    parser.add_argument(
        '--log_dirpath', help='log dir to hold test and nuvo logs', default=None)
    parser.add_argument(
        '--kubernetes_version', help='version of kubernetes to deploy', default='1.14.8')
    parser.add_argument(
        '--image', help='AMI Image for all instances', default=DEFAULT_AMI)
    parser.add_argument(
        '--node_volume_size', help='volume size for slave nodes of k8s cluster', type=int,
        default=10)
    parser.add_argument(
        '--master_volume_size', help='volume size for master node of k8s cluster', type=int,
        default=20)
    parser.add_argument(
        '--account_name', help='Nuvoloso account name', default='Demo Tenant')
    parser.add_argument(
        '--create_only', help='Create cluster only, skip Nuvoloso config', action='store_true')

    args = parser.parse_args()
    assert(args.kops_cluster_name and args.region and args.kops_state_store and args.aws_access_key
           and args.aws_secret_access_key), "Some/all input parameters are not filled. Aborting"

    if not args.create_only and args.nuvo_kontroller_hostname == None:
        logging.error("Must specify nuvo kontroller hostname")
        return

    home_dir = pathlib.Path.home()
    args.log_dirpath = args.log_dirpath if args.log_dirpath else str(home_dir.joinpath("logs-%s" % str(uuid.uuid4())[:5]))
    pathlib.Path(args.log_dirpath).mkdir(parents=True, exist_ok=True)
    logging.basicConfig(format='%(asctime)s %(message)s', filename=pathlib.Path(args.log_dirpath).joinpath(
        "%s.log" % os.path.basename(__file__)), level=logging.INFO)

    print("Script to deploy an application cluster")
    test = CreateAppCluster(args)
    nuvo_cluster_name = test.create_application_cluster()

    if not args.create_only:
        with open(NUVO_CLUSTER_NAME, 'w') as fd:
            fd.write(nuvo_cluster_name)

        logging.info("Application cluster name created: %s", nuvo_cluster_name)
    else:
        logging.info("Application cluster created: %s", args.kops_cluster_name)

if __name__ == '__main__':
    main()


from generators import display_grid, clear
from turn_handler import cpu_turn, player_turn
from file_handlers import save_game, clear_save_data

def game_loop(board_size, player_ships, player_attack, cpu_ships, cpu_attack,
              player_ship_count, cpu_ship_count, consecutive_hits, rounds):
    """
    Run the main game loop for a game of battleship.

    @param board_size: An integer representing the size of the game board.
    @type board_size: int

    @param player_ships: A list representing the locations of the player's ships on the game board.
    @type player_ships: List[List[str, int]]

    @param player_attack: A list representing the locations that the player has attack on the game board.
    @type player_attack: List[List[str, int]]

    @param cpu_ships: A list representing the locations of the CPU's ships on the game board.
    @type cpu_ships: List[List[str, int]]

    @param cpu_attack: A list representing the locations that the CPU has attack on the game board.
    @type cpu_attack: List[List[str, int]]

    @param player_ship_count: An integer representing the number of ships remaining for the player.
    @type player_ship_count: int

    @param cpu_ship_count: An integer representing the number of ships remaining for the CPU.
    @type cpu_ship_count: int

    @param consecutive_hits: An integer representing the number of consecutive hits made by the player.
    @type consecutive_hits: List[List[int, int]]

    @param rounds: An integer representing the number of rounds played in the game.
    @type rounds: int
    """

    options = []
    # print the initial game board
    game_over = False

    # loop until the game is over
    while not game_over:
        # switch the current player
        if rounds % 2 == 1:
            current_player = "cpu"
            display_grid(cpu_attack)
            # handle the CPU's turn
            cpu_attack, player_ship_count, consecutive_hits = cpu_turn(player_ships, cpu_attack,
                                                                       board_size, player_ship_count,
                                                                       consecutive_hits)
        else:
            current_player = "player"
            # handle the player's turn
            display_grid(player_attack)
            player_attack, cpu_ship_count = player_turn(cpu_ships, player_attack, board_size, cpu_ship_count)

        # print the updated game board

        rounds += 1
        game_over = (player_ship_count == 0) or (cpu_ship_count == 0)
        save_game(player_ships, player_attack, cpu_ships, cpu_attack, consecutive_hits, player_ship_count,
                  cpu_ship_count, rounds)
        clear(3)

    # print a message to indicate who won the game
    if current_player == "player":
        print("You won!")
        clear(5)
    else:
        print("The CPU won :(")
        clear(5)
    clear_save_data()


import numpy as np

from onerl.utils.import_module import get_class_from_str
from onerl.nodes.node import Node
from onerl.utils.shared_array import SharedArray
from onerl.utils.batch.shared import BatchShared

class EnvNode(Node):
    @staticmethod
    def node_preprocess_ns_config(node_class: str, num: int, ns_config: dict):
        Node.node_preprocess_ns_config(node_class, num, ns_config)

        # create sample env
        sample_env = EnvNode.create_env(ns_config)
        # obs
        ns_config["env"]["obs_shape"] = sample_env.observation_space.shape
        ns_config["env"]["obs_dtype"] = sample_env.observation_space.dtype
        # act
        if hasattr(sample_env.action_space, "n"):
            # discrete
            ns_config["env"]["act_shape"] = ()
            ns_config["env"]["act_dtype"] = np.int64
            ns_config["env"]["act_n"] = sample_env.action_space.n
        else:
            # continuous
            ns_config["env"]["act_shape"] = sample_env.action_space.shape
            ns_config["env"]["act_dtype"] = sample_env.action_space.dtype
            ns_config["env"]["act_max"] = sample_env.action_space.high
            assert np.isclose(sample_env.action_space.low, -sample_env.action_space.high).all(), \
                "Action range must be symmetric"

        # batch
        ns_config["env"]["batch"] = {
            "obs": (ns_config["env"]["obs_shape"], ns_config["env"]["obs_dtype"]),
            "act": (ns_config["env"]["act_shape"], ns_config["env"]["act_dtype"]),
            "rew": ((), np.float32),
            "done": ((), np.float32)
        }

        # offline visualization
        if hasattr(sample_env, "load_state") and hasattr(sample_env, "save_state"):
            sample_vis_state = sample_env.save_state()
            ns_config["env"]["vis_state_shape"] = sample_vis_state.shape
            ns_config["env"]["vis_state_dtype"] = sample_vis_state.dtype

    @staticmethod
    def node_create_shared_objects(node_class: str, num: int, ns_config: dict):
        objects = Node.node_create_shared_objects(node_class, num, ns_config)
        for obj in objects:
            # env
            obj["obs"] = SharedArray((ns_config["env"]["frame_stack"], *ns_config["env"]["obs_shape"]),
                                     ns_config["env"]["obs_dtype"])
            obj["act"] = SharedArray(ns_config["env"]["act_shape"], ns_config["env"]["act_dtype"])

            # log to replay
            obj["log"] = BatchShared(ns_config["env"]["batch"], init_ready=True)

            # offline visualization
            if ("vis_state_shape" in ns_config["env"]) and (Node.node_count("VisualizerNode", ns_config) > 0):
                obj["vis_state"] = SharedArray(ns_config["env"]["vis_state_shape"], ns_config["env"]["vis_state_dtype"])

        return objects

    @staticmethod
    def create_env(ns_config: dict):
        env_config = ns_config["env"]
        env_class = get_class_from_str(env_config.get("import", ""), env_config["name"])
        env = env_class(**env_config.get("params", {}))
        return env

    def run(self):
        # acquire shared objects
        shared_obs = self.objects["obs"].get()
        shared_act = self.objects["act"].get()
        shared_log = self.objects["log"].get()

        shared_vis_state = self.objects["vis_state"].get() if "vis_state" in self.objects else None

        # find nodes
        node_scheduler = self.find("SchedulerNode")
        node_replay_buffer = self.find("ReplayBufferNode")
        if node_replay_buffer is None:
            self.log("ReplayBufferNode not found, skip storing experience.")

        # create and reset env
        env = self.create_env(self.ns_config)
        obs = env.reset()
        tot_reward = 0
        while True:
            # copy obs to shared mem
            self.setstate("copy_obs")
            shared_obs[:-1] = shared_obs[1:]
            shared_obs[-1] = obs

            # request act
            self.setstate("wait_act")
            self.send(node_scheduler, self.node_name)
            self.recv()

            # step env
            self.setstate("step")
            obs_next, rew, done, info = env.step(shared_act)
            # ignore time limit induced done
            log_done = done and (not info.get("TimeLimit.truncated", False))
            tot_reward += rew
            # log
            if node_replay_buffer is not None:
                self.setstate("wait_log")
                self.objects["log"].wait_ready()

                self.setstate("copy_log")
                shared_log.obs[...] = obs
                shared_log.act[...] = shared_act
                shared_log.rew[...] = rew
                shared_log.done[...] = log_done
                self.send(node_replay_buffer, self.node_name)

            # update obs & reset
            obs = obs_next
            if done:
                self.setstate("reset")
                obs = env.reset()

                self.log_metric({"{}@episode_reward".format(self.node_ns): tot_reward})
                tot_reward = 0

            # offline visualization
            if shared_vis_state is not None:
                shared_vis_state[...] = env.save_state()