from tkinter import *
from tkinter.messagebox import *
import  copy
root = Tk()
root.title("box")

imgs= [PhotoImage(file='boxbox\\Wall.gif'),
       PhotoImage(file='boxbox\\Worker.gif'),
       PhotoImage(file='boxbox\\Box.gif'),
       PhotoImage(file='boxbox\\Passageway.gif'),
       PhotoImage(file='boxbox\\Destination.gif'),
       PhotoImage(file='boxbox\\WorkerInDest.gif'),
       PhotoImage(file='boxbox\\RedBox.gif') ]

Wall = 0
Worker = 1
Box = 2
Passageway = 3
Destination = 4
WorkerInDest = 5
RedBox = 6
sum = False
#原始地图
myArray1 = [[0,3,1,4,3,3,3],
           [0,3,3,2,3,3,0],
           [0,0,3,0,3,3,0],
           [3,3,2,3,0,0,0],
           [3,4,3,3,3,0,0],
           [0,0,3,3,3,3,0],
           [0,0,0,0,0,0,0]]
#绘制整个游戏区域图形
def drawGameImage( ):
    global x,y

    for i in range(0,7) :#0--6
       for j in range(0,7) :#0--6
            if myArray[i][j] == Worker :
               x=i  #工人当前位置(x,y)
               y=j

            img1= imgs[myArray[i][j]]
            cv.create_image((i*32+20,j*32+20),image=img1)
            cv.pack()
    #print (myArray)

def callback(event) :#按键处理
    global x,y,myArray

    KeyCode = event.keysym
    #工人当前位置(x,y)
    if KeyCode=="Up":#分析按键消息
    #向上
            x1 = x;
            y1 = y - 1;
            x2 = x;
            y2 = y - 2;
            #将所有位置输入以判断并作地图更新
            MoveTo(x1, y1, x2, y2);
    #向下
    elif KeyCode=="Down":
            x1 = x;
            y1 = y + 1;
            x2 = x;
            y2 = y + 2;
            MoveTo(x1, y1, x2, y2);
    #向左
    elif KeyCode=="Left":
            x1 = x - 1;
            y1 = y;
            x2 = x - 2;
            y2 = y;
            MoveTo(x1, y1, x2, y2);
    #向右
    elif KeyCode=="Right":
            x1 = x + 1;
            y1 = y;
            x2 = x + 2;
            y2 = y;
            MoveTo(x1, y1, x2, y2);
    elif KeyCode=="space": #空格键
       print ("按下键：空格", event.char)
       myArray=copy.deepcopy(myArray1)  #恢复原始地图
       drawGameImage( )

#判断是否在游戏区域
def IsInGameArea(row, col) :
    return (row >= 0 and row < 7 and col >= 0 and col < 7)
def MoveTo(x1, y1, x2, y2) :
        global x,y
        P1=None
        P2=None
        if IsInGameArea(x1, y1) : #判断是否在游戏区域
            P1=myArray[x1][y1];
        if IsInGameArea(x2, y2) :
            P2 = myArray[x2][y2]
        if P1 ==  Passageway :#P1处为通道
            MoveMan(x,y);
            x = x1; y = y1;
            myArray[x1][y1] =  Worker;
        if P1 ==  Destination :#P1处为目的地
            MoveMan(x, y);
            x = x1; y = y1;
            myArray[x1][y1] =  WorkerInDest;
        if P1 ==  Wall or  not IsInGameArea(x1, y1) :
            #P1处为墙或出界
            return;

        #以下P1处为箱子
        if P1 ==  Box  :#P1处为箱子
           if P2 ==  Wall or  not IsInGameArea(x1, y1) or P2 ==  Box :##P2处为墙或出界
              return;

        #P1处为箱子,P2处为通道
        if P1 ==  Box and P2 ==  Passageway :
            MoveMan(x, y);
            x = x1; y = y1;
            myArray[x2][y2]= Box;
            myArray[x1][y1] =  Worker;
        if P1 ==  Box and P2 ==  Destination :
            MoveMan(x, y);
            x = x1; y = y1;
            myArray[x2][y2]= RedBox;
            myArray[x1][y1] =  Worker;
        #P1处为放到目的地的箱子,P2处为通道
        if P1 ==  RedBox and P2 ==  Passageway :
            MoveMan(x, y);
            x = x1; y = y1;
            myArray[x2][y2] =  Box;
            myArray[x1][y1] =  WorkerInDest;
        #P1处为放到目的地的箱子,P2处为目的地
        if P1 ==  RedBox and P2 ==  Destination :
            MoveMan(x, y);
            x = x1; y = y1;
            myArray[x2][y2] =  RedBox;
            myArray[x1][y1] =  WorkerInDest;
        drawGameImage()
        #这里要验证是否过关
        if IsFinish() :
            showinfo(title="successful",message=" Congratulations!" )
            import terrorist

def  MoveMan(x, y) :
    if myArray[x][y] == Worker :
        myArray[x][y] = Passageway;
    elif myArray[x][y] == WorkerInDest :
        myArray[x][y] = Destination;

def IsFinish( ):#验证是否过关
    bFinish = True;
    for i in range(0,7) :#0--6
       for j in range(0,7) :#0--6
            if  (myArray[i][j] == Destination
                   or myArray[i][j] == WorkerInDest) :
                bFinish = False;
    return bFinish;

def drawQiPan( ) :#画棋盘
    for i in range(0,15) :
        cv.create_line(20,20+40*i,580,20+40*i,width=2)
    for i in range(0,15) :
        cv.create_line(20+40*i,20,20+40*i,580,width=2)
    cv.pack()

def print_map( ) :#输出map地图
    for i in range(0,15) :#0--14
       for j in range(0,15) :#0--14
           print (map[i][j],end=' ')
       print ('w')

cv = Canvas(root, bg = 'green', width = 226, height = 226)
#drawQiPan( )
myArray=copy.deepcopy(myArray1)
drawGameImage()

cv.bind("<KeyPress>", callback)
cv.pack()
cv.focus_set() #将焦点设置到cv上
root.mainloop()


from Models import *

with open('captcha_data', 'rb') as f:
    data = torch.load(f)
with open('captcha_label', 'rb') as f:
    labels = torch.load(f)
with open('Net_param', 'rb') as f:
    net_param = torch.load(f)
with open('Decode_param', 'rb') as f:
    dec_param = torch.load(f)
with open('Cls_param', 'rb') as f:
    cls_param = torch.load(f)

class Tester(nn.Module):

    def __init__(self, net_p, cls_p, dec_p):
        super(Tester, self).__init__()
        self.Net = Encoder()
        self.Net.load_state_dict(net_p)
        self.Dec = Decoder()
        self.Dec.load_state_dict(dec_p)
        self.Cls = Classifier()
        self.Cls.load_state_dict(cls_p)

    def forward(self, x):
        x = self.Net(x)
        y = self.Cls(x)
        r = self.Dec(x)
        return y, r

torch.backends.cudnn.benchmark = True
data = data.cuda()
labels = labels.cuda()
model = Tester(net_param, cls_param, dec_param).cuda()

c, x = model(data[691:711])
c = torch.argmax(c, dim=1)
la = labels[691:711]
accuracy = float((la == c).sum())
print('Accuracy:', accuracy / (20 * 5))


# -*- coding: utf-8 -*-

import sys
import socket
import threading

def server_loop(localhost, localport, remote_host, remote_port, receive_first):
    server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        server.bind((localhost, localport))
    except:
        print("[!!] Failed to listen on %s:%d" % (localhost, localport))
        print("[!!] Check for other listening sockets or correct permissions")
        sys.exit()

    print("[*] Listening on %s:%d" % (localhost, localport))

    server.listen(5)

    while True:
        client_socket, addr = server.accept()

        #show connect information from local host
        print("[==>] Received incoming connection from %s:%d" % (addr[0], addr[1]))

        #start thread to connect to remote_host
        proxy_thread = threading.Thread(
                target=proxy_handler,
                args=(client_socket,remote_host,remote_port,receive_first))
        proxy_thread.start()

def main():
    #read commandline arguments
    if len(sys.argv[1:]) != 5:
        print("Usage: ./proxy.py [localhost] [localport] [remote_host] [remote_port] [receive_first]")
        print("Example: ./proxy.py 127.0.0.1 9000 10.12.132.1 9000 True")
        sys.exit(0)

    #local host setting to receive transmission
    local_host = sys.argv[1]
    local_port = int(sys.argv[2])

    #setting to remote
    remote_host = sys.argv[3]
    remote_port = int(sys.argv[4])

    #setting whether or not to receive data before submit it to remote host
    receive_first = sys.argv[5]

    if "True" in receive_first:
        receive_first = True
    else:
        receive_first = False

    #start socket to wait connections
    server_loop(local_host, local_port, remote_host, remote_port, receive_first)

def proxy_handler(client_socket, remote_host, remote_port, receive_first):
    #connect to remote host
    remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    remote_socket.connect((remote_host, remote_port))

    #if need , receive data from remote host
    if receive_first:
        remote_buffer = receive_from(remote_socket)
        hexdump(remote_buffer)

        #give data to function processing received data
        remote_buffer = response_handler(remote_buffer)

        #if exist data to submit to local host
        if len(remote_buffer):
            print("[<==] Sending %d bytes to localhost." % len(remote_buffer))
            client_socket.send(remote_buffer)

    #start loop to repeat receiving data from localhost, submitting data to remote host and submitting to local host
    while True:
        #receive data from local host
        local_buffer = receive_from(client_socket)

        if len(local_buffer):
            print("[==>] Received %d bytes from localhost." % len(local_buffer))
            hexdump(local_buffer)

            #give data to function processing submitting data
            local_buffer = request_handler(local_buffer)

            #submit data to remote host
            remote_socket.send(local_buffer)
            print("[==>] Sent to remote.")

        #receive response
        remote_buffer = receive_from(remote_socket)

        if len(remote_buffer):
            print("[<==] Received %d bytes from remote." % len(remote_buffer))
            hexdump(remote_buffer)

            #give data to function processing received data
            remote_buffer = response_handler(remote_buffer)

            #submit response data to local host
            client_socket.send(remote_buffer)
            print("[<==] Sent to localhost.")
            # if don't come data from remote host or local host, close connection
            if not len(local_buffer) or not len(remote_buffer):
                client_socket.close()
                remote_socket.close()
                print("[*] No more data. Closing connections.")
                break

main()


# -*- coding: utf-8 -*-
#BEGIN_HEADER
import logging
import os
import sys
import subprocess
import shutil
from jinja2 import Template
from installed_clients.WorkspaceClient import Workspace as workspaceService
import requests
from pprint import pprint, pformat
from datetime import datetime
import re
from glob import glob
from zipfile import ZipFile

from installed_clients.KBaseReportClient import KBaseReport
from installed_clients.ReadsUtilsClient import ReadsUtils
from installed_clients.DataFileUtilClient import DataFileUtil
from installed_clients.SetAPIServiceClient import SetAPI
#END_HEADER

class gottcha2:
    '''
    Module Name:
    gottcha2

    Module Description:
    A KBase module: gottcha2
    '''

    ######## WARNING FOR GEVENT USERS ####### noqa
    # Since asynchronous IO can lead to methods - even the same method -
    # interrupting each other, you must be *very* careful when using global
    # state. A method could easily clobber the state set by another while
    # the latter method is running.
    ######################################### noqa
    VERSION = "0.0.6"
    GIT_URL = "git@github.com:LANL-Bioinformatics/kbase-gottcha2.git"
    GIT_COMMIT_HASH = "6ae998a2dc9a771381f290622789a8c182c5cdea"

    #BEGIN_CLASS_HEADER
    def _generate_DataTable(self, infile, outfile):
        f =  open(infile, "r")
        wf = open(outfile,"w")

        header = f.readline().strip()
        headerlist = [ x.strip() for x in header.split('\t')]

        wf.write("\n")
        wf.write("<link rel='stylesheet' type='text/css' href='https://cdn.datatables.net/1.10.19/css/jquery.dataTables.css'>\n")
        wf.write("\n")
        wf.write("\n")
        wf.write("\n")
        wf.write("\n")
        wf.write("""""")
        wf.write("
\n")
        wf.write('

' + ''.join("{0}	".format(t) for t in headerlist) + '

\n')
        wf.write("
\n")
        for line in f:
            if not line.strip():continue
            wf.write("
\n")
            temp = [ x.strip() for x in line.split('\t')]
            wf.write(''.join("{0}	".format(t) for t in temp))
            wf.write("
\n")
        wf.write("
\n")
        wf.write("
")
        wf.write("\n")

    def create_report(self, ws, output_list, html_folder):
        output_html_files = []
        output_zip_files = []
        first_file = ""
        html_count = 0
        with open('/kb/data/index_start.txt', 'r') as start_file:
            html_string = start_file.read()

        # Make HTML folder
        # html_folder = os.path.join(self.scratch, 'gottcha2_output', 'html_report')
        os.mkdir(html_folder) if not os.path.exists(html_folder) else None
        for files_dict in output_list:
            zip_files = [d['name'] for d in files_dict['output_files'] if d['name'].endswith('.zip')]
            logging.info(f'zip files {zip_files}')
            for file in zip_files:
                desc = 'Zip file generated by GOTTCHA2 that contains ' + \
                       'original images seen in the report'
                output_zip_files.append({'path': os.path.join(html_folder, file),
                                         'name': file,
                                         'label': ".".join(file.split(".")[1:]),
                                         'description': desc})
            logging.info(f"files_dict {files_dict['output_files']}")
            # each sample run through GOTTCHA2 has a file
            first_file = [d['name'] for d in files_dict['output_files'] if d['name'].endswith('_index.html')][0]
            html_string+="            <button data-button=\"page "+str(html_count) + \
                         "\" data-page=\""+first_file+"\">Page "+str(html_count+1)+"</button>\n"
            html_count += 1

        html_string += "

\n"
        html_string += "        \n
"

        with open('/kb/data/index_end.txt', 'r') as end_file:
            html_string += end_file.read()
        assert os.path.exists(html_folder)
        with open(os.path.join(html_folder, "index.html"), 'w') as index_file:
            index_file.write(html_string)

        shock = self.dfu.file_to_shock({'file_path': html_folder,
                                        'make_handle': 0,
                                        'pack': 'zip'})
        desc = 'HTML files generated by GOTTCHA2 that contains report on ' + \
               'taxonomic profile of reads'
        output_html_files.append({'shock_id': shock['shock_id'],
                                  'name': 'index.html',
                                  'label': 'html files',
                                  'description': desc})
        l = [p['output_files'] for p in output_list]
        flat_list = [item for sublist in l for item in sublist]
        logging.info(f"file links {flat_list}")
        logging.info(f'output_zip_files {output_zip_files}')
        logging.info(f'output_html_files {output_html_files}')
        report_params = {
            'direct_html_link_index': 0,
            'file_links': flat_list,
            'html_links': output_html_files,
            'workspace_name': ws
        }
        kbase_report_client = KBaseReport(self.callback_url)
        output = kbase_report_client.create_extended_report(report_params)
        logging.info(f'output {output}')
        return output

    def package_folder(self, folder_path, zip_file_name, zip_file_description):
        ''' Simple utility for packaging a folder and saving to shock '''
        if folder_path == self.scratch:
            raise ValueError ("cannot package scatch itself.  folder path: "+folder_path)
        elif not folder_path.startswith(self.scratch):
            raise ValueError ("cannot package folder that is not a subfolder of scratch.  folder path: "+folder_path)

        if not os.path.exists(folder_path):
            raise ValueError ("cannot package folder that doesn't exist: "+folder_path)
        output = self.dfu.file_to_shock({'file_path': folder_path,
                                    'make_handle': 0,
                                    'pack': 'zip'})
        return {'shock_id': output['shock_id'],
                'name': zip_file_name,
                'label': zip_file_description}

    def log(self, target, message):
        if target is not None:
            target.append(message)
        print(message)
        sys.stdout.flush()

    @staticmethod
    def fill_template(template_file, params_dict, output_file):
        """
        Fill template
        :param template_file: path to template file
        :param params_dict: parameters to fill template with
        :param output_file: path to output file
        :return:
        """
        with open(template_file, 'r') as fp:
            doc = fp.read()
        t = Template(doc)
        template_string = t.render(params_dict)
        with open(output_file, 'w') as fp:
            fp.write(template_string)

    #END_CLASS_HEADER

    # config contains contents of config file in a hash or None if it couldn't
    # be found
    def __init__(self, config):
        #BEGIN_CONSTRUCTOR
        self.callback_url = os.environ['SDK_CALLBACK_URL']
        self.scratch = config['scratch']
        self.workspaceURL = config['workspace-url']
        self.shockURL = config['shock-url']
        self.handleURL = config['handle-service-url']
        self.serviceWizardURL = config['service-wizard-url']
        self.dfu = DataFileUtil(self.callback_url)
        logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
                            level=logging.INFO)

        if not os.path.exists(self.scratch):
            os.makedirs(self.scratch)
        os.chdir(self.scratch)

        #END_CONSTRUCTOR
        pass

    def run_gottcha2(self, ctx, params):
        """
        This example function accepts any number of parameters and returns results in a KBaseReport
        :param params: instance of mapping from String to unspecified object
        :returns: instance of type "ReportResults" -> structure: parameter
           "report_name" of String, parameter "report_ref" of String
        """
        # ctx is the context object
        # return variables are: output
        #BEGIN run_gottcha2

        # Step 2 - Download the input data as a FASTQ and
        # We can use the ReadsUtils module to download a FASTQ file from our Reads data object.
        # The return object gives us the path to the file that was created.
        logging.info('Downloading Reads data as a Fastq file.')
        output_dir = os.path.join(self.scratch, f'gottcha2_output')
        report_dir = os.path.join(output_dir, 'html_report')
        params['output_dir'] = output_dir
        params['report_dir'] = report_dir
        logging.info(f'params {params}')
        token = ctx['token']
        wsClient = workspaceService(self.workspaceURL, token=token)
        env = os.environ.copy()
        env['KB_AUTH_TOKEN'] = token

        # object info
        [OBJID_I, NAME_I, TYPE_I, SAVE_DATE_I, VERSION_I, SAVED_BY_I, WSID_I, WORKSPACE_I, CHSUM_I, SIZE_I,
         META_I] = range(11)  # object_info tuple

        Set_types = ["KBaseSets.ReadsSet", "KBaseRNASeq.RNASeqSampleSet"]
        PE_types = ["KBaseFile.PairedEndLibrary", "KBaseAssembly.PairedEndLibrary"]
        SE_types = ["KBaseFile.SingleEndLibrary", "KBaseAssembly.SingleEndLibrary"]
        acceptable_types = Set_types + PE_types + SE_types

        # Determine whether read library or read set is input object
        #
        try:
            input_reads_obj_info = wsClient.get_object_info([{'ref': params['input_refs'][0]}], 1)[0]
            input_reads_obj_type = input_reads_obj_info[TYPE_I]
            input_reads_obj_type = re.sub('-[0-9]+\.[0-9]+$', "", input_reads_obj_type)  # remove trailing version
            # input_reads_obj_version = input_reads_obj_info[VERSION_I]  # this is object version, not type version
        except Exception as e:
            raise ValueError('Unable to get read library object from workspace: (' + str(
                params['input_refs']) + ')' + str(e))

        if input_reads_obj_type not in acceptable_types:
            raise ValueError(
                "Input reads of type: '" + input_reads_obj_type + "'.  Must be one of " + ", ".join(acceptable_types))

        # auto-detect reads type
        read_type = None
        if input_reads_obj_type in PE_types:
            read_type = 'PE'
        elif input_reads_obj_type in SE_types:
            read_type = 'SE'

        # get set
        #
        readsSet_ref_list = []
        readsSet_names_list = []
        if input_reads_obj_type in Set_types:
            try:
                # self.log (console, "INPUT_READS_REF: '"+input_params['input_refs']+"'")  # DEBUG
                # setAPI_Client = SetAPI (url=self.callbackURL, token=ctx['token'])  # for SDK local.  doesn't work for SetAPI
                setAPI_Client = SetAPI(url=self.serviceWizardURL, token=ctx['token'],
                                       service_ver='beta')  # for dynamic service
                input_readsSet_obj = setAPI_Client.get_reads_set_v1(
                    {'ref': params['input_refs'][0], 'include_item_info': 1})

            except Exception as e:
                raise ValueError('SetAPI FAILURE: Unable to get read library set object from workspace: (' + str(
                    params['input_refs']) + ")\n" + str(e))
            logging.info(f"reads set items {input_readsSet_obj['data']['items']}")
            for readsLibrary_obj in input_readsSet_obj['data']['items']:
                readsSet_ref_list.append(readsLibrary_obj['ref'])
                readsSet_names_list.append(readsLibrary_obj['info'][NAME_I])
                reads_item_type = readsLibrary_obj['info'][TYPE_I]
                reads_item_type = re.sub('-[0-9]+\.[0-9]+$', "", reads_item_type)  # remove trailing version
                if reads_item_type in PE_types:
                    this_read_type = 'PE'
                elif reads_item_type in SE_types:
                    this_read_type = 'SE'
                else:
                    raise ValueError(
                        "Can't handle read item type '" + reads_item_type + "' obj_name: '" + readsLibrary_obj['info'][
                            NAME_I] + " in Set: '" + str(params['input_refs']) + "'")
                if read_type != None and this_read_type != read_type:
                    raise ValueError("Can't handle read Set: '" + str(params[
                                                                          'input_refs']) + "'.  Unable to process mixed PairedEndLibrary and SingleEndLibrary.  Please split into separate ReadSets")
                elif read_type == None:
                    read_type = this_read_type
        else:
            readsSet_ref_list = params['input_refs']
            readsSet_names_list = input_reads_obj_info[NAME_I]

        output_list = []
        for reads_item_i,input_reads_library_ref in enumerate(readsSet_ref_list):
            params['input_ref'] = input_reads_library_ref
            params['read_type']= read_type
            params['reads_item_i'] = reads_item_i
            output_list.append(self.exec_gottcha2(ctx, params)[0])
        report_output = self.create_report(params.get('workspace_name'), output_list, params['report_dir'])
        logging.info(f'report_output {report_output}')
        output = {'report_name': report_output['name'], 'report_ref': report_output['ref']}
        #END run_gottcha2

        # At some point might do deeper type checking...
        if not isinstance(output, dict):
            raise ValueError('Method run_gottcha2 return value ' +
                             'output is not type dict as required.')
        # return the results
        return [output]

    def exec_gottcha2(self, ctx, params):
        """
        :param params: instance of mapping from String to unspecified object
        :returns: instance of type "ReportResults" -> structure: parameter
           "report_name" of String, parameter "report_ref" of String
        """
        # ctx is the context object
        # return variables are: output
        #BEGIN exec_gottcha2
        console = []
        self.log(console, 'Running GOTTCHA2 with parameters: ')
        self.log(console, "\n" + pformat(params))
        report = ''
        retVal = dict()
        token = ctx['token']
        wsClient = workspaceService(self.workspaceURL, token=token)
        headers = {'Authorization': 'OAuth ' + token}
        env = os.environ.copy()
        env['KB_AUTH_TOKEN'] = token
        # object info
        [OBJID_I, NAME_I, TYPE_I, SAVE_DATE_I, VERSION_I, SAVED_BY_I, WSID_I, WORKSPACE_I, CHSUM_I, SIZE_I,
         META_I] = range(11)  # object_info tuple

        # Set_types = ["KBaseSets.ReadsSet", "KBaseRNASeq.RNASeqSampleSet"]
        PE_types = ["KBaseFile.PairedEndLibrary", "KBaseAssembly.PairedEndLibrary"]
        SE_types = ["KBaseFile.SingleEndLibrary", "KBaseAssembly.SingleEndLibrary"]
        acceptable_types = PE_types + SE_types
        # Determine whether read library is of correct type
        #
        try:
            input_reads_obj_info = \
                wsClient.get_object_info_new({'objects': [{'ref': params['input_ref']}]})[0]
            logging.info(f'input_reads_obj_info {input_reads_obj_info}')
            logging.info(f"params input_ref {params['input_ref']}")
            input_reads_obj_type = input_reads_obj_info[TYPE_I]
            # input_reads_obj_version = input_reads_obj_info[VERSION_I]  # this is object version, not type version

        except Exception as e:
            raise ValueError('Unable to get read library object from workspace: (' + str(
                params['input_ref']) + ')' + str(e))

        input_reads_obj_type = re.sub('-[0-9]+\.[0-9]+$', "", input_reads_obj_type)  # remove trailing version
        acceptable_types = PE_types + SE_types
        if input_reads_obj_type not in acceptable_types:
            raise ValueError(
                "Input reads of type: '" + input_reads_obj_type + "'.  Must be one of " + ", ".join(acceptable_types))

        # Ensure that all libraries in ReadsSet are either single-ended or pair ended
        #
        if params['read_type'] == 'PE' and not input_reads_obj_type in PE_types:
            raise ValueError("read_type set to 'Paired End' but object is SingleEndLibrary")
        if params['read_type'] == 'SE' and not input_reads_obj_type in SE_types:
            raise ValueError("read_type set to 'Single End' but object is PairedEndLibrary")

        # Instatiate ReadsUtils
        #
        try:
            readsUtils_Client = ReadsUtils(url=self.callback_url, token=ctx['token'])  # SDK local

            download_reads_output = readsUtils_Client.download_reads({'read_libraries': [params['input_ref']],
                                                                      'interleaved': 'false'
                                                                      })
        except Exception as e:
            raise ValueError('Unable to get read library object from workspace: (' + str(
                params['input_ref']) + ")\n" + str(e))

        logging.info(
            f"Input parameters {params['input_refs']}, {params['db_type']} download_reads_output {download_reads_output}")
        fastq_files = []
        fastq_files_name = []
        for key, val in download_reads_output['files'].items():
            if 'fwd' in val['files'] and val['files']['fwd']:
                fastq_files.append(val['files']['fwd'])
                fastq_files_name.append(val['files']['fwd_name'])
            if 'rev' in val['files'] and val['files']['rev']:
                fastq_files.append(val['files']['rev'])
                fastq_files_name.append(val['files']['rev_name'])
        logging.info(f"fastq files {fastq_files}")
        fastq_files_string = ' '.join(fastq_files)
        # label = params['reads_item_i']
        if len(input_reads_obj_info[NAME_I].split('.')) > 1 :
            label = '_'.join(input_reads_obj_info[NAME_I].split('.')[:-1])
        else:
            label = input_reads_obj_info[NAME_I]
        logging.info(f'label {label}')
        output_dir = params['output_dir']
        report_dir = params['report_dir']
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        ## default options
        if 'min_coverage' not in params:
            params['min_coverage'] = 0.005
        if 'min_reads' not in params:
            params['min_reads'] = 3
        if 'min_length' not in params:
            params['min_length'] = 60
        if 'max_zscore' not in params:
            params['max_zscore'] = 10
        outprefix = "gottcha2"

        cmd = ['/kb/module/lib/gottcha2/src/uge-gottcha2.sh', '-i', fastq_files_string, '-t', '4', '-o', output_dir,
               '-p',
               outprefix, '-d', '/data/gottcha2/RefSeq90/' + params['db_type'], '-c', str(params['min_coverage']), '-r',
               str(params['min_reads']), '-s', str(params['min_length']), '-z', str(params['max_zscore'])]
        logging.info(f'cmd {cmd}')
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        logging.info(f'subprocess {p.communicate()}')
        summary_file = os.path.join(report_dir, f'{label}_{outprefix}.summary.tsv')
        lineage_file = os.path.join(output_dir, outprefix + '.lineage.tsv')

        with open("/kb/module/work/tmp/gottcha2_output/gottcha2.gottcha_species.log", "r" ) as f:
            logging.info(f.readlines())

        if os.stat(lineage_file).st_size == 0:
            msg = 'There is no taxonomy classification assignment against ' + params[
                'db_type'] + ' reference DB. You may try another reference DB.'
            logging.error(msg)
            raise ValueError(msg)

        # generate report directory and html file
        if not os.path.exists(report_dir):
            os.makedirs(report_dir)
        output_files = []
        [output_files.append(f) for f in os.listdir(output_dir) if not os.path.isdir(os.path.join(output_dir, f))]

        # zf = ZipFile(os.path.join(output_dir, f'{label}.zip'), mode='w')
        # [zf.write(os.path.join(output_dir, f)) for f in output_files if os.path.splitext(f)[1] != '.sam']
        # output_files.append(f'{label}.zip')
        print(f"output_files list {output_files}")
        # copy files from output_dir to report_dir if file is not .sam so it doesn't end up in the zip file
        [shutil.copy2(os.path.join(output_dir, f), os.path.join(report_dir, f'{label}_{f}')) for f in output_files if os.path.splitext(f)[1] != '.sam']

        summary_file_dt = os.path.join(report_dir,  f'{label}_gottcha2.datatable.html')

        self._generate_DataTable(summary_file, summary_file_dt)
        self.fill_template('/kb/module/lib/gottcha2/src/index.html.tmpl', {'label': label}, os.path.join(report_dir, f'{label}_index.html'))
        # shutil.copy2('/kb/module/lib/gottcha2/src/index.html', os.path.join(report_dir, f'{label}_index.html'))
        shutil.copy2(os.path.join(output_dir, outprefix + '.krona.html'),
                     os.path.join(report_dir, f'{label}_gottcha2.krona.html'))
        if os.path.exists(os.path.join(output_dir, outprefix + '.tree.svg')):
            shutil.move(os.path.join(output_dir, outprefix + '.tree.svg'),
                        os.path.join(report_dir, f'{label}_gottcha2.tree.svg'))
        # html_zipped = self.package_folder(report_dir, f'{label}.zip', f'{label}.zip')
        # logging.info(f'html_zipped {html_zipped}')

        # copy .sam file to report_dir so that it's in the file_links
        [shutil.copy2(os.path.join(output_dir, f), os.path.join(report_dir, f'{label}_{f}')) for f in output_files if
         os.path.splitext(f)[1] == '.sam']

        # Step 5 - Build a Report and return
        objects_created = []
        output_files = glob(report_dir + f'/{label}*')

        logging.info(f'glob output_files {output_files}')
        output_files_list = []
        for file in output_files:
            if not os.path.isdir(file):
                # logging.info(f'file {file}')
                output_files_list.append({'path': file,
                                          'name': file.split('/')[-1]
                                          })
        logging.info(f'output_files_list {output_files_list}')
        zip_file = os.path.join(report_dir, f'{label}.zip')
        with ZipFile(zip_file, 'w', allowZip64=True) as myzip:
            [myzip.write(f['path']) for f in output_files_list if not f['path'].endswith('.sam')]
        output_files_list.append({'path': zip_file,
                                  'name': zip_file.split('/')[-1]
                                  })
        # not used

        output = {'output_files': output_files_list, 'file_links': output_files_list}
        #END exec_gottcha2

        # At some point might do deeper type checking...
        if not isinstance(output, dict):
            raise ValueError('Method exec_gottcha2 return value ' +
                             'output is not type dict as required.')
        # return the results
        return [output]
    def status(self, ctx):
        #BEGIN_STATUS
        returnVal = {'state': "OK",
                     'message': "",
                     'version': self.VERSION,
                     'git_url': self.GIT_URL,
                     'git_commit_hash': self.GIT_COMMIT_HASH}
        #END_STATUS
        return [returnVal]


import json
import subprocess
import time

import sys

import base64
import click
import os
import random
from collections import namedtuple
from two1.commands.config import pass_config
from two1.lib.bitcoin.block import CompactBlock
from two1.lib.bitcoin.txn import Transaction
from two1.lib.server import rest_client, message_factory, login
from two1.lib.server.analytics import capture_usage
import two1.commands.config as cmd_config
from two1.commands import status
from two1.commands.status import has_bitcoinkit
from two1.lib.bitcoin.hash import Hash
from two1.lib.server.rest_client import ServerRequestError
from two1.lib.util.exceptions import MiningDisabledError
from two1.lib.util.uxstring import UxString
import two1.lib.bitcoin.utils as utils
import two1.commands.config as app_config

@click.command()
@click.option('--dashboard', default=False, is_flag=True,
              help="Dashboard with mining details")
@click.pass_context
def mine(ctx, dashboard):
    """Mine bitcoin at the command line.

\b
Usage
-----
Invoke this with no arguments to start mining.
$ 21 mine

\b
Then view your new balance.
$ 21 status

\b
View aggregated logs to see your mining progress.
$ 21 log

\b
See a mining dashboard for low-level mining details.
$ 21 mine --dashboard
"""
    config = ctx.obj['config']
    _mine(config, dashboard=dashboard)

@capture_usage
def _mine(config, dashboard=False):
    if has_bitcoinkit():
        if not is_minerd_running():
            start_minerd(config, dashboard)
        elif dashboard:
            show_minertop(dashboard)
        # if minerd is running and we have not specified a dashboard flag
        # do a cpu mine
        else:
            start_cpu_mining(config)
    else:
        config.log(UxString.buy_ad, fg="magenta")
        start_cpu_mining(config)

def is_minerd_running():
    rc = True
    import socket
    try:
        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        s.connect("/tmp/minerd.sock")
        s.close()
    except Exception:
        rc = False
    return rc

def show_minertop(show_dashboard):
    if show_dashboard:
        click.pause(UxString.mining_show_dashboard_prompt)
        subprocess.call("minertop")
    else:
        click.echo(UxString.mining_show_dashboard_context)

def start_minerd(config, show_dashboard=False):
    # Check if it's already up and running by checking pid file.
    minerd_pid_file = "/run/minerd.pid"
    config.log(UxString.mining_chip_start)
    # Read the PID and check if the process is running
    if os.path.isfile(minerd_pid_file):
        pid = None
        with open(minerd_pid_file, "r") as f:
            pid = int(f.read().rstrip())

        if pid is not None:
            if check_pid(pid):
                # Running, so fire up minertop...
                click.echo(UxString.mining_chip_running)
                show_minertop(show_dashboard)
                return
            else:
                # Stale PID file, so delete it.
                subprocess.call(["sudo", "minerd", "--stop"])

    # Not running, let's start it
    # TODO: make sure config exists in /etc
    # TODO: replace with sys-ctrl command
    minerd_cmd = ["sudo", "minerd", "-u", config.username,
                  app_config.TWO1_POOL_URL]
    try:
        o = subprocess.check_output(minerd_cmd, universal_newlines=True)
    except subprocess.CalledProcessError as e:
        config.log("\nError starting minerd: %r" % e)

    # Now call minertop after it's started
    show_minertop(show_dashboard)

def start_cpu_mining(config):
    """Mine at the CPU.
    >>> from two1.commands.config import Config
    >>> config = Config()
    >>> start_cpu_mining(config)
    """

    client = rest_client.TwentyOneRestClient(cmd_config.TWO1_HOST,
                                             config.machine_auth,
                                             config.username)

    enonce1, enonce2_size, reward = set_payout_address(config, client)

    start_time = time.time()
    config.log(UxString.mining_start.format(config.username, reward))

    work = get_work(config, client)

    found_share = mine_work(work, enonce1=enonce1, enonce2_size=enonce2_size)

    paid_satoshis = save_work(client, found_share, config.username)

    end_time = time.time()
    duration = end_time - start_time

    config.log(
        UxString.mining_success.format(config.username, paid_satoshis, duration),
        fg="magenta")

    click.echo(UxString.mining_status)
    status.status_wallet(config, client)

    click.echo(UxString.mining_finish.format(
        click.style("21 status", bold=True), click.style("21 buy", bold=True)))

def set_payout_address(config, client):
    # set a new address from the HD wallet for payouts
    payout_address = config.wallet.current_address
    auth_resp = client.account_payout_address_post(payout_address)

    user_info = json.loads(auth_resp.text)
    enonce1_base64 = user_info["enonce1"]
    enonce1 = base64.decodebytes(enonce1_base64.encode())
    enonce2_size = user_info["enonce2_size"]
    reward = user_info["reward"]

    return enonce1, enonce2_size, reward

# Copied from: http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a
# -process-with-a-given-pid
def check_pid(pid):
    if pid < 0:
        return False
    if pid == 0:
        # According to "man 2 kill" PID 0 refers to every process
        # in the process group of the calling process.
        # On certain systems 0 is a valid PID but we have no way
        # to know that in a portable fashion.
        raise ValueError('invalid PID 0')
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        # There is a process but we don't have permissions
        return True
    except:
        raise

    return True

def get_work(config, client):
    try:
        work_msg = client.get_work()
    except ServerRequestError as e:
        if e.status_code == 404 or e.status_code == 403:
            click.echo(UxString.mining_limit_reached)
            raise MiningDisabledError(UxString.mining_limit_reached)
        else:
            raise e

    msg_factory = message_factory.SwirlMessageFactory()
    msg = base64.decodebytes(work_msg.content)
    work = msg_factory.read_object(msg)
    return work

Share = namedtuple('Share', ['enonce2', 'nonce', 'otime', 'work_id'])
Work = namedtuple('Work', ['work_id', 'enonce2', 'cb'])

def mine_work(work_msg, enonce1, enonce2_size):
    pool_target = utils.bits_to_target(work_msg.bits_pool)
    for enonce2_num in range(0, 2 ** (enonce2_size * 8)):
        enonce2 = enonce2_num.to_bytes(enonce2_size, byteorder="big")

        cb_txn, _ = Transaction.from_bytes(
            work_msg.coinb1 + enonce1 + enonce2 + work_msg.coinb2)
        cb = CompactBlock(work_msg.height,
                          work_msg.version,
                          Hash(work_msg.prev_block_hash),
                          work_msg.ntime,
                          work_msg.nbits,  # lower difficulty work for testing
                          work_msg.merkle_edge,
                          cb_txn)

        row_counter = 0
        for nonce in range(0xffffffff):

            if nonce % 6e3 == 0:
                click.echo(click.style(u'█', fg='green'), nl=False)
                row_counter += 1
            if row_counter > 40:
                row_counter = 0
                click.echo("")

            cb.block_header.nonce = nonce
            h = cb.block_header.hash.to_int('little')
            if h < pool_target:
                share = Share(
                    enonce2=enonce2,
                    nonce=nonce,
                    work_id=work_msg.work_id,
                    otime=int(time.time()))
                # adds a new line at the end of progress bar
                click.echo("")
                return share

        click.echo("Exhausted enonce1 space. Changing enonce2")

def save_work(client, share, username):
    message_id = random.randint(1, 1e5)
    msg_factory = message_factory.SwirlMessageFactory()
    req_msg = msg_factory.create_submit_share_request(message_id=message_id,
                                                      work_id=share.work_id,
                                                      enonce2=share.enonce2,
                                                      otime=share.otime,
                                                      nonce=share.nonce)

    payment_result = client.send_work(data=req_msg)
    payment_details = json.loads(payment_result.text)
    amount = payment_details["amount"]
    return amount


from __future__ import print_function
import fitsio
import scipy as sp
import healpy
import glob
import sys
import time
import os.path

from picca.data_lens import forest # SY 7/2/19
from picca.data_lens import delta # SY 7/2/19
from picca.data_lens import qso # SY 7/2/19

from picca.prep_Pk1D import exp_diff
from picca.prep_Pk1D import spectral_resolution
from picca.prep_Pk1D import spectral_resolution_desi

def read_dlas(fdla):
    f=open(os.path.expandvars(fdla))
    dlas={}
    nb_dla = 0
    col_names=None
    for l in f:
        l = l.split()
        if len(l)==0:continue
        if l[0][0]=="#":continue
        if l[0]=="ThingID":
            col_names = l
            continue
        if l[0][0]=="-":continue
        thid = int(l[col_names.index("ThingID")])
        if thid not in dlas:
            dlas[thid]=[]
        zabs = float(l[col_names.index("z_abs")])
        nhi = float(l[col_names.index("NHI")])
        dlas[thid].append((zabs,nhi))
        nb_dla += 1

    print("")
    print(" In catalog: {} DLAs".format(nb_dla) )
    print(" In catalog: {} forests have a DLA".format(len(dlas)) )
    print("")

    return dlas

def read_absorbers(file_absorbers):
    f=open(file_absorbers)
    absorbers={}
    nb_absorbers = 0
    col_names=None
    for l in f:
        l = l.split()
        if len(l)==0:continue
        if l[0][0]=="#":continue
        if l[0]=="ThingID":
            col_names = l
            continue
        if l[0][0]=="-":continue
        thid = int(l[col_names.index("ThingID")])
        if thid not in absorbers:
            absorbers[thid]=[]
        lambda_absorber = float(l[col_names.index("lambda")])
        absorbers[thid].append(lambda_absorber)
        nb_absorbers += 1
    f.close()

    print("")
    print(" In catalog: {} absorbers".format(nb_absorbers) )
    print(" In catalog: {} forests have absorbers".format(len(absorbers)) )
    print("")

    return absorbers

def read_drq(drq,zmin,zmax,keep_bal,bi_max=None):
    vac = fitsio.FITS(drq)

    ## Redshift
    try:
        zqso = vac[1]["Z"][:]
    except:
        sys.stderr.write("Z not found (new DRQ >= DRQ14 style), using Z_VI (DRQ <= DRQ12)\n")
        zqso = vac[1]["Z_VI"][:]

    ## Info of the primary observation
    thid  = vac[1]["THING_ID"][:]
    ra    = vac[1]["RA"][:].astype('float64')
    dec   = vac[1]["DEC"][:].astype('float64')
    plate = vac[1]["PLATE"][:]
    mjd   = vac[1]["MJD"][:]
    fid   = vac[1]["FIBERID"][:]
    z     = vac[1]["Z"][:]                       #SY 8/2/19
    ra0   = vac[1]["RA0"][:].astype('float64')   #SY 8/2/19
    dec0   = vac[1]["DEC0"][:].astype('float64') #SY 8/2/19

    print("")
    ## Sanity
    print(" start               : nb object in cat = {}".format(ra.size) )
    w = (thid>0)
    print(" and thid>0          : nb object in cat = {}".format(ra[w].size) )
    w = w & (ra!=dec)
    print(" and ra!=dec         : nb object in cat = {}".format(ra[w].size) )
    w = w & (ra!=0.)
    print(" and ra!=0.          : nb object in cat = {}".format(ra[w].size) )
    w = w & (dec!=0.)
    print(" and dec!=0.         : nb object in cat = {}".format(ra[w].size) )
    w = w & (zqso>0.)
    print(" and z>0.            : nb object in cat = {}".format(ra[w].size) )

    ## Redshift range
    if not zmin is None:
        w &= zqso>zmin
        print(" and z>zmin          : nb object in cat = {}".format(ra[w].size) )
    if not zmax is None:
        w &= zqso<zmax
        print(" and z<zmax          : nb object in cat = {}".format(ra[w].size) )

    ## BAL visual
    if not keep_bal and bi_max==None:
        try:
            bal_flag = vac[1]["BAL_FLAG_VI"][:]
            w = w & (bal_flag==0)
            print(" and BAL_FLAG_VI == 0  : nb object in cat = {}".format(ra[w].size) )
        except:
            sys.stderr.write("BAL_FLAG_VI not found\n")
    ## BAL CIV
    if bi_max is not None:
        try:
            bi = vac[1]["BI_CIV"][:]
            w = w & (bi<=bi_max)
            print(" and BI_CIV<=bi_max  : nb object in cat = {}".format(ra[w].size) )
        except:
            sys.stderr.write("--bi-max set but no BI_CIV field in vac")
            sys.exit(1)
    print("")

    ra    = ra[w]*sp.pi/180.
    dec   = dec[w]*sp.pi/180.
    zqso  = zqso[w]
    thid  = thid[w]
    plate = plate[w]
    mjd   = mjd[w]
    fid   = fid[w]
    ra0   = ra0[w]*sp.pi/180.
    dec0  = dec0[w]*sp.pi/180.
    vac.close()

    return ra,dec,zqso,thid,plate,mjd,fid,dec0,ra0

target_mobj = 500
nside_min = 8

def read_data(in_dir,drq,mode,zmin = 2.1,zmax = 3.5,nspec=None,log=None,keep_bal=False,bi_max=None,order=1, best_obs=False, single_exp=False, pk1d=None):

    sys.stderr.write("mode: "+mode)
    ra,dec,zqso,thid,plate,mjd,fid = read_drq(drq,zmin,zmax,keep_bal,bi_max=bi_max)

    if nspec != None:
        ra = ra[:nspec]
        dec = dec[:nspec]
        zqso = zqso[:nspec]
        thid = thid[:nspec]
        plate = plate[:nspec]
        mjd = mjd[:nspec]
        fid = fid[:nspec]

    if mode == "pix":
        try:
            fin = in_dir + "/master.fits.gz"
            h = fitsio.FITS(fin)
        except IOError:
            try:
                fin = in_dir + "/master.fits"
                h = fitsio.FITS(fin)
            except IOError:
                try:
                    fin = in_dir + "/../master.fits"
                    h = fitsio.FITS(fin)
                except:
                    print("error reading master")
                    sys.exit(1)
        nside = h[1].read_header()['NSIDE']
        h.close()
        pixs = healpy.ang2pix(nside, sp.pi / 2 - dec, ra)
    elif mode in ["spec","corrected-spec","spcframe","spplate","spec-mock-1D"]:
        nside = 256
        pixs = healpy.ang2pix(nside, sp.pi / 2 - dec, ra)
        mobj = sp.bincount(pixs).sum()/len(sp.unique(pixs))

        ## determine nside such that there are 1000 objs per pixel on average
        sys.stderr.write("determining nside\n")
        while mobj<target_mobj and nside >= nside_min:
            nside //= 2
            pixs = healpy.ang2pix(nside, sp.pi / 2 - dec, ra)
            mobj = sp.bincount(pixs).sum()/len(sp.unique(pixs))
        sys.stderr.write("nside = {} -- mean #obj per pixel = {}\n".format(nside,mobj))
        if log is not None:
            log.write("nside = {} -- mean #obj per pixel = {}\n".format(nside,mobj))

    elif mode=="desi":
        nside = 8
        sys.stderr.write("Found {} qsos\n".format(len(zqso)))
        data = read_from_desi(nside,in_dir,thid,ra,dec,zqso,plate,mjd,fid,order)
        return data,len(data),nside,"RING"

    else:
        sys.stderr.write("I don't know mode: {}".format(mode))
        sys.exit(1)

    data ={}
    ndata = 0

    if mode=="spcframe":
        pix_data = read_from_spcframe(in_dir,thid, ra, dec, zqso, plate, mjd, fid, order, mode=mode, log=log, best_obs=best_obs, single_exp=single_exp)
        ra = [d.ra for d in pix_data]
        ra = sp.array(ra)
        dec = [d.dec for d in pix_data]
        dec = sp.array(dec)
        pixs = healpy.ang2pix(nside, sp.pi / 2 - dec, ra)
        for i, p in enumerate(pixs):
            if p not in data:
                data[p] = []
            data[p].append(pix_data[i])

        return data, len(pixs),nside,"RING"

    if mode=="spplate":
        pix_data = read_from_spplate(in_dir,thid, ra, dec, zqso, plate, mjd, fid, order, log=log, best_obs=best_obs)
        ra = [d.ra for d in pix_data]
        ra = sp.array(ra)
        dec = [d.dec for d in pix_data]
        dec = sp.array(dec)
        pixs = healpy.ang2pix(nside, sp.pi / 2 - dec, ra)
        for i, p in enumerate(pixs):
            if p not in data:
                data[p] = []
            data[p].append(pix_data[i])

        return data, len(pixs), nside, "RING"

    upix = sp.unique(pixs)

    for i, pix in enumerate(upix):
        w = pixs == pix
        ## read all hiz qsos
        if mode == "pix":
            t0 = time.time()
            pix_data = read_from_pix(in_dir,pix,thid[w], ra[w], dec[w], zqso[w], plate[w], mjd[w], fid[w], order, log=log)
            read_time=time.time()-t0
        elif mode == "spec" or mode =="corrected-spec":
            t0 = time.time()
            pix_data = read_from_spec(in_dir,thid[w], ra[w], dec[w], zqso[w], plate[w], mjd[w], fid[w], order, mode=mode,log=log, pk1d=pk1d)
            read_time=time.time()-t0
        elif mode == "spec-mock-1D":
            t0 = time.time()
            pix_data = read_from_mock_1D(in_dir,thid[w], ra[w], dec[w], zqso[w], plate[w], mjd[w], fid[w], order, mode=mode,log=log)
            read_time=time.time()-t0
        if not pix_data is None:
            sys.stderr.write("{} read from pix {}, {} {} in {} secs per spectrum\n".format(len(pix_data),pix,i,len(upix),read_time/(len(pix_data)+1e-3)))
        if not pix_data is None and len(pix_data)>0:
            data[pix] = pix_data
            ndata += len(pix_data)

    return data,ndata,nside,"RING"

def read_from_spec(in_dir,thid,ra,dec,zqso,plate,mjd,fid,order,mode,log=None,pk1d=None):
    pix_data = []
    for t,r,d,z,p,m,f in zip(thid,ra,dec,zqso,plate,mjd,fid):
        try:
            fid = str(f)
            if f<10:
                fid='0'+fid
            if f<100:
                fid = '0'+fid
            if f<1000:
                fid = '0'+fid
            fin = in_dir + "/{}/{}-{}-{}-{}.fits".format(p,mode,p,m,fid)
            h = fitsio.FITS(fin)
        except IOError:
            log.write("error reading {}\n".format(fin))
            continue

        log.write("{} read\n".format(fin))
        ll = h[1]["loglam"][:]
        fl = h[1]["flux"][:]
        iv = h[1]["ivar"][:]*(h[1]["and_mask"][:]==0)

        if(pk1d is not None) :
            # compute difference between exposure
            diff = exp_diff(h,ll)
            # compute spectral resolution
            wdisp =  h[1]["wdisp"][:]
            reso = spectral_resolution(wdisp,True,f,ll)
            d = forest(ll,fl,iv, t, r, d, z, p, m, f,order,diff,reso)
        else :
            d = forest(ll,fl,iv, t, r, d, z, p, m, f,order)

        pix_data.append(d)
        h.close()
    return pix_data

def read_from_mock_1D(in_dir,thid,ra,dec,zqso,plate,mjd,fid, order,mode,log=None):
    pix_data = []

    try:
        fin = in_dir
        hdu = fitsio.FITS(fin)
    except IOError:
        log.write("error reading {}\n".format(fin))

    for t,r,d,z,p,m,f in zip(thid,ra,dec,zqso,plate,mjd,fid):
        h = hdu["{}".format(t)]
        log.write("file: {} hdu {} read  \n".format(fin,h))
        lamb = h["wavelength"][:]
        ll = sp.log10(lamb)
        fl = h["flux"][:]
        error =h["error"][:]
        iv = 1.0/error**2

        # compute difference between exposure
        diff = sp.zeros(len(lamb))
        # compute spectral resolution
        wdisp =  h["psf"][:]
        reso = spectral_resolution(wdisp)

        # compute the mean expected flux
        f_mean_tr = h.read_header()["MEANFLUX"]
        cont = h["continuum"][:]
        mef = f_mean_tr * cont
        d = forest(ll,fl,iv, t, r, d, z, p, m, f,order, diff,reso, mef)
        pix_data.append(d)

    hdu.close()

    return pix_data

def read_from_pix(in_dir,pix,thid,ra,dec,zqso,plate,mjd,fid,order,log=None):
        try:
            fin = in_dir + "/pix_{}.fits.gz".format(pix)
            h = fitsio.FITS(fin)
        except IOError:
            try:
                fin = in_dir + "/pix_{}.fits".format(pix)
                h = fitsio.FITS(fin)
            except IOError:
                print("error reading {}".format(pix))
                return None

        ## fill log
        if log is not None:
            for t in thid:
                if t not in h[0][:]:
                    log.write("{} missing from pixel {}\n".format(t,pix))
                    sys.stderr.write("{} missing from pixel {}\n".format(t,pix))

        pix_data=[]
        thid_list=list(h[0][:])
        thid2idx = {t:thid_list.index(t) for t in thid if t in thid_list}
        loglam  = h[1][:]
        flux = h[2].read()
        ivar = h[3].read()
        andmask = h[4].read()
        for (t, r, d, z, p, m, f) in zip(thid, ra, dec, zqso, plate, mjd, fid):
            try:
                idx = thid2idx[t]
            except:
                ## fill log
                if log is not None:
                    log.write("{} missing from pixel {}\n".format(t,pix))
                sys.stderr.write("{} missing from pixel {}\n".format(t,pix))
                continue
            d = forest(loglam,flux[:,idx],ivar[:,idx]*(andmask[:,idx]==0), t, r, d, z, p, m, f,order)

            if log is not None:
                log.write("{} read\n".format(t))
            pix_data.append(d)
        h.close()
        return pix_data

def read_from_spcframe(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, mode=None, log=None, best_obs=False, single_exp = False):
    pix_data={}
    plates = sp.unique(plate)
    print("reading {} plates".format(len(plates)))

    for p in plates:
        wplate = plate==p
        plate_mjd = "{}-*".format(p)

        ##if best_obs then select only the given mjd
        if best_obs:
            the_mjd = sp.unique(mjd[wplate])
            print(the_mjd)
            m = the_mjd[0]
            plate_mjd = "{}-{}".format(p, m)

        ## find out exposures from all the spPlates
        fi = in_dir+"/{}/spPlate-{}.fits".format(p, plate_mjd)
        print(fi)
        fi = glob.glob(fi)
        fi = sorted(fi)
        exps = []
        for f in fi:
            print("INFO: reading plate {}".format(f))
            h=fitsio.FITS(f)
            head = h[0].read_header()
            iexp = 1
            for c in ["B1", "B2", "R1", "R2"]:
                card = "NEXP_{}".format(c)
                if card in head:
                    nexp = head["NEXP_{}".format(c)]
                else:
                    continue
                for _ in range(nexp):
                    str_iexp = str(iexp)
                    if iexp<10:
                        str_iexp = '0'+str_iexp

                    card = "EXPID"+str_iexp
                    if not card in head:
                        continue

                    exps.append(head["EXPID"+str_iexp][:11])
                    iexp += 1

        print("INFO: found {} exposures in plate {}".format(len(exps), p))

        if len(exps) == 0:
            continue

        exp_num = [e[3:] for e in exps]
        exp_num = sp.unique(exp_num)
        sp.random.shuffle(exp_num)
        exp_num = exp_num[0]
        for exp in exps:
            if single_exp:
                if not exp_num in exp:
                    continue
            t0 = time.time()
            ## find the spectrograph number:
            spectro = int(exp[1])
            assert spectro == 1 or spectro == 2

            ## find out the fibers where the qsos are:
            if spectro == 1:
                wfib = wplate & (fid <= 500)
            if spectro == 2:
                wfib = wplate & (fid > 500)

            if wfib.sum()==0:
                continue

            spcframe = fitsio.FITS(in_dir+"/{}/spCFrame-{}.fits".format(p, exp))

            flux = spcframe[0].read()
            ivar = spcframe[1].read()*(spcframe[2].read()==0)
            llam = spcframe[3].read()

            ## now convert all those fluxes into forest objects
            for index, (t, r, d, z, p, m, f) in enumerate(zip(thid[wfib], ra[wfib], dec[wfib], zqso[wfib], plate[wfib], mjd[wfib], fid[wfib])):
                index =(f-1)%500
                d = forest(llam[index],flux[index],ivar[index], t, r, d, z, p, m, f, order)
                if t in pix_data:
                    pix_data[t] += d
                else:
                    pix_data[t] = d
                if log is not None:
                    log.write("{} read from exp {} and mjd {}\n".format(t, exp, m))

            print("INFO: read {} from {} in {} per spec. Progress: {} of {} \n".format(wfib.sum(), exp, (time.time()-t0)/(wfib.sum()+1e-3), len(pix_data), len(thid)))
            spcframe.close()

    data = list(pix_data.values())
    return data

def read_from_spplate(in_dir, thid, ra, dec, zqso, plate, mjd, fid, order, log=None, best_obs=False):
    pix_data={}
    unique_plates = sp.unique(plate)
    print("reading {} plates".format(len(unique_plates)))

    for p in unique_plates:
        wplate = plate==p
        plate_mjd = "{}-*".format(p)
        mjd_in_plate = sp.unique(mjd[wplate])

        spplates = glob.glob(in_dir+"/{}/spPlate-{}.fits".format(p, plate_mjd))

        mjds_found = sp.array([spfile.split("-")[-1].replace(".fits",'') for spfile in spplates]).astype(int)
        wmissing = ~sp.in1d(mjd_in_plate, mjds_found)
        if wmissing.sum()>0:
            for m in mjd_in_plate[wmissing]:
                print("INFO: can't find spplate {} {}".format(p,m))
                if log is not None:
                    log.write("INFO: can't find spplate {} {}\n".format(p,m))

        for spplate in spplates:
            h = fitsio.FITS(spplate)
            head0 = h[0].read_header()
            MJD = head0["MJD"]

            t0 = time.time()

            wfib = wplate
            if best_obs:
                ## select only the objects which have specified mjd within this plate
                wmjd = mjd == MJD
                wfib = wplate & wmjd

            coeff0 = head0["COEFF0"]
            coeff1 = head0["COEFF1"]

            flux = h[0].read()
            ivar = h[1].read()*(h[2].read()==0)
            llam = coeff0 + coeff1*sp.arange(flux.shape[1])

            ## now convert all those fluxes into forest objects
            for (t, r, d, z, p, m, f) in zip(thid[wfib], ra[wfib], dec[wfib], zqso[wfib], plate[wfib], mjd[wfib], fid[wfib]):
                index = f-1
                d = forest(llam,flux[index],ivar[index], t, r, d, z, p, m, f, order)
                if t in pix_data:
                    pix_data[t] += d
                else:
                    pix_data[t] = d
                if log is not None:
                    log.write("{} read from file {} and mjd {}\n".format(t, spplate, m))

            print("INFO: read {} from {} in {} per spec. Progress: {} of {} \n".format(wfib.sum(), os.path.basename(spplate), (time.time()-t0)/(wfib.sum()+1e-3), len(pix_data), len(thid)))
            h.close()

    data = list(pix_data.values())
    return data

def read_from_desi(nside,in_dir,thid,ra,dec,zqso,plate,mjd,fid,order):

    in_nside = int(in_dir.split('spectra-')[-1].replace('/',''))
    nest = True
    data = {}
    ndata = 0

    ztable = {t:z for t,z in zip(thid,zqso)}
    in_pixs = healpy.ang2pix(in_nside, sp.pi/2.-dec, ra,nest=nest)
    fi = sp.unique(in_pixs)

    for i,f in enumerate(fi):
        path = in_dir+"/"+str(int(f/100))+"/"+str(f)+"/spectra-"+str(in_nside)+"-"+str(f)+".fits"

        sys.stderr.write("\rread {} of {}. ndata: {}".format(i,len(fi),ndata))
        try:
            h = fitsio.FITS(path)
        except IOError:
            sys.stderr.write("Error reading pix {}\n".format(f))
            continue

        ## get the quasars
        tid_qsos = thid[(in_pixs==f)]
        ra = h["FIBERMAP"]["RA_TARGET"][:]*sp.pi/180.
        de = h["FIBERMAP"]["DEC_TARGET"][:]*sp.pi/180.
        pixs = healpy.ang2pix(nside, sp.pi / 2 - de, ra)
        exp = h["FIBERMAP"]["EXPID"][:]
        night = h["FIBERMAP"]["NIGHT"][:]
        fib = h["FIBERMAP"]["FIBER"][:]

        b_ll = sp.log10(h["B_WAVELENGTH"].read())
        b_iv = h["B_IVAR"].read()*(h["B_MASK"].read()==0)
        b_fl = h["B_FLUX"].read()
        r_ll = sp.log10(h["R_WAVELENGTH"].read())
        r_iv = h["R_IVAR"].read()*(h["R_MASK"].read()==0)
        r_fl = h["R_FLUX"].read()
        z_ll = sp.log10(h["Z_WAVELENGTH"].read())
        z_iv = h["Z_IVAR"].read()*(h["Z_MASK"].read()==0)
        z_fl = h["Z_FLUX"].read()
        b_reso = h["B_RESOLUTION"].read()
        r_reso = h["R_RESOLUTION"].read()
        z_reso = h["Z_RESOLUTION"].read()

        for t in tid_qsos:
            wt = h[1]["TARGETID"][:] == t
            if wt.sum()==0:
                sys.stderr.write("\nError reading thingid {}\n".format(t))
                continue
            ### B
            iv = b_iv[wt]
            fl = (iv*b_fl[wt]).sum(axis=0)
            iv = iv.sum(axis=0)
            w = iv>0
            fl[w]/=iv[w]
            reso_sum = b_reso[wt].sum(axis=0)
            reso_in_km_per_s=spectral_resolution_desi(reso_sum,b_ll)
            diff = sp.zeros(b_ll.shape)
            d  = forest(b_ll,fl,iv,t,ra[wt][0],de[wt][0],ztable[t],exp[wt][0],night[wt][0],fib[wt][0],order,diff,reso_in_km_per_s)
            ### R
            iv = r_iv[wt]
            fl = (iv*r_fl[wt]).sum(axis=0)
            iv = iv.sum(axis=0)
            w = iv>0
            fl[w]/=iv[w]
            reso_sum = r_reso[wt].sum(axis=0)
            reso_in_km_per_s=spectral_resolution_desi(reso_sum,r_ll)
            diff = sp.zeros(r_ll.shape)
            d += forest(r_ll,fl,iv,t,ra[wt][0],de[wt][0],ztable[t],exp[wt][0],night[wt][0],fib[wt][0],order,diff,reso_in_km_per_s)
            ### Z
            iv = z_iv[wt]
            fl = (iv*z_fl[wt]).sum(axis=0)
            iv = iv.sum(axis=0)
            w = iv>0
            fl[w]/=iv[w]
            reso_sum = z_reso[wt].sum(axis=0)
            reso_in_km_per_s=spectral_resolution_desi(reso_sum,z_ll)
            diff = sp.zeros(z_ll.shape)
            d += forest(z_ll,fl,iv,t,ra[wt][0],de[wt][0],ztable[t],exp[wt][0],night[wt][0],fib[wt][0],order,diff,reso_in_km_per_s)

            pix = pixs[wt][0]
            if pix not in data:
                data[pix]=[]
            data[pix].append(d)
            ndata+=1

    sys.stderr.write("found {} quasars in input files\n".format(ndata))

    return data

def read_deltas(indir,nside,lambda_abs,alpha,zref,cosmo,nspec=None,no_project=False,from_image=None):
    '''
    reads deltas from indir
    fills the fields delta.z and multiplies the weights by (1+z)^(alpha-1)/(1+zref)^(alpha-1)
    returns data,zmin_pix
    '''

    fi = []
    if from_image is None or len(from_image)==0:
        if len(indir)>8 and indir[-8:]=='.fits.gz':
            fi += glob.glob(indir)
        elif len(indir)>5 and indir[-5:]=='.fits':
            fi += glob.glob(indir)
        else:
            fi += glob.glob(indir+'/*.fits') + glob.glob(indir+'/*.fits.gz')
    else:
        for arg in from_image:
            if len(arg)>8 and arg[-8:]=='.fits.gz':
                fi += glob.glob(arg)
            elif len(arg)>5 and arg[-5:]=='.fits':
                fi += glob.glob(arg)
            else:
                fi += glob.glob(arg+'/*.fits') + glob.glob(arg+'/*.fits.gz')
    fi = sorted(fi)

    dels = []
    ndata = 0
    for i,f in enumerate(fi):
        sys.stderr.write("\rread {} of {} {}".format(i,len(fi),ndata))
        if from_image is None:
            hdus = fitsio.FITS(f)
            dels += [delta.from_fitsio(h) for h in hdus[1:]]
            hdus.close()
        else:
            dels += delta.from_image(f)

        ndata = len(dels)
        if not nspec is None:
            if ndata>nspec:break

    sys.stderr.write("\n")

    phi = [d.ra for d in dels]
    th = [sp.pi/2.-d.dec for d in dels]
    pix = healpy.ang2pix(nside,th,phi)
    if pix.size==0:
        raise AssertionError()

    data = {}
    zmin = 10**dels[0].ll[0]/lambda_abs-1.
    zmax = 0.
    for d,p in zip(dels,pix):
        if not p in data:
            data[p]=[]
        data[p].append(d)

        z = 10**d.ll/lambda_abs-1.
        zmin = min(zmin,z.min())
        zmax = max(zmax,z.max())
        d.z = z
        if not cosmo is None: d.r_comov = cosmo.r_comoving(z)
        d.we *= ((1+z)/(1+zref))**(alpha-1)

        if not no_project:
            d.project()

    return data,ndata,zmin,zmax

def read_objects(drq,nside,zmin,zmax,alpha,zref,cosmo,keep_bal=True):
    objs = {}
    ra,dec,zqso,thid,plate,mjd,fid,dec_delens,ra_delens = read_drq(drq,zmin,zmax,keep_bal=True) #SY 8/2/19
    phi = ra
    th = sp.pi/2.-dec
    pix = healpy.ang2pix(nside,th,phi)
    if pix.size==0:
        raise AssertionError()
    print("reading qsos")

    upix = sp.unique(pix)
    for i,ipix in enumerate(upix):
        sys.stderr.write("\r{} of {}".format(i,len(upix)))
        w=pix==ipix
        objs[ipix] = [qso(t,r,d,z,p,m,f,dec_delens=d0,ra_delens=r0) for t,r,d,z,p,m,f,d0,r0 in zip(thid[w],ra[w],dec[w],zqso[w],plate[w],mjd[w],fid[w],dec_delens[w],ra_delens[w])] #SY 9/2/19
        for q in objs[ipix]:
            q.we = ((1.+q.zqso)/(1.+zref))**(alpha-1.)
            q.r_comov = cosmo.r_comoving(q.zqso)

    sys.stderr.write("\n")

    return objs,zqso.min()


# import libraries
import os
import sys
import itertools
from ROOT import *
from AtlasStyle import *

# make pyROOT not display the silly canvases
gROOT.SetBatch(True)

# use official ATLAS style
SetAtlasStyle();

gr_obs = TGraph("mA_vs_mZ_obs.dat", "%lg %lg", " \t,;")
gr_obs.SetLineWidth(4)
gr_obs.SetLineColor(kRed + 2)
gr_obs.SetMarkerStyle(20)
gr_obs.SetMarkerColor(kBlue)

gr_exp = TGraph("mA_vs_mZ_exp.dat", "%lg %lg", " \t,;")
gr_exp.SetLineStyle(7)
gr_exp.SetLineWidth(4)
gr_exp.SetLineColor(kBlue)

gr_exp_lo_err = TGraph("mA_vs_mZ_exp_lo_err.dat", "%lg %lg", " \t,;")
gr_exp_lo_err.SetFillStyle(1001)
gr_exp_lo_err.SetFillColor(kOrange)

gr_exp_hi_err = TGraph("mA_vs_mZ_exp_hi_err.dat", "%lg %lg", " \t,;")
gr_exp_hi_err.SetFillColor(0)

l = TLegend(0.67, 0.77, 0.92, 0.92, "")
l.SetFillStyle(0)
l.SetBorderSize(0)
l.SetTextSize(0.04)
l.SetTextFont(42)
l.AddEntry(gr_obs, "Observed Limits", "lp")
l.AddEntry(gr_exp, "Expected Limits", "l")
l.AddEntry(gr_exp_lo_err, "Expected #pm1#sigma", "f")

c = TCanvas("c", "c", 800, 600)

gr_exp_lo_err.GetXaxis().SetTitle("m_{Z'} [GeV]")
gr_exp_lo_err.GetYaxis().SetTitle("m_{A} [GeV]")
gr_exp_lo_err.GetXaxis().SetLimits(600, 1400)
gr_exp_lo_err.GetYaxis().SetRangeUser(300, 800)

gr_exp_lo_err.Draw("af")
gr_exp_hi_err.Draw("f")
gr_obs.Draw("lp")
gr_exp.Draw("l")
l.Draw("same")
gPad.RedrawAxis()

# draw additional text
ATLASLabel(0.20, 0.88, "Internal", 1)
additional_text = TLatex()
additional_text.SetNDC()
additional_text.SetTextColor(1)
additional_text.SetTextSize(0.04)
additional_text.DrawLatex(0.20, 0.83, "#sqrt{s} = 8 TeV, 20.3 fb^{-1}")
additional_text.DrawLatex(0.20, 0.78, "Z'-2HDM Simplified Model")
additional_text.DrawLatex(0.20, 0.73, "tan(#beta) = 1, m_{DM} = 100 GeV")

c.SaveAs("mA_vs_mZ.pdf")


#Finding a Spliced Motif
#solution for rosalind_sseq
#solved by yjkim

import os
os.chdir("C://Users/김용진/Documents/Git")

import itertools

#문제를 불러온다.
with open("rosalind_sseq.txt","r") as f :
  lines=f.readlines()

#빈 딕셔너리를 생성한다.
sequence={}

#for문으로 각 line을 읽어 첫 시작에 따라 key 값으로 설정할지, value 값으로 설정할지 정하여 딕셔너리를 완성한다.
for line in lines :
  if line[0]==">":
    value=""
    key=line.strip()
    continue
  else :
    value+=line.strip()
  sequence[key]=value

s=sequence[list(sequence.keys())[0]]
t=sequence[list(sequence.keys())[1]]

def get_all_index(t):
  result=[]
  for i in list(t):
    for j in re.finditer(i,s):
      result.append(j.start())
  return result

def get_spliced_index(all_index):
  result=[]
  for index in all_index:
    if len(result)==0:
      result.append(index)
    elif result[-1] < index-1 and s[result[-1]]!=s[index]:
      result.append(index)

  add_one=[]
  for i in result :
    add_one.append(str(i+1))
  return add_one

all_index=get_all_index(t)
result=get_spliced_index(all_index)

print(" ".join(result))


"""Script with functions to read the learning metrics from classifier outputs.
"""

import pandas
import os
import re

def learning_single_file(filename):
    result = pandas.read_csv(
        filename, sep='\t', header=None,
        names=['epoch', 'model_name', 'dev_score', 'test_score',
               'max_dev_score', 'max_test_score'])
    return result

def learning_filenames(dirname):
    return [filename for filename in os.listdir(dirname)
            if os.path.isfile(os.path.join(dirname, filename))
                and 'results' in filename]

def classifier_learning(classifier_dirpath, keys=None):
    """Reads all learning files in classifier dirpath.

    Args:
        classifier_dirpath: [str] the full path to the directory with prediction
        files.
        keys: [dict] map from column name to a function that receives
        the prediction filename and returns the key value. This is use in case
        there are multiple partitions of the same datasets on the directory.

    Returns:
        A DataFrame with the following columns:
        - epoch: the epoch number
        - Metric Value
        - Dataset: test or dev
        - Classifier: name of the classifier's directory
        - Extra keys
    """
    result = []
    for index, result_file in enumerate(learning_filenames(classifier_dirpath)):
        learning_metrics = learning_single_file(
            os.path.join(classifier_dirpath, result_file)).drop(
            columns=['model_name', 'max_dev_score', 'max_test_score'])
        learning_metrics = learning_metrics.set_index(
            ['epoch']).stack().reset_index().rename(
            columns={0: 'Metric Value', 'level_1': 'Dataset'})
        learning_metrics['Dataset'] = learning_metrics['Dataset'].replace(
            to_replace='dev_score', value='dev').replace(
            to_replace='test_score', value='test')
        learning_metrics['Classifier'] = os.path.basename(
            os.path.normpath(classifier_dirpath))
        if keys is not None:
            for key_name, key_function in keys.items():
                learning_metrics.loc[:, key_name] = key_function(result_file)
        result.append(learning_metrics)
    return pandas.concat(result)

def architecture_learning(experiment_dirs, keys=None):
    """Reads learning metrics for all classifiers

    Args:
         classifier_dirpath: [str] the full path to the directory with prediction
         files.
         keys: [dict] map from column name to a function that receives
         the prediction filename and returns the key value. This is used when
         there are multiple partitions of the same datasets on the directory.

    Returns:
        A DataFrame with the following columns:
        - epoch: the epoch number
        - Metric Value
        - Dataset: test or dev
        - Classifier: name of the classifier's directory
        - Extra keys
    """
    result = []
    for classifier_path in experiment_dirs:
        result.append(classifier_learning(classifier_path, keys))
    return pandas.concat(result)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np
import tensorflow as tf
import pandas as pd
import argparse
import os
import time
import sys
import pwd
import csv
import re
import deepchem
import pickle
import dcCustom
from dcCustom.molnet.preset_hyper_parameters import hps
from dcCustom.molnet.run_benchmark_models import model_regression, model_classification
from dcCustom.molnet.check_availability import CheckFeaturizer, CheckSplit

def load_davis(featurizer = 'Weave', cross_validation=False, test=False, split='random',
  reload=True, K = 5, mode = 'regression', predict_cold = False, cold_drug=False,
  cold_target=False, split_warm=False, filter_threshold=0, prot_seq_dict=None,
  oversampled=False):
  # The last parameter means only splitting into training and validation sets.

  if cross_validation:
    assert not test

  if mode == 'regression' or mode == 'reg-threshold':
    mode = 'regression'
    tasks = ['davis']
    file_name = "restructured.csv"
  elif mode == 'classification':
    tasks = ['davis_bin']
    file_name = "restructured_bin.csv"

  data_dir = "davis_data/"
  if reload:
    delim = "/"
    if filter_threshold > 0:
      delim = "_filtered" + delim
    if predict_cold:
      delim = "_cold" + delim
    elif split_warm:
      delim = "_warm" + delim
    elif cold_drug:
      delim = "_cold_drug" + delim
    elif cold_target:
      delim = "_cold_target" + delim
    if cross_validation:
      delim = "_CV" + delim
      save_dir = os.path.join(data_dir, featurizer + delim + mode + "/" + split)
      loaded, all_dataset, transformers = dcCustom.utils.save.load_cv_dataset_from_disk(
          save_dir, K)
    else:
      save_dir = os.path.join(data_dir, featurizer + delim + mode + "/" + split)
      loaded, all_dataset, transformers = deepchem.utils.save.load_dataset_from_disk(
          save_dir)
    if loaded:
      return tasks, all_dataset, transformers

  dataset_file = os.path.join(data_dir, file_name)
  if featurizer == 'Weave':
    featurizer = dcCustom.feat.WeaveFeaturizer()
  elif featurizer == 'ECFP':
    featurizer = dcCustom.feat.CircularFingerprint(size=1024)
  elif featurizer == 'GraphConv':
    featurizer = dcCustom.feat.ConvMolFeaturizer()

  loader = dcCustom.data.CSVLoader(
      tasks = tasks, smiles_field="smiles", protein_field = "proteinName",
      source_field = 'protein_dataset', featurizer=featurizer, prot_seq_dict=prot_seq_dict)
  dataset = loader.featurize(dataset_file, shard_size=8192)

  if mode == 'regression':
    transformers = [
          dcCustom.trans.NormalizationTransformer(
              transform_y=True, dataset=dataset)
    ]
  elif mode == 'classification':
    transformers = [
        dcCustom.trans.BalancingTransformer(transform_w=True, dataset=dataset)
    ]

  print("About to transform data")
  for transformer in transformers:
    dataset = transformer.transform(dataset)

  splitters = {
      'index': deepchem.splits.IndexSplitter(),
      'random': dcCustom.splits.RandomSplitter(split_cold=predict_cold, cold_drug=cold_drug,
        cold_target=cold_target, split_warm=split_warm, prot_seq_dict=prot_seq_dict,
        threshold=filter_threshold),
      'scaffold': deepchem.splits.ScaffoldSplitter(),
      'butina': deepchem.splits.ButinaSplitter(),
      'task': deepchem.splits.TaskSplitter()
  }
  splitter = splitters[split]
  if test:
    train, valid, test = splitter.train_valid_test_split(dataset)
    all_dataset = (train, valid, test)
    if reload:
      deepchem.utils.save.save_dataset_to_disk(save_dir, train, valid, test,
                                               transformers)
  elif cross_validation:
    fold_datasets = splitter.k_fold_split(dataset, K)
    all_dataset = fold_datasets
    if reload:
      dcCustom.utils.save.save_cv_dataset_to_disk(save_dir, all_dataset, K, transformers)

  else:
    # not cross validating, and not testing.
    train, valid, test = splitter.train_valid_test_split(dataset, frac_valid=0.2,
      frac_test=0)
    all_dataset = (train, valid, test)
    if reload:
      deepchem.utils.save.save_dataset_to_disk(save_dir, train, valid, test,
                                               transformers)

  return tasks, all_dataset, transformers


import logging
import time
import random
from locust import User, events
from client import MilvusClient

url = 'http://192.168.1.238:19121'

class MilvusTask(object):
    def __init__(self, url):
        self.request_type = "http"
        self.m = MilvusClient(url)
        # logging.getLogger().info(id(self.m))

    def __getattr__(self, name):
        func = getattr(self.m, name)

        def wrapper(*args, **kwargs):
            start_time = time.time()
            try:
                result = func(*args, **kwargs)
                total_time = int((time.time() - start_time) * 1000)
                if result:
                    events.request_success.fire(request_type=self.request_type, name=name, response_time=total_time,
                                                response_length=0)
                else:
                    events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time,
                                            exception=e, response_length=0)

            except Exception as e:
                total_time = int((time.time() - start_time) * 1000)
                events.request_failure.fire(request_type=self.request_type, name=name, response_time=total_time,
                                            exception=e, response_length=0)

        return wrapper


##########################
# Import relevant modules
##########################
import ROOT
import numpy as np
from glob import glob
import time
from array import array
import sys

##########################
# Additional tools
##########################

# prints out a progressbar which will be flushed at the terminal
def progressbar(count, total, status=''):
        sys.stdout.flush()
        bar_len = 60
        filled_len = int(round(bar_len * count / float(total)))
        percents = round(100.0 * count / float(total), 1)
        bar = '=' * (filled_len-1) + '>' +'-' * (bar_len - filled_len)
        sys.stdout.write('[%s] %s%s ...%s\r' % (bar, percents, '%', status))

def showlumi(title):
    latex = ROOT.TLatex()
    latex.SetNDC()
    latex.SetTextAlign(31)
    latex.SetTextFont(42)
    latex.SetTextSize(0.04)
    latex.DrawLatex(0.83,0.94,title)
    # 'xxx fb^{-1} (xxx TeV)'

def showlogo(text):
    logo = ROOT.TLatex()
    logo.SetNDC()
    logo.SetTextAlign(11)
    logo.SetTextFont(61)
    logo.SetTextSize(0.045)
    logo.DrawLatex(0.15,0.94,text)

def showTitle(text):
    logo = ROOT.TLatex()
    logo.SetNDC()
    logo.SetTextAlign(31)
    logo.SetTextFont(42)
    logo.SetTextSize(0.04)
    logo.DrawLatex(0.81,0.87,text)

def showlogopreliminary():
    logo = ROOT.TLatex()
    logo.SetNDC()
    logo.SetTextAlign(11)
    logo.SetTextFont(61)
    logo.SetTextSize(0.045)
    logo.DrawLatex(0.15,0.94,'CMS')

    preliminary = ROOT.TLatex()
    preliminary.SetNDC()
    preliminary.SetTextAlign(11)
    preliminary.SetTextFont(52)
    preliminary.SetTextSize(0.038)
    preliminary.DrawLatex(0.24,0.94,'Preliminary')

def showlogoprelimsim(text):
    logo = ROOT.TLatex()
    logo.SetNDC()
    logo.SetTextAlign(11)
    logo.SetTextFont(61)
    logo.SetTextSize(0.045)
    logo.DrawLatex(0.15,0.94,text)

    preliminary = ROOT.TLatex()
    preliminary.SetNDC()
    preliminary.SetTextAlign(11)
    preliminary.SetTextFont(52)
    preliminary.SetTextSize(0.038)
    preliminary.DrawLatex(0.24,0.94,'Simulation Preliminary')

########################################
# Style settings (based on CMS TDRStyle)
########################################
def setpfstyle():
    pfstyle = ROOT.TStyle('pfstyle','pfstyle')

    pfstyle.SetOptStat(0)
    pfstyle.SetPalette(ROOT.kBird) # look up the color palette options in https://root.cern.ch/doc/master/classTColor.html

    # Canvas
    pfstyle.SetCanvasDefH(500)
    pfstyle.SetCanvasDefW(550)

    # Use plain black on white colors
    icol = 0
    pfstyle.SetFrameBorderMode(icol)
    pfstyle.SetCanvasBorderMode(icol)
    pfstyle.SetPadBorderMode(icol)
    pfstyle.SetPadColor(icol)
    pfstyle.SetCanvasColor(icol)
    pfstyle.SetStatColor(icol)

    # Set the paper & margin sizes
    pfstyle.SetPaperSize(20,26)
    pfstyle.SetPadTopMargin(0.08)
    pfstyle.SetPadRightMargin(0.17)
    pfstyle.SetPadBottomMargin(0.12)
    pfstyle.SetPadLeftMargin(0.15)

    # Use large fonts
    font = 42
    # font = 62
    tsize = 0.045
    pfstyle.SetTextFont(font)

    # Global Title Properties
    pfstyle.SetOptTitle(0)
    pfstyle.SetTitleFont(font)
    pfstyle.SetTitleSize(tsize)
    pfstyle.SetTitleBorderSize(0)
    pfstyle.SetTitleColor(1)
    pfstyle.SetTitleTextColor(1)
    pfstyle.SetTitleFillColor(0)
    pfstyle.SetTitleFontSize(tsize)
    pfstyle.SetTitleH(0.05)
    pfstyle.SetTitleW(0.)
    pfstyle.SetTitleStyle(1001)
    pfstyle.SetTitleAlign(13)

    # Axis Titles and Labels
    ROOT.TGaxis.SetMaxDigits(3)
    pfstyle.SetTextSize(tsize)
    pfstyle.SetLabelFont(font,"x")
    pfstyle.SetTitleFont(font,"x")
    pfstyle.SetLabelFont(font,"y")
    pfstyle.SetTitleFont(font,"y")
    pfstyle.SetLabelFont(font,"z")
    pfstyle.SetTitleFont(font,"z")

    pfstyle.SetLabelSize(tsize,"x")
    pfstyle.SetTitleSize(tsize,"x")
    pfstyle.SetLabelSize(tsize,"y")
    pfstyle.SetTitleSize(tsize,"y")
    pfstyle.SetLabelSize(tsize,"z")
    pfstyle.SetTitleSize(tsize,"z")

    pfstyle.SetTitleOffset(1.1,"x")
    pfstyle.SetTitleOffset(1.3,"y")
    pfstyle.SetTitleOffset(1.35,"z")

    pfstyle.SetMarkerStyle(20)
    pfstyle.SetMarkerSize(0.5)
    pfstyle.SetLineWidth(1)
    # pfstyle.SetHistLineWidth(2.)
    pfstyle.SetLineStyleString(2,'[12 12]') # postscript dashes

    # Draw horizontal and vertical grids
    pfstyle.SetPadGridX(ROOT.kTRUE)
    pfstyle.SetPadGridY(ROOT.kTRUE)
    pfstyle.SetGridStyle(3)
    pfstyle.SetPadTickX(1)
    pfstyle.SetPadTickY(1)

    # Legend
    pfstyle.SetLegendBorderSize(1)
    pfstyle.SetLegendFont(font)
    # pfstyle.SetFillColor(0) # White
    # pfstyle.SetfillStyle(4000) # Transparent

    #Statistics
    pfstyle.SetOptFit(111)
    pfstyle.SetStatX(.80)
    pfstyle.SetStatY(0.26)
    pfstyle.SetStatBorderSize(1)
    pfstyle.SetStatW(0.16)
    pfstyle.SetStatH(0.15)
    pfstyle.SetStatFont(font)
    pfstyle.SetStatFontSize(0.01)

    # When this static function is called with sumw2=kTRUE, all new histograms will automatically activate the storage of the sum of squares of errors
    pfstyle.SetPaintTextFormat('4.2f')
    ROOT.TH1.SetDefaultSumw2()

    ROOT.gROOT.SetStyle('pfstyle')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Jan 18 00:04:38 2020

@author: nancyscarlet
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
dataset=pd.read_csv('Salary_Data.csv')
dataset.head()
X=dataset.iloc[:,:-1].values
Y=dataset.iloc[:,1].values

from sklearn.cross_validation import train_test_split
X_train, X_test , Y_train, Y_test = train_test_split(X,Y,test_size=0.25,random_state=0)
#test_size = 0.25-> 20% is test and 80% is train

#Importing Linear regressor from scikit learn
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train,Y_train,sample_weight=None)

Y_predicted = regressor.predict(X_test)

#plt.scatter(X_train,Y_train,color='red')
plt.scatter(X_train,Y_train,color='red')
plt.plot(X_train,regressor.predict(X_train),color='blue')
plt.title('Salary based on experience')
plt.xlabel('Experience in Years')
plt.ylabel('Salary')
plt.show()

plt.scatter(X_test,Y_test,color='red')
plt.plot(X_train,regressor.predict(X_train),color='blue')
plt.title('Salary based on experience')
plt.xlabel('Experience in Years')
plt.ylabel('Salary')
plt.show()

Machine Learning A-Z Template Folder/Part 2 - Regression/Section 4 - Simple Linear Regression/Simple_Linear_Regression/practiceLinearRegression.py


inp=0
wish='B'
product=1
sum=0
while wish!='q':
    wish=input("enter q to quit or any other key to continue>>")
    if wish=='q':
        break
    else:
        inp=int(input("enter value"))
        if inp%2==0:
            sum=sum+inp
        else:
            product=product*inp
print("the sum of even numbers is >>", sum)
print("the product of odd numbers is >>", product)


"""
this code checks if all open parenthesis have closing pair on right side, uses loop, is almost the same as loop.py,
but have only one counter for both parenthesis

"""
str = '((((((((((((((2, 3)))))))))))))'
diff = 0

# go through each element and increase counter if find left parenthesis or decrease counter if right.
for x in str:
    if x == "(":
        diff += 1
    elif x == ")":
        diff -= 1

# recommend user add appropriate parenthesis
if diff > 0:
    print("you have to add {} parenthesis on the right side".format(abs(diff)))
elif diff < 0:
    print("you have to add {} parenthesis on the right side".format(abs(diff)))
else:
    print("its correct expression, amount of left and right parenthesis is equal")


import os
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import dates
import numpy as np
import numpy.ma as ma
from scipy.io import loadmat
import datetime
import pandas as pd
#import seawater as sw
import gsw

# Some infos:
mab_adcp = 1.58
mooring_depth = 83.0
adcpdir = 'up'

#XLIM = [pd.Timestamp('2010-03-01 15:30:00'), pd.Timestamp('2010-03-01 16:30:00')]
XLIM = [pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00')]
#fig_name = 'S1_outlook_wind.png'
fig_name = 'S1_outlook_meteo.png'

#### ---------- Meteo data --------- ####
meteo_dict = loadmat('SMHI_meteo_55570_201002_201003.mat',squeeze_me=True)
wave_dict = loadmat('SMHI_wave_33004_201002_201003.mat',squeeze_me=True)
wind_dir = { k: meteo_dict[k] for k in ['wind_dir']}
wind_mag = { k: meteo_dict[k] for k in ['wind_mag']}
wave_height = { k: wave_dict[k] for k in ['waveheight_max','waveheight_signif']}
wave_dir = { k: wave_dict[k] for k in ['wavedir_maxenergy']}
wave_period= { k: wave_dict[k] for k in ['waveperiod_mean']}
def matlab2datetime(matlab_datenum):
    day = datetime.datetime.fromordinal(int(matlab_datenum))
    dayfrac = datetime.timedelta(days=matlab_datenum%1) - datetime.timedelta(days = 366)
    return day + dayfrac

# convert Matlab time into list of python datetime objects and put in dictionary
wind_dir['date_time'] = [matlab2datetime(tval) for tval in meteo_dict['wind_matlabtime']]
wind_mag['date_time'] = [matlab2datetime(tval) for tval in meteo_dict['wind_matlabtime']]
wave_height['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]
wave_dir['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]
wave_period['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]

# Wind data from R/V Alkor
#Alkor_dict = loadmat('/media/cyrf0006/Fred32/IOW/IOWdata/AL351_DDL_wind.mat',squeeze_me=True, struct_as_record=False)
Alkor_dict = loadmat('./data/AL351_DDL_wind.mat',squeeze_me=True, struct_as_record=False)
Ualkor =  Alkor_dict['DDL'].wind.speed.data
yearday = Alkor_dict['DDL'].time.data+1
pdTimeAlkor = pd.to_datetime('2010') + pd.to_timedelta(yearday - 1, unit='d')
alkor = pd.DataFrame(Ualkor, index=pdTimeAlkor)
alkor = alkor.resample('30Min').mean()
# --------------------------------------#

#### ---------- IOW SBE Mcats --------- ####
sbe_dict = loadmat('/home/cyrf0006/research/IOW/iow_mooring/AL351_TSC2_S1_sbe_all.mat',squeeze_me=True)
Tmat = sbe_dict['Temperature']
Smat = sbe_dict['Salinity']
Dmat = sbe_dict['Density']
zVecSBE = sbe_dict['SBE_depth']
yearday = sbe_dict['SBE_decday']
pdTimeCats = pd.to_datetime('2010') + pd.to_timedelta(yearday - 1, unit='d')
TT = pd.DataFrame(Tmat, index=pdTimeCats, columns=zVecSBE) # now in m/s
S = pd.DataFrame(Smat, index=pdTimeCats, columns=zVecSBE)
D = pd.DataFrame(Dmat, index=pdTimeCats, columns=zVecSBE)

## NOTE: Use CTD cast from MSS to get profile after storm!!!
# --------------------------------------#

#### ---------- CTD cast --------- ####
# CTD casts before storm (pd is powerful!))
lat = 55
lon = 16
ctd_fname = './data/ctd/0001_01_bav_buo.cnv'
data = pd.read_csv(ctd_fname, header=None, delim_whitespace=True)
Zctd = data[:][7]
Tctd = data[:][1]
Sctd = data[:][5]

Ibtm = np.argmax(Zctd)
Pbin = np.arange(0.5, Zctd[Ibtm], 1)
digitized = np.digitize(Zctd[0:Ibtm], Pbin) #<- this is awesome!
Tbin = np.array([Tctd[0:Ibtm][digitized == i].mean() for i in range(0, len(Pbin))])
Sbin = np.array([Sctd[0:Ibtm][digitized == i].mean() for i in range(0, len(Pbin))])
Z = np.abs(gsw.z_from_p(Pbin,lat))
SA = gsw.SA_from_SP_Baltic(Sbin,lon,lat)
CT = gsw.CT_from_t(SA,Tbin,Pbin)
SIG0 = gsw.sigma0(SA,CT)
N2 = gsw.Nsquared(SA,CT,Pbin,lat)
# ------------------------------------#

#### --------- NIOZ Tsensors --------- ####
T_dict = loadmat('./data/Titp_S1_p4std_10minAve.mat',squeeze_me=True)
T = T_dict['Tbin'].T
zVecT = mooring_depth - np.array(T_dict['habVec'])
pdTimeT = pd.date_range('2010-02-28 11:07:00', '2010-03-04 07:37:00', freq='10min')

# Dataframe
T = pd.DataFrame(T, index=pdTimeT, columns=zVecT)
# --------------------------------------#

#### ---------- MSS cast --------- ####
# CTD casts before storm (pd is powerful!))
lat = 55
lon = 16
MSS_S1_1_dict = loadmat('./data/MSS_DATA/S1_1.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_1_dict['CTD'][2].P
Tmss =  MSS_S1_1_dict['CTD'][2].T
Smss =  MSS_S1_1_dict['CTD'][2].S
timemss1 = pd.Timestamp(MSS_S1_1_dict['STA'][2].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_01 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_01 = gsw.CT_from_t(SA_MSS_01,TTbin,Pbin)
SIG0_MSS_01 = gsw.sigma0(SA_MSS_01,CT_MSS_01)
N2_MSS_01 = gsw.Nsquared(SA_MSS_01,CT_MSS_01,Pbin,lat)
import SW_extras as swe
N2_01 = N2_MSS_01[0]
zN2_01 = N2_MSS_01[1]
N2_period_01 = 60.0/swe.cph(N2_01)

MSS_S1_2_dict = loadmat('./data/MSS_DATA/S1_2.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_2_dict['CTD'][0].P
Tmss =  MSS_S1_2_dict['CTD'][0].T
Smss =  MSS_S1_2_dict['CTD'][0].S
timemss2 = pd.Timestamp(MSS_S1_2_dict['STA'][0].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_02 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_02 = gsw.CT_from_t(SA_MSS_02,TTbin,Pbin)
SIG0_MSS_02 = gsw.sigma0(SA_MSS_02,CT_MSS_02)
N2_MSS_02 = gsw.Nsquared(SA_MSS_02,CT_MSS_02,Pbin,lat)
N2_02 = N2_MSS_02[0]
zN2_02 = N2_MSS_02[1]
N2_period_02 = 60.0/swe.cph(N2_02)

MSS_S1_5_dict = loadmat('./data/MSS_DATA/S1_5.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_5_dict['CTD'][-10].P
Tmss =  MSS_S1_5_dict['CTD'][-10].T
Smss =  MSS_S1_5_dict['CTD'][-10].S
timemss5 = pd.Timestamp(MSS_S1_5_dict['STA'][-10].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_05 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_05 = gsw.CT_from_t(SA_MSS_05,TTbin,Pbin)
SIG0_MSS_05 = gsw.sigma0(SA_MSS_05,CT_MSS_05)
N2_MSS_05 = gsw.Nsquared(SA_MSS_05,CT_MSS_05,Pbin,lat)
N2_05 = N2_MSS_05[0]
zN2_05 = N2_MSS_05[1]
N2_period_05 = 60.0/swe.cph(N2_05)
## # ------------------------------------#

#### --------- plots ---------- ####
days = dates.DayLocator()
hours = dates.HourLocator(interval=6)
dfmt = dates.DateFormatter('%d')
storm2 = pd.Timestamp('2010-03-01 12:00:00') # onset langmuir
zoomx = [pd.Timestamp('2010-03-01 11:00:00'), pd.Timestamp('2010-03-02 1:45:00')]
zoomy = [19,78]

fig = plt.figure(2)

# AX1 - Wind mag.
ax1 = plt.subplot2grid((7, 7), (0, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wind_mag)
df = df.set_index('date_time')
df.plot(ax=ax1, grid='on', legend=False)
alkor.plot(ax=ax1, grid='on', legend=False)
#ax1.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00 '))
ax1.set_xlim(XLIM)
ax1.set_ylim(0,20)
ax1.tick_params(labelbottom='off')
ax1.xaxis.label.set_visible(False)
ax1.xaxis.set_major_locator(days)
ax1.xaxis.set_major_formatter(dfmt)
ax1.xaxis.set_minor_locator(hours)
ax1.set_ylabel(r'$\overline{U} (m/s)$')
ax1.legend(['SMHI','ship'], bbox_to_anchor=(1., 1.), loc=2)
ax1.text(XLIM[0], 15, '  a', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [0, 20], '--k')
# add patch
import matplotlib.dates as mdates
start = mdates.date2num(zoomx[0])
end = mdates.date2num(zoomx[1])
rect_x = [start, end, end, start, start]
zoomy = [0, 20]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax1.add_patch(Rgon)

# AX2 - Wave heights
ax2 = plt.subplot2grid((7, 7), (1, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wave_height)
df = df.set_index('date_time')
df.plot(ax=ax2, grid='on')
#XTICKS = axes[0].get_xticks()
#ax2.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax2.set_xlim(XLIM)
ax2.set_ylim(0,7.5)
ax2.tick_params(labelbottom='off')
ax2.xaxis.label.set_visible(False)
ax2.set_ylabel('H(m)')
ax2.xaxis.set_major_locator(days)
ax2.xaxis.set_major_formatter(dfmt)
ax2.xaxis.set_minor_locator(hours)
ax2.legend([r'$\rm H_{max}$', r'$\rm H_{s}$'], bbox_to_anchor=(1., 1.), loc=2)
# bbox_to_anchor is origin; loc=1 is inside; bordaxesoad=0 means directly on the axis
ax2.text(XLIM[0], 6, '  b', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [0, 7.5], '--k')
# add patch
zoomy = [0, 7.5]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax2.add_patch(Rgon)

# AX3 - Wind/wave direction
ax3 = plt.subplot2grid((7, 7), (2, 0), rowspan=1, colspan=6)
df1 = pd.DataFrame(wind_dir)
df2= pd.DataFrame(wave_dir)
df1 = df1.set_index('date_time')
df2 = df2.set_index('date_time')
df1.plot(ax=ax3, grid='on', label="winddir", legend=True)
df2.plot(secondary_y=False, ax=ax3, grid='on', label="wavedir", legend=True)
#ax3.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax3.set_xlim(XLIM)
ax3.set_ylim(85,365)
ax3.set_ylabel(r'$\rm \theta$')
ax3.xaxis.label.set_visible(False)
ax3.tick_params(labelbottom='off')
#ax3.legend([r'$\rm \theta_{wind}$', r'$\rm \theta_{wave}$'], bbox_to_anchor=(-0.18, 0.8), loc=1, borderaxespad=0)
ax3.legend([r'$\rm wind$', r'$\rm wave$'], bbox_to_anchor=(1., 1.), loc=2)
ax3.xaxis.set_major_locator(days)
ax3.xaxis.set_major_formatter(dfmt)
ax3.xaxis.set_minor_locator(hours)
plt.yticks([90, 180, 270, 360], ['E', 'S', 'W', 'N'], rotation='horizontal')
ax3.text(XLIM[0], 340, '  c', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [85, 365], '--k')
# add patch
zoomy = [0, 360]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax3.add_patch(Rgon)

# AX4 - Wave period
ax4 = plt.subplot2grid((7, 7), (3, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wave_period)
df = df.set_index('date_time')
df.plot(ax=ax4, grid='on', legend=False)
#ax4.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax4.set_xlim(XLIM)
ax4.set_ylim(2,7)
ax4.xaxis.label.set_visible(False)
ax4.tick_params(labelbottom='off')
ax4.set_ylabel(r'$\rm \overline{T}_s (s)$')
ax4.xaxis.set_major_locator(days)
ax4.xaxis.set_major_formatter(dfmt)
ax4.xaxis.set_minor_locator(hours)
ax4.text(XLIM[0], 6, '  d', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [2, 7], '--k')
# add patch
zoomy = [0, 7]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax4.add_patch(Rgon)

# AX5 - Temperature
ax5 = plt.subplot2grid((7, 7), (4, 0), rowspan=3, colspan=6)
levels = np.linspace(0,10, 21)
levels2 = np.linspace(0,10, 11)
c = plt.contourf(T.index, T.columns, T.T, levels, cmap=plt.cm.RdBu_r)
#plt.plot([storm, storm], [53, 83], '--k')
plt.plot([timemss1, timemss1], [20, 83], '--')
plt.plot([timemss2, timemss2], [20, 83], '--')
plt.plot([timemss5, timemss5], [20, 83], '--')
SBE_time = pd.Timestamp('2010-03-02 00:00:00')
plt.plot([SBE_time, SBE_time], [20, 83], 'r--')
#plt.plot([storm2, storm2], [20, 83], '--k')
cax = plt.axes([0.88,0.08,0.02,0.3])
plt.colorbar(c, cax=cax)
ax5.set_xlim(XLIM[0], XLIM[1])
#ax5.set_ylim(1, 83)
ax5.set_ylim(53, 83)
ax5.tick_params(labelbottom='on')
ax5.tick_params(labelleft='on')
ax5.invert_yaxis()
ax5.xaxis.set_major_locator(days)
ax5.xaxis.set_major_formatter(dfmt)
ax5.xaxis.set_minor_locator(hours)
ax5.set_ylabel('Depth (m)')
ax5.text(XLIM[1], 54, r'  T($^{\circ}$C)', horizontalalignment='left', verticalalignment='center', fontsize=14, color='k')
ax5.text(XLIM[0], 55, '  e', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
# add patch
zoomy = [53, 83]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax5.add_patch(Rgon)

#### --------- Save Figure ------------- ####
fig.set_size_inches(w=8, h=10)
fig.set_dpi(300)
fig.tight_layout()
fig.savefig(fig_name)

#plt.show()