# This file contains code to aggregate data
import pandas as pd
from tqdm import tqdm
import glob
import dask.dataframe as dd
from datetime import datetime
import traceback
import os
import csv

#custom_date_parser = lambda x: datetime.strptime(x, "%a %b %d %H:%M:%S +0000 %Y")
custom_date_parser = lambda x: date_parser(x)

columns = ['id','created_at','text','source','tweet_type','source_tweet_id','retweeted_tweet_id',
           'quoted_tweet_id','quote_count','retweet_count','favorite_count','hashtags','most_unrolled_urls',
           'tweet_links','user_screen_name','user_id','user_mentions','user_followers_count',
           'user_following_count','user_listed_count','user_created_at','user_favourites_count','user_verified','user_statuses_count']

data_type_dict = {'id':str,'created_at':object,'text':str,'source':str,'tweet_type':str,'source_tweet_id':str,'retweeted_tweet_id':str,
           'quoted_tweet_id':str,'quote_count':object,'retweet_count':object,'favorite_count':object,'hashtags':str,'most_unrolled_urls':str,
           'tweet_links':str,'user_screen_name':str,'user_id':str,'user_mentions':str,'user_followers_count':object,
           'user_following_count':object,'user_listed_count':object,'user_created_at':object,'user_favourites_count':object,
           'user_verified':object,'user_statuses_count':object}
target = './output/data/*-json.csv'
files = glob.glob(target)

processed_files = []
files_to_process = [file for file in files]
unroccessed_files = [os.path.basename(file) for file in files]

def date_parser(str_date):
    try:
        #'Fri Dec 21 22:53:46 +0000 2012'
        custom_date = datetime.strptime(str_date, "%a %b %d %H:%M:%S +0000 %Y") if str_date else None
        #print(custom_date_parser)
        return custom_date
    except:
        return None

def process_file(file):
    base_name = os.path.basename(file)
    try:
        print('[[{}]] Starting to load....'.format(base_name))
        df = pd.read_csv(file,sep='\t', encoding='iso-8859-1', warn_bad_lines=True, error_bad_lines=False)
        # df = pd.read_csv(file,sep='\t', encoding='iso-8859-1', names=columns, warn_bad_lines=True, error_bad_lines=False, dtype=data_type_dict)
        print('[[{}]] Ended loading....{}'.format(base_name, df.shape))
        processed_files.append(base_name)
        unroccessed_files.remove(base_name)
        return df
    except Exception as e:
        print('[[{}]] Failed loading....{}'.format(base_name, str(e)))
        traceback.print_exc()

def process_file_in_loop(file, wf_good, wf_bad):
    base_name = os.path.basename(file)
    good =0
    bad = 0
    print('[[{}]] Starting to load....'.format(base_name))
    try:
        with open(file, 'r') as r:
            next(r)                   # SKIP HEADERS
            reader = csv.reader(r, delimiter='\t')
            try:
                for row in reader:
                    if (row and len(row)==24):
                        wf_good.writerow(row)
                        good += 1
                    else:
                        wf_bad.writerow(row)
                        bad += 1
                        print('[[{}]] error line @{}'.format(base_name, reader.line_num))

            except csv.Error as e:
                print('[[{}]] Failed line @{}'.format(base_name, reader.line_num))
                traceback.print_exc()

        print('[[{}]] Ended loading....valid:{} invalid:{}'.format(base_name, good, bad))
        processed_files.append(base_name)
        unroccessed_files.remove(base_name)

    except Exception as e:
        print('[[{}]] Failed loading....{}'.format(base_name, str(e)))
        traceback.print_exc()

def do_combine_in_loop():
    print('Concatanating files...')
    start_time = datetime.now()
    try:
        with open('./output/data/combined_data.csv', 'w', newline='\n') as fout:
            wf = csv.writer(fout, delimiter='\t')
            ## add header row
            wf.writerow(columns)
            with open('./output/data/combined_data_invalid.csv', 'w', newline='\n') as fout_invalid:
                wf_invalid = csv.writer(fout_invalid, delimiter='\t')
                ## add header row
                wf_invalid.writerow(columns)
                [process_file_in_loop(file, wf, wf_invalid) for file in tqdm(files_to_process)]
    except:
        traceback.print_exc()
    total_time = datetime.now() - start_time
    print('total time for concatanating data:{}'.format(total_time))
    print('proccessed files {}:{}'.format(len(processed_files),'\n'.join(processed_files)))
    print('unprocessed files {}:{}'.format(len(unroccessed_files),'\n'.join(unroccessed_files)))

def do_combine():

    print('Concatanating dataframes...')

    # try:
    #     start_time = datetime.now()
    #     df = dd.read_csv(target, encoding='iso-8859-1', names=columns, warn_bad_lines=True, error_bad_lines=False, dtype=data_type_dict, sep='\t')
    #     # dd.to_csv(df,'./output/data/combined_data.gzip', single_file=True,columns=columns, index=False, compression="gzip", sep='\t')
    #     dd.to_parquet(df,'./output/data/parquet/', write_index=False )
    #     total_time = datetime.now() - start_time
    #     print('total time for concatanating data:{}'.format(total_time))
    # except:
    #     traceback.print_exc()
    start_time = datetime.now()
    try:

        frames = [ process_file(file) for file in tqdm(files_to_process)]
        print('concatenating.....')
        merged_df = pd.concat(frames, ignore_index=True)
        #merged_df.to_csv('./output/data/combined_data.gzip',columns=columns, index=False, compression="gzip")
        merged_df.to_csv('./output/data/combined_data.gzip', index=False, compression="gzip")
        print('merged_df.info():{}'.format(merged_df.info()))
    except:
        traceback.print_exc()
    total_time = datetime.now() - start_time
    print('total time for concatanating data:{}'.format(total_time))
    print('proccessed files {}:{}'.format(len(processed_files),'\n'.join(processed_files)))
    print('unprocessed files {}:{}'.format(len(unroccessed_files),'\n'.join(unroccessed_files)))

if __name__ == "__main__":
    #do_combine()
    do_combine_in_loop()

    #date_parsing('Fri Dec 21 22:53:46 +0000 2012')


'''
--- Day 8: Treetop Tree House ---

1. Consider your map; how many trees are visible from outside the grid?
2. Consider each tree on your map. What is the highest scenic score possible for any tree?
'''

from santas_helpers import file_lines, multiply_list, two_axis_directions

def parse_trees(s):
  try:
    return [int(c) for c in s]
  except:
    return None

# Part 1, generator to deduce duplicates
def determine_visible(d, x, y, dx, dy):
  t = data[y][x]
  while True:
    # Peek ahead
    nx, ny = (x + dx, y + dy)
    if nx < 1 or nx >= len(d[0]) - 1:
      break
    if ny < 1 or ny >= len(d) - 1:
      break
    # Next visible?
    if d[ny][nx] > t:
      # Add and step
      # print((nx, ny), "visible:", d[ny][nx])
      yield((nx, ny))
      x, y = (nx, ny)
      t = d[y][x]
    else:
      x, y = (nx, ny)

# Part 2, get view distance from point
def viewing_distance(d, x, y, dx, dy):
  t = data[y][x]
  n = 0 # viewing distance
  while True:
    # Peek ahead
    nx, ny = (x + dx, y + dy)
    if nx < 0 or nx > len(d[0]) - 1:
      break
    if ny < 0 or ny > len(d) - 1:
      break
    n += 1
    # View obstructed?
    if d[ny][nx] >= t:
      break
    else:
      # Step further
      x, y = (nx, ny)
  return n

data = []
directions = two_axis_directions.copy()

for line in file_lines("./input_8.txt", parser=parse_trees):
  data.append(line)

'''
Part 1
'''

# Count trees visible on the edge
visible = len(data) * 2 + (len(data[0]) - 2) * 2
# Set to hold interior trees (visible)
interior_trees_visible = set()

# From the top, down
for x in range(1, len(data[0]) - 1):
  y = 0
  dx, dy = directions['D']
  for t in determine_visible(data, x, y, dx, dy):
    interior_trees_visible.add(t)

# From the bottom, up
for x in range(1, len(data[0]) - 1):
  y = len(data) - 1
  dx, dy = directions['U']
  for t in determine_visible(data, x, y, dx, dy):
    interior_trees_visible.add(t)

# From the left, right
for y in range(1, len(data) - 1):
  x = 0
  dx, dy = directions['R']
  for t in determine_visible(data, x, y, dx, dy):
    interior_trees_visible.add(t)

# From the right, left
for y in range(1, len(data) - 1):
  x = len(data[0]) - 1
  dx, dy = directions['L']
  for t in determine_visible(data, x, y, dx, dy):
    interior_trees_visible.add(t)

'''
Part 2
'''

scenic_score_max = 0

for y in range(len(data)):
  for x in range(len(data[0])):
    tree_viewing_distances = {}
    for d in directions.keys():
      dx, dy = directions[d]
      tree_viewing_distances[d] = viewing_distance(data, x, y, dx, dy)
    scenic_score = multiply_list(tree_viewing_distances.values())
    if scenic_score > scenic_score_max:
      scenic_score_max = scenic_score

'''
Output
'''

visible += len(interior_trees_visible)

print(f"Part 1: {visible}")
print(f"Part 2: {scenic_score_max}")


from dataclasses import dataclass, field
from typing import Optional

import torch
from datasets import load_dataset
from peft import LoraConfig
from tqdm import tqdm
from transformers import AutoTokenizer, HfArgumentParser, pipeline

from trl import AutoModelForCausalLMWithValueHead, AutoModelForSeq2SeqLMWithValueHead, PPOConfig, PPOTrainer, set_seed
from trl.core import LengthSampler

tqdm.pandas()

@dataclass
class ScriptArguments:
    """
    The name of the Casual LM model we wish to fine with PPO
    """

    # NOTE: gpt2 models use Conv1D instead of Linear layers which are not yet supported in 8 bit mode
    # models like gpt-neo* models are more suitable.
    model_name: Optional[str] = field(default="lvwerra/gpt2-imdb", metadata={"help": "the model name"})
    log_with: Optional[str] = field(default=None, metadata={"help": "use 'wandb' to log with wandb"})
    learning_rate: Optional[float] = field(default=1.41e-5, metadata={"help": "the learning rate"})
    mini_batch_size: Optional[int] = field(default=128, metadata={"help": "the PPO minibatch size"})
    batch_size: Optional[int] = field(default=128, metadata={"help": "the batch size"})
    gradient_accumulation_steps: Optional[int] = field(
        default=1, metadata={"help": "the number of gradient accumulation steps"}
    )
    early_stopping: Optional[bool] = field(default=False, metadata={"help": "whether to early stop"})
    target_kl: Optional[float] = field(default=6, metadata={"help": "kl target for early stopping"})
    use_peft: Optional[bool] = field(default=False, metadata={"help": "whether to use peft"})
    use_seq2seq: Optional[bool] = field(default=False, metadata={"help": "whether to use seq2seq models"})
    kl_penalty: Optional[str] = field(
        default="kl",
        metadata={
            "help": "kl penalty options: 'kl': model_logp - ref_logp,  'abs': abs(kl),  'mse': mean squared error mse(kl) and 'full': the actual kl for all tokens in the distribution"
        },
    )
    target_kl: Optional[float] = field(default=0.1, metadata={"help": "kl target for early stopping"})
    seed: Optional[int] = field(default=0, metadata={"help": "the random seed"})

parser = HfArgumentParser(ScriptArguments)
script_args = parser.parse_args_into_dataclasses()[0]

config = PPOConfig(
    model_name=script_args.model_name,
    learning_rate=script_args.learning_rate,
    log_with=script_args.log_with,
    mini_batch_size=script_args.mini_batch_size,
    batch_size=script_args.batch_size,
    gradient_accumulation_steps=script_args.gradient_accumulation_steps,
    early_stopping=script_args.early_stopping,
    target_kl=script_args.target_kl,
    kl_penalty=script_args.kl_penalty,
    seed=script_args.seed,
)

# We then define the arguments to pass to the sentiment analysis pipeline.
# We set `return_all_scores` to True to get the sentiment score for each token.
sent_kwargs = {"return_all_scores": True, "function_to_apply": "none", "batch_size": 16}

trl_model_class = (
    AutoModelForCausalLMWithValueHead if not script_args.use_seq2seq else AutoModelForSeq2SeqLMWithValueHead
)

# Below is an example function to build the dataset. In our case, we use the IMDB dataset
# from the `datasets` library. One should customize this function to train the model on
# its own dataset.
def build_dataset(config, dataset_name="imdb", input_min_text_length=2, input_max_text_length=8):
    """
    Build dataset for training. This builds the dataset from `load_dataset`, one should
    customize this function to train the model on its own dataset.

    Args:
        dataset_name (`str`):
            The name of the dataset to be loaded.

    Returns:
        dataloader (`torch.utils.data.DataLoader`):
            The dataloader for the dataset.
    """
    tokenizer = AutoTokenizer.from_pretrained(config.model_name)
    tokenizer.pad_token = tokenizer.eos_token
    # load imdb with datasets
    ds = load_dataset(dataset_name, split="train")
    ds = ds.rename_columns({"text": "review"})
    ds = ds.filter(lambda x: len(x["review"]) > 200, batched=False)

    input_size = LengthSampler(input_min_text_length, input_max_text_length)

    def tokenize(sample):
        sample["input_ids"] = tokenizer.encode(sample["review"])[: input_size()]
        sample["query"] = tokenizer.decode(sample["input_ids"])
        return sample

    ds = ds.map(tokenize, batched=False)
    ds.set_format(type="torch")
    return ds

# We retrieve the dataloader by calling the `build_dataset` function.
dataset = build_dataset(config)

def collator(data):
    return dict((key, [d[key] for d in data]) for key in data[0])

# set seed before initializing value head for deterministic eval
set_seed(config.seed)

# Now let's build the model, the reference model, and the tokenizer.
if not script_args.use_peft:
    ref_model = trl_model_class.from_pretrained(config.model_name)
    device_map = None
    peft_config = None
else:
    peft_config = LoraConfig(
        r=16,
        lora_alpha=16,
        bias="none",
        task_type="CAUSAL_LM",
    )
    ref_model = None
    device_map = {"": 0}

model = trl_model_class.from_pretrained(
    config.model_name,
    device_map=device_map,
    peft_config=peft_config,
)

tokenizer = AutoTokenizer.from_pretrained(config.model_name)

# GPT-2 tokenizer has a pad token, but it is not eos_token by default. We need to set it to eos_token.
# only for this model.
tokenizer.pad_token = tokenizer.eos_token

# We then build the PPOTrainer, passing the model, the reference model, the tokenizer
ppo_trainer = PPOTrainer(config, model, ref_model, tokenizer, dataset=dataset, data_collator=collator)

# We then build the sentiment analysis pipeline, passing the model name and the
# sentiment analysis pipeline arguments. Let's also make sure to set the device
# to the same device as the PPOTrainer.
device = ppo_trainer.accelerator.device
if ppo_trainer.accelerator.num_processes == 1:
    device = 0 if torch.cuda.is_available() else "cpu"  # to avoid a `pipeline` bug
sentiment_pipe = pipeline("sentiment-analysis", model="lvwerra/distilbert-imdb", device=device)

# We then define the arguments to pass to the `generate` function. These arguments
# are passed to the `generate` function of the PPOTrainer, which is a wrapper around
# the `generate` function of the trained model.
generation_kwargs = {
    "min_length": -1,
    "top_k": 0.0,
    "top_p": 1.0,
    "do_sample": True,
    "pad_token_id": tokenizer.eos_token_id,
    "max_new_tokens": 32,
}

for epoch, batch in tqdm(enumerate(ppo_trainer.dataloader)):
    query_tensors = batch["input_ids"]

    # Get response from gpt2
    response_tensors = ppo_trainer.generate(query_tensors, return_prompt=False, **generation_kwargs)
    batch["response"] = tokenizer.batch_decode(response_tensors)

    # Compute sentiment score
    texts = [q + r for q, r in zip(batch["query"], batch["response"])]
    pipe_outputs = sentiment_pipe(texts, **sent_kwargs)
    rewards = [torch.tensor(output[1]["score"]) for output in pipe_outputs]

    # Run PPO step
    stats = ppo_trainer.step(query_tensors, response_tensors, rewards)
    ppo_trainer.log_stats(stats, batch, rewards)


import time
import copy

with open('Day05/input') as file:
    inputList = file.read()

start_time = time.time()

inputList = inputList.split("\n\n")
#this gets me a list where the list can actually be seperated into '' or [_] easily.
inputList[0] = inputList[0].replace("    ", ' ')
#List comp takes the first aprt of the input, and seperates each row into a list consisiting of a value for each col of the input, for all rows
#resulting in a 2d list representing the input as in the file
inputList[0] = [part for x in inputList[0].split('\n')[:-1] for part in [x.split(" ")]]
#This takes the latter part of the input, and puts the values from the instructions into a tuple, where 1 = # boxes, 2 = Origin, 3 = Dest,
#then puts all the tuples into a list
craneInstructions = [(int(parts[1]),int(parts[3])-1,int(parts[5])-1) for instruction in inputList[1].split('\n') for parts in [instruction.split(" ")]]

#this unholy peice of code takes the "boxes" input, and parses it into a lost of stacks that can be easily popped/pushed
boxesList = [[] for _ in inputList[0][0]]
for j in range(len(inputList[0][0])):
    for i in range(len(inputList[0])-1, -1, -1):
        if inputList[0][i][j] != '':
            boxesList[j].append(inputList[0][i][j])

#make a list for the part2 problem
problem2BoxList = copy.deepcopy(boxesList)

for instruction in craneInstructions:
    #part 1 solution, pop box off top of origin, append to dest, repeat however many times the instruction wants.
    for move in range(instruction[0]):
        boxesList[instruction[2]].append(boxesList[instruction[1]].pop())

    #part 2
    #slice the last X elements from the orgin stack, and put on the dest stack. This would also work for part1, if we reverse the list, but I like keeping the pop/place
    [problem2BoxList[instruction[2]].append(_) for _ in problem2BoxList[instruction[1]][-instruction[0]:]]
    #remove the boxes from the origin stack
    del problem2BoxList[instruction[1]][-instruction[0]:]

print(''.join([x[-1] for x in boxesList]).replace('[', '').replace(']', ''))
print(''.join([x[-1] for x in problem2BoxList]).replace('[', '').replace(']', ''))
print("--- %s seconds ---" % (time.time() - start_time))


#To read the details of a telephone data file and print systematically
#telepy.py
#25-4-23
#By Anupam Kanoongo

with open("tele.txt", "r") as t:
    dt = t.readlines()

print("S.No\t\tName\t\tTelephone")
for i in range(len(dt)):
    print(i+1,"\t\t" ,dt[i].split()[0] , "\t\t", dt[i].split()[1])

##Output
##S.No		Name		Telephone
##1 		 Arvind 		 1234567
##2 		 Sachin 		 7654321
##3 		 Anmol 		 1356723


#Лаба №5 Cортировка вставками

from random import randint

#создаем случайную последовательность
x = int(input("Введите размер списка \n"))
nums = [randint(1, 100) for i in range(0, x)]
print(*nums)

for i in range(1, len(nums)):
    for j in range(i,0,-1):
        if nums[j] < nums[j-1]:
            nums[j], nums[j-1] = nums[j-1], nums[j]
        else:
            break
print(*nums)


import numpy as np

inputs = np.array([[0, 0, 1], [0, 1, 0], [0, 1, 1], [1, 0, 0], [1, 0, 1], [1, 1, 0], [1, 1, 1]])
expectedOutputs = np.array([[1], [2], [3], [4], [5], [6], [7]])

weights = np.array([0.84, 0.394, 0.783])
learningRate = 0.3

epochs = 7
amountOfData = 3

def calculateSumOfWeight(inputs, weights, row):
    sum = 0;
    for column in range(amountOfData):
        sum += inputs[row][column] * weights[column]
    return sum

def isNeedLearning(sumOfWeight, expectedOutputs, row):
    return sumOfWeight != expectedOutputs[row][0]

def calculateError(sumOfWeight, expectedOutputs, row):
    return expectedOutputs[row][0] - sumOfWeight

def modifiedWeights(inputs, row, error):
    for column in range(amountOfData):
        errorOfWeight = learningRate * error * inputs[row][column]
        print("row: " + str(row) + ", column: " + str(column) + ", before weight:" + str(weights[column]))
        weights[column] = errorOfWeight + weights[column]
        print("row: " + str(row) + ", column: " + str(column) + ", after weight:" + str(weights[column]))

def learning():
    for row in range(epochs):
        sumOfWeight = calculateSumOfWeight(inputs, weights, row)
        if (isNeedLearning(sumOfWeight, expectedOutputs, row)):
            error = calculateError(sumOfWeight, expectedOutputs, row)
            modifiedWeights(inputs, row, error)

learning()


import requests
from mysecrets import MySecrets

secrets = MySecrets()

SHEET_TOKEN = secrets.sheet_token
SHEET_ENDPOINT = secrets.sheet_endpoint
SHEET_USER_ENDPOINT = secrets.sheet_user_endpoint

SHEET_HEADER = {
    'Authorization': f'Bearer {SHEET_TOKEN}'
}

class DataManager:
    def __init__(self):
        self.destination_data = {}

    def get_destination_data(self):
        response = requests.get(url=SHEET_ENDPOINT)
        data = response.json()
        self.destination_data = data['prices']
        return self.destination_data

    def update_destination_iata(self):
        for city in self.destination_data:
            new_data = {
                'price': {
                    'iataCode': city['iataCode']
                }
            }
            response = requests.put(url=f"{SHEET_ENDPOINT}/{city['id']}", json=new_data)
            print(response.text)

    def get_user_data(self):
        response = requests.get(url=SHEET_USER_ENDPOINT)
        self.user_data = response.json()['users']
        return self.user_data


from django.urls import path
from . import views
urlpatterns = [
    path('wel', views.index1),
    path('', views.index),
    path('create_user', views.create_user),
    path('success', views.success),
    path('login', views.login),
    path('log_out', views.log_out),
    path('process_message', views.post_mess),
    path('add_comment/<int:id>', views.post_comment),
    path('edit_profile/<int:id>', views.edit_profile),
    path('update/<int:id>', views.update),
    path('user_profile/<int:id>', views.profile),
    path('like/<int:id>', views.add_like),
    path('delete/<int:id>', views.delete_comment)
]


# Goal Parser Interpretation
# You own a Goal Parser that can interpret a string command. The command consists of an alphabet of "G", "()" and/or "(al)" in some order. The Goal Parser will interpret "G" as the string "G", "()" as the string "o", and "(al)" as the string "al". The interpreted strings are then concatenated in the original order.
# Given the string command, return the Goal Parser's interpretation of command.

def interpret (command) :
    newStr = ""
    for i in range(len(command)) :
        if command[i] == "G" :
            newStr += "G"
        if command[i] == "(" :
            if command[i+1:i+3] == "al" :
                newStr += "al"
            else :
                newStr += "o"
    return newStr

print(interpret(command="(al)G(al)()()G"))


from __future__ import with_statement
import eg

eg.RegisterPlugin(
    name = "UIRT2",
    author = "Bitmonster",
    version = "1.1",
    kind = "remote",
    guid = "{DEB2BDB1-F7BD-44B1-B5F7-153B54C3A245}",
    canMultiLoad = True,
    description = (
        'Hardware plugin for the '
        'Universal InfraRed Transceiver V2.'
    ),
    icon = (
        "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAaElEQVR42mNkoBAwDgMD"
        "/jMw/IeaRLJhIL1gAw4eOMBg7+AANwQsgYWNrrkR2QWkGALTXE+uAQ1APkgziguQQpU8"
        "F5BiAE4XNEIkGYkJRJghcANgmkmJRpAhjA1QA0jVjORlysDAGwAAHWBIBf4cTRAAAAAA"
        "SUVORK5CYII="
    ),
)

import wx
from threading import Thread, Event
from time import sleep, clock
from binascii import hexlify, unhexlify
from Queue import Queue, Full
from string import hexdigits

# currently unused stuff

#def GetStructTime(code, start):
#    tvalue = (ord(code[start+1]) * 256) + ord(code[start+2])
#    bBits = ord(code[start+3])
#    bHdr1 = ord(code[start+4])
#    bHdr0 = ord(code[start+5])
#    bOff0 = ord(code[start+6])
#    bOff1 = ord(code[start+7])
#    bOn0 = ord(code[start+8])
#    bOn1 = ord(code[start+9])
#    tvalue += bHdr1 + bHdr0
#    for i in range(0, bBits):
#        bit = (ord(code[start + 10 + (i / 8)]) >> (i % 8)) & 1
#        if (i % 2) == 0:
#            if bit:
#                tvalue += bOn1
#            else:
#                tvalue += bOn0
#        else:
#            if bit:
#                tvalue += bOff1
#            else:
#                tvalue += bOff0
#    return tvalue
#
#
#def CalcTime(code):
#    tvalue = 0
#    if code[0] == "\x36":
#        # RAW
#        length = ord(code[1])
#        tvalue = (ord(code[2]) * 256) + ord(code[3])
#        for i in range(4, length + 4):
#            tvalue += ord(code[i])
#        tvalue = tvalue * (ord(code[-2]) & 0x1F)
#        #tvalue -= (ord(code[2]) * 256) + ord(code[3])
#    elif (ord(code[0]) & 0x1F) > 0:
#        # REMSTRUCT1
#        repeat = ord(code[0]) & 0x1F
#        tvalue = GetStructTime(code, 0) * repeat
#        #tvalue -= (ord(code[1]) * 256) + ord(code[2])
#
#    else:
#        # REMSTRUCT2
#        tvalue = GetStructTime(code, 0)
#        repeat = ord(code[26]) & 0x1F
#        tvalue = tvalue + (GetStructTime(code, 26) * repeat)
#    return tvalue * SAMPLE_TIME

def CalcChecksum(data):
    checksum = 0
    for byte in data:
        checksum += ord(byte)
        checksum %= 256
    checksum = (0x100 - checksum) % 256
    return chr(checksum)

class HexValidator(wx.PyValidator):

    def __init__(self, maxLength=100, allowRaw=False):
        self.maxLength = maxLength
        self.allowRaw = allowRaw
        wx.PyValidator.__init__(self)
        self.Bind(wx.EVT_CHAR, self.OnChar)

    def Clone(self):
        return HexValidator(self.maxLength, self.allowRaw)

    def TransferToWindow(self):
        return True

    def TransferFromWindow(self):
        return True

    @eg.LogIt
    def Validate(self, win):
        val = win.GetValue()
        for x in val:
            if x not in hexdigits:
                return False
        return True

    def OnChar(self, event):
        key = event.KeyCode

        if key < wx.WXK_SPACE or key == wx.WXK_DELETE or key > 255:
            event.Skip()
            return

        textCtrl = self.GetWindow()
        startPos, endPos = textCtrl.GetSelection()
        length = textCtrl.GetLastPosition() - (endPos - startPos)
        if length >= self.maxLength:
            if not wx.Validator_IsSilent():
                wx.Bell()
            return

        if(
            self.allowRaw
            and textCtrl.GetInsertionPoint() == 0
            and chr(key).upper() == "R"
        ):
            textCtrl.WriteText("R")
            return

        if chr(key) in hexdigits:
            textCtrl.WriteText(chr(key).upper())
            return

        if not wx.Validator_IsSilent():
            wx.Bell()

        # Returning without calling event.Skip eats the event before it
        # gets to the text control
        return

class UIRT2(eg.IrDecoderPlugin):

    def __init__(self):
        eg.IrDecoderPlugin.__init__(self, 50.0)
        self.buffer = ""
        self.AddAction(TransmitIR)

    def __start__(self, comport=0):
        serialThread = self.serialThread = eg.SerialThread()
        serialThread.Open(comport, 115200)
        serialThread.Start()
        serialThread.SetRts()
        for dummyCounter in range(3):
            sleep(0.05)
            serialThread.Flush()
            serialThread.Write("\x23\xdd") # get version
            if serialThread.Read(3, 0.2) != "\x01\x04\xFB":
                continue
            serialThread.Write("\x21\xdf") # set RAW mode
            if serialThread.Read(1, 0.1) != "\x21":
                continue
            break
        else:
            serialThread.Close()
            raise self.Exceptions.InitFailed
        self.buffer = ""
        serialThread.SetReadEventCallback(self.OnReceive)
        self.sendQueue = Queue(50)
        self.alive = True
        Thread(target=self.SendLoop).start()

    @eg.LogIt
    def __stop__(self):
        self.alive = False
        self.sendQueue.put((StopIteration, None))
        self.serialThread.SuspendReadEvents()
        self.serialThread.Write("\x20\xe0")
        self.serialThread.Close()

    def __close__(self):
        self.irDecoder.Close()

    #@eg.LogIt
    def OnReceive(self, serialThread):
        self.buffer += serialThread.Read(1024)
        while True:
            terminatorPos = self.buffer.find("\xff")
            #print terminatorPos
            if terminatorPos < 0:
                break
            data = self.buffer[2:terminatorPos+1]
            self.buffer = self.buffer[terminatorPos+1:]
            if len(data) < 2:
                continue
            self.irDecoder.Decode(data, len(data))

    @eg.LogItWithReturn
    def SendLoop(self):
        while self.alive:
            code, event = self.sendQueue.get()
            if code == StopIteration:
                return
            with self.serialThread as serial:
                serial.Write(code)
                sleep(0.05)
                data = serial.Read(1)
                if data != " ":
                    self.PrintError("Error sending IR code to UIRT2")
                data = ""
                startTime = clock()
                while data != "\x21":
                    if (clock() - startTime) > 5.0:
                        self.PrintError("Transmitting timed out")
                        break
                    serial.Write("\x21\xdf")
                    sleep(0.05)
                    data = serial.Read(1)
            event.set()

    def Configure(self, comport=0):
        panel = eg.ConfigPanel()
        portCtrl = panel.SerialPortChoice(comport)
        panel.AddLine("COM-Port:", portCtrl)
        while panel.Affirmed():
            panel.SetResult(portCtrl.GetValue())

class TransmitIR(eg.ActionBase):
    name = "Transmit IR"

    def __call__(self, code, waitUntilFinished=True):
        finishEvent = Event()
        try:
            self.plugin.sendQueue.put((code, finishEvent), False)
        except Full:
            self.PrintError("Too many transmissions pending")
            return
        if waitUntilFinished:
            finishEvent.wait(10.0)

    def GetLabel(self, *dummyArgs):
        return self.name

    def Configure(self, code="", waitUntilFinished=True):
        panel = eg.ConfigPanel()
        code1 = ""
        code2 = ""
        repeatCount = 4
        carrier = 0
        if code:
            code += (48 * "\x00")
            if code[0] == "\x36":
                length = ord(code[1])
                code1 = "R" + hexlify(code[2:length]).upper()
                repeatCount = ord(code[length]) & 0x1F
                carrier = ord(code[length]) >> 6
            else:
                repeatCount = ord(code[0]) & 0x1F
                if repeatCount > 0:
                    carrier = ord(code[0]) >> 6
                    code1 = hexlify(code[1:26]).upper()
                else:
                    repeatCount = ord(code[26]) & 0x1F
                    carrier = ord(code[0]) >> 6
                    code1 = hexlify(code[1:26]).upper()
                    code2 = hexlify(code[27:48]).upper()
        if carrier < 0:
            carrier = 0
        elif carrier > 3:
            carrier = 3
        if repeatCount < 1:
            repeatCount = 1
        elif repeatCount > 31:
            repeatCount = 31
        sizer = wx.FlexGridSizer(4, 2, 5, 5)
        sizer.AddGrowableCol(1)

        sizer.Add(panel.StaticText("Code 1:"), 0, wx.ALIGN_CENTER_VERTICAL)
        code1Ctrl = panel.TextCtrl(
            code1,
            size=(325, -1),
            validator=HexValidator(allowRaw=True),
        )
        sizer.Add(code1Ctrl)

        sizer.Add(panel.StaticText("Code 2:"), 0, wx.ALIGN_CENTER_VERTICAL)
        code2Ctrl = panel.TextCtrl(
            code2,
            size=(275, -1),
            validator=HexValidator()
        )
        sizer.Add(code2Ctrl)

        sizer.Add(panel.StaticText("Repeat:"), 0, wx.ALIGN_CENTER_VERTICAL)
        repeatCtrl = eg.SpinIntCtrl(panel, -1, repeatCount, 1, 31)
        repeatCtrl.SetInitialSize((50, -1))
        sizer.Add(repeatCtrl, 0)

        sizer.Add(panel.StaticText("Carrier:"), 0, wx.ALIGN_CENTER_VERTICAL)
        choices = ('35.7 kHz', '37.0 kHz', '38.4 kHz', '40.0 kHz')
        carrierCtrl = wx.Choice(panel, -1, choices=choices)
        carrierCtrl.SetSelection(3 - carrier)
        sizer.Add(carrierCtrl, 0)

        panel.sizer.Add(sizer, 0, wx.EXPAND)

        panel.sizer.Add((5, 5))
        waitUntilFinishedCtrl = panel.CheckBox(
            waitUntilFinished,
            "Pause till transmission is finished"
        )
        panel.sizer.Add(waitUntilFinishedCtrl)

        while panel.Affirmed():
            code1 = code1Ctrl.GetValue()
            if len(code1) == 0:
                panel.SetResult("", waitUntilFinishedCtrl.GetValue())
                continue
            code2 = code2Ctrl.GetValue()
            repeatCount = repeatCtrl.GetValue()
            carrier = 3 - carrierCtrl.GetSelection()
            if code1[0] == "R":
                data = unhexlify(code1[1:])
                bCmd = repeatCount | (carrier << 6)
                code = "\x36" + chr(len(data) + 2) + data + chr(bCmd)
            elif len(code2) == 0:
                data = unhexlify(code1)
                bCmd = repeatCount | (carrier << 6)
                code = chr(bCmd) + data
            else:
                bCmd = 0 | (carrier << 6)
                bCmd2 = repeatCount | (carrier << 6)
                code = chr(bCmd) + unhexlify(code1) \
                       + chr(bCmd2) + unhexlify(code2)
            panel.SetResult(
                code + CalcChecksum(code),
                waitUntilFinishedCtrl.GetValue()
            )


import aiosqlite
from aiogram import types

from loader import dp

@dp.callback_query_handler(callable('add'))
async def call_inn(call: types.CallbackQuery):
    if call.data == 'add':  # Ошибка 'ValueError: no active connection' надо исправить
        info = []
        i = call.message.text.split('\n')
        for n in i:
            k = n.split(":")
            info.append(str(k[1]))
        async with aiosqlite.connect('test.sql') as db:
            cursor = await db.execute('''CREATE TABLE IF NOT EXISTS test(
                Название TEXT NOT NULL,
                ОГРН INTEGER NOT NULL,
                ИНН INTEGER NOT NULL,
                Регион TEXT,
                Адресс TEXT,
                Директор TEXT,
                ОКВЭД TEXT,
                Уставной капилал INTEGER NOT NULL,
                Дата_рег TEXT)''')
            await cursor.close()
            await db.close()
        async with aiosqlite.connect('test.sql') as conn:
            await conn.execute(f'''INSERT INTO test VALUES (?,?,?,?,?,?,?,?,?);''', info)
            await conn.commit()
            await conn.close()
        await call.answer(f'')


import time

import numpy as np
import torch
from torchvision.transforms import functional as tvtf
import imageio
import PIL

from COTR.inference.inference_helper import BASE_ZOOM, THRESHOLD_PIXELS_RELATIVE, get_patch_centered_at, two_images_side_by_side, find_prediction_loop
from COTR.utils import debug_utils, utils
from COTR.utils.constants import MAX_SIZE
from COTR.utils.utils import ImagePatch

class RefinementTask():
    def __init__(self, image_from, image_to, loc_from, loc_to, area_from, area_to, converge_iters, zoom_ins, identifier=None):
        self.identifier = identifier
        self.image_from = image_from
        self.image_to = image_to
        self.loc_from = loc_from
        self.best_loc_to = loc_to
        self.cur_loc_to = loc_to
        self.area_from = area_from
        self.area_to = area_to
        if self.area_from < self.area_to:
            self.s_from = BASE_ZOOM
            self.s_to = BASE_ZOOM * np.sqrt(self.area_to / self.area_from)
        else:
            self.s_to = BASE_ZOOM
            self.s_from = BASE_ZOOM * np.sqrt(self.area_from / self.area_to)

        self.cur_job = {}
        self.status = 'unfinished'
        self.result = 'unknown'

        self.converge_iters = converge_iters
        self.zoom_ins = zoom_ins
        self.cur_zoom_idx = 0
        self.cur_iter = 0
        self.total_iter = 0

        self.loc_to_at_zoom = []
        self.loc_history = [loc_to]
        self.all_loc_to_dict = {}
        self.job_history = []
        self.submitted = False

    @property
    def cur_zoom(self):
        return self.zoom_ins[self.cur_zoom_idx]

    @property
    def confidence_scaling_factor(self):
        if self.cur_zoom_idx > 0:
            conf_scaling = float(self.cur_zoom) / float(self.zoom_ins[0])
        else:
            conf_scaling = 1.0
        return conf_scaling

    def peek(self):
        assert self.status == 'unfinished'
        patch_from = get_patch_centered_at(None, self.loc_from, scale=self.s_from * self.cur_zoom, return_content=False, img_shape=self.image_from.shape)
        patch_to = get_patch_centered_at(None, self.cur_loc_to, scale=self.s_to * self.cur_zoom, return_content=False, img_shape=self.image_to.shape)
        top_job = {'patch_from': patch_from,
                    'patch_to': patch_to,
                    'loc_from': self.loc_from,
                    'loc_to': self.cur_loc_to,
                    }
        return top_job

    def get_task_pilot(self, pilot):
        assert self.status == 'unfinished'
        patch_from = ImagePatch(None, pilot.cur_job['patch_from'].x, pilot.cur_job['patch_from'].y, pilot.cur_job['patch_from'].w, pilot.cur_job['patch_from'].h, pilot.cur_job['patch_from'].ow, pilot.cur_job['patch_from'].oh)
        patch_to   = ImagePatch(None, pilot.cur_job['patch_to'].x, pilot.cur_job['patch_to'].y, pilot.cur_job['patch_to'].w, pilot.cur_job['patch_to'].h, pilot.cur_job['patch_to'].ow, pilot.cur_job['patch_to'].oh)
        query = torch.from_numpy((np.array(self.loc_from) - np.array([patch_from.x, patch_from.y])) / np.array([patch_from.w * 2, patch_from.h]))[None].float()
        self.cur_job = {'patch_from': patch_from,
                        'patch_to': patch_to,
                        'loc_from': self.loc_from,
                        'loc_to': self.cur_loc_to,
                        'img': None,
                        }
        self.job_history.append((patch_from.h, patch_from.w, patch_to.h, patch_to.w))
        assert self.submitted == False
        self.submitted = True
        return None, query

    def get_task_fast(self):
        assert self.status == 'unfinished'
        patch_from = get_patch_centered_at(self.image_from, self.loc_from, scale=self.s_from * self.cur_zoom, return_content=False)
        patch_to = get_patch_centered_at(self.image_to, self.cur_loc_to, scale=self.s_to * self.cur_zoom, return_content=False)
        query = torch.from_numpy((np.array(self.loc_from) - np.array([patch_from.x, patch_from.y])) / np.array([patch_from.w * 2, patch_from.h]))[None].float()
        self.cur_job = {'patch_from': patch_from,
                        'patch_to': patch_to,
                        'loc_from': self.loc_from,
                        'loc_to': self.cur_loc_to,
                        'img': None,
                        }

        self.job_history.append((patch_from.h, patch_from.w, patch_to.h, patch_to.w))
        assert self.submitted == False
        self.submitted = True

        return None, query

    def get_task(self):
        assert self.status == 'unfinished'
        patch_from = get_patch_centered_at(self.image_from, self.loc_from, scale=self.s_from * self.cur_zoom)
        patch_to = get_patch_centered_at(self.image_to, self.cur_loc_to, scale=self.s_to * self.cur_zoom)

        query = torch.from_numpy((np.array(self.loc_from) - np.array([patch_from.x, patch_from.y])) / np.array([patch_from.w * 2, patch_from.h]))[None].float()

        img_from = patch_from.patch
        img_to = patch_to.patch
        assert img_from.shape[0] == img_from.shape[1]
        assert img_to.shape[0] == img_to.shape[1]

        img_from = np.array(PIL.Image.fromarray(img_from).resize((MAX_SIZE, MAX_SIZE), resample=PIL.Image.BILINEAR))
        img_to = np.array(PIL.Image.fromarray(img_to).resize((MAX_SIZE, MAX_SIZE), resample=PIL.Image.BILINEAR))
        img = two_images_side_by_side(img_from, img_to)
        img = tvtf.normalize(tvtf.to_tensor(img), (0.485, 0.456, 0.406), (0.229, 0.224, 0.225)).float()

        self.cur_job = {'patch_from': ImagePatch(None, patch_from.x, patch_from.y, patch_from.w, patch_from.h, patch_from.ow, patch_from.oh),
                        'patch_to': ImagePatch(None, patch_to.x, patch_to.y, patch_to.w, patch_to.h, patch_to.ow, patch_to.oh),
                        'loc_from': self.loc_from,
                        'loc_to': self.cur_loc_to,
                        }

        self.job_history.append((patch_from.h, patch_from.w, patch_to.h, patch_to.w))
        assert self.submitted == False
        self.submitted = True

        return img, query

    def next_zoom(self):
        if self.cur_zoom_idx >= len(self.zoom_ins) - 1:
            self.status = 'finished'
            if self.conclude() is None:
                self.result = 'bad'
            else:
                self.result = 'good'
        self.cur_zoom_idx += 1
        self.cur_iter = 0
        self.loc_to_at_zoom = []

    def scale_to_loc(self, raw_to_loc):
        raw_to_loc = raw_to_loc.copy()
        patch_b = self.cur_job['patch_to']
        raw_to_loc[0] = (raw_to_loc[0] - 0.5) * 2
        loc_to = raw_to_loc * np.array([patch_b.w, patch_b.h])
        loc_to = loc_to + np.array([patch_b.x, patch_b.y])
        return loc_to

    def step(self, raw_to_loc):
        assert self.submitted == True
        self.submitted = False
        loc_to = self.scale_to_loc(raw_to_loc)
        self.total_iter += 1
        self.loc_to_at_zoom.append(loc_to)
        self.cur_loc_to = loc_to
        zoom_finished = False
        if self.cur_zoom_idx == len(self.zoom_ins) - 1:
            # converge at the last level
            if len(self.loc_to_at_zoom) >= 2:
                zoom_finished = np.prod(self.loc_to_at_zoom[:-1] == loc_to, axis=1, keepdims=True).any()
            if self.cur_iter >= self.converge_iters - 1:
                zoom_finished = True
            self.cur_iter += 1
        else:
            # finish immediately for other levels
            zoom_finished = True
        if zoom_finished:
            self.all_loc_to_dict[self.cur_zoom] = np.array(self.loc_to_at_zoom).copy()
            last_level_loc_to = self.all_loc_to_dict[self.cur_zoom]
            if len(last_level_loc_to) >= 2:
                has_loop = np.prod(last_level_loc_to[:-1] == last_level_loc_to[-1], axis=1, keepdims=True).any()
                if has_loop:
                    loop = find_prediction_loop(last_level_loc_to)
                    loc_to = loop.mean(axis=0)
            self.loc_history.append(loc_to)
            self.best_loc_to = loc_to
            self.cur_loc_to = self.best_loc_to
            self.next_zoom()

    def conclude(self, force=False):
        loc_history = np.array(self.loc_history)
        if (force == False) and (max(loc_history.std(axis=0)) >= THRESHOLD_PIXELS_RELATIVE * max(*self.image_to.shape)):
            return None
        return np.concatenate([self.loc_from, self.best_loc_to])

    def conclude_intermedia(self):
        return np.concatenate([np.array(self.loc_history), np.array(self.job_history)], axis=1)


##----------------------------------------------------------------------------
## author:Moira Shooter
## filename: process_img.py
## build: is one of the scripts we used to process the image to calculate the porosity factor
##----------------------------------------------------------------------------
import numpy as np
import cv2
from matplotlib import pyplot as plt
import os

#src = cv2.imread("/Users/moirashooter/RDProject/dataset/tibia-mouse2-jpg/26Oim__rec0002.jpg",0)
#src = cv2.imread("/Users/moirashooter/RDProject/dataset/ts5-jpg/118Oim__rec0200.jpg",0)
##----------------------------------------------------------------------------
# function for preparing the data set
# param : path to the picture
##----------------------------------------------------------------------------
def createImage(pathToImage):
    # read image
    # image is a matrix
    src = cv2.imread(pathToImage, 0)
    cv2.imshow("src", src)
    # blur to get rid of noise
    blur = cv2.blur(src, (8, 8))
    cv2.imshow("blur", blur)
    # thresholding
    ret, im_thresh1 = cv2.threshold(blur, 50, 255, cv2.THRESH_BINARY)
    cv2.imshow("threshold", im_thresh1)
    blur2= cv2.blur(im_thresh1, (8,8))
    cv2.imshow("blur2", blur)
    ret, im_thresh2 = cv2.threshold(blur2, 50, 255, cv2.THRESH_BINARY)
    mask_alpha = im_thresh2
    cont = im_thresh1
    # opening contour
    im2,  contours1, hierarchy = cv2.findContours(cont, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    hull = []
    for i in range(len(contours1)):
        hull.append(cv2.convexHull(contours1[i], False))
    drawing = np.zeros((cont.shape[0], cont.shape[1], 3), np.uint8)
    for i in range(len(contours1)):
        color_contour = (0,255,0)
        color = (255, 0, 0)
        #cv2.drawContours(drawing, contours1, i, color_contour, 1, 8, hierarchy)
        cv2.drawContours(drawing, hull, i, color, 1, 8)
    cv2.imshow("t", drawing)
    # creating a mask_alpha
    img, contour, hier = cv2.findContours(mask_alpha, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)
    for cnt in contour:
        cv2.drawContours(mask_alpha,[cnt],0,255, -1)
    cv2.imshow("mask_alpha",mask_alpha)
    # outline of the bone
    #im_outline = cv2.Canny(mask_alpha, 0, 255, 3)
    # merge the two images together
    # img_merge = cv2.addWeighted(src, 0.8, im_outline, 0.5, 0)
#    ret, img_thresh_final = cv2.threshold(src, 90, 255, cv2.THRESH_BINARY)
#    img_merge2 = cv2.addWeighted(img_thresh_final, 0.8, cv2.bitwise_not(mask_alpha), 0.2, 0)
#    return img_merge2
 # calculate the porosity of an image
##----------------------------------------------------------------------------
## function that calculates the porosity
##----------------------------------------------------------------------------
def calculatePorosity(img):
    # amount of black and white pixels
    n_white_pix = np.sum(img == 204)
    n_black_pix = np.sum(img == 0)
    # error on the thingie - fix it.
    n_total_black_pix = n_black_pix - (n_black_pix*0.20)
    print('amount of white pixels: ', n_white_pix)
    print('amount of black pixels: ',n_total_black_pix)
    if(n_total_black_pix != 0):
        porosity = n_white_pix / n_total_black_pix
    else:
        porosity = 0
        return porosity
    print('porosity factor: ', porosity)
    return porosity * 100

##----------------------------------------------------------------------------
## function that gets the filename path
##----------------------------------------------------------------------------
def getFileName(path):
    base = os.path.basename(path)
    return base
original_path = '/Users/moirashooter/RDProject/dataset_final/'
# if porosity is a certain range - store it in directory
def saveToFolder(porosity, pathToImage, img):
    dest_path =  original_path
    lower_bound = int(porosity / 5) * 5
    upper_bound = lower_bound + 5
    path = dest_path+'[{}-{}]'.format(lower_bound, upper_bound)+'/'+getFileName(pathToImage)
    cv2.imwrite(path, img)

##----------------------------------------------------------------------------
## path variables, this was to test
##----------------------------------------------------------------------------
paths = "/Users/moirashooter/RDProject/dataset/tibia-mouse2-jpg/26Oim__rec0002.jpg"
pathss = "/Users/moirashooter/RDProject/dataset/tibia-mouse6-jpg/142Oim__rec0120.jpg"
pathst = "/Users/moirashooter/RDProject/dataset/iom3-jpg/22Oim__rec0010.jpg"
patt ="/Users/moirashooter/RDProject/dataset/tibia-mouse5-jpg/141Oim__rec0991.jpg"
##----------------------------------------------------------------------------
## This function process the image, to find the area we are interested in,
## in creates the alpha mask - another function
##----------------------------------------------------------------------------
def process_imgage(path):
    print(path)
    # create the alpha mask
    if os.path.exists(paths):
        img = cv2.imread(path, cv2.IMREAD_GRAYSCALE)

        ret, img_thresh_final = cv2.threshold(img, 90, 255, cv2.THRESH_BINARY)
#         ret, th = cv2.threshold(blur, 40, 255, cv2.THRESH_BINARY)
#         output = np.hstack([img, cv2.bitwise_not(th)])

        blur = cv2.blur(img, (6,6))
        ret, threshold = cv2.threshold(blur , 50, 255, cv2.THRESH_BINARY)

        blur1 = cv2.blur(threshold, (25,25))
        ret, threshold1 = cv2.threshold(blur1 , 50, 255, cv2.THRESH_BINARY)
        blur2 = cv2.blur(threshold1,(3,3))
        ret, threshold2 = cv2.threshold(blur2, 50, 255, cv2.THRESH_BINARY)

        img2, contour, hier = cv2.findContours(threshold2, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_SIMPLE)

        for cnt in contour:
            cv2.drawContours(threshold2,[cnt],0,255, -10)
        finalImg = cv2.addWeighted(cv2.bitwise_not(threshold2), 0.2, img_thresh_final,0.8, 0)
        porosity = calculatePorosity(finalImg)
        img = cv2.resize(img, (150, 150))
        return [img, porosity]


from copy import deepcopy
import random
import ipdb
from openprompt.data_utils.utils import InputExample
import os
import json, csv
from abc import ABC, abstractmethod
from collections import defaultdict, Counter
from typing import List, Dict, Callable,Optional
import re
import spacy
spacier = spacy.load('en_core_web_sm')
def set_custom_boundaries(doc):
    '''Sometimes, spacy set < and > as end of sentence.
    This custom boundary will fix that bug.  '''
    for token in doc[:-1]:
        if ">" in token.text or "<" in token.text :
            doc[token.i].is_sent_start = False
            doc[token.i+1].is_sent_start = False
    return doc
#add custom boundary once
spacier.add_pipe(set_custom_boundaries, before="parser")

from openprompt.utils.logging import logger
from openprompt.data_utils.data_processor import DataProcessor

from DocumentalAMR import DocumentalAmrGraph,load_amrs_to_graph,parse_RAMS,load_document_amr

class RAMSProcesser (DataProcessor):

    def __init__(self):
        super().__init__()
        self.labels = None

    def get_event_type(self,ex):
        evt_type = []
        for evt in ex['evt_triggers']:
            for t in evt[2]:
                evt_type.append( t[0])

        return evt_type

    def load_ontology(self, ontology_file):
        # read ontology
        ontology_dict = {}
        with open(ontology_file,'r',encoding='utf8') as f:
            for lidx, line in enumerate(f):
                if lidx == 0:  # header
                    continue
                fields = line.strip().split(',')
                if len(fields) < 2:
                    break
                evt_type = fields[0]
                args = fields[2:]

                ontology_dict[evt_type] = {
                    'template': fields[1]
                }

                for i, arg in enumerate(args):
                    if arg != '':
                        ontology_dict[evt_type]['arg{}'.format(i + 1)] = arg
                        ontology_dict[evt_type][arg] = 'arg{}'.format(i + 1)

        return ontology_dict

    def create_gold_gen(self, ex, ontology_dict, mark_trigger=True):
        '''assumes that each line only contains 1 event.
        Input: <s> Template with special <arg> placeholders </s> </s> Passage </s>
        Output: <s> Template with arguments and <arg> when no argument is found.
        '''

        evt_type = self.get_event_type(ex)[0]
        context_words = [w for sent in ex['sentences'] for w in sent]
        template = ontology_dict[evt_type.replace('n/a', 'unspecified')]['template'].replace('.',',')
        input_template = re.sub(r'<arg\d>', '<arg>', template)
        for triple in ex['gold_evt_links']:
            trigger_span, argument_span, arg_name = triple
            try:
                arg_num = ontology_dict[evt_type.replace('n/a', 'unspecified')][arg_name]
            except:
                continue # err annotation
            arg_text = ' '.join(context_words[argument_span[0]:argument_span[1] + 1])
            template = re.sub('<{}>'.format(arg_num), arg_text, template)
        template = re.sub(r'<arg\d>', '<arg>', template) # to guarantee no arg\d remains in templates
        # replace <arg> with arg_type
        # for empty_arg_num in re.findall(r'<arg\d>', template):
        #     arg_name=ontology_dict[evt_type.replace('n/a', 'unspecified')][empty_arg_num[1:-1]]
        #     arg_name=re.split(r'\d+',arg_name)[-1]
        #     template = template.replace(f'{empty_arg_num}', arg_name)
            # arg_endpos=template.find(arg_name) + len(arg_name)+1
            # if arg_endpos<=len(template)-1 and arg_name.startswith(template[arg_endpos:arg_endpos+min(len(empty_arg_num),3)]):
            #     template=re.sub(empty_arg_num,'', template)
            # else:
            #     template=template.replace(f'{empty_arg_num}',arg_name)

        return input_template,template

    def get_examples(self, data_dir: str, split: str,**kwargs) -> List[InputExample]:
        examples,ex_dicts,amrs,= [],[],[]
        path = os.path.join(data_dir, "{}.jsonl".format(split))

        ontology_dict=self.load_ontology(kwargs['ontology_file'])
        doc_keys=json.load(open(os.path.join(data_dir,'doc_keys', "{}.json".format(split)))) # for few_shot learning
        with open(path) as f:
            all_dict = f.readlines()
            cnt=0
            for i, line in enumerate(all_dict):
                instance = json.loads(line)
                if instance['doc_key'] not in doc_keys or ('evt_key' in instance.keys() and instance['evt_key'] not in doc_keys):
                    continue
                event_type = self.get_event_type(instance)[0]
                trigger_id = instance["evt_triggers"][0][0]

                evt_links=deepcopy(instance['gold_evt_links'])
                sentences_tk = sum(instance['sentences'], [])
                try:
                    trigger_text=sentences_tk[trigger_id]
                except:
                    ipdb.set_trace()
                    trigger_text=instance['trigger_text']

                if kwargs['data_part']=='full':
                    input_template, tgt_text = self.create_gold_gen(instance, ontology_dict)

                else: # curriculum learnig

                    ex = deepcopy(instance)
                    if evt_links == []:
                        input_template, tgt_text = self.create_gold_gen(instance, ontology_dict)
                        if kwargs['data_part']=='easy':
                            sentences_tk=instance['sentences'][0]
                    else : # to find sent with trigger inside
                        sent_len = 0
                        sent_w_trg = 0
                        for sentid in range(len(instance['sentences'])):
                            if sent_len < trigger_id and (sent_len + len(instance['sentences'][sentid])) > trigger_id:
                                boundery = (sent_len, sent_len + len(instance['sentences'][sentid]))
                                sent_w_trg = sentid
                                break
                            sent_len += len(instance['sentences'][sentid])

                        if kwargs['data_part']=='easy':# intra sentence learning
                            try:
                                ex['sentences'] = [instance['sentences'][sent_w_trg]]
                            except:
                                ipdb.set_trace()
                            sentences_tk=instance['sentences'][sent_w_trg]

                            for ent in ex["ent_spans"]:
                                ent[0]-=boundery[0]
                                ent[1] -= boundery[0]
                            for trg in ex["evt_triggers"]:
                                trg[0]-=boundery[0]
                                trg[1] -= boundery[0]

                            for lkid,link in enumerate(evt_links): # modify links inside boundary
                                arg_spn=link[1]
                                if arg_spn[1]<boundery[1] and arg_spn[0]>=boundery[0]:
                                    ex['gold_evt_links'][lkid][1]=[arg_spn[0]-boundery[0],arg_spn[1]-boundery[0]]
                                    ex['gold_evt_links'][lkid][0]=[link[0][0]-boundery[0],link[0][1]-boundery[0]]

                            for lkid, link in enumerate(evt_links): # remove links outside boundary
                                arg_spn = link[1]
                                if arg_spn[1] < boundery[0] or arg_spn[0] >= boundery[1]:
                                    ex['gold_evt_links'].remove(link)

                            input_template, tgt_text = self.create_gold_gen(ex, ontology_dict)

                        elif kwargs['data_part']=='Given2For13': # sentence1,3 learning
                            ex['sentences'] = instance['sentences'][sent_w_trg - 1:sent_w_trg + 2]
                            newbdry0 = boundery[0] -  (len(instance['sentences'][sent_w_trg - 1]) if sent_w_trg-1 >=0 else 0)
                            newbdry1 = boundery[1] +  (len(instance['sentences'][sent_w_trg + 1]) if sent_w_trg + 1 < len(instance['sentences']) else 0)
                            boundery=(newbdry0,newbdry1)

                            inner_boundary = (len(sum(instance['sentences'][:sent_w_trg], [])),
                                              len(sum(instance['sentences'][:sent_w_trg + 1], [])))

                            arg_texts=[]
                            for lkid,link in enumerate(evt_links): # modify links inside boundary
                                arg_spn = link[1]
                                if arg_spn[1]<boundery[1] and arg_spn[0]>boundery[0]:
                                    ex['gold_evt_links'][lkid][1]=[arg_spn[0]-boundery[0],arg_spn[1]-boundery[0]]
                                    ex['gold_evt_links'][lkid][0]=[link[0][0]-boundery[0],link[0][1]-boundery[0]]

                                    if arg_spn[1]<inner_boundary[0] or arg_spn[0]>inner_boundary[1]:
                                        arg_texts.append(' '.join(sentences_tk[arg_spn[0]:arg_spn[1] + 1]))

                            for lkid, link in enumerate(evt_links): # remove links outside boundary
                                arg_spn = link[1]
                                if arg_spn[1] < boundery[0] or arg_spn[0] > boundery[1]:
                                    ex['gold_evt_links'].remove(link)

                            _, tgt_text = self.create_gold_gen(ex, ontology_dict)
                            input_template = tgt_text
                            for arg in arg_texts:
                                input_template = input_template.replace(arg, '<arg>')

                            sentences_tk = sum(ex['sentences'], [])

                        elif kwargs['data_part']=='Given123For04': # sentence1,3 learning
                            inner_boundary = (len(sum(instance['sentences'][:sent_w_trg-1], [])),
                                              len(sum(instance['sentences'][:sent_w_trg + 2], [])))

                            arg_texts=[]
                            # record=False
                            for lkid,link in enumerate(evt_links): # record spans outside inner boundery
                                arg_spn = link[1]
                                if arg_spn[1]<inner_boundary[0] or arg_spn[0] >= inner_boundary[1]:
                                    arg_texts.append(' '.join(sentences_tk[arg_spn[0]:arg_spn[1] + 1]))
                                    # record=True
                            # if len(evt_links)<3: record=False

                            _, tgt_text = self.create_gold_gen(ex, ontology_dict)
                            input_template = tgt_text
                            for arg in arg_texts:
                                input_template = input_template.replace(arg, '<arg>')

                        elif kwargs['data_part']=='Given2For0134': # inter sentence learning
                            inter_args=[]
                            for link in evt_links:# collect args outside boundary
                                arg_spn=link[1]
                                if arg_spn[1]<boundery[0] or arg_spn[0]>boundery[1]:
                                    inter_args.append(' '.join(sentences_tk[arg_spn[0]:arg_spn[1]]))

                            _, tgt_text = self.create_gold_gen(ex, ontology_dict)
                            input_template=tgt_text
                            for arg in inter_args:
                                input_template=input_template.replace(arg,'<arg>')

                doc = ' '.join(sentences_tk)
                doc = re.sub(r'[\u201c\u201d]', '\"', doc)
                doc = re.sub(r'[\u2019\u2018]', '\'', doc)
                doc = re.sub(r'[—–]', '-', doc)
                try:
                    doc = re.sub(trigger_text, '<trg>' + trigger_text + '<trg>', doc)
                except:
                    ipdb.set_trace()
                example = InputExample(guid=str(i) + '.' + split, text_a=doc,
                                       text_b=input_template, tgt_text=tgt_text, label=1,
                                       meta={'event_type': event_type, 'evt_links': evt_links,
                                             # 'sentences': instance['sentences'],
                                             'trigger_text': trigger_text, 'trigger_id': trigger_id,
                                             'doc_key': instance['doc_key']})
                # ex_dicts.append(ex)
                # if record:
                examples.append(example)

            if 'k_shot' in kwargs and kwargs['k_shot']!=None:
                random.shuffle(examples)
                examples=examples[:kwargs['k_shot']]

        print(cnt)
        return examples#,ex_dicts

    def get_src_tgt_len_ratio(self, ):
        pass

class WikiEventProcesser(DataProcessor):
    def __init__(self):
        super(WikiEventProcesser, self).__init__()
        self.MAX_CONTEXT_LENGTH = 300  # measured in words
        self.MAX_LENGTH = 512
        self.MAX_TGT_LENGTH = 70

        PRONOUN_FILE = 'pronoun_list.txt'
        self.pronoun_set = set()
        with open(PRONOUN_FILE, 'r') as f:
            for line in f:
                self.pronoun_set.add(line.strip())

    def check_pronoun(self,text):
        if text.lower() in self.pronoun_set:
            return True
        else:
            return False

    def clean_mention(self,text):
        '''
        Clean up a mention by removing 'a', 'an', 'the' prefixes.
        '''
        prefixes = ['the ', 'The ', 'an ', 'An ', 'a ', 'A ']
        for prefix in prefixes:
            if text.startswith(prefix):
                return text[len(prefix):]
        return text

    def load_ontology(self, ontology_file):
        '''
        Read ontology file for event to argument mapping.
        '''
        ontology_dict = {}
        with open(ontology_file, 'r') as f:
            ontology_dict = json.load(f)

        for evt_name, evt_dict in ontology_dict.items():
            for i, argname in enumerate(evt_dict['roles']):
                evt_dict['arg{}'.format(i + 1)] = argname
                # argname -> role is not a one-to-one mapping
                if argname in evt_dict:
                    evt_dict[argname].append('arg{}'.format(i + 1))
                else:
                    evt_dict[argname] = ['arg{}'.format(i + 1)]

        return ontology_dict

    def create_gold_gen(self, ex, ontology_dict, mark_trigger=True, index=0, ent2info=None, use_info=False,span_range=4,inner_boundary=0):
        '''
        If there are multiple events per example, use index parameter.

        Input: <s> Template with special <arg> 2 </s> </s> Passage </s>
        Output: <s> Template with arguments and <arg> when no argument is found.
        '''
        if use_info and ent2info == None:
            raise ValueError('entity to informative mention mapping required.')

        evt_type = ex['event_mentions'][index]['event_type']
        sentences=ex['sentences']
        sentID_w_trigger=ex['event_mentions'][index]['trigger']['sent_id']
        template = ontology_dict[evt_type]['template']
        # input_template = re.sub(r'<arg\d>', '<arg>', template)

        role2arg = defaultdict(list)

        for argument in ex['event_mentions'][index]['arguments']:
            role2arg[argument['role']].append(argument)

        role2arg = dict(role2arg)
        # create output template
        arg_idx2text = defaultdict(list)

        for role in role2arg.keys():
            if role not in ontology_dict[evt_type]:
                # annotation error
                continue
            for i, argument in enumerate(role2arg[role]):
                use_arg = True
                if use_info:
                    ent_id = argument['entity_id']
                    if ent_id in ent2info:
                        arg_text = self.clean_mention(ent2info[ent_id])
                        if self.check_pronoun(arg_text):
                            # skipping this argument
                            use_arg = False
                            # if arg_text != argument['text']:
                            # print('Original mention:{}, Informative mention:{}'.format(argument['text'], arg_text))
                    else:
                        arg_text = argument['text']
                else:
                    arg_text = argument['text']

                # assign the argument index
                if i < len(ontology_dict[evt_type][role]):
                    # enough slots to fill in
                    arg_idx = ontology_dict[evt_type][role][i]

                else:
                    # multiple participants for the same role
                    arg_idx = ontology_dict[evt_type][role][-1]

                if use_arg:
                    arg_idx2text[arg_idx].append(arg_text)

        ## select contents
        trigger = ex['event_mentions'][index]['trigger']
        offset = 0
        # trigger span does not include last index
        context_words = ex['tokens']
        center_sent = trigger['sent_idx']
        document=[]
        if len(context_words) > self.MAX_CONTEXT_LENGTH:
            cur_len = len(ex['sentences'][center_sent][0])
            context_words = [tup[0] for tup in ex['sentences'][center_sent][0]]
            if cur_len > self.MAX_CONTEXT_LENGTH:
                # one sentence is very long
                trigger_start = trigger['start']
                start_idx = max(0, trigger_start - self.MAX_CONTEXT_LENGTH // 2)
                # end_idx = min(len(context_words), trigger_start + self.MAX_CONTEXT_LENGTH // 2)
                context_words = context_words[start_idx: start_idx+self.MAX_CONTEXT_LENGTH]
                # context_words = context_words[start_idx: end_idx]
                offset = start_idx

            else:
                # take a sliding window
                left = center_sent - 1
                right = center_sent + 1

                total_sents = len(ex['sentences'])
                prev_len = 0
                while cur_len > prev_len:
                    prev_len = cur_len
                    # try expanding the sliding window
                    if left >= 0:
                        left_sent_tokens = [tup[0] for tup in ex['sentences'][left][0]]
                        if cur_len + len(left_sent_tokens) <= self.MAX_CONTEXT_LENGTH:
                            context_words = left_sent_tokens + context_words
                            left -= 1
                            cur_len += len(left_sent_tokens)

                    if right < total_sents:
                        right_sent_tokens = [tup[0] for tup in ex['sentences'][right][0]]
                        if cur_len + len(right_sent_tokens) <= self.MAX_CONTEXT_LENGTH:
                            context_words = context_words + right_sent_tokens
                            right += 1
                            cur_len += len(right_sent_tokens)
                # for sid in range(left+1,right):
                #     document.append([tup[0] for tup in ex['sentences'][sid][0]])

                # update trigger offset
                offset = sum([len(ex['sentences'][idx][0]) for idx in range(left + 1)])

        assert (len(context_words) <= self.MAX_CONTEXT_LENGTH)

        trigger['start'] = trigger['start'] - offset
        trigger['end'] = trigger['end'] - offset
        if mark_trigger:
            prefix = ' '.join(context_words[:trigger['start']])
            tgt = ' '.join(context_words[trigger['start']: trigger['end']])
            suffix = ' '.join(context_words[trigger['end']:])
            context = prefix + ' <trg>' + tgt + '<trg> '+ suffix
        else:
            context = ' '.join(context_words)

        doc=spacier(context)
        trg_pos=-1
        contextls=[]
        clues = ''
        for sid,sent in enumerate(doc.sents):
            if '<trg>' in sent.text:
                trg_pos=sid
            if '<trg>' not in sent.text and 'trg' in sent.text :
                ipdb.set_trace()
        for sid,sent in enumerate(doc.sents):
            if sid>=trg_pos-span_range and sid<=trg_pos+span_range:
                contextls.append(sent.text)
            if inner_boundary>=0 and sid >= trg_pos - inner_boundary and sid <= trg_pos + inner_boundary:
                clues+=(' '+sent.text)

        context=' '.join(contextls)
        if context== '':ipdb.set_trace()

        # measure the impact of context_len
        omitted_num=0
        arg_mentions=sum(list(arg_idx2text.values()),[])
        for arg in arg_mentions:
            if arg not in context:
                omitted_num+=1

        # develop clue prompts
        input_template=template
        for arg_idx, text_list in arg_idx2text.items():
            tmpls=text_list.copy()
            for argtxt in text_list:
                if argtxt not in clues:
                    tmpls.remove(argtxt)
            if tmpls==[]:continue
            text=' and '.join(tmpls)
            input_template = re.sub('<{}>'.format(arg_idx), text, input_template)

        # develop gold template
        output_template=template
        argtp2argtxt=dict()
        for arg_idx, text_list in arg_idx2text.items():
            text = ' and '.join(text_list)
            output_template = re.sub('<{}>'.format(arg_idx), text, output_template)
            arg_type=ontology_dict[evt_type][arg_idx]
            argtp2argtxt[arg_type] = text_list

        input_template = re.sub(r'<arg\d>', '<arg>', input_template)
        output_template = re.sub(r'<arg\d>', '<arg>', output_template)

        return input_template, output_template, contextls,context_words, omitted_num, argtp2argtxt

    def get_examples(self, data_dir: Optional[str] = None, split: Optional[str] = None,data_part:str='full',recordAsRAMS=False,**kwarg) -> List[InputExample]:
        ontology_dict = self.load_ontology(kwarg['ontology_file'])
        coref_split = 'dev' if split == 'val' else split
        coref_dir=os.path.join(data_dir,'coref')
        coref_reader = open(os.path.join(coref_dir, '{}.jsonlines'.format(coref_split)))
        examples=[]
        omitted_num,ttl_arg_num,ttl_doc_tks,skipped_ent=0,0,0,0
        doc_keys=json.load(open(os.path.join(data_dir,'doc_keys', "{}.json".format(split)))) # for few_shot learning
        with open(os.path.join(data_dir, '{}.jsonl'.format(split)), 'r') as reader:
            for step,(line, coref_line) in enumerate(zip(reader, coref_reader)):
                ex = json.loads(line.strip())
                corefs = json.loads(coref_line.strip())
                assert (ex['doc_id'] == corefs['doc_key'])
                if ex['doc_id'] not in doc_keys:
                    continue
                # mapping from entity id to information mention
                ent2info = {}
                for cidx, cluster in enumerate(corefs['clusters']):
                    for eid in cluster:
                        ent2info[eid] = corefs['informative_mentions'][cidx]

                for i in range(len(ex['event_mentions'])):
                    if split == 'train' and len(ex['event_mentions'][i]['arguments']) == 0:
                        continue
                    evt_type = ex['event_mentions'][i]['event_type']

                    if evt_type not in ontology_dict:  # should be a rare event type
                        continue

                    if data_part=='easy':
                        span_range=0
                        inner_boundary=-1 # no inner boundary
                    elif data_part=='Given2For13':
                        span_range=1
                        inner_boundary=0
                    elif data_part=='Given123For04' :
                        span_range=4
                        inner_boundary=1
                    elif data_part=='full':
                        span_range=4
                        inner_boundary=-1 # no inner boundary

                    input_template, output_template, contextls,context_words,omitted_args,argtp2argtxt = self.create_gold_gen(ex, ontology_dict,
                                                                                index=i, ent2info=ent2info,
                                                                                use_info=kwarg['use_info'],
                                                                                span_range=span_range,inner_boundary=inner_boundary)
                    if recordAsRAMS:
                        trigger_span=[ex['event_mentions'][i]['trigger']['start'],ex['event_mentions'][i]['trigger']['end']-1] # -1 to fit the RAMS form, no need to -offset cause we hanled in create_gold_gen already
                        evt_triggers=[[trigger_span[0],trigger_span[1],[[ex['event_mentions'][i]['event_type']]]]]
                        entys={}
                        for enty in ex['entity_mentions']:
                            entys[enty['id']]=deepcopy(enty)

                        ent_spans=[]
                        gold_evt_links=[]
                        for arg in ex['event_mentions'][i]['arguments']:
                            start=entys[arg['entity_id']]['start']
                            end=entys[arg['entity_id']]['end']
                            ent_spans.append([start,end,[[arg['role'],1.0]]])
                            gold_evt_links.append([trigger_span,[start,end],arg['role']])

                        example = InputExample(guid=str(step) + '.' + str(i) + '.' + split, text_a=' '.join(contextls),
                                               label=1,
                                               tgt_text=output_template,
                                               text_b=input_template,
                                               meta={'doc_key':ex['event_mentions'][i]['id'],
                                                     'gold_evt_links':gold_evt_links,
                                                     'evt_triggers': evt_triggers, 'ent_spans': ent_spans,
                                                     'sentences': context_words,
                                                     }
                                               )

                    else:
                        trigger_text=ex['event_mentions'][i]['trigger']['text']
                        ttl_arg_num+=ex['event_mentions'][i]['arguments'].__len__()
                        ttl_doc_tks+=len((' '.join(contextls)).split(' '))
                        omitted_num+=omitted_args

                        example = InputExample(guid=str(step)+'.'+ str(i) + '.' + split, text_a=' '.join(contextls), label=1,
                                               tgt_text=output_template,
                                               text_b=input_template,
                                               meta={'evt_type':evt_type,'doc_key':ex['doc_id'],'evt_key':ex['event_mentions'][i]['id'],
                                                    'arguments':argtp2argtxt,'trigger_text':trigger_text,'sentences':contextls,
                                                    }
                                               )
                    examples.append(example)
                if (step+1)%100==0:
                    print(f'{step+1} documents handled.')

        print(f'{omitted_num} args are discarded due to the limitation of context len.')
        print(f'{ttl_arg_num} args in total. Avg doc_len ={ttl_doc_tks/len(examples)}')

        return examples

class AmrGraphProcesser(DataProcessor):

    def __init__(self):
        super().__init__()
        self.labels = None

    def print_path(self,path,sentences_tk):
        path_text=[]
        intersentence = False
        last_pos = 0

        for nid, node in enumerate(path):
            if node == {}:  continue
            # TODO: currently we give up parallel path and only choose one of the shortest paths, future work is to decode all paths
            for n, pathdir in node:
                if nid == 0:
                    begin = ' '.join(sentences_tk[n[0]:n[1]])
                if nid == len(path) - 1:
                    end = ' '.join(sentences_tk[n[0]:n[1]])
                try:
                    if n[1] == 'c' or n[1] == 's':
                        intersentence = True

                    else:
                        if pathdir == 1 or pathdir[0] != '-':
                            path_text.insert(last_pos, ' '.join(sentences_tk[n[0]:n[1]]))
                        elif pathdir[0] == '-':
                            last_pos += 1
                            path_text.insert(last_pos, ' '.join(sentences_tk[n[0]:n[1]]))
                except:
                    ipdb.set_trace()
                break  # we only decode one path

        return path_text,intersentence,begin,end

    def get_examples(self, data_dir: str, split: str, dataset='RAMS',**kwargs) -> List[InputExample]:
        assert 'train' in split , 'amr graphs are only contained in training data'
        examples, amrs = [], []
        path = os.path.join(data_dir, "{}.jsonl".format(split))
        if not os.path.exists(path):
            path = os.path.join(data_dir, "{}.jsonlines".format(split))

        with open(kwargs['amr_path'], encoding='utf8') as f:
            amrs = f.read()
        amrs = amrs.split('\n\n')
        amrs = amrs[:-1] if amrs[-1] == "" else amrs

        amr_id = 0
        doc_keys=json.load(open(os.path.join(data_dir,'doc_keys', "{}.json".format(split)))) # for few_shot learning
        with open(path) as f:
            all_dict = f.readlines()
            err,err_doc = 0,0
            for i, line in enumerate(all_dict):
                instance = json.loads(line)
                if instance['doc_key'] not in doc_keys:
                    continue
                if dataset=='RAMS':
                    event_type = instance["evt_triggers"][0][2][0][0]
                    trg_span=(instance['evt_triggers'][0][0],instance['evt_triggers'][0][1])
                    arg_span2type=dict()
                    for links in instance["gold_evt_links"]:
                        arg_span2type[tuple(links[1])]=re.split('[\d]+', links[2])[-1]
                    keys=[ sp[0] for sp in sorted(arg_span2type.items(), key=lambda x: abs((x[0][0] + x[0][1]) / 2 - trg_span[0]))] # sort by distance from trg to arg
                    values=[ sp[1] for sp in sorted(arg_span2type.items(), key=lambda x: abs((x[0][0] + x[0][1]) / 2 - trg_span[1]))]
                    arg_span2type=dict(zip(keys,values))

                    sentences_tk = sum(instance['sentences'], [])
                    doc=' '.join(sentences_tk)
                    doc = re.sub(r'[\u201c\u201d]', '\"', doc)
                    doc = re.sub(r'[\u2019\u2018]', '\'', doc)
                    doc = re.sub(r'[—–]', '-', doc)
                    sentences_tk = (doc).split(' ')
                    trigger_text = ' '.join(sentences_tk[trg_span[0]:trg_span[1] + 1])

                    documental_amrs = amrs[amr_id:amr_id + len(instance['sentences'])]
                    tokens = documental_amrs[0].split('\n')[0]
                    if not tokens.startswith('# ::tok '):
                        if tokens.startswith('WrongSample'):
                            amr_id += 1
                            while not amrs[amr_id].startswith('evt_mention'):
                                amr_id += 1
                            err_doc += 1
                            continue
                        else:
                            ipdb.set_trace()

                    tokens = tokens[len("# ::tok "):].split(' ')
                    if tokens[:min(5,len(tokens)-1)]!=sentences_tk[:min(5,len(tokens)-1)] or 'clusters' not in instance: # <ROOT> <trg> # or
                        if 'clusters'  in instance:
                            # print(tokens[:7],sentences_tk[:7])
                            amr_id += len(json.loads(all_dict[i - 1])['sentences'])  # due to the errs in amr parser are solved by copying previous doc's amr, so we + previous len(sentences)
                            # ipdb.set_trace()
                        else:
                            amr_id += len(instance['sentences'])
                        err_doc+=1
                        continue

                    amr_id += len(instance['sentences'])

                    graph = load_amrs_to_graph(documental_amrs)
                    graph.merge_coref({"clusters": instance['clusters'], 'document': instance['document']})
                    if not graph.tokens[0][0] == sentences_tk[0]: ipdb.set_trace()

                elif dataset=='wikievents':

                    event_type = instance["evt_triggers"][0][2][0][0]
                    trg_span=(instance['evt_triggers'][0][0],instance['evt_triggers'][0][1])
                    arg_span2type=dict()
                    for links in instance["gold_evt_links"]:
                        arg_span2type[tuple(links[1])]=re.split('[\d]+', links[2])[-1]
                    keys=[ sp[0] for sp in sorted(arg_span2type.items(), key=lambda x: abs((x[0][0] + x[0][1]) / 2 - trg_span[0]))] # sort by distance from trg to arg
                    values=[ sp[1] for sp in sorted(arg_span2type.items(), key=lambda x: abs((x[0][0] + x[0][1]) / 2 - trg_span[1]))]
                    arg_span2type=dict(zip(keys,values))

                    sentences_tk = sum(instance['sentences'], [])
                    doc=' '.join(sentences_tk)
                    doc = re.sub(r'[\u201c\u201d]', '\"', doc)
                    doc = re.sub(r'[\u2019\u2018]', '\'', doc)
                    doc = re.sub(r'[—–]', '-', doc)
                    sentences_tk = (doc).split(' ')
                    trigger_text = ' '.join(sentences_tk[trg_span[0]:trg_span[1] + 1])

                    documental_amrs = amrs[amr_id:amr_id + len(instance['sentences'])]
                    evt_key = documental_amrs[0].split('\n')[0]

                    if not evt_key.startswith('evt_mention'): ipdb.set_trace()
                    documental_amrs[0] = '\n'.join(documental_amrs[0].split('\n')[1:])
                    tokens = documental_amrs[0].split('\n')[0]
                    if not tokens.startswith('# ::tok '):
                        if tokens.startswith('WrongSample'):
                            amr_id +=1
                            while not amrs[amr_id].startswith('evt_mention'):
                                amr_id+=1
                            err_doc += 1
                            continue
                        else:
                            ipdb.set_trace()

                    tokens = tokens[len("# ::tok "):].split(' ')
                    if tokens[:min(5,len(tokens)-1)]!=sentences_tk[:min(5,len(tokens)-1)] or 'clusters' not in instance: # <ROOT> <trg> # or
                        if 'clusters' in instance:
                            ipdb.set_trace()
                            print(tokens[:7],sentences_tk[:7])
                            amr_id += len(json.loads(all_dict[i - 1])['sentences'])  # due to the errs in amr parser are solved by copying previous doc's amr, so we + previous len(sentences)
                            # ipdb.set_trace()
                        else:
                            amr_id += len(instance['sentences'])
                        err_doc+=1
                        continue

                    amr_id += len(instance['sentences'])

                    graph = load_amrs_to_graph(documental_amrs)
                    graph.merge_coref({"clusters": instance['clusters'], 'document': instance['document']})
                    if not graph.tokens[0][0] == sentences_tk[0]: ipdb.set_trace()
                if kwargs['path_method']=='all_T2As':
                    for links in instance['gold_evt_links']:
                        arg_span = links[1]
                        arg_text = ' '.join(sentences_tk[arg_span[0]:arg_span[1] + 1])
                        arg_type = re.split('[\d]+', links[2])[-1]
                        path = graph.BFS((trg_span[0], trg_span[1] + 1), (arg_span[0], arg_span[1] + 1))

                        if path == None or len(path) == 1:
                            err += 1
                            continue

                        path_text,intersentence,begin,end=self.print_path(path,sentences_tk)
                        path_text=' '.join(path_text)
                        example = InputExample(guid=str(i) + '.' + split,label=1,
                                               text_a=' '.join(sentences_tk).replace(trigger_text,'<trg>'+trigger_text+'<trg>'),
                                               tgt_text= path_text.replace(begin,begin+' '+arg_type),
                                               text_b=path_text.replace(begin,"<arg>"+' '+arg_type),
                                               meta={'event_type':event_type, 'trigger_text': trigger_text,
                                                     'arg_text': arg_text, 'arg_type': arg_type,
                                                     'trigger_span': trg_span, 'arg_span': arg_span,
                                                     'path_begin': ' '.join(begin), 'path_end': ' '.join(end)
                                                     ,'doc_key': instance['doc_key'],})
                        examples.append(example)

                elif kwargs['path_method']=='SteinerTree': # too many bugs in this Lib
                    NotFound = True
                    if trigger_text != instance['trigger_text']: continue
                    # arg_span_1st as the nearest
                    arg_span_1st=list(arg_span2type.keys())[0]
                    NotFound=False

                    if NotFound:
                        if instance['gold_evt_links']!=[]:
                            arg_span_1st=list(arg_span2type.keys())[0]
                        else:
                            err_doc += 1
                            continue

                    A2ApathExist = False
                    for arg_span in list(arg_span2type.keys()):
                        if arg_span==arg_span_1st: continue
                        try:
                            steinerPath=graph.steinerTree(trigger_span=(trg_span[0], trg_span[1]+1), clue_arg_span=(arg_span_1st[0], arg_span_1st[1] + 1),
                                              target_arg_span=(arg_span[0], arg_span[1] + 1))
                        except:
                            ipdb.set_trace()
                            graph.steinerTree(trigger_span=(trg_span[0], trg_span[1]+1), clue_arg_span=(arg_span_1st[0], arg_span_1st[1] + 1),
                                              target_arg_span=(arg_span[0], arg_span[1] + 1))
                        if steinerPath != None and len(steinerPath) != 1:
                            A2ApathExist=True
                            steinerPath_text, intersentence, _ , _ = self.print_path(steinerPath, sentences_tk)
                            arg_text=' '.join(sentences_tk[graph.find_substitute(arg_span)[0],graph.find_substitute(arg_span)[1]])
                            arg_1st_text=' '.join(sentences_tk[graph.find_substitute(arg_span_1st)[0],graph.find_substitute(arg_span_1st)[1]])
                            steinerPath_text=' '.join(steinerPath_text).replace(arg_text,arg_text+' '+ arg_span2type[arg_span])
                            steinerPath_text=steinerPath_text.replace(arg_1st_text,arg_1st_text+' '+ arg_span2type[arg_span_1st])
                            ipdb.set_trace()
                            maskedsteinerPath_text=steinerPath_text.replace(arg_text,'<arg>')
                            # full_masked_path=maskedT2Apath_text+' ' +maskedA2Apath_text
                            example = InputExample(guid=str(i) + '.' + split, label=1,
                                                   text_a=' '.join(sentences_tk).replace(trigger_text,
                                                                                '<trg>' + trigger_text + '<trg>'),
                                                   tgt_text=steinerPath_text,
                                                   text_b=maskedsteinerPath_text,
                                                   meta={'event_type': event_type, 'trigger_text': trigger_text,
                                                         'trigger_span': trg_span, 'arg_1st':arg_1st,
                                                         'arg_2nd':arg_text,'doc_key': instance['doc_key'],
                                                         'evt_key':instance['evt_key'] # denote if dataset==RAMS
                                                         })
                            examples.append(example)

                elif kwargs['path_method']=='T2A2A':
                    NotFound = True
                    # if trigger_text != instance['trigger_text']: continue
                    # arg_span_1st as the nearest
                    if instance['gold_evt_links']!=[]:
                        arg_span_1st=list(arg_span2type.keys())[0]
                        NotFound=False

                    if NotFound:
                        if instance['gold_evt_links']!=[]:
                            arg_span_1st=list(arg_span2type.keys())[0]
                        else:
                            err_doc += 1
                            continue

                    T2Apath = graph.BFS((trg_span[0],trg_span[1] + 1), (arg_span_1st[0], arg_span_1st[1] + 1))
                    if T2Apath==None or len(T2Apath) == 1:
                        # err_doc += 1
                        # generate splited paths , useless
                        # for arg_span in list(arg_span2type.keys()):
                        #     T2Apath = graph.BFS((trg_id, trg_id + 1), (arg_span[0], arg_span[1] + 1))
                        #     if T2Apath == None or len(T2Apath) == 1:
                        #         err+=1
                        #         continue
                        #     T2Apath_text, intersentence, arg_1st, end = self.print_path(T2Apath, sentences_tk)
                        #     T2Apath_text = ' '.join(T2Apath_text).replace(arg_1st,
                        #                                                   arg_1st + ' ' + arg_span2type[arg_span])
                        #     example = InputExample(guid=str(i) + '.' + split, label=1,
                        #                            text_a=' '.join(sentences_tk).replace(trigger_text,
                        #                                                                  '<trg>' + trigger_text + '<trg>'),
                        #                            tgt_text=T2Apath_text,
                        #                            text_b=T2Apath_text.replace(arg_1st, '<arg>'),
                        #                            meta={'event_type': event_type, 'trigger_text': trigger_text,
                        #                                  'trigger_span': [trg_id, trg_id], 'arg_1st': arg_1st,
                        #                                 'doc_key': instance['doc_key'], })
                        #     examples.append(example)
                        continue

                    else:
                        T2Apath_text,intersentence,arg_1st,end = self.print_path(T2Apath,sentences_tk)
                        T2Apath_text=' '.join(T2Apath_text).replace(arg_1st,arg_1st+' '+ arg_span2type[arg_span_1st])
                        # pred arg_1st
                        # maskedT2Apath_text=T2Apath_text.replace(arg_1st,'<arg>')

                        A2ApathExist = False
                        for arg_span in list(arg_span2type.keys()):
                            if arg_span==arg_span_1st: continue
                            A2Apath = graph.BFS((arg_span_1st[0], arg_span_1st[1] + 1), (arg_span[0], arg_span[1] + 1))
                            if A2Apath != None and len(A2Apath) != 1:
                                A2ApathExist=True
                                A2Apath_text, intersentence, arg_text, end = self.print_path(A2Apath, sentences_tk)
                                full_path = T2Apath_text
                                for text in A2Apath_text:
                                    if text not in T2Apath_text:
                                        full_path += ' ' + text
                                full_path = full_path.replace(arg_text, arg_text + ' ' + arg_span2type[arg_span])
                                full_masked_path=full_path.replace(arg_text, '<arg>')
                                example = InputExample(guid=str(i) + '.' + split, label=1,
                                                       text_a=' '.join(sentences_tk).replace(trigger_text,
                                                                                '<trg>' + trigger_text + '<trg>'),
                                                       tgt_text=full_path,
                                                       text_b=full_masked_path,
                                                       meta={'event_type': event_type, 'trigger_text': trigger_text,
                                                             'trigger_span': trg_span, 'arg_1st':arg_1st,
                                                             'arg_2nd':arg_text,'doc_key': instance['doc_key'],
                                                             # 'evt_key':instance['evt_key'] # denote if dataset==RAMS
                                                             })
                                examples.append(example)
                            else:
                                # expand to 6991 samples
                                T2Apath = graph.BFS((trg_span[0], trg_span[1]+1), (arg_span[0], arg_span[1] + 1))
                                if T2Apath == None or len(T2Apath) == 1:
                                    err += 1
                                    continue
                                new_T2Apath_text, intersentence, arg_text, end = self.print_path(T2Apath, sentences_tk)
                                full_path=T2Apath_text
                                for text in new_T2Apath_text:
                                    if text not in T2Apath_text:
                                        full_path+=' '+text
                                full_path=full_path.replace(arg_text,arg_text + ' ' + arg_span2type[arg_span])

                                example = InputExample(guid=str(i) + '.' + split, label=1,
                                                       text_a=' '.join(sentences_tk).replace(trigger_text,
                                                                                '<trg>' + trigger_text + '<trg>'),
                                                       tgt_text=full_path, # TODO: how does it work?
                                                       text_b=full_path.replace(arg_text, '<arg>'),
                                                       meta={'event_type': event_type, 'trigger_text': trigger_text,
                                                             'trigger_span': trg_span, 'arg_1st': arg_1st,
                                                             'arg_2nd': arg_text
                                                           , 'doc_key': instance['doc_key'], })
                                examples.append(example)

                elif kwargs['path_method']=='maskedT2As':
                    for links in instance['gold_evt_links']:
                        arg_span = links[1]
                        # if trigger_span==arg_span: continue
                        trigger_text = ' '.join(sentences_tk[trg_span[0]:trg_span[1] + 1])
                        arg_text = ' '.join(sentences_tk[arg_span[0]:arg_span[1] + 1])
                        arg_type = re.split('[\d]+', links[2])[-1]
                        path = graph.BFS((trg_span[0], trg_span[1] + 1), (arg_span[0], arg_span[1] + 1))

                        if path == None or len(path) == 1:
                            err += 1
                            continue

                        path_text,intersentence,begin,end=self.print_path(path,sentences_tk)

                        raw_path_text=' '.join(path_text)
                        mask_poses=random.choices(path_text,k=1)
                        masked_path_text=raw_path_text
                        for mask in mask_poses:
                            if mask==begin:continue
                            masked_path_text=masked_path_text.replace(mask,'<arg>')

                        example = InputExample(guid=str(i) + '.' + split,label=1,
                                               text_a=' '.join(sentences_tk).replace(trigger_text,'<trg>'+trigger_text+'<trg>'),
                                               tgt_text= raw_path_text.replace(begin,begin+' '+arg_type),
                                               text_b=masked_path_text.replace(begin,"<arg>"+' '+arg_type),
                                               meta={'event_type':event_type, 'trigger_text': trigger_text,
                                                     'arg_text': arg_text, 'arg_type': arg_type,
                                                     'trigger_span': trg_span, 'arg_span': arg_span,
                                                     'path_begin': ' '.join(begin), 'path_end': ' '.join(end)
                                                     ,'doc_key': instance['doc_key'],})
                        examples.append(example)

                if (i + 1) % 500 == 0:
                    print(f'have handled {i + 1} documents')

        print('err num is ', err, f',err doc num is {err_doc}, ttl number of examples is ', len(examples))

        return examples

if __name__=='__main__':
    examples=WikiEventProcesser().get_examples(data_dir='./data/wikievents/informative', split='train', use_info=True,
                                               ontology_file='event_role_wikievents.json',data_part='full',recordAsRAMS=True)
    exampleInspection=[]
    ttl_sent=0
    for ex in examples:
        instance=ex.meta
        exampleInspection.append(instance)
    with open('./data/wikievents/informative/train.jsonl','w') as f:
        f.writelines((json.dumps(ex)+'\n' for ex in exampleInspection))