import csv
import datetime as dt
import os

import numpy as np
from matplotlib import pyplot
from pandas import datetime
from pandas import read_csv
from sklearn.metrics import mean_squared_error
from statsmodels.tsa.arima_model import ARIMA
from statsmodels.tsa.statespace.sarimax import SARIMAX

from prepare_historical_data import PREDICTION_END_YEAR, months, DATA_FOLDER, PYTHON_LANGUAGE, R_LANGUAGE, \
    CSV_FILE_SUFFIX

SARIMAX_ = "sarimax"

PREDICTIONS_FOLDER = "results"

def write_to_csv(dates, predictions, language, model):
    data_file_name = os.path.join(PREDICTIONS_FOLDER, language + "_" + model + "_predictions.csv")

    with open(data_file_name, 'w', newline='') as f:
        writer = csv.writer(f)
        writer.writerows(zip(dates, predictions))
    return True

def create_predictions_folder():
    if not os.path.exists(PREDICTIONS_FOLDER):
        os.makedirs(PREDICTIONS_FOLDER)

def RMSE(actual, pred):
    mse = mean_squared_error(actual, pred)
    rmse = np.sqrt(mse)
    return rmse

def parser(x):
    return datetime.strptime(x, '%Y-%m')

def get_future_date_list():
    future_dates = [str(year) + "-" + month for year in range(2019, PREDICTION_END_YEAR + 1) for month in
                    months]

    dates_list = [dt.datetime.strptime(date, '%Y-%m').date() for date in future_dates]
    print(dates_list)
    return dates_list

def run_arima(language):
    create_predictions_folder()
    series = read_csv(os.path.join(DATA_FOLDER, language + CSV_FILE_SUFFIX), header=0, parse_dates=[0], index_col=0,
                      squeeze=True,
                      date_parser=parser)

    data = series.values.tolist()
    train, test = data[:-12], data[-12:]
    model = ARIMA(train, order=(1, 1, 1))
    model_fit = model.fit(disp=False,
                          start_params=[1, .1, .1])  # , start_params=[np.mean(data), .1, np.mean(data)]
    dates_list = get_future_date_list()
    test_pred = model_fit.predict(len(train) + 1, len(data) - 1, typ='levels')
    future_pred = model_fit.predict(len(data), len(data) + 59, typ='levels')
    future_pred = future_pred[12:]
    pyplot.figure()
    pyplot.title("Predictions based on ARIMA model : " + language + " repositories")
    pyplot.plot(series, label='Historical data')
    pyplot.plot(series.keys().tolist(), [None for i in range(len(train))] + test_pred.tolist(),
                label='Predictions - Test data')
    pyplot.plot(dates_list, future_pred, label='Predictions - 2019 to 2023')
    pyplot.legend()
    pyplot.savefig(os.path.join(PREDICTIONS_FOLDER, language + "_predictions_arima.png"))

    rmse = RMSE(test, test_pred)
    print('ARIMA RMSE: %.3f' % rmse + " for " + language + " repos test set")
    write_to_csv([str(date_)[:-3] for date_ in dates_list], future_pred, language, "arima")
    return future_pred

def run_sarimax(language):
    create_predictions_folder()
    series = read_csv(os.path.join(DATA_FOLDER, language + CSV_FILE_SUFFIX), header=0, parse_dates=[0], index_col=0,
                      squeeze=True,
                      date_parser=parser)

    data = series.values.tolist()
    train, test = data[:-12], data[-12:]
    model_fit = SARIMAX(train, order=(2, 1, 4), seasonal_order=(1, 1, 1, 12)).fit()
    dates_list = get_future_date_list()
    print(len(dates_list))
    test_pred = model_fit.predict(len(train) + 1, len(data), dynamic=True)
    future_pred = model_fit.predict(len(data), len(data) + 59, dynamic=True)
    pyplot.figure()
    pyplot.title("Predictions based on SARIMAX model : " + language + " repositories")
    pyplot.plot(series, label='Historical data')
    pyplot.plot(series.keys().tolist(), [None for i in range(len(train))] + test_pred.tolist(),
                label='Predictions - Test data')
    pyplot.plot(dates_list, future_pred, label='Predictions - 2019 to 2023')
    pyplot.legend()
    pyplot.savefig(os.path.join(PREDICTIONS_FOLDER, language + "_predictions_SARIMAX.png"))

    rmse = RMSE(test, test_pred)
    print('SARIMAX RMSE: %.3f' % rmse + " for " + language + " repos test set")
    write_to_csv([str(date_)[:-3] for date_ in dates_list], future_pred, language, SARIMAX_)
    return future_pred

if __name__ == '__main__':
    run_arima(PYTHON_LANGUAGE)
    run_arima(R_LANGUAGE)
    run_sarimax(PYTHON_LANGUAGE)
    run_sarimax(R_LANGUAGE)


from collections import defaultdict

class TrieNode:
    def __init__(self):
        self.word = -1
        self.children = defaultdict(TrieNode)
        self.palindrome_word = []

class Trie:
    def __init__(self):
        self.root = TrieNode()

    @staticmethod
    def is_palindrome(word: str) -> bool:
        return word[::] == word[::-1]

    def insert(self, word: str, index: int) -> None:
        node = self.root
        for i, w in enumerate(reversed(word)):
            if self.is_palindrome(word[:len(word) - i]):
                node.palindrome_word.append(index)
            node = node.children[w]
        node.word = index

    def search(self, word: str, index: int) -> list[list[int]]:
        node = self.root
        result = []
        # word 를 따라 내려갔을 때 중간에 id가 있고 word 가 palindrome 일때
        while word:
            if node.word != -1:
                if self.is_palindrome(word):
                    result.append([index, node.word])
            if word[0] not in node.children:
                return result
            node = node.children[word[0]]
            word = word[1:]

        # word ID
        if node.word != -1 and node.word != index:
            result.append([index, node.word])
        # palindrome ID
        for palindrome in node.palindrome_word:
            result.append([index, palindrome])

        return result

    def print_trie(self) -> None:
        # dfs 출력
        node = self.root
        stack = [(node,"")]
        visited = []

        while stack:
            new_node, val = stack.pop()
            if new_node not in visited:
                visited.append(new_node)
                print(val, new_node.word, new_node.palindrome_word)
                for ke in new_node.children.keys():
                    stack.append((new_node.children[ke], ke))

from typing import List

class Solution:
    # brute force -> time limit exceed

    # def palindromePairs(self, words: List[str]) -> List[List[int]]:
    #     length = len(words)
    #     result = []
    #     for i in range(length):
    #         for j in range(length):
    #             if i == j:
    #                 continue
    #             concat = words[i] + words[j]
    #             if concat[:] == concat[::-1]:
    #                 result.append([i, j])
    #     return result

    def palindromePairs(self, words: List[str]) -> List[List[int]]:
        trie = Trie()
        result = []
        for idx, word in enumerate(words):
            trie.insert(word, idx)

        # trie.print_trie()
        for idx, word in enumerate(words):
            result.extend(trie.search(word, idx))

        return result

sol = Solution()
print(sol.palindromePairs(["abcd", "dcba", "lls", "s", "sssll"]))
# print(sol.palindromePairs(["a",""]))


from inspect import stack, getmodule
from os.path import exists

def inline_snippet(
    example_path: str,
    write_mode: str = 'w+',
    error_message: str = "# Attempted inline snippet failed."
) -> None:
    """Writes the content in `example_path` to the calling file.

    Args:
        example_path (str): Path to the example file.
        write_mode (str, optional): The mode in which the file is opened. Defaults to 'w+'.
        error_message (str, optional): What to insert if something goes wrong.
            Defaults to "# Attempted inline example failed.".
    """

    # open example and get contents
    example = None
    if exists(example_path):
        with open(example_path, 'r') as i:
            example = i.read()

    # set error message if example not found
    if not example:
        example = error_message
        write_mode = 'a+'

    # write to caller's current file
    module = getmodule(stack()[-1][0])
    with open(module.__file__, write_mode) as out:
        if 'a' in write_mode:
            example = '\n\n' + example
        out.write(example)


import time
import requests

# Takes in a schedule dictionary and sends instructions at specified times
# Exits when mode is changed to manual
def send_instructions(schedule):
    manual = False
    while not manual:
        for scheduled_time in schedule.keys():
            if int(time.strftime('%H')) == int(scheduled_time.split(':')[0]) and int(time.strftime('%M')) == int(scheduled_time.split(':')[1]):
                new_state = schedule[scheduled_time]
                print('Posting... ' + str(new_state))
                requests.post(
                    'https://blind-control-299118.ue.r.appspot.com/flip', json={'new_state': new_state})
                print('Posted.')

        manual = requests.get(
            'https://blind-control-299118.ue.r.appspot.com/get-mode').json()['mode'] == 'manual'
        schedule = requests.get(
            'https://blind-control-299118.ue.r.appspot.com/get-schedule').json()
    return

def create_schedule(request):
    """Responds to any HTTP request.
    Args:
        request (flask.Request): HTTP request object.
    Returns:
        The response text or any set of values that can be turned into a
        Response object using
        `make_response <http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>`.
    """
    request_json = request.get_json()

    send_instructions(request_json)

    return 'Success!'


from selenium import webdriver
import time
import json
import os
from selenium.webdriver.common.by import By
import subprocess
import difflib
import re
from urllib.parse import unquote
from colorama import *

class Voltaire:

    def __init__(self):
        options = webdriver.ChromeOptions()
        options.add_experimental_option('excludeSwitches', ['enable-logging'])

        ch_output = subprocess.Popen(["chromedriver"])
        self.driver = webdriver.Chrome(options=options)
        self.driver.implicitly_wait(1)

    def Header(self):
        os.system('cls')

        print(f"""
{Fore.GREEN}__   __   _ _        {Fore.RED}_{Fore.GREEN}
\ \ / /__| | |_ __ _{Fore.RED}(_){Fore.GREEN}_ _ ___
 \ V / {Fore.WHITE}_{Fore.GREEN} \ |  _/ {Fore.WHITE}_{Fore.GREEN}` | | '_/ -_)
  \_/\___/_|\__\__,_|_|_| \___|{Fore.RESET} [{Fore.RED}solved{Fore.RESET}]
""")

        self.par = input(f' [{Fore.GREEN}*{Fore.RESET}] Supérieur / Excellence ? => ')
        self.nb = input(f' [{Fore.GREEN}*{Fore.RESET}] Test number => ')

        try:
            int(self.nb)
            pass
        except ValueError:
            print(f' [{Fore.RED}!{Fore.RESET}] INT is required !')
            time.sleep(3.5)
            self.Header()

    def solver(self):
        data_filename = f"Module{self.nb}.txt"

        directory = f"./src/{self.par}/"
        reponses = []

        for filename in os.listdir(directory): #Credit => https://github.com/sylvain-reynaud/projet-voltaire-solver
            if filename.endswith(data_filename):
                with open(directory + filename, 'r', encoding="utf-8") as f:
                    data = f.read()
                    try:
                        data = data[data.index("[\"java.util.ArrayList"):data.index("]")] + "]"
                        data = data.replace("\\", "\\\\")
                        reponses += json.loads(data)
                    except:
                        pass
        reponses = [x for x in reponses if "\\x3C" in x]

        input(f' [{Fore.GREEN}*{Fore.RESET}] Ready ? (Enter to start)')
        print(f' [{Fore.GREEN}*{Fore.RESET}] Test {self.nb} Started with 1 attempt every 5 seconds !')

        while True:
            try:
                time.sleep(5)
                items = self.driver.find_elements(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[1]/div/div/div[2]/div")
                for item in items:
                    phrase = item.text

                possibilites = difflib.get_close_matches(phrase, reponses)
                if len(possibilites) != 0:
                    toPrint = re.sub(r"<B>(.*)<\/B>", fr"{Fore.GREEN}\1{Fore.RESET}", unquote(possibilites[0].replace("\\x", "%")))
                    print(f' [{Fore.GREEN}*{Fore.RESET}]', toPrint)

                else:
                    print(f' [{Fore.GREEN}*{Fore.RESET}] Il n\'y a pas de faute')
                    self.driver.find_element(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[2]/div[1]/div[1]/div/button").click()
                    self.driver.find_element(By.XPATH, "/html/body/div[5]/div[2]/div[3]/div/div[3]/div[2]/button").click()

            except:
                print(f' [{Fore.RED}*{Fore.RESET}] Attempt Failed !')

Voltaire_solved = Voltaire()
Voltaire_solved.Header()
Voltaire_solved.solver()


import argparse
import json
import os

import cv2
import imageio
import numpy as np
import pims

def _get_box(annot_box):
    x, y, w, h = annot_box["x"], annot_box["y"], annot_box["width"], annot_box["height"]
    return (int(x), int(y), int(x + w), int(y + h))

def extract_crop_from_image(image, box):
    x1, y1, x2, y2 = box
    crop = image[y1:y2, x1:x2]
    h, w = crop.shape[:2]
    # Add padding to make it square
    pad_l, pad_r, pad_t, pad_b = 0, 0, 0, 0
    if h > w:
        pad_l = (h - w) // 2
        pad_r = (h - w) - pad_l
    if w > h:
        pad_t = (w - h) // 2
        pad_b = (w - h) - pad_t
    crop = np.pad(
        crop,
        ((pad_t, pad_b), (pad_l, pad_r), (0, 0)),
        mode="constant",
        constant_values=255,
    )
    return crop

def draw_box_on_image(image, box, color=(255, 0, 0), thickness=5):
    x1, y1, x2, y2 = [int(b) for b in box]
    image = cv2.rectangle(image, (x1, y1), (x2, y2), color=color, thickness=thickness)
    return image

def draw_border(image, color=(255, 0, 0), thickness=5):
    x1, y1 = 0, 0
    x2, y2 = image.shape[1] - 1, image.shape[0] - 1
    return draw_box_on_image(image, [x1, y1, x2, y2], color=color, thickness=thickness)

def scale_im_height(image, H):
    im_H, im_W = image.shape[:2]
    W = int(1.0 * H * im_W / im_H)
    return cv2.resize(image, (W, H))

def visualize_query_set(video_reader, qset, save_height=640):
    qf_fno = qset["query_frame"]
    vc_fno = qset["visual_crop"]["frame_number"]
    last_fno = max(qf_fno, vc_fno)
    vc_box = _get_box(qset["visual_crop"])
    rt_fnos = [rf["frame_number"] for rf in qset["response_track"]]
    rt_boxes = [_get_box(rf) for rf in qset["response_track"]]
    oW = qset["visual_crop"]["original_width"]
    oH = qset["visual_crop"]["original_height"]
    # Visualize visual crop
    vc_frame = np.copy(video_reader[vc_fno])
    ## Scale up to original resolution
    vc_frame = scale_im_height(vc_frame, oH)
    vc_im = extract_crop_from_image(vc_frame, vc_box)
    ## Add text header
    tx_height = 50
    ob_title = qset["object_title"]
    tx_im = get_text_box(ob_title, (tx_height, vc_im.shape[1]))
    vc_im = np.concatenate([tx_im, vc_im], axis=0)
    # Visualize frames in the response track
    rt_ims = []
    for rf_fno, rf_box in zip(rt_fnos, rt_boxes):
        rf_frame = np.copy(video_reader[rf_fno])
        ## Scale up to original resolution
        rf_frame = scale_im_height(rf_frame, oH)
        rf_im = draw_box_on_image(rf_frame, rf_box, color=(0, 255, 0), thickness=8)
        rf_im = draw_border(rf_im, color=(0, 255, 0), thickness=15)
        ## Scale down to save height
        rf_im = scale_im_height(rf_im, save_height)
        rt_ims.append(rf_im)
    # Visualize frames after the response track till the query frame
    post_rt_ims = []
    for i in range(rt_fnos[-1], qf_fno):
        # Concatenate vc_plot_im to the right
        frame = scale_im_height(video_reader[i], save_height)
        post_rt_ims.append(frame)
    rt_ims = rt_ims + post_rt_ims
    return rt_ims, vc_im

def get_mp4_writer(path, fps, output_params=["-crf", "31"]):
    writer = imageio.get_writer(
        path,
        codec="h264",
        fps=fps,
        quality=None,
        pixelformat="yuv420p",
        bitrate=0,  # Setting bitrate to 0 is required to activate -crf
        output_params=output_params,
    )
    return writer

def save_video(frames, path, fps):
    writer = get_mp4_writer(path, fps)
    for frame in frames:
        writer.append_data(frame)
    writer.close()

def get_text_box(text, shape, fg_color=(255, 255, 255), bg_color=(0, 0, 0)):
    text_im = np.zeros((*shape, 3), dtype=np.uint8)
    text_im[:, :] = bg_color
    font = cv2.FONT_HERSHEY_SIMPLEX
    font_scale = 1.0
    line_thickness = 2
    text_im = cv2.putText(
        text_im,
        text,
        (2, shape[0] - 10),
        font,
        font_scale,
        fg_color,
        line_thickness,
        cv2.LINE_AA,
    )
    return text_im

def visualize_annotation(clip_path, rt_save_path, crop_save_path, qset):
    """
    Visualizes an annotation from the visual-queries task
    """
    video_reader = pims.Video(clip_path)
    # Visualize annotations for 3 query sets
    rt_frames, vc_im = visualize_query_set(video_reader, qset)
    save_video(rt_frames, rt_save_path, 5.0)
    imageio.imwrite(crop_save_path, vc_im)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--annot-path", type=str, required=True)
    parser.add_argument("--clips-root", type=str, required=True)
    parser.add_argument("--vis-save-root", type=str, default="./videos")

    args = parser.parse_args()

    with open(args.annot_path, "r") as fp:
        annotations = json.load(fp)

    os.makedirs(args.vis_save_root, exist_ok=True)

    for v in annotations["videos"]:
        for c in v["clips"]:
            cuid = c["clip_uid"]
            for a_idx, a in enumerate(c["annotations"]):
                for qset_id, qset in a["query_sets"].items():
                    if not qset["is_valid"]:
                        continue
                    qf_fno = qset["query_frame"]
                    rt_last_fno = max(
                        [rf["frame_number"] for rf in qset["response_track"]]
                    )
                    rtsp = f"{args.vis_save_root}/{cuid}_{a_idx:05d}_{qset_id}_rt.mp4"
                    csp = f"{args.vis_save_root}/{cuid}_{a_idx:05d}_{qset_id}_crop.png"
                    clip_path = f"{args.clips_root}/{cuid}.mp4"
                    if not os.path.isfile(clip_path):
                        print(f"======> Clip {clip_path} is missing...")
                        continue
                    visualize_annotation(clip_path, rtsp, csp, qset)


import torch
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler,
                              TensorDataset)
from tqdm import tqdm
from transformers import ElectraForTokenClassification, ElectraConfig, WEIGHTS_NAME, CONFIG_NAME
from transformers import ElectraTokenizer
import time
import pandas as pd
import spacy
import re
import _pickle as cPickle
import numpy as np
from extract_hpi import get_hpi_output
import argparse

#load pickle file which is a label to id mapping required for predictions
with open(r"./label2id.pkl", "rb") as input_file:
    label2id = cPickle.load(input_file)

_nlp = spacy.load('en_core_sci_lg', disable=['ner', 'parser', 'tagger', 'lemmatizer'])
_nlp.create_pipe('sentencizer')
_nlp.add_pipe('sentencizer')

tokenizer = ElectraTokenizer.from_pretrained('./electra/')
model = ElectraForTokenClassification.from_pretrained('./electra/')

def processing_hpi_output(data):
    """
    Clean the HPI section - text analysis

    :return: text
    :rtype: text

    """
    final = []
    for fileid, record in enumerate(data):

        text = re.sub('[\r\n]{2}', '. ', record['text'])
        text = re.sub('\r|\n|\W', ' ', text)
        text = re.sub('\\n', '', text)
        text = re.sub('\r|\n', ' ', text)
        esc = re.escape('.\\n\\n')
        text = re.sub(esc, '', text)
        esc2 = re.escape('\\n')
        text = re.sub(esc2, '', text)
        tokens = {tok.idx: tok.text for tok in _nlp.tokenizer(text)}

        out = ['-DOCSTART- -X- O O', '<s> _ _ O']
        sentence_sep = '</s> _ _ O'

        # get tokens and tags for documents
        for idx, txt in tokens.items():
            if re.search('\n+|\s+', txt):
                out.append('<blank> _ _ ')
            else:
                out.append(txt + ' _ _ ')

        # split documents into sentences and apply HPI sectioning if required
        text = ' '.join(o.split(' ')[0] for o in out)
        doc = _nlp(text)
        sentences = [sent.text.strip() for sent in doc.sents]

        start = 2
        for sent in sentences:
            start += len(sent.split())
            out.insert(start, sentence_sep)
            start += 1

        out.append(sentence_sep)
        final += out
    return final

def get_essentials(filepath):
    """
    Read train/valid/test data from input_dir and
    convert data to features (input_ids, label_ids, attention_masks)

    :return: label2id, num_labels, label2id[pad_token], id2label
    :rtype: dict, number, list, dict

    """

    pad_token = 0
    pad_token_label_id = -100
    with open(filepath, 'r') as f:
        lines = f.readlines()
        sentences, labels, idx, sent, lab, id = [], [], [], [], [], []
        tags = set()
        for line in lines:
            if '-DOCSTART-' in line or '</s>' in line or '<s>' in line or line.rstrip() == '':
                if sent and lab:
                    sentences.append(sent)
                    labels.append(lab)
                    idx.append(id)
                if '-DOCSTART-' in line:
                    sent, lab, id = ['D'], ['O'], ['D{}'.format(line.split()[-2])]
                else:
                    sent, lab, id = [], [], []
            else:
                sent.append(line.split()[0])
                id.append(line.split()[-2])
                lab.append(line.split()[-1])
                tags.add(line.split()[-1])

    # label_map
    label2id = {t: i for i, t in enumerate(list(tags))}
    num_labels = len(label2id)
    label2id[pad_token] = pad_token_label_id
    id2label = {v: k for k, v in label2id.items()}
    return label2id, num_labels, label2id[pad_token], id2label

def prepare_data(filepath):
    """
    Read train/valid/test data from input_dir and
    convert data to tokens

    :return: processed data, tokenized_token, tokenized_sentences
    :rtype: dataframe, list, list

    """

    pad_token = 0
    with open(filepath, 'r') as f:
        lines = f.readlines()
        sentences, sent = [], []
        for line in lines:
            if '-DOCSTART-' in line or '</s>' in line or '<s>' in line or line.rstrip() == '':
                if sent:
                    sentences.append(sent)
                if '-DOCSTART-' in line:
                    sent = ['D']
                else:
                    sent = []
            else:
                sent.append(line.split()[0])

    # tokenize the sentences and save the start offset of each subwords
    tokenized_sentences, tokenized_token = [], []
    for sent in sentences:
        tokenized_sent, tokenized_tok = [], []
        for word in sent:
            tokenized_word = tokenizer.tokenize(word)
            tokenized_sent.extend(tokenized_word)
            tokenized_tok.extend([word] * len(tokenized_word))
        # truncate the subword tokens longer than maxium sequence length
        if len(tokenized_sent) > 512:
            tokenized_sent = tokenized_sent[: 512]
            tokenized_tok = tokenized_tok[: 512]
        tokenized_sentences.append(tokenized_sent)
        tokenized_token.append(tokenized_tok)

    input_ids, attention_masks = [], []
    for sent in tokenized_sentences:
        # get token's id and label's id
        input_id = tokenizer.convert_tokens_to_ids(sent)
        # The mask has 1 for real tokens and 0 for padding tokens. Only real tokens are attended to
        input_mask = [1] * len(input_id)
        # Zero-pad up to the sequence length (pad on right)
        padding_length = 512 - len(input_id)
        input_id += [pad_token] * padding_length
        input_mask += [0] * padding_length
        input_ids.append(input_id)
        attention_masks.append(input_mask)

    data = TensorDataset(torch.tensor(input_ids), torch.tensor(attention_masks))
    return data, tokenized_token, tokenized_sentences

def format_tags(predictions,tokenized_token,id2label):
    """
    convert ids to original labels and create formatted output prediction

    :return: predicted tags, labels, indices
    :rtype: list, list, list

    """

    pred_tags, out = [], []
    docid =0
    start=0
    indices = []
    for prediction, token in zip(predictions,tokenized_token):
        i,j = False,False
        docid+=1
        if token == ['D']:
            i=True
        for index, (pred, tok) in enumerate(zip(prediction, token)):
            pred_tags.append(id2label[pred])
            text = '{} {}'.format(tok, id2label[pred])
            if text == 'D O':
                j=True
            out.append(text)
            if i==False and j==True:
                indices.append(start)
            j=False
            start+=1
        out.append('')
        start+=1
    return pred_tags, out,indices

def get_text_file(hpi_ip,col_name):
    """
    perform HPI processing

    """
    hpi_output = get_hpi_output(hpi_ip,col_name)
    processed_hpi_output = processing_hpi_output(hpi_output)
    with open('processed_file.txt', 'w') as f:
        f.writelines("%s\n" % o for o in processed_hpi_output)

def predict(file):
    """
    Load the saved model and perform prediction

    :return: labels dataframe
    :rtype: dataframe

    """
    hpi_ip = pd.read_csv(file)
    get_text_file(hpi_ip)
    te_data, tt, ts = prepare_data('./processed_file.txt')
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    test_dataloader = DataLoader(te_data, sampler=SequentialSampler(te_data), batch_size=32)
    model.to(device)
    model.eval()
    test_loss = 0
    test_steps = 0
    predictions, pr_label = [], []
    start = time.time()
    for batch in tqdm(test_dataloader, desc='Prediction'):
        # move batch to gpu
        b_input_ids, b_input_mask = tuple(t.to(device) for t in batch)
        with torch.no_grad():
            # Forward pass
            outputs = model(b_input_ids, attention_mask=b_input_mask)
            logits = outputs.logits
        logits = logits.detach().cpu().numpy()
        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])

        test_steps += 1
    test_loss /= test_steps
    test_time = time.time() - start
    id2label = {v: k for k, v in label2id.items()}
    pred_tags, out, indices = format_tags(predictions, tt, id2label)
    index_pos_list = [i for i in range(len(out)) if out[i] == 'D O']
    for i in tqdm(indices, desc='indices'):
        index_pos_list.pop(index_pos_list.index(i))
    final = []
    for i in range(len(index_pos_list)):
        start = index_pos_list[i]
        if i < len(index_pos_list) - 1:
            end = index_pos_list[i + 1]
            final.append(out[start:end])
        else:
            final.append(out[start:])
    tags = []
    for i in tqdm(final):
        s = []
        for j in i:
            s.append(j.split(' ')[-1])
        tags.append(list(set(s)))
    labels = []
    for i in tqdm(tags, desc="Tags"):
        k = []
        for j in i:
            if j != '' or j != 'O':
                k.append(j.split('-')[-1])

        labels.append(k)
    hpi_ip['labels'] = pd.Series(labels)
    return hpi_ip

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Predict labels for clinical text')
    parser.add_argument('-ipf', '--input_file', dest='ipf', help='Path to input file', required=True)
    parser.add_argument('-cn', '--column_name', dest='cn', help='Name of the column containing the note', required=True)
    parser.add_argument('-opf', '--output_file', dest='opf', help='Path to output file', required=True)
    args = parser.parse_args()
    final_op = predict(args.ipf)
    final_op.to_csv(args.opf, index=False)


from typing import List

def insert_at(original: List, value: int, target_index: int) -> List:
    #return original[:i] + [value] + original[i:] (python version)
    new_list = [0] * (len(original)+1) #(java version)
    index = -1
    for index in range(target_index):
        new_list[index] = original[index]

    new_list[index+1] = value
    print(value)

    for index in range(index+2, len(new_list)):
        new_list[index] = original[index-1]

    return new_list

def insert(original: List, value: int) -> List:
    for i, num in enumerate(original):
        if num > value:
            break
        else:
            i += 1

    return insert_at(original, value, i)

def remove_at(original: List, i: int) -> List:
    return original[:i] + original[i+1:]


# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE

from opus_core.variables.variable import Variable
from numpy import ma, clip, where
from opus_core.logger import logger, log_block

class total_SSS_job_space(Variable):
    """ number of job spaces that is vacant/unoccupied"""

    _return_type = "int32"

    def __init__(self, sector):
        Variable.__init__(self)
        self.sector = sector

    def dependencies(self):
        return [
                ]

    @log_block(name='total_SSS_job_space.compute')
    def compute(self,  dataset_pool):
        with logger.block('Analyzing sector'):
            sectors = dataset_pool.get_dataset("sector")
            name_equals_sector = sectors.get_attribute("name") == self.sector
            name_equals_sector_indexes = where(name_equals_sector)
            assert(len(name_equals_sector_indexes) == 1)
            name_equals_sector_index = name_equals_sector_indexes[0]
            sector_ids = sectors.get_attribute("sector_id")
            sector_id = sector_ids[name_equals_sector_index][0]

        with logger.block('Analyzing buildings'):
            buildings = self.get_dataset()
            sqm_our_sector = buildings.get_attribute("sqm_sector%s" % sector_id) #get column of observed jobs
            logger.log_note("sqm_sector%s: %s" % (sector_id, sum(sqm_our_sector)))

        return sqm_our_sector

    def post_check(self,  values, dataset_pool=None):
        self.do_check("x >= 0", values)

from opus_core.tests import opus_unittest
from opus_core.datasets.dataset_pool import DatasetPool
from opus_core.storage_factory import StorageFactory
from numpy import array
from opus_core.tests.utils.variable_tester import VariableTester

class Tests(opus_unittest.OpusTestCase):
    def test_my_inputs(self):
        tester = VariableTester(
            __file__,
            package_order=['urbansim_parcel','urbansim'],
            test_data={
            "building":{"building_id":         array([1,2,3,4,5,6,7,8,9,10]),
                       "zone_id":              array([1,1,2,2,1,3,3,3,2,2]),
                       "building_type_id":     array([1,3,1,2,2,1,2,3,2,4]),
                       "non_residential_sqft": array([1,2,2,1,7,0,3,5,4,6])*1000,
             "occupied_building_sqft_by_jobs": array([500, 2100, 0, 0, 6000, 100, 3000, 1000, 0, 5999])
                                       #vacant sqft:  500, 0,  2000, 1000, 1000, 0, 0, 4000, 4000, 1
                },
            "building_sqft_per_job":{
                       "zone_id":              array([1,  1, 1,  2, 2, 2,  3, 3]),
                       "building_type_id":     array([1,  2, 3,  1, 2, 3,  1, 3]),
                       "building_sqft_per_job":array([100,50,200,80,60,500,20,10]),
                },
        }
        )

        should_be = array([5, 0, 2000/80, 1000/60, 1000/50, 0, 0, 4000/10, 4000/60, 0])

        tester.test_is_close_for_variable_defined_by_this_module(self, should_be)

if __name__=='__main__':
    opus_unittest.main()


import os, tempfile, shutil
from . import utilityFunctions as uF
from .output import message

def medToGeo(medFile, geoFile, tmpdir, opt=[], verbose=0):
  medLoc=os.path.dirname(medFile)
  medName=os.path.basename(medFile)
  inpFile=os.path.join(tmpdir,'import.inp')
  zfile = open(inpFile,'w')
  zfile.write('****mesher\n')
  zfile.write(' ***mesh %s\n' %(geoFile.replace('.geo','')+'.geo'))
  zfile.write('  **import med %s\n' %medName)
  for x in opt:
    zfile.write(x+'\n')
  zfile.write('****return\n')
  zfile.close()
  commande='cd %s; Zrun -m %s' %(medLoc,inpFile)
  if verbose!=0: commande+=' >> %s' %os.path.join(tmpdir,'log.msg')
  res=os.system(commande)

  return(res)

  #os.remove(inpFile)

def geoToMed(medFile, geoFile, tmpdir, opt=[], verbose=0):
  medLoc=os.path.dirname(medFile)
  medName=os.path.basename(medFile)
  inpFile=os.path.join(tmpdir,'export.inp')
  zfile = open(inpFile,'w')
  zfile.write('****mesher\n')
  zfile.write(' ***mesh\n')
  zfile.write('  **open %s\n' %geoFile)
  for x in opt:
    zfile.write(x+'\n')
  zfile.write('  **export med %s\n' %medName)
  zfile.write('****return\n')
  zfile.close()
  commande='cd %s; Zrun -m %s' %(medLoc,inpFile)
  if verbose!=0: commande+=' >> %s' %os.path.join(tmpdir,'log.msg')
  res=os.system(commande)
  #print ' -------- '
  #print res
  #print ' -------- '
  return(res)

def launchZcrack(minS, maxS,
                saneN, crackN, crackedN,
                grad, quad, extrL,
                nbLay, nbIter,
                Gvol, Gfac, Gedg, Gnod,
                surfOpt, tmpdir, cas2D, refine, verbose=0, ):

  zfile = open(os.path.join(tmpdir,'insert.z7p'),'w')
  zfile.write('   #include <Zcracks_base.z7p> \n')
  zfile.write(' int main()\n{\n')
  zfile.write('   init_var();\n')

  if cas2D==True:
    zfile.write('   if_2D=1;\n')
    zfile.write('   thickness.resize(1);\n')
    zfile.write('   thickness[0]=-1.;\n')

  zfile.write('   format="geo";\n')
  zfile.write('   gradation=%e;\n' %grad)
  zfile.write('   min_size= %e;\n' %minS)
  zfile.write('   max_size=%e;\n' %maxS)
  zfile.write('   nb_velem=%d;\n' %(nbLay*2))
  zfile.write('   nb_iter=%d;\n' %nbIter)
  zfile.write('   sane_name="%s";\n' %saneN.replace('.geo',''))
  #zfile.write('   crack_name="%s";\n' %crackN.replace('.geo',''))
  zfile.write('   convert_surface("%s");\n' %crackN.replace('.geo',''))
  zfile.write('   cracked_name="%s";\n' %crackedN.replace('.geo',''))

  if Gfac!='': zfile.write('   faset_names="%s";\n' %(Gfac[0] if isinstance(Gfac, list) else Gfac))
  if Gnod!='': zfile.write('   nset_names="%s";\n' %(Gnod[0] if isinstance(Gnod, list) else Gnod))
  if Gvol!='': zfile.write('   elset_names="%s";\n' %(Gvol[0] if isinstance(Gvol, list) else Gvol))
  if Gedg!='': zfile.write('   liset_names="%s";\n' %(Gedg[0] if isinstance(Gedg, list) else Gedg))
  if surfOpt!='':
    zfile.write('   yams_options="%s";\n' %surfOpt)

  if refine: zfile.write('   if_must_refine=1;\n')

  if extrL<=1.E-12:
    zfile.write('   if_must_define_elset=0;\n')
  else:
    zfile.write('   if_must_define_elset=1;\n')
    if extrL==[]:
      zfile.write('   elset_radius=%e;\n' %maxS)
    else:
      zfile.write('   elset_radius=%e;\n' %extrL)

  zfile.write('   nice_cut(20.0);\n}\n\n')
  zfile.close()
  commande='Zrun -zp %s' %(os.path.join(tmpdir,'insert.z7p'))
  if verbose!=0: commande+=' >> %s' %os.path.join(tmpdir,'log.msg')
  res=os.system(commande)
  #print ' -------- '
  #print res
  #print ' -------- '
  return(res)

def insertCrack(data, names, tmpdir='./zcracks_temp', verbose=0):

  saneN=names['saneGeoName']
  crackN=names['crackGeoName']
  crackedN=names['crackedGeoName']

  minS=data['minSize'][0]
  maxS=data['maxSize'][0]
  extrL=data['extractLength'][0]

  grad = data['gradation'][0] if 'gradation' in list(data.keys()) else 1.3
  quad = data['quad'] if 'quad' in list(data.keys()) else False
  cas2D = data['is2D'] if 'is2D' in list(data.keys()) else False
  refine = data['refine'] if 'refine' in list(data.keys()) else False
  nbLay = data['layers'][0] if 'layers' in list(data.keys()) else 5
  nbIter = data['iterations'][0] if 'iterations' in list(data.keys()) else 2

  Gvol = data['grVol'] if 'grVol' in list(data.keys()) else ''
  Gfac = data['grFace'] if 'grFace' in list(data.keys()) else ''
  Gedg = data['grEdge'] if 'grEdge' in list(data.keys()) else ''
  Gnod = data['grNodes'] if 'grNodes' in list(data.keys()) else ''
  surfOpt = data['surfopt'] if 'surfopt' in list(data.keys()) else ''

  if not os.path.isdir(tmpdir): os.mkdir(tmpdir)
  curDir=os.getcwd()
  os.chdir(tmpdir)
  res=launchZcrack(minS, maxS, saneN, crackN, crackedN,grad, quad, extrL,
                nbLay, nbIter,Gvol, Gfac, Gedg, Gnod,surfOpt, tmpdir, cas2D, refine, verbose)
  os.chdir(curDir)
  return(res)

#def TUI(data, names, tmpdir='./zcracks_temp', verbose=0)


import sys, random, string, poplib
from PyQt5 import QtCore, uic
from PyQt5.QtWidgets import QApplication, QComboBox, \
    QPushButton, QLineEdit, QLabel

def on_cross_pushbutton_clicked():
    if method_combo_box.currentText() == "Corte Simples":
        offsprings = simple_cut_crossover()
        son1_label_3.setVisible(False)
        son2_label_3.setVisible(False)
    else:
        offsprings = pmx_crossover()
        son1_label_3.setVisible(True)
        son2_label_3.setVisible(True)

    son1_label_1.setText(offsprings[0])
    son1_label_2.setText(offsprings[1])
    son1_label_3.setText(offsprings[2])
    son2_label_1.setText(offsprings[3])
    son2_label_2.setText(offsprings[4])
    son2_label_3.setText(offsprings[5])

def on_method_combobox_current_text_changed():
    if method_combo_box.currentText()=="PMX":
        father_line_edit.setInputMask('AAAAAAAAAA')
        father_line_edit.setText('ABCDEFGHIJ')
        mother_line_edit.setInputMask('AAAAAAAAAA')
        mother_line_edit.setText('JHIGFEDCBA')
    else:
        father_line_edit.setInputMask('BBBBBBBBBB')
        father_line_edit.setText('0000000000')
        mother_line_edit.setInputMask('BBBBBBBBBB')
        mother_line_edit.setText('1111111111')
    print('combo box changed')

def pmx_crossover():
    cromossomopai= father_line_edit.text()
    cromossomomae= mother_line_edit.text()
    corte1 = random.randint(0,len(cromossomopai)-2)
    corte2 = random.randint(corte1+1,len(cromossomomae)-1)
    cortepai=''
    cortemae=''
    corte1pai=''
    corte1mae=''
    corte2pai=''
    corte2mae=''
    for i in range(0,corte1):
        cortepai=cortepai+(cromossomopai[i])
        corte1mae= corte1mae+(cromossomomae[i])
    for i in range(corte1,corte2):
        cortemae=cortemae+(cromossomomae[i])
        corte1pai= corte1pai+(cromossomopai[i])
    for i in range(corte2,len(cromossomopai)):
        corte2pai=corte2pai+(cromossomopai[i])
        corte2mae= corte2mae+(cromossomomae[i])
    # Troca de caracteres repetidos
    for i in range(0,len(cortemae)):
        for j in range(0,len(cortepai)):
            if (cortemae[i]==cortepai[j]):
                cortepai=cortepai.replace(cortepai[j],corte1pai[i])

            if (corte1pai[i]==corte1mae[j]):
                corte1mae=corte1mae.replace(corte1mae[j],cortemae[i])
        for h in range(0,len(corte2pai)):
            if (cortemae[i]==corte2pai[h]):
                corte2pai=corte2pai.replace(corte2pai[h],corte1pai[i])

            if (corte1pai[i]==corte2mae[h]):
                corte2mae=corte2mae.replace(corte2mae[h],cortemae[i])
    # Esta função está retornando 6 valores.
    # Ao criar o corpo da função você deve ordená-los de acordo com
    # as linhas 17 a 22 deste arquivo.
    return cortepai,cortemae,corte2pai,corte1mae,corte1pai,corte2mae

def simple_cut_crossover():
    cromossomopai= father_line_edit.text()
    cromossomomae= mother_line_edit.text()
    corte= random.randint(0,len(cromossomopai)-1)
    cortepai=''
    cortemae=''
    corte1pai=''
    corte1mae=''
    for i in range(0,corte):
        cortepai=cortepai+(cromossomopai[i])
        corte1mae= corte1mae+(cromossomomae[i])
    for i in range(corte,len(cromossomopai)):
        cortemae= cortemae+(cromossomomae[i])
        corte1pai= corte1pai+(cromossomopai[i])
    return cortepai,cortemae,'',corte1mae,corte1pai,''

if __name__ == "__main__":
    QtCore.QCoreApplication.setAttribute(QtCore.Qt.AA_ShareOpenGLContexts)
    app = QApplication(sys.argv)

    # Loading widgets elements from ui file
    window = uic.loadUi("crossover_operation.ui")
    window.show()

    # Getting widgets elements
    father_line_edit = window.findChild(QLineEdit, 'fatherLineEdit')
    mother_line_edit = window.findChild(QLineEdit, 'motherLineEdit')
    son1_label_1 = window.findChild(QLabel, 'son1Label1')
    son1_label_2 = window.findChild(QLabel, 'son1Label2')
    son1_label_3 = window.findChild(QLabel, 'son1Label3')
    son2_label_1 = window.findChild(QLabel, 'son2Label1')
    son2_label_2 = window.findChild(QLabel, 'son2Label2')
    son2_label_3 = window.findChild(QLabel, 'son2Label3')
    method_combo_box = window.findChild(QComboBox, 'methodComboBox')
    cross_push_button = window.findChild(QPushButton, 'crossPushButton')

    # Connecting
    cross_push_button.clicked.connect(on_cross_pushbutton_clicked)
    method_combo_box.currentTextChanged.connect(on_method_combobox_current_text_changed)

    sys.exit(app.exec_())


import json
import requests
from bs4 import BeautifulSoup

URL = 'https://www.zr.ru/news/'
HEADERS = {'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:93.0) Gecko/20100101 Firefox/93.0', 'accept': '*/*'}
HOST = 'https://www.zr.ru'

# Функция получения данных с сервера
def get_html(url, params=None):                                             # params - для определения числа страниц
    get_request = requests.get(url, headers=HEADERS, params=params)         # При помощи requests делается get-запрос к серверу
    return get_request                                                      # Обьект r будет возвращён и использован в функции parse()

# Функция сбора данных с сайта
def get_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = soup.find_all('article', class_='story-short')
    news = []
    for item in items[:10]:
        details = get_article_content(HOST + item.find('a', class_='link').get('href'))
        # Ограничился десятью постами, чтобы собрать больше статей требуется пагинация
        news.append({
            'title': item.find('a', class_='link').get_text(strip=True),
            'link': HOST + item.find('a', class_='link').get('href'),
            'article': item.find('div', class_='articles__item-desc').get_text(strip=True),
            'picture': HOST + item.find('img').get('src'),
            'autor': details['autor'],
            'date': details['date']
        })
    print(news)
    with open("db_zr.json", "w") as jfile:
        json.dump(news, jfile, indent=4, ensure_ascii=False)

# Функция сбора данных внутри статьи
def get_article_content(article_url):
    html = get_html(article_url)
    soup = BeautifulSoup(html.text, 'html.parser')
    items = soup.find('body', class_='zr')
    return {
        'autor': items.find('span', class_='link_pink').get_text(strip=True),
        'date': items.find('div', class_='info__date').get_text(strip=True)
        }

# Основная функция парсинга первой страницы сайта
def parse():
    html = get_html(URL)
    # Проверяем связь со страицей
    if html.status_code == 200:
        get_content(html.text)
    else:
        print('Error')

parse()


# 标准库
import io as _io
import os as _os
import sys as _sys
import imp as _imp
import codecs as _codecs
import traceback as _traceback
import pathlib as _pathlib

def enhance_init(work_dir=__file__, python_version_require=0, check_module_list=[]):
    """
    :param pythonVersionRequire(int): 最低python所需版本
    :param checkModuleList: 示例: ("requests", "requests_toolbelt", "urllib3", "bs4", "Crypto", "pyDes", "yaml", "lxml", "rsa")
    """
    # ==========检查python版本==========
    if not (
        _sys.version_info[0] == 3 and _sys.version_info[1] >= python_version_require
    ):
        raise Exception(
            "!!!!!!!!!!!!!!Python版本错误!!!!!!!!!!!!!!\n请使用python3.%d及以上版本，而不是[python %s]"
            % (python_version_require, _sys.version)
        )
    # ==========环境变量初始化==========
    try:
        print("==========开始初始化==========")
    except UnicodeEncodeError:
        # 设置默认输出编码为utf-8, 但是会影响腾讯云函数日志输出。
        _sys.stdout = _codecs.getwriter("utf-8")(_sys.stdout.detach())
        print("==========开始初始化(utf-8输出)==========")
    script_abs_path = _pathlib.Path(work_dir).absolute().parent
    _os.chdir(script_abs_path)  # 将工作路径设置为脚本位置
    if _os.name == "posix":
        # 如果是linux系统, 增加TZ环境变量
        _os.environ["TZ"] = "Asia/Shanghai"
    _sys.path.append(script_abs_path)  # 将脚本路径加入模块搜索路径
    # ==========检查第三方模块==========
    try:
        for i in check_module_list:
            _imp.find_module(i)
    except ImportError as e:  # 腾讯云函数在初始化过程中print运作不正常，所以将信息丢入异常中
        raise ImportError(
            f"""!!!!!!!!!!!!!!缺少第三方模块(依赖)!!!!!!!!!!!!!!
    请使用pip3命令安装或者手动将依赖拖入文件夹
    错误信息: [{e}]"""
        )

class Decorators:
    @staticmethod
    def catch_exception(func):
        def wrapper(*args, **kwargs):
            try:
                return func(*args, **kwargs)
            except Exception:
                print(_traceback.format_exc())
                return None

        return wrapper

class FileOut:
    """
    代替stdout和stderr, 使print同时输出到文件和终端中。
    start()方法可以直接用自身(self)替换stdout和stderr
    close()方法可以还原stdout和stderr
    """

    stdout = _sys.stdout
    stderr = _sys.stderr
    log: str = ""  # 同时将所有输出记录到log字符串中
    logFile: _io.TextIOWrapper = None

    @classmethod
    def set_file_out(cla, path: str = None):
        """
        设置日志输出文件
        :params path: 日志输出文件路径, 如果为空则取消日志文件输出
        """
        # 关闭旧文件
        if cla.logFile:
            cla.logFile.close()
            cla.logFile = None

        # 更新日志文件输出
        if path:
            try:
                path = _os.path.abspath(path)
                logDir = _os.path.dirname(path)
                if not _os.path.isdir(logDir):
                    _os.makedirs(logDir)
                cla.logFile = open(path, "w+", encoding="utf-8")
                cla.logFile.write(cla.log)
                cla.logFile.flush()
                return
            except Exception as e:
                print(2, f"设置日志文件输出失败, 错误信息: [{e}]")
                cla.logFile = None
                return
        else:
            cla.logFile = None
            return

    @classmethod
    def start(cla):
        """开始替换stdout和stderr"""
        if type(_sys.stdout) != cla and type(_sys.stderr) != cla:
            _sys.stdout = cla
            _sys.stderr = cla
        else:
            print("sysout/syserr已被替换为FileOut")

    @classmethod
    def write(cla, str_):
        r"""
        :params str: print传来的字符串
        :print(s)等价于sys.stdout.write(s+"\n")
        """
        str_ = str(str_)
        cla.log += str_
        if cla.logFile:
            cla.logFile.write(str_)
        cla.stdout.write(str_)
        cla.flush()

    @classmethod
    def flush(cla):
        """刷新缓冲区"""
        cla.stdout.flush()
        if cla.logFile:
            cla.logFile.flush()

    @classmethod
    def close(cla):
        """关闭"""
        if cla.logFile:
            cla.logFile.close()
        cla.log = ""
        _sys.stdout = cla.stdout
        _sys.stderr = cla.stderr


#coding:utf-8
import urllib
import http.cookiejar
import json
class Qqpush:
    pushurl='https://wx.scjtqs.com/qq/push/pushMsg'
    def push(self,qq,token,data):
        url = self.pushurl+"?token="+token
        post={}
        post['qq']=qq
        post['content']=[{"msgtype":"text","text":data}]
        postdata=bytes(json.dumps(post),'utf8')
        header = {
            "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0",
            'Content-Type': 'application/json',
        }
        req = urllib.request.Request(url=url, data=postdata, headers=header)
        cj = http.cookiejar.CookieJar()
        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
        r = opener.open(req)
        response = r.read().decode('utf-8')
        jsonret=json.loads(response)
        return jsonret


__all__ = [
	"EBCOTCodec"
]

from copy import deepcopy
from multiprocessing import Pool
import numpy as np

from fpeg.base import Codec
from fpeg.config import read_config
from fpeg.funcs import parse_marker, cat_arrays_2d

config = read_config()

D = config.get("jpeg2000", "D")
G = config.get("jpeg2000", "G")
QCD = config.get("jpeg2000", "QCD")
mq_table = config.get("jpeg2000", "mq_table")

min_task_number = config.get("accelerate", "codec_min_task_number")
max_pool_size = config.get("accelerate", "codec_max_pool_size")

class EBCOTCodec(Codec):
	"""
	EBCOT codec
	"""

	def __init__(self,
							 name="EBCOT codec",
							 mode="encode",
							 D=D,
							 G=G,
							 QCD=QCD,
							 accelerated=False
							 ):
		"""
		Init and set attributes of a canonical EBCOT codec.

		Explicit Attributes
		-------------------
		name: str, optional
			Name of the codec.
		mode: str, optional
			Mode of the codec, must in ["encode", "decode"].
		G: integer, optional
			a parameter for calculate Kmax
		D: integer, optional
			Depth of graphic.
		epsilon_b:integer, must
			a parameter for calculate Kmax
		accelerated: bool, optional
			Whether the process would be accelerated by subprocess pool.

		"""
		super().__init__()

		self.name = name
		self.mode = mode
		# based on equation 10.22 in jpeg2000
		self.D = D
		self.G = G
		self.QCD = QCD
		self.accelerated = accelerated

		self.epsilon_b, _ = parse_marker(self.QCD)
		self.Kmax = max(0, self.G + self.epsilon_b - 1)
		self.min_task_number = min_task_number
		self.max_pool_size = max_pool_size

	def encode(self, X, **params):
		self.logs[-1] += self.formatter.message("Trying to encode received data.")
		try:
			self.epsilon_b = params["epsilon_b"]
		except KeyError:
			pass

		if self.accelerated:
			self.logs[-1] += self.formatter.message("Using multiprocess pool to accelerate EBCOT encoding.")
			inputs = [[x, self.D] for x in X]
			with Pool(min(self.task_number, self.max_pool_size)) as p:
				bitcodes = p.starmap(_EBCOT_encode, inputs)
		else:
			bitcodes = [_EBCOT_encode(x, self.D) for x in X]

		return bitcodes

	def decode(self, bitcodes, **params):
		self.logs[-1] += self.formatter.message("Trying to decode received data.")

		if self.accelerated:
			inputs = [[bitcode, self.D] for bitcode in bitcodes]
			with Pool(min(self.task_number, self.max_pool_size)) as p:
				X = p.starmap(_tile_decode, inputs)
		else:
			X = [_tile_decode(bitcode, self.D) for bitcode in bitcodes]

		# print(X[0])
		return X

def _EBCOT_encode(tile, D):
	"""
	EBCOT encode and decode part
	encode part:
	| _EBCOT_encode
		| _tile_encode
			| _band_encode
				| _embeddedBlockEncoder _MQencode
					 | 三个通道过程

	｜ _EBCOT_decode
		| _tile_decode
			| _band_decode
				| _block_decode
					| three decode passes and MQdecode
						|signdecode and runlengthdecode
	"""

	bitcode = list(_tile_encode(tile, D))

	# with open('test.bin', 'wb') as f:
	#   f.write(struct.pack(str(l)+'i', *bitcode))
	# streamonly = [int(i) for i in streamonly]
	# l = len(streamonly)
	# with open('streamonly.bin', 'wb') as f:
	#   f.write(struct.pack(str(l)+'i', *streamonly))

	return bitcode

def _MQencode(CX, D):
	"""
	MQ encode and decode part
	|MQencode
		|transferbyte
			|putbyte
	|MQdecode
		|fill_lsp

	"""
	PETTable, CXTable = deepcopy(mq_table)
	encoder = EBCOTparam()
	for i in range(len(D)):
		symbol = D[i][0]
		cxLabel = CX[i][0]
		expectedSymbol = CXTable[cxLabel][1]
		p = PETTable[CXTable[cxLabel][0]][3]  # PETTable [CXTable[cxLabel][0]]---[3]
		encoder.A = encoder.A - p
		if encoder.A < p:
			# Conditional exchange of MPS and LPS
			expectedSymbol = 1 - expectedSymbol
		if symbol == expectedSymbol:
			# assign MPS the upper sub-interval
			encoder.C = encoder.C + np.uint32(p)
		else:
			# assign LPS the lower sub-interval
			encoder.A = np.uint32(p)
		if encoder.A < 32768:
			if symbol == CXTable[cxLabel][1]:
				CXTable[cxLabel][0] = PETTable[CXTable[cxLabel][0]][0]
			else:
				CXTable[cxLabel][1] = CXTable[cxLabel][1] ^ PETTable[CXTable[cxLabel][0]][2]
				CXTable[cxLabel][0] = PETTable[CXTable[cxLabel][0]][1]
			while encoder.A < 32768:
				encoder.A = 2 * encoder.A
				encoder.C = 2 * encoder.C
				encoder.t = encoder.t - 1
				if encoder.t == 0:
					encoder = _transferbyte(encoder)
	encoder = _encode_end(encoder)
	return encoder

def _transferbyte(encoder):
	CPartialMask = np.uint32(133693440)  # 00000111111110000000000000000000
	CPartialCmp = np.uint32(4161273855)  # 11111000000001111111111111111111
	CMsbsMask = np.uint32(267386880)  # 27-20msbs标志位  00001111111100000000000000000000
	CMsbsCmp = np.uint32(4027580415)  # CMsbs的补码      11110000000011111111111111111111
	CCarryMask = np.uint32(2 ** 27)  # 取进位              00001000000000000000000000000000
	if encoder.T == 255:
		# 不能将任何进位传给T，需要位填充
		encoder = _putbyte(encoder)
		encoder.T = np.uint8((encoder.C & CMsbsMask) >> 20)  # 27-20位
		encoder.C = encoder.C & CMsbsCmp  #
		encoder.t = 7
	else:
		# 从C将任何进位传到T
		encoder.T = encoder.T + np.uint8((encoder.C & CCarryMask) >> 27)
		encoder.C = encoder.C ^ CCarryMask
		encoder = _putbyte(encoder)
		if encoder.T == 255:
			encoder.T = np.uint8((encoder.C & CMsbsMask) >> 20)
			encoder.C = encoder.C & CMsbsCmp
			encoder.t = 7
		else:
			encoder.T = np.uint8((encoder.C & CPartialMask) >> 19)
			encoder.C = encoder.C & CPartialCmp
			encoder.t = 8
	return encoder

def _putbyte(encoder):
	# 将T中的内容写入字节缓存
	if encoder.L >= 0:
		encoder.stream = np.append(encoder.stream, encoder.T)
	encoder.L = encoder.L + 1
	return encoder

def _MQ_decode(stream, CX):
	PETTable, CXTable = deepcopy(mq_table)
	# MQ decode initializtion
	encoder = EBCOTparam()
	encoder.A = np.uint16(0)
	encoder.C = np.uint32(0)
	encoder.t = np.uint8(0)
	encoder.T = np.uint8(0)
	encoder.L = np.int32(0)
	encoder.stream = stream
	encoder = _fill_lsb(encoder)
	encoder.C = encoder.C << encoder.t
	encoder = _fill_lsb(encoder)
	encoder.C = encoder.C << 7
	encoder.t = encoder.t - 7
	encoder.A = np.uint16(2 ** 15)

	# MQ decode procedure
	CActiveMask = np.uint32(16776960)  # 00000000111111111111111100000000
	CActiveCmp = np.uint32(4278190335)  # 11111111000000000000000011111111
	decodeD = []
	for i in range(len(CX)):
		cxLabel = CX[i][0]
		expectedSymbol = CXTable[cxLabel][1]
		p = PETTable[CXTable[cxLabel][0]][3]
		encoder.A = encoder.A - np.uint16(p)
		if encoder.A < np.uint16(p):
			expectedSymbol = 1 - expectedSymbol
		if ((encoder.C & CActiveMask) >> 8) < p:
			symbol = 1 - expectedSymbol
			encoder.A = np.uint16(p)
		else:
			symbol = expectedSymbol
			temp = ((encoder.C & CActiveMask) >> 8) - np.uint32(p)
			encoder.C = encoder.C & CActiveCmp
			encoder.C = encoder.C + np.uint32((np.uint32(temp << 8)) & CActiveMask)
		if encoder.A < 2 ** 15:
			if symbol == CXTable[cxLabel][1]:
				CXTable[cxLabel][0] = PETTable[CXTable[cxLabel][0]][0]
			else:
				CXTable[cxLabel][1] = CXTable[cxLabel][1] ^ PETTable[CXTable[cxLabel][0]][2]
				CXTable[cxLabel][0] = PETTable[CXTable[cxLabel][0]][1]
			while encoder.A < 2 ** 15:
				if encoder.t == 0:
					encoder = _fill_lsb(encoder)
				encoder.A = 2 * encoder.A
				encoder.C = 2 * encoder.C
				encoder.t = encoder.t - 1
		# print(i, symbol)
		decodeD.append([symbol])
	return decodeD

def _fill_lsb(encoder):
	encoder.t = 8
	if encoder.L == len(encoder.stream) or \
			(encoder.T == 255 and encoder.stream[encoder.L] > 143):
		encoder.C = encoder.C + 255
	else:
		if encoder.T == 255:
			encoder.t = 7
		encoder.T = encoder.stream[encoder.L]
		encoder.L = encoder.L + 1
		encoder.C = encoder.C + np.uint32((encoder.T) << (8 - encoder.t))
	return encoder

def _tile_encode(tile, D, h=64, w=64):
	_depthOfDwt = D

	tile_cA = tile[0][:, :, 0]
	newBit, newStream = _band_encode(tile_cA, 'LL', h, w)
	bitcode = newBit
	streamOnly = newStream
	for i in range(1, _depthOfDwt+1):
		newBit, newStream = _band_encode(tile[i][0][:, :, 0], 'LH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][1][:, :, 0], 'HL', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][2][:, :, 0], 'HH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))

	tile_cA = tile[0][:, :, 1]
	newBit, newStream = _band_encode(tile_cA, 'LL', h, w)
	bitcode = np.hstack((bitcode, newBit))
	streamOnly = np.hstack((streamOnly, newStream))
	for i in range(1, _depthOfDwt+1):
		newBit, newStream = _band_encode(tile[i][0][:, :, 1], 'LH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][1][:, :, 1], 'HL', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][2][:, :, 1], 'HH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))

	tile_cA = tile[0][:, :, 2]
	newBit, newStream = _band_encode(tile_cA, 'LL', h, w)
	bitcode = np.hstack((bitcode, newBit))
	streamOnly = np.hstack((streamOnly, newStream))
	for i in range(1, _depthOfDwt+1):
		newBit, newStream = _band_encode(tile[i][0][:, :, 2], 'LH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][1][:, :, 2], 'HL', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
		newBit, newStream = _band_encode(tile[i][2][:, :, 2], 'HH', h, w)
		bitcode = np.hstack((bitcode, newBit))
		streamOnly = np.hstack((streamOnly, newStream))
	bitcode = np.hstack((bitcode, [2051]))
	return bitcode

def _band_encode(tile, bandMark, h=64, w=64, num=8):
	# 码流：[h, w, CX1, 2048, stream1, 2048, ..., CXn, streamn, 2048, 2049,CXn+1, streamn+1, 2048, ...,2050]
	h_cA, w_cA = np.shape(tile)
	h_left_over = h_cA % h
	w_left_over = w_cA % w
	cA_extend = np.pad(tile, ((0, h - h_left_over), (0, w - w_left_over)), 'constant')
	bitcode = [h_cA, w_cA]
	streamOnly = []
	for i in range(0, h_cA, h):
		for j in range(0, w_cA, w):
			codeBlock = cA_extend[i:i + h, j:j + w]
			CX, D, bitplanelength= _embeddedBlockEncoder(codeBlock, bandMark, h, w, num)
			encoder = _MQencode(CX, D)
			bitcode = np.hstack((bitcode, CX.flatten(), [2048], encoder.stream, [2048], bitplanelength,[2048]))
			streamOnly = np.hstack((streamOnly, encoder.stream))
		bitcode = np.hstack((bitcode, [2049]))
	bitcode = np.hstack((bitcode, [2050]))
	return (bitcode, streamOnly)

def _embeddedBlockEncoder(codeBlock, bandMark, h=64, w=64, num=8):
	S1 = np.zeros((h, w))
	S2 = np.zeros((h, w))
	S3 = np.zeros((h, w))
	MaxInCodeBlock = len(bin(int(np.max(abs(codeBlock)))))-2
	signs = (- np.sign(codeBlock) + 1) // 2  # positive: 0, negative: 1
	bitPlane = np.zeros((h,w,MaxInCodeBlock),dtype=np.uint8)
	for i in range(h):
		for j in range(w):
			# number = bin(np.abs(codeBlock[i][j], dtype=np.int64))[2:]
			number = bin(np.abs(codeBlock[i][j]))[2:]
			temp = [0] * (MaxInCodeBlock-len(number)) + [int(num) for num in number]
			bitPlane[i][j] = np.array(temp)
	bitPlane = np.transpose(bitPlane, (2, 0, 1))
	# For Test
	"""
	signs = np.zeros((8,8))
	bitPlane = np.zeros((2,8,8))
	bitPlane[0][1][1] = 1
	bitPlane[0][4][4] = 1
	bitPlane[1][0][2] = 1
	bitPlane[1][1] = np.array([0,1,0,0,1,1,0,0])
	bitPlane[1][2][2] = 1
	bitPlane[1][3][3] = 1
	bitPlane[1][4][5] = 1
	bitPlane[1][5] = np.array([0,0,0,0,1,1,0,1])
	bitPlane[1][6][6] = 1
	"""
	sizeofCXandD = h*w*MaxInCodeBlock *5
	CX = np.zeros((sizeofCXandD, 1), dtype=np.uint32)
	D = np.zeros((sizeofCXandD, 1), dtype=np.uint32)
	pointer = 0
	for i in range(MaxInCodeBlock ):
		######
		# three function need rename
		D, CX, S1, S3, pointer = _SignifiancePropagationPass(D, CX, S1, S3, pointer, bitPlane[i], bandMark, signs, w, h)
		D, CX, S2, pointer = _MagnitudeRefinementPass(D, CX, S1, S2, S3, pointer, bitPlane[i], w, h)
		D, CX, pointer, S1 = _CLeanUpPass(D, CX, S1, S3, pointer, bitPlane[i], bandMark, signs, w, h)
		S3 = np.zeros((h, w))
	CX_final = CX[0:pointer]
	D_final = D[0:pointer]
	return CX_final, D_final, MaxInCodeBlock

# three encode pass start here
# in the sequence of significancePass,magnitudepass,_cleanuppass.

def _SignifiancePropagationPass(D, CX, S1, S3, pointer, plane, bandMark, signs, w=64, h=64):
	# input S1:  list of significance, size 64*64
	# input CX: the list of context
	# plane: the value of bits at this plane
	# bandMark: LL, HL, HH, or LH
	# pointer: the pointer of the CX
	# S3: denote that the element has been coded
	# output: D, CX, S1, S3, pointer
	S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			for ii in range(4):
				row = 4 * i + ii
				if S1[row][col] != 0:
					continue  # is significant
				temp = S1extend[row][col] + S1extend[row + 1][col] + S1extend[row + 2][col] + S1extend[row][col + 1] + \
							 S1extend[row + 2][col + 1] + S1extend[row][col + 2] + S1extend[row + 1][col + 2] + S1extend[row + 2][
								 col + 2]
				if temp == 0:
					continue  # is insignificant
				tempCx = _ZeroCoding(S1extend[row:row + 3, col:col + 3], bandMark)
				D[pointer][0] = plane[row][col]
				CX[pointer][0] = tempCx
				pointer = pointer + 1
				S3[row][col] = 1  # mark that plane[row][col] has been coded
				if plane[row][col] == 1:  # _signcoding
					signComp, tempCx = _SignCoding(S1extend[row:row + 3, col:col + 3], signs[row][col])
					D[pointer][0] = signComp
					CX[pointer][0] = tempCx
					pointer = pointer + 1
					S1[row][col] = 1  # mark as significant
					S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
	return D, CX, S1, S3, pointer

def _MagnitudeRefinementPass(D, CX, S1, S2, S3, pointer, plane, w=64, h=64):
	S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			for ii in range(4):
				row = 4 * i + ii
				if S1[row][col] != 1 or S3[row][col] != 0:
					continue
				tempCx = _MagnitudeRefinementCoding(S1extend[row:row + 3, col:col + 3], S2[row][col])
				S2[row][col] = 1  # Mark that the element has been refined
				D[pointer][0] = plane[row][col]
				CX[pointer][0] = tempCx
				pointer = pointer + 1
	return D, CX, S2, pointer

def _CLeanUpPass(D, CX, S1, S3, pointer, plane, bandMark, signs, w=64, h=64):
	S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			ii = 0
			row = 4 * i
			tempSum = np.sum(S1extend[row:row + 6, col:col + 3]) + np.sum(S3[row:row + 4, col])
			# 整一列未被编码，都为非重要，且领域非重要
			if tempSum == 0:
				ii, tempD, tempCx = _RunLengthCoding(plane[row:row + 4, col])
				if len(tempD) == 1:
					D[pointer] = tempD
					CX[pointer] = tempCx
					pointer = pointer + 1
				else:
					D[pointer], D[pointer + 1], D[pointer + 2] = tempD[0], tempD[1], tempD[2]
					CX[pointer], CX[pointer + 1], CX[pointer + 2] = tempCx[0], tempCx[1], tempCx[2]
					pointer = pointer + 3
					# sign coding
					row = i * 4 + ii - 1
					signComp, tempCx = _SignCoding(S1extend[row:row + 3, col:col + 3], signs[row][col])
					D[pointer] = signComp
					CX[pointer] = tempCx
					pointer = pointer + 1
					S1[row][col] = 1
					S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
			while ii < 4:
				row = i * 4 + ii
				ii = ii + 1
				if S1[row][col] != 0 or S3[row][col] != 0:
					continue
				tempCx = _ZeroCoding(S1extend[row:row + 3, col:col + 3], bandMark)
				D[pointer] = plane[row][col]
				CX[pointer] = tempCx
				pointer = pointer + 1
				if plane[row][col] == 1:  # _signcoding
					signComp, tempCx = _SignCoding(S1extend[row:row + 3, col:col + 3], signs[row][col])
					D[pointer][0] = signComp
					CX[pointer][0] = tempCx
					pointer = pointer + 1
					S1[row][col] = 1  # mark as significant
					S1extend = np.pad(S1, ((1, 1), (1, 1)), 'constant')
	return D, CX, pointer, S1

# here is some function used by three passes

def _SignCoding(neighbourS1, sign):
	# input neighbourS1: size 3*3, matrix of significance
	# input sign
	# output: signComp,(equal: 0, not equal: 1) context
	if len(neighbourS1) == 3 and len(neighbourS1[0]) == 3:
		hstr = str(int(neighbourS1[1][0])) + str(int(neighbourS1[1][2]))
		vstr = str(int(neighbourS1[0][1])) + str(int(neighbourS1[2][1]))
		dict = {
			'00': 0, '1-1': 0, '-11': 0, '01': 1, '10': 1, '11': 1,
			'0-1': -1, '-10': -1, '-1-1': -1
		}
		h = dict[hstr]
		v = dict[vstr]
		hAndv = str(h) + str(v)
		hv2Sign = {
			'11': 0, '10': 0, '1-1': 0, '01': 0, '00': 0,
			'0-1': 1, '-11': 1, '-10': 1, '-1-1': 1
		}
		hv2Context = {
			'11': 13, '10': 12, '1-1': 11, '01': 10, '00': 9,
			'0-1': 10, '-11': 11, '-10': 12, '-1-1': 13
		}
		signPredict = hv2Sign[hAndv]
		context = hv2Context[hAndv]
		signComp = int(sign) ^ signPredict
	else:
		# self.logs[-1] += self.formatter.warning("_SignCoding: Size of neighbourS1 not valid")
		signComp = -1
		context = -1
		"""
		try:
			raise ValidationError('_SignCoding: Size of neighbourS1 not valid')
		except ValidationError as e:
			print(e.args)
			signComp = -1
			context = -1
		"""
	return signComp, context

def _ZeroCoding(neighbourS1, bandMark):
	# input neighbourS1: size 3*3, matrix of significance
	# input s2: whether it is the first time for Magnitude Refinement Coding
	# output: context
	if len(neighbourS1) == 3 and len(neighbourS1[0]) == 3:
		h = neighbourS1[1][0] + neighbourS1[1][2]
		v = neighbourS1[0][1] + neighbourS1[2][1]
		d = neighbourS1[0][0] + neighbourS1[0][2] + neighbourS1[2][0] + neighbourS1[2][2]
		if bandMark == 'LL' or bandMark == 'LH':
			if h == 2:
				cx = 8
			elif h == 1 and v >= 1:
				cx = 7
			elif h == 1 and v == 0 and d >= 1:
				cx = 6
			elif h == 1 and v == 0 and d == 0:
				cx = 5
			elif h == 0 and v == 2:
				cx = 4
			elif h == 0 and v == 1:
				cx = 3
			elif h == 0 and v == 0 and d >= 2:
				cx = 2
			elif h == 0 and v == 0 and d == 1:
				cx = 1
			else:
				cx = 0
		elif bandMark == 'HL':
			if v == 2:
				cx = 8
			elif v == 1 and h >= 1:
				cx = 7
			elif v == 1 and h == 0 and d >= 1:
				cx = 6
			elif v == 1 and h == 0 and d == 0:
				cx = 5
			elif v == 0 and h == 2:
				cx = 4
			elif v == 0 and h == 1:
				cx = 3
			elif v == 0 and h == 0 and d >= 2:
				cx = 2
			elif v == 0 and h == 0 and d == 1:
				cx = 1
			else:
				cx = 0
		elif bandMark == 'HH':
			hPlusv = h + v
			if d >= 3:
				cx = 8
			elif d == 2 and hPlusv >= 1:
				cx = 7
			elif d == 2 and hPlusv == 0:
				cx = 6
			elif d == 1 and hPlusv >= 2:
				cx = 5
			elif d == 1 and hPlusv == 1:
				cx = 4
			elif d == 1 and hPlusv == 0:
				cx = 3
			elif d == 0 and hPlusv >= 2:
				cx = 2
			elif d == 0 and hPlusv == 1:
				cx = 1
			else:
				cx = 0
		else:
			# self.logs[-1] += self.formatter.warning('_ZeroCoding: bandMark not valid')
			cx = -1
			"""
			try:
				raise ValidationError('_ZeroCoding: bandMark not valid')
			except ValidationError as e:
				print(e.args)
				cx = -1
			"""
	else:
		# self.logs[-1] += self.formatter.warning('_ZeroCoding: Size of neighbourS1 not valid')
		cx = -1
		"""
		try:
			raise ValidationError('_ZeroCoding: Size of neighbourS1 not valid')
		except ValidationError as e:
			print(e.args)
			cx = -1
		"""
	return cx

def _RunLengthCoding(listS1):
	# input listS1: size 1*4, list of significance
	# output n: number of elements encoded
	# output d: 0 means the _RunLengthCoding does not end.
	# [1, x, x] means the _RunLengthCoding ends and the position is indicated.
	# output cx: context
	if listS1.__len__() == 4:
		if listS1[0] == 0 and listS1[1] == 0 and listS1[2] == 0 and listS1[3] == 0:
			n = 4
			d = [0]
			cx = [17]
		elif listS1[0] == 1:
			n = 1
			d = [1, 0, 0]
			cx = [17, 18, 18]
		elif listS1[0] == 0 and listS1[1] == 1:
			n = 2
			d = [1, 0, 1]
			cx = [17, 18, 18]
		elif listS1[0] == 0 and listS1[1] == 0 and listS1[2] == 1:
			n = 3
			d = [1, 1, 0]
			cx = [17, 18, 18]
		elif listS1[0] == 0 and listS1[1] == 0 and listS1[2] == 0 and listS1[3] == 1:
			n = 4
			d = [1, 1, 1]
			cx = [17, 18, 18]
		else:
			# self.logs[-1] += self.formatter.warning('_RunLengthCoding: listS1 not valid')
			n, d, cx = 0, -1, -1
			"""
			try:
				raise ValidationError('_RunLengthCoding: listS1 not valid')
			except ValidationError as e:
				print(e.args)
				n, d, cx = 0, -1, -1
			"""
	else:
		# self.logs[-1] += self.formatter.warning('_RunLengthCoding: length of listS1 not valid')
		n, d, cx = 0, -1, -1
		"""
		try:
			raise ValidationError('_RunLengthCoding: length of listS1 not valid')
		except ValidationError as e:
			print(e.args)
			n, d, cx = 0, -1, -1
		"""
	return n, d, cx

def _MagnitudeRefinementCoding(neighbourS1, s2):
	# input neighbourS1: size 3*3, matrix of significance
	# input s2: whether it is the first time for Magnitude Refinement Coding
	# output: context
	if len(neighbourS1) == 3 and len(neighbourS1[0]) == 3:
		temp = np.sum(neighbourS1) - neighbourS1[1][1]
		if s2 == 1:
			cx = 16
		elif s2 == 0 and temp >= 1:
			cx = 15
		else:
			cx = 14
	else:
		# self.logs[-1] += self.formatter.warning('_MagnitudeRefinementCoding: Size of neighbourS1 not valid')
		cx = -1
		"""
		try:
			raise ValidationError('_MagnitudeRefinementCoding: Size of neighbourS1 not valid')
		except ValidationError as e:
			print(e.args)
			cx = -1
		"""
	return cx

def _encode_end(encoder):
	nbits = 27 - 15 - encoder.t
	encoder.C = encoder.C * np.uint32(2 ** encoder.t)
	while nbits > 0:
		encoder = _transferbyte(encoder)
		nbits = nbits - encoder.t
		encoder.C = encoder.C * np.uint32(2 ** encoder.t)
	encoder = _transferbyte(encoder)

	return encoder

def _tile_decode(codestream, D):
	_depthOfDWT = D
	temp = []
	for i in range(0, 9 * _depthOfDWT + 3):
		_index = codestream.index(2050)
		deStream = codestream[0:_index + 1]
		temp.append(_band_decode(deStream))
		codestream = codestream[_index + 1:]

	start1 = _depthOfDWT*3+1
	start2 = _depthOfDWT*6+2
	tile = [cat_arrays_2d([temp[0],
												temp[start1],
												temp[start2]])]

	for i in range(_depthOfDWT):

		tile.append((cat_arrays_2d([temp[3 * i + 1],
															 temp[3 * i + start1+1],
															 temp[3 * i + start2+1]]),
								 cat_arrays_2d([temp[3 * i + 2],
															 temp[3 * i + start1+2],
															 temp[3 * i + start2+2]]),
								 cat_arrays_2d([temp[3 * i + 3],
															 temp[3 * i + start1+3],
															 temp[3 * i + start2+3]])))

	return tile

def _band_decode(codestream, h=64, w=64, num=32):
	h_cA = codestream[0]
	w_cA = codestream[1]
	codestream = codestream[2:]
	h_num = h_cA // h + 1
	w_num = w_cA // w + 1
	band_extend = np.zeros((h_num * h, w_num * w))
	for i in range(0, h_num):
		for j in range(0, w_num):
			_index = codestream.index(2048)
			deCX = codestream[0:_index]
			deCX = np.resize(deCX, (_index + 1, 1))
			codestream = codestream[_index + 1:]
			_index = codestream.index(2048)
			deStream = codestream[0:_index]
			codestream = codestream[_index + 1:]
			num = codestream[0]
			codestream = codestream[2:]
			decodeD = _MQ_decode(deStream, deCX)
			band_extend[i * h:(i + 1) * h, j * w:(j + 1) * w] = _decode_block(decodeD, deCX, h, w, num)
		if codestream[0] != 2049:
			print("Error!")
		codestream = codestream[1:]
	if codestream[0] != 2050:
		print("Error!")
	print("finish one band")
	return band_extend[0:h_cA, 0:w_cA]

def _decode_block(D, CX, h=64, w=64, num=32):
	deS1 = np.uint32(np.zeros((h, w)))
	deS2 = np.uint32(np.zeros((h, w)))
	deS3 = np.uint32(np.zeros((h, w)))
	signs = np.uint32(np.zeros((h, w)))
	V = np.uint32(np.zeros((num, h, w)))
	deCode = np.zeros((h, w))
	pointer = 0
	for i in range(num):
		V[i, :, :], signs, deS1, deS3, pointer = _SignificancePassDecoding(V[i, :, :], D, CX, deS1, deS3, pointer, signs, w,
																																			 h)
		V[i, :, :], deS2, pointer = _MagnitudePassDecoding(V[i, :, :], D, deS1, deS2, deS3, pointer, w, h)
		V[i, :, :], deS1, deS3, signs, pointer = _CleanPassDecoding(V[i, :, :], D, CX, deS1, deS3, pointer, signs, w, h)
		deS3 = np.zeros((h, w))
	V = np.transpose(V, (1, 2, 0))
	tempV = np.zeros((h,w))
	for i in range(h):
		for j in range(w):
			tempV[i][j] = sum([V[i][j][k]*2**(num-1-k) for k in range(num)])

	for i in range(h):
		for j in range(w):
			deCode[i][j] = (1 - 2 * signs[i][j]) * tempV[i][j]
	return deCode

def _SignificancePassDecoding(V, D, CX, deS1, deS3, pointer, signs, w=64, h=64):
	S1extend = np.pad(deS1, ((1, 1), (1, 1)), 'constant')
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			for ii in range(4):
				row = 4 * i + ii
				temp = np.sum(S1extend[row:row + 3, col:col + 3]) - S1extend[row + 1][col + 1]
				if deS1[row][col] != 0 or temp == 0:
					continue
				###
				if pointer>=len(D):
					continue
				V[row][col] = D[pointer][0]
				pointer = pointer + 1
				deS3[row][col] = 1
				if V[row][col] == 1:
					signs[row][col] = _SignDecoding(D[pointer], CX[pointer], S1extend[row:row + 3, col:col + 3])
					pointer = pointer + 1
					deS1[row][col] = 1
					S1extend = np.pad(deS1, ((1, 1), (1, 1)), 'constant')
	return V, signs, deS1, deS3, pointer

def _MagnitudePassDecoding(V, D, deS1, deS2, deS3, pointer, w=64, h=64):
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			for ii in range(4):
				row = 4 * i + ii
				if deS1[row][col] != 1 or deS3[row][col] != 0:
					continue
				###
				if pointer>=len(D):
					continue
				V[row][col] = D[pointer][0]
				pointer = pointer + 1
				deS2[row][col] = 1
	return V, deS2, pointer

def _CleanPassDecoding(V, D, CX, deS1, deS3, pointer, signs, w=64, h=64):
	a = pointer
	S1extend = np.pad(deS1, ((1, 1), (1, 1)), 'constant')
	rounds = h // 4
	for i in range(rounds):
		for col in range(w):
			ii = 0
			row = 4 * i
			tempSum = np.sum(S1extend[row:row + 6, col:col + 3]) + np.sum(deS3[row:row + 4, col])
			# 整一列未被编码，都为非重要，且领域非重要
			if tempSum == 0:
				if CX.__len__() < pointer + 3:
					CXextend = np.pad(CX, (0, 2), 'constant')
					Dextend = np.pad(D, (0, 2), 'constant')
					tempCx = CXextend[pointer:pointer + 3]
					tempD = Dextend[pointer:pointer + 3]
				else:
					tempCx = CX[pointer:pointer + 3]
					tempD = D[pointer:pointer + 3]
				ii, tempV = _RunLengthDecoding(tempCx, tempD)
				if tempV == [0, 0, 0, 0]:
					V[row][col] = 0
					V[row + 1][col] = 0
					V[row + 2][col] = 0
					V[row + 3][col] = 0
					pointer = pointer + 1
				else:
					if tempV == [1]:
						V[row][col] = 1
						pointer = pointer + 3
					elif tempV == [0, 1]:
						V[row][col] = 0
						V[row + 1][col] = 1
						pointer = pointer + 3
					elif tempV == [0, 0, 1]:
						V[row][col] = 0
						V[row + 1][col] = 0
						V[row + 2][col] = 1
						pointer = pointer + 3
					elif tempV == [0, 0, 0, 1]:
						V[row][col] = 0
						V[row + 1][col] = 0
						V[row + 2][col] = 0
						V[row + 3][col] = 1
						pointer = pointer + 3
					# sign coding
					row = row + ii - 1
					###
					if pointer>=len(D):
						continue
					signs[row][col] = _SignDecoding(D[pointer], CX[pointer], S1extend[row:row + 3, col:col + 3])
					pointer = pointer + 1
					deS1[row][col] = 1
					S1extend = np.pad(deS1, ((1, 1), (1, 1)), 'constant')
			while ii < 4:
				row = i * 4 + ii
				ii = ii + 1
				if deS1[row][col] != 0 or deS3[row][col] != 0:
					continue
				###
				if pointer>=len(D):
					continue
				V[row][col] = D[pointer][0]
				pointer = pointer + 1
				deS3[row][col] = 1
				if V[row][col] == 1:
					signs[row][col] = _SignDecoding(D[pointer], CX[pointer], S1extend[row:row + 3, col:col + 3])
					pointer = pointer + 1
					deS1[row][col] = 1
					S1extend = np.pad(deS1, ((1, 1), (1, 1)), 'constant')
	return V, deS1, deS3, signs, pointer

def _RunLengthDecoding(CX, D):
	n = CX.__len__()
	wrong = 1
	if CX[0][0] == 17 and D[0][0] == 0 or CX[0][0] == 17 and CX[1][0] == 18 and CX[2][0] == 18 and D[0][0] == 1:
		wrong = 0
	if wrong == 0:
		if D[0][0] == 0:
			deLen = 4
			V = [0, 0, 0, 0]
		elif D[0][0] == 1 and D[1][0] == 0 and D[2][0] == 0:
			deLen = 1
			V = [1]
		elif D[0][0] == 1 and D[1][0] == 0 and D[2][0] == 1:
			deLen = 2
			V = [0, 1]
		elif D[0][0] == 1 and D[1][0] == 1 and D[2][0] == 0:
			deLen = 3
			V = [0, 0, 1]
		elif D[0][0] == 1 and D[1][0] == 1 and D[2][0] == 1:
			deLen = 4
			V = [0, 0, 0, 1]
		else:
			# self.logs[-1] += self.formatter.warning('_RunLengthDecoding: D not valid')
			deLen = -1
			V = [-1]
			"""
			try:
				raise ValidationError('_RunLengthDecoding: D not valid')
			except ValidationError as e:
				print(e.args)
				deLen = -1
				V = [-1]
			"""
	else:
		# self.logs[-1] += self.formatter.warning('_RunLengthDecoding: CX not valid')
		deLen = -1
		V = [-1]
		"""
		try:
			raise ValidationError('_RunLengthDecoding: CX not valid')
		except ValidationError as e:
			print(e.args)
			deLen = -1
			V = [-1]
		"""
	return deLen, V

def _SignDecoding(D, CX, neighbourS1):
	if neighbourS1.__len__() == 3 and neighbourS1[0].__len__() == 3:
		hstr = str(int(neighbourS1[1][0])) + str(int(neighbourS1[1][2]))
		vstr = str(int(neighbourS1[0][1])) + str(int(neighbourS1[2][1]))
		dict = {
			'00': 0, '1-1': 0, '-11': 0, '01': 1, '10': 1, '11': 1,
			'0-1': -1, '-10': -1, '-1-1': -1
		}
		h = dict[hstr]
		v = dict[vstr]
		hAndv = str(h) + str(v)
		hv2Sign = {
			'11': 0, '10': 0, '1-1': 0, '01': 0, '00': 0,
			'0-1': 1, '-11': 1, '-10': 1, '-1-1': 1
		}
		hv2Context = {
			'11': 13, '10': 12, '1-1': 11, '01': 10, '00': 9,
			'0-1': 10, '-11': 11, '-10': 12, '-1-1': 13
		}
		temp = hv2Sign[hAndv]
		deCX = hv2Context[hAndv]
		if deCX == CX:
			deSign = D[0] ^ temp
		else:
			# self.logs[-1] += self.formatter.warning('_SignDecoding: Context does not match. Error occurs.')
			deSign = -1
			"""
			try:
				raise ValidationError('_SignDecoding: Context does not match. Error occurs.')
			except ValidationError as e:
				print(e.args)
				deSign = -1
			"""
	else:
		# self.logs[-1] += self.formatter.warning('_SignDecoding: Size of neighbourS1 not valid')
		deSign = -1
		"""
		try:
			raise ValidationError('_SignDecoding: Size of neighbourS1 not valid')
		except ValidationError as e:
			print(e.args)
			deSign = -1
		"""
	return deSign

class EBCOTparam(object):
	"""
	EBCOT parameter is parameter used by MQ encode and decode processes

	initialized in MQ encoding
	interval length         A = 8000H
	Lower bound register        C = 0
	Current code byte number    L = -1
	Temporary byte buffer     T = 0
	Down-counter          t = 12

	"""

	def __init__(self):
		self.C = np.uint32(0)
		self.A = np.uint16(32768)
		self.t = np.uint8(12)
		self.T = np.uint8(0)
		self.L = np.int32(-1)
		self.stream = np.uint8([])

'''
改了decodeblock，_embeddedBlockEncoder，banddecode和bandencode改了num的预设值
'''

# if __name__ == "__main__":
# 	h, w = 64, 64
# 	testblock = np.zeros((h, w), dtype=np.uint8)
# 	for i in range(h):
# 		for j in range(w):
# 			testblock[i][j] = i*4
# 	#(bitcode, _) = _band_encode(testblock, "LL", h=64, w=64, num=8)
# 	#decodeblock = _band_decode(list(bitcode), h=64, w=64, num=32)
# 	CX, D, bitplanelength = _embeddedBlockEncoder(testblock, "LL", h, w, num=8)
# 	encoder = _MQencode(CX, D)
# 	decodeD = _MQ_decode(encoder.stream, CX)
# 	decodeblock = _decode_block(D, CX, h, w, num=8)

# 	a = 1


import tqdm
from tensorboardX import SummaryWriter
import cv2
import numpy as np
import os
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from models.lbs import batch_rodrigues
from utils import misc
from pytorch3d.io import save_obj
from opt_params import OptParams

def process_visuals(visuals):
    visual_list = []
    keys_visuals = list(visuals.keys())
    for visual_key in keys_visuals:
        visual_list += misc.prepare_visual(visuals, visual_key, preprocessing_op=None)
    visual_list = torch.cat(visual_list, 3) # cat w.r.t. width
    visual_list = visual_list.clamp(0, 1)
    visual_list = torch.cat(visual_list.split(1, 0), 2)[0] # cat batch dim in lines w.r.t. height
    visual_list = visual_list.cpu()
    return visual_list

def tensor2image(tensor):
    image = tensor.detach().cpu().numpy()
    image = image * 255.
    image = np.maximum(np.minimum(image, 255), 0)
    image = image.transpose(1, 2, 0)
    return image.astype(np.uint8).copy()

class Runner:
    def __init__(
        self,
        dataset,
        losses,
        smplx_model,
        device,
        save_path,
        cut_flame_head,
        loss_weights,
        train_rotation,
        train_pose,
        train_shape,
        checkpoint_path
    ):

        self.dataset = dataset
        self.losses = losses
        self.smplx = smplx_model
        self.opt_params = OptParams(
                                    device,
                                    dataset,
                                    train_rotation,
                                    train_pose,
                                    train_shape,
                                    checkpoint_path
                                    )
        self.device = device
        self.save_path = save_path

        self.eye_pose = torch.eye(3, requires_grad=False, device=self.device).unsqueeze(0).repeat(self.dataset.nimages, 2, 1, 1)
        self.left_hand_pose = torch.eye(3, requires_grad=False, device=self.device).unsqueeze(0).repeat(self.dataset.nimages, 15, 1, 1)
        self.right_hand_pose = torch.eye(3, requires_grad=False, device=self.device).unsqueeze(0).repeat(self.dataset.nimages, 15, 1, 1)

        self.cut_flame_head = cut_flame_head
        self.loss_weights = loss_weights
        self.img_size = dataset.images.shape[2]

        os.makedirs(os.path.join(save_path, 'mesh'), exist_ok=True)
        os.makedirs(os.path.join(save_path, 'opt_params'), exist_ok=True)

    @torch.no_grad()
    def get_visuals(self, lm_pred, lm_gt, label, visuals, batch):
        name = 'target_stickman' + str(label)
        image_size = batch['img'].shape[1]
        if label == 'openpose_face':
            visuals[name] = misc.draw_stickman(lm_gt / (image_size / 2) - 1, image_size, images=batch['img']) #lmks[-1, 1]
            visuals['pred_' + name] = misc.draw_stickman(lm_pred / (image_size / 2) - 1, image_size, images=batch['img'])
        elif label =='openpose_body':
            visuals[name] = misc.draw_stickman_body(lm_gt / (image_size / 2) - 1, image_size, images=batch['img'])
            visuals['pred_' + name] = misc.draw_stickman_body(lm_pred / (image_size / 2) - 1, image_size, images=batch['img'])
        else:
            visuals[name] = misc.draw_stickman_fa(lm_gt/ (image_size / 2) - 1, image_size, images=batch['img'])
            visuals['pred_' + name] = misc.draw_stickman_fa(lm_pred / (image_size / 2) - 1, image_size, images=batch['img'])
        return visuals

    def fit(self, epochs, lr, max_iter, tol=1e-9):

        dataloader = DataLoader(
            self.dataset,
            batch_size=self.dataset.nimages,
            shuffle=False,
            num_workers=0
        )

        for batch_idx, batch in enumerate(dataloader):
            if epochs > 0:
                param_lst = self.opt_params.get_train_params_list()
                # need to create optimizer for every batch so that data for prev batches is not changed
                optimizer = optim.LBFGS(
                    param_lst,
                    lr=lr, max_iter=max_iter,
                    line_search_fn='strong_wolfe',
                    tolerance_grad=tol, tolerance_change=tol)
            frame_id_0 = batch['frame_ids'][0]
            writer = SummaryWriter(log_dir=os.path.join(self.save_path, f'logs/{frame_id_0:06d}'))
            step = 0

            for epoch in range(0, epochs):
                tq = tqdm.tqdm(ncols=100)
                tq.set_description(f'Epoch {epoch}')
                tq.refresh()

                def closure():
                    nonlocal step, optimizer, tq, epoch, writer

                    def log(loss_str, label, value_gpu):
                        loss_value_cpu = value_gpu.detach().cpu().numpy()
                        loss_str += f'\n\t{label}: {loss_value_cpu:.1E} '
                        writer.add_scalar(os.path.join('train', label), loss_value_cpu, step)
                        return loss_str, loss_value_cpu

                    if torch.is_grad_enabled():
                        optimizer.zero_grad()

                    pred = self.forward(batch)
                    writer.add_scalar(os.path.join('train', 'epoch'), epoch, step)
                    loss_str = 'losses:'
                    step += 1
                    loss_values = []

                    visuals = {}
                    for loss in self.losses:
                        loss_value, lm_gt, lm_pred = loss.compute(pred, batch, img_size=self.img_size, weight=self.loss_weights[loss.label])
                        loss_str, _ = log(loss_str, loss.label, loss_value)
                        loss_values.append(loss_value)
                        if loss.label ==  'openpose_face' or loss.label ==  'openpose_body' or loss.label == 'fa_kpts':
                            visuals = self.get_visuals(lm_pred, lm_gt, loss.label, visuals, batch)

                    if step % 10 == 0:
                        self.dump_results(pred, step, epoch, batch_idx)

                    visual_list = process_visuals(visuals)
                    writer.add_image(f'images', visual_list, step)

                    total_loss = torch.stack(loss_values).sum()
                    if step % 20 == 0:
                        print(total_loss)

                    loss_str, loss_value_cpu = log(loss_str, 'total_loss', total_loss)
                    total_loss.backward()

                    tq.update()
                    tq.set_postfix({'loss': f'{loss_value_cpu:.3E}'})
                    tq.refresh()

                    return total_loss

                optimizer.step(closure)

                tq.close()

    def dump_results(self, pred, step, epoch, batch_idx):
        save_obj(os.path.join(self.save_path, 'mesh', f'mesh{epoch}_{step}_{batch_idx}.obj'), pred['verts_world'][0], pred['faces_world'])
        self.opt_params.dump_json_dict(os.path.join(self.save_path, 'opt_params', f'opt_params_{epoch}_{step}_{batch_idx}'))

    def obtain_global_matx(self, batch, ds_ids):
        b = len(ds_ids)
        scale = (torch.eye(3, device=self.device) * self.opt_params.global_scale.repeat(3)).repeat(b, 1, 1)
        global_rot = torch.bmm(batch_rodrigues(self.opt_params.global_rot.repeat(b, 1)), scale) #[b, 3, 3]
        global_trans = self.opt_params.global_trans.repeat(b, 1).reshape(b, 3, 1)  #[b, 3, 1]
        return global_rot.unsqueeze(1), global_trans.reshape(b, 3).unsqueeze(1)

    def forward(self, batch):
        ds_ids = batch['frame_ids']
        b = len(ds_ids)

        pose_jaw_rotmtx =  torch.index_select(self.opt_params.pose_jaw, 0, ds_ids)
        pose_body_rotmtx  =  torch.index_select(self.opt_params.pose_body, 0, ds_ids)
        face_expression = torch.index_select(self.opt_params.face_expression, 0, ds_ids)
        betas = self.opt_params.beta.reshape(1, -1).repeat(b, 1)

        global_rotmtx, global_trans = self.obtain_global_matx(batch, ds_ids)

        #  use default pose
        eye_pose = torch.index_select(self.eye_pose, 0, ds_ids)
        left_hand_pose = torch.index_select(self.left_hand_pose, 0, ds_ids)
        right_hand_pose = torch.index_select(self.right_hand_pose, 0, ds_ids)

        verts, landmarks, joints = self.smplx(shape_params=betas, expression_params=face_expression,  global_pose=global_rotmtx, body_pose=pose_body_rotmtx, jaw_pose=pose_jaw_rotmtx, eye_pose=eye_pose, left_hand_pose=left_hand_pose, right_hand_pose=right_hand_pose)

        # Add translation
        verts += global_trans
        joints += global_trans
        landmarks += global_trans

#        Change axis to be in NEUS space
        verts[:, :, :0] = -verts[:, :, :0]
        joints[:, :, :0]  = -joints[:, :, :0]
        landmarks[:, :, :0]  = -landmarks[:, :, :0]

        result = {}

        if self.cut_flame_head:
            flame_verts, flame_faces = self.smplx.cut_flame_head(verts)
            result['verts_world'] = flame_verts
            result['faces_world'] = flame_faces
        else:
            result['verts_world'] = verts
            result['faces_world'] = self.smplx.faces_tensor

#       get extrinsics
        extrinsics_rot = batch['extrinsics_rvec'].unsqueeze(1)
        extrinsics_trans = batch['extrinsics_tvec'].unsqueeze(1)

#         world to camera transform
        joints = torch.matmul(extrinsics_rot.repeat(1, joints.shape[1], 1, 1), joints.unsqueeze(-1)).squeeze(-1)
        verts = torch.matmul(extrinsics_rot.repeat(1, verts.shape[1], 1, 1), verts.unsqueeze(-1)).squeeze(-1)
        landmarks = torch.matmul(extrinsics_rot.repeat(1, landmarks.shape[1], 1, 1), landmarks.unsqueeze(-1)).squeeze(-1)

        Jtr = joints + extrinsics_trans
        verts_trans = verts + extrinsics_trans
        landmarks += extrinsics_trans

        result['verts_extrinsics'] = verts #[bs, 10475, 3]
        result['verts'] = verts_trans
        result['Jtr'] = Jtr #[166, 3]
        result['betas'] = self.opt_params.beta
        result['face_kpt'] = landmarks
        return result


import copy
import numpy as np
import Applications.general_ci.config as config
import Applications.general_ci.state  as state

def one_elec_hamiltonian(state_obj, h_mat, rec_num_states):
	#
	# h1 = \sum_pq h_pq p+ q
	#
	#    h_pq = Kinetic E + Nuclear Attraction E
	#
	num_mol = len(rec_num_states)
	# print("num spin orb =", num_spin_orb)
	new_state = state.state(state_obj.get_configs())
	#
	#
	idx_shift = 0
	for mol in range(num_mol):
		for p in range(rec_num_states[mol]):
			for q in range(rec_num_states[mol]):
				op_list    = [ [config.create, p+idx_shift],[config.annihilate, q+idx_shift] ]
				temp_state = state.operate(op_list, state_obj)
				new_state.add_to( h_mat(p+idx_shift, q+idx_shift), temp_state )
		idx_shift += rec_num_states[mol]

	return new_state

def two_elec_hamiltonian(state_obj, V_mat, rec_num_states):
	#
	# h2 = \sum_pqrs 1/4 V_pqrs p+ q+ r s
	#
	#    V_pqrs = V_mat[pqsr] - V_mat[pqrs]
	#
	num_mol = len(rec_num_states)
	#
	# print("num spin orb =", num_spin_orb)
	new_state = state.state(state_obj.get_configs())
	#
	#

	first_idx_shift  = 0
	for i in range(num_mol):               # Mol 1
		# Compute the second index shift
		second_idx_shift = 0
		for num in rec_num_states[:i+1]:
			second_idx_shift += num
		#
		for j in range(i+1, num_mol):      # Mol 2
			#
			#
			#
			#
			for p in range(rec_num_states[i]):      # All States in Mol 1
				for q in range(rec_num_states[j]):  # All States in Mol 2
					#
					#
					for r in range(rec_num_states[i]):   # All States in Mol 1
						for s in range(rec_num_states[j]):  # All States in Mol 2
							#
							op_list    = [ [config.create, p+first_idx_shift], [config.create, q+second_idx_shift], \
										   [config.annihilate, r+first_idx_shift],[config.annihilate, s+second_idx_shift] ]
							temp_state = state.operate(op_list, state_obj)
							V_value = V_mat(i, j, p, q, r, s)
							new_state.add_to( V_value , temp_state )
							# print(this_config, first_idx_shift, second_idx_shift, '==>', first_idx_shift+k+1, second_idx_shift+l+1)
			#
			#
			second_idx_shift += rec_num_states[j]
		first_idx_shift += rec_num_states[i]
	#
	#
	#
	return new_state

def hamiltonian(state_obj, t_amp_obj, h_mat, V_mat, rec_num_states):
	new_state = state.state(state_obj.get_configs())
	new_state = state.add_state( new_state, one_elec_hamiltonian(state_obj, h_mat, rec_num_states) )
	new_state = state.add_state( new_state, two_elec_hamiltonian(state_obj, V_mat, rec_num_states) )
	return new_state

def h_wrapper(this_input_list):
	return hamiltonian(*this_input_list)


import matplotlib.pyplot as graph
from usrfuncs import *
from time import *

def desmos(FUNC, X_MIN, X_MAX):
    error = 0
    if (len(FUNC) != len(X_MIN)) or (len(FUNC) != len(X_MAX)) or (len(X_MIN) != len(X_MAX)):
        exit('Недопустимые вводные данные!')
    for c in range(len(FUNC)):
        t = time()
        if (X_MAX[c] <= X_MIN[c]):
            print('Недопустимый интервал сканирования для %s-й функи, пропускаем её построение.' % (c + 1))
            error += 1
            continue
        X, Y = [], []; x, percents = X_MIN[c], 0
        while X_MIN[c] <= x <= X_MAX[c]:
            X.append(x)
            Y.append(eval(streplace(FUNC[c], 'there')))
            percents += 0.1
            print('Построение ' + str(c + 1) + '-й функи: ' + f'{round(percents, 2)}' + '%\r', end = '', flush = True)
            x += (X_MAX[c] - X_MIN[c]) / 10 ** 3
        graph.plot(X, Y)
        print('Построение ' + str(c + 1) + '-й функи завершено за %s сек.' % round((time() - t), 3))
    if error != len(FUNC):
        graph.title('График фунок: ' + str(FUNC))
        graph.show()
    return 0

try:
    count = int(input('Количество исследуемых фунок: '))
    FUNC, X_MIN, X_MAX = [], [], []
    for c in range(count):
        FUNC.append(str(input('Введите %s-ю функу: ' % (c + 1))))
        X_MIN.append(float(input('Введите начало области определения %s-й функи: ' % (c + 1))))
        X_MAX.append(float(input('Введите конец области определения %s-й функи: ' % (c + 1))))
    desmos(FUNC, X_MIN, X_MAX)
except KeyboardInterrupt:
    desmos(FUNC, X_MIN, X_MAX)
except OverflowError:
    exit('При вычислении невозможно обработать слишком большие числа ...')
except ValueError as error:
    if str(error) == 'math domain error':
        exit('Заданная область определения выходит за рамки определенной области определения для заданной функи ...')
    exit('Произошла ошибка при вычислении ...')
except NameError:
    exit('Поддерживается только следующие функции и константы: pi, e, pf(x, m), relat(x) и другие стандартные функции ...')


def get_bed_files(config):
    if 'genotype_bed' not in config:
        raise ValueError('No genotype_bed in config.')
    file_prefix = config['genotype_bed']
    files = [ file_prefix + '.' + ss for ss in ['bim', 'bed', 'fam'] ]
    command = '--bfile ' + file_prefix
    return files, command

def get_parquet_files(config):
    if 'genotype_parquet' not in config:
        raise ValueError('No genotype_parquet in config.')
    file_prefix = config['genotype_parquet']
    files = [ file_prefix + '.' + ss for ss in ['variants_metadata.parquet', 'variants.parquet'] ]
    command = file_prefix
    return files, file_prefix

def get_plink_filters(config):
    if 'plink_filters' not in config:
        return ''
    filters = config['plink_filters']
    return ' '.join(filters)


from __future__ import print_function

import re
import sys
from infi.execute import execute_assert_success
import pkg_resources
from ..depends.dependencies import get_dependencies

def run_easy_install(package_name, package):
    cmd = "easy_install -U \"{}\"".format(package)
    print("Running:", cmd, end=' ')
    sys.stdout.flush()
    output = execute_assert_success(cmd, shell=True).get_stdout().decode("ascii")

    try:
        # try to get the name of the package from what is already installed
        # e.g. setuptools will become distribute
        package_name = pkg_resources.get_distribution(package_name).project_name
    except pkg_resources.DistributionNotFound:
        pass

    added = re.search("Adding ({}.*?) to".format(package_name), output)
    removed = re.search("Removing ({}.*?) from".format(package_name), output)
    stayed = re.search("({}.*?) is already".format(package_name), output)
    best_match = re.search("Best match: ({}.*)".format(package_name), output)
    if stayed:
        print("(Already on latest version {})".format(stayed.group(1)))
    elif removed and added:
        print("(Updated from {} to {})".format(removed.group(1), added.group(1)))
    elif best_match:
        print("(Installed version {})".format(best_match.group(1)))
    else:
        # status unknown
        print("(updated)")

def hard_install(name):
    dependencies = list(reversed(list(get_dependencies(name))))
    print("Found {} dependencies.".format(len(dependencies)))
    for parent, package_name, package in dependencies:
        run_easy_install(package_name, package)

def main():
    import sys
    hard_install(sys.argv[1])


from __future__ import unicode_literals
import six

def python_2_unicode_compatible(klass):
    """
    A decorator that defines __unicode__ and __str__ methods under Python 2.
    Under Python 3 it does nothing.

    To support Python 2 and 3 with a single code base, define a __str__ method
    returning text and apply this decorator to the class.

    This was taken from the Django source code.

    """
    if six.PY2:
        if '__str__' not in klass.__dict__:
            raise ValueError("@python_2_unicode_compatible cannot be applied "
                             "to %s because it doesn't define __str__()." %
                             klass.__name__)
        klass.__unicode__ = klass.__str__
        klass.__str__ = lambda self: self.__unicode__().encode('utf-8')
    return klass

def force_text(s, encoding='utf-8', strings_only=False, errors='strict'):
    """ Forces the argument s into either a str object (Python 3) or a unicode
        object (Python 2). Taken and modified from the Django source code. """
    if isinstance(s, six.text_type):
        return s
    if not isinstance(s, six.string_types[0]):
        if six.PY3:
            if isinstance(s, bytes):
                s = six.text_type(s, encoding, errors)
            else:
                s = six.text_type(s)
        elif hasattr(s, '__unicode__'):
            s = six.text_type(s)
        else:
            s = six.text_type(bytes(s), encoding, errors)
    else:
        s = s.decode(encoding, errors)
    return s