# Figure 6.4
# Plot standardized coefficients as function of lambda (alpha in sklearn)
# and ratio of l-2 norm of ridge regression coefficients and l-2 norm
# of OLS regression coefficients

import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import Ridge, LinearRegression
from statsmodels import datasets
import pandas as pd

credit = datasets.get_rdataset('Credit', 'ISLR').data
credit.drop(columns='ID', inplace=True)

X_numeric = credit[['Income', 'Limit', 'Rating',
                    'Cards', 'Age', 'Education']].copy()
X_num_std = (X_numeric - X_numeric.mean()) / X_numeric.std()
# X_num_std = X_numeric / X_numeric.std()
X_cat = credit[['Gender', 'Student', 'Married', 'Ethnicity']].copy()
X_cat = pd.get_dummies(X_cat)
X = pd.concat((X_num_std, X_cat), axis=1)
y = credit['Balance']
alpha_vals = np.logspace(-3, 4, 49)
beta_ridge = []
intercept_ridge = []
for alpha in alpha_vals:
    ridge_model = Ridge(alpha=alpha)
    ridge_model.fit(X, y)
    beta_ridge.append(ridge_model.coef_)
    intercept_ridge.append(ridge_model.intercept_)

income_coef = [beta_ridge[i][0] for i in range(len(alpha_vals))]
limit_coef = [beta_ridge[i][1] for i in range(len(alpha_vals))]
rating_coef = [beta_ridge[i][2] for i in range(len(alpha_vals))]
student_coef = [beta_ridge[i][9] for i in range(len(alpha_vals))]

fig = plt.figure(figsize=(8, 4))
ax1 = fig.add_subplot(121)
ax1.plot(alpha_vals, income_coef, label='Income', c='k')
ax1.plot(alpha_vals, limit_coef, label='Limit', c='r', linestyle='--')
ax1.plot(alpha_vals, rating_coef, label='Rating', c='b', linestyle=':')
ax1.plot(alpha_vals, student_coef, label='Student', color='brown',
         linestyle='-.')
ax1.legend(loc='lower left', frameon=False)
ax1.set_xscale('log')
ax1.set_xlabel(r'$\lambda$')

ols_model = LinearRegression()
ols_model.fit(X, y)
beta_l2_ols = np.sqrt(np.sum([beta ** 2 for beta in ols_model.coef_]))

beta_l2_ratio = []
for i in range(len(beta_ridge)):
    beta_l2_ridge = np.sqrt(np.sum(beta_ridge[i] ** 2))
    beta_l2_ratio.append(beta_l2_ridge / beta_l2_ols)

ax2 = fig.add_subplot(122)
ax2.plot(beta_l2_ratio, income_coef, c='k')
ax2.plot(beta_l2_ratio, limit_coef, c='r', linestyle='--')
ax2.plot(beta_l2_ratio, rating_coef, c='b', linestyle=':')
ax2.plot(beta_l2_ratio, student_coef, color='brown', linestyle='-.')
# ax2.set_xscale('log')
ax2.set_xlabel(r'$\|| \hat{\beta}_\lambda^R \||_2 / \|| \hat{\beta} \||_2$')

for ax in fig.axes:
    ax.set_ylabel('Standardized Coefficients')

fig.tight_layout()


#!/usr/bin/env python
# coding: utf-8
from __future__ import (absolute_import, print_function,
                        division, unicode_literals)

from pkg_resources import resource_string, resource_listdir
from datetime import datetime
import os
import sys
import re
import argparse

join = lambda a, b: os.path.join(os.path.abspath(a), b)

TODAY = datetime.today().strftime("%Y%m%d")
GENERATOR_MESSAGES = {
    'error': 'Generator failed to generate files',
    'message': "Generating Folder...",
    'lang': "Language found: %s",
    'lang_error': "%s generator wasn't found",
    'exists': "Dojo path already exists!",
    'create': "New %s!",
}
GENERATORS_PATH = join(os.path.dirname(__file__), 'generators')

class GeneratorException(Exception):
    value = GENERATOR_MESSAGES['error']
    def __str__(self):
        return repr(self.value)

class GeneratorNotFoundError(GeneratorException):
    def __init__(self, language):
        self.value = GENERATOR_MESSAGES['lang_error'] % language

class GeneratorPathExistsError(GeneratorException):
    value = GENERATOR_MESSAGES['exists']

def sprint(text, show):
    if show:
        print(text)

def list_to_args(args, show=True):
    opts = args.split(' ')
    if len(opts) < 2:
        raise GeneratorException()
    n = argparse.Namespace()
    n.language, n.problem = opts[:2]
    n.extra = opts[2:]
    n.path = os.path.curdir
    return n

class Generator(object):

    def __init__(self, args):
        self.language = args.language
        self.problem = args.problem
        self.extra =  ('_' + '_'.join(args.extra)) if args.extra else ''
        self.path = args.path
        self.ignore = args.ignore

        self.today = TODAY

        self.folder_name = "{date}{_extra}_{language}_{problem}".format(
            date=TODAY,
            _extra=self.extra,
            language=self.language,
            problem=self.problem,
        )

        self.folder_path = join(self.path, self.folder_name)
        self.generator_path = join(GENERATORS_PATH, self.language)

        splitted_name =  self.problem.split('_')

        snake_case = self.problem
        pascal_case = ''.join(part.capitalize() for part in splitted_name)
        down_case = ''.join(splitted_name)
        camel_case = (
            splitted_name.pop(0) +
            ''.join(part.capitalize() for part in splitted_name)
        )

        self.cases = {
            '___dojogen___' : snake_case,
            '___class_dojogen___' : pascal_case,
            '___down_dojogen___' : down_case,
            '___camel_dojogen___' : camel_case,
        }
        print(self.cases)

        self.generated = False

    def replace(self, text):
        for sub, replace in self.cases.items():
            text = re.sub(sub, replace, text)
        return text

    def copy_and_rename(self, current, folder_name, original):
        isdir = lambda x: os.path.isdir(x) and not os.path.islink(x)

        folder_path = join(current, folder_name)
        os.mkdir(folder_path)
        file_list = os.listdir(original)
        for infile in file_list:
            gen_path = join(original, infile)
            if isdir(gen_path):
                self.copy_and_rename(folder_path, infile, gen_path)
            elif not '.pyc' in gen_path:
                new_path = join(folder_path,self.replace(infile))
                with open(new_path, 'w') as w:
                    with open(gen_path, 'r') as r:
                        for line in r:
                            w.write(self.replace(line))

    def generate(self, show=True):
        if not os.path.exists(self.folder_path):
            sprint(GENERATOR_MESSAGES['message'], show)
            if self.ignore:
                os.mkdir(join(self.path, self.folder_name))
            elif os.path.exists(self.generator_path):
                sprint(GENERATOR_MESSAGES['lang'] % (self.language), show)
                self.copy_and_rename(
                    self.path, self.folder_name, self.generator_path)
                self.generated = True
            else:
                raise GeneratorNotFoundError(self.language)
            sprint(GENERATOR_MESSAGES['create'] % self.folder_name, show)
        else:
            self.generated = True
            raise GeneratorPathExistsError()

def generate(generate, directory):
    if generate:
        generator = Generator(list_to_args(generate))
        try:
            generator.generate()
        finally:
            if generator.generated:
                return generator.folder_path
    return directory

def generate_mode(args):
    generator = Generator(args)
    generator.generate()

def help_mode(args):
    help_path = join(join(GENERATORS_PATH, "help"), args.language)
    if os.path.exists(help_path):
        with open(help_path, 'r') as f:
            print(f.read())
    else:
        raise GeneratorNotFoundError(args.language)

def lang_mode(args):
    for name in os.listdir(GENERATORS_PATH):
        if os.path.isdir(join(GENERATORS_PATH, name)) and name != 'help':
            print(name)

def main():
    gen_msg = '''Generates directory for coding dojo following the pattern:
        {}[_extra]_language_problem\n'''.format(TODAY)
    help_msg = 'Describes how to prepare the environment for a language'
    lang_msg = 'Shows existing generators'
    dgen_msg, dhelp_msg, dlang_msg = gen_msg, help_msg, lang_msg

    parser = argparse.ArgumentParser()
    subparsers = parser.add_subparsers()
    for name in ['generate', 'gen', 'g']: # Aliases on Python 2.x
        parser_gen = subparsers.add_parser(
            name, description=gen_msg, help=dgen_msg)
        parser_gen.add_argument(
            'language', type=str, help='Programming Language')
        parser_gen.add_argument(
            'problem', type=str, help='Problem Name')
        parser_gen.add_argument(
            'extra', type=str, nargs='*', default='', help='Extra identifier')
        parser_gen.add_argument(
            '--path', '-p', type=str, default=os.path.curdir,
            help='Custom path')
        parser_gen.add_argument(
            '--ignore', '-i', action='store_true',
            help='Do not generate files. Create only the folder')
        parser_gen.set_defaults(func=generate_mode)
        dgen_msg = '...'

    for name in ['help', 'man', 'h']: # Aliases on Python 2.x
        parser_help = subparsers.add_parser(
            name, description=help_msg, help=dhelp_msg)
        parser_help.add_argument(
            'language', type=str, help='Programming Language')
        parser_help.set_defaults(func=help_mode)
        dhelp_msg = '...'

    for name in ['language', 'l']: # Aliases on Python 2.x
        parser_lang = subparsers.add_parser(
            name, description=lang_msg, help=dlang_msg)
        parser_lang.set_defaults(func=lang_mode)
        dlang_msg = '...'

    args, _ = parser.parse_known_args()
    try:
        args.func(args)
    except GeneratorException as err:
        print("Error: {0}".format(err))

if __name__ == '__main__':
    main()


# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next
class Solution:
    def rotateRight(self, head: Optional[ListNode], k: int) -> Optional[ListNode]:
        if not head or not head.next or k == 0: return head
        cur = head
        length = 0
        while cur:
            cur = cur.next
            length += 1
        k %= length
        if k == 0: return head

        dummy = ListNode(-1, head)
        slow, fast = head, head
        pre = dummy
        for i in range(k):
            fast = fast.next
        while fast.next:
            pre = pre.next
            slow = slow.next
            fast = fast.next
        head2 = slow.next
        slow.next = None
        head2_tail = fast
        head2_tail.next = dummy.next
        dummy.next = head2
        return dummy.next


import os,json,sys,glob
import sqlite3
import urllib3

import sys
import base64
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
import psutil

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def dpapi_decrypt(encrypted):  # 老版本Chrome解密方法
    import ctypes
    import ctypes.wintypes

    class DATA_BLOB(ctypes.Structure):
        _fields_ = [('cbData', ctypes.wintypes.DWORD),
                    ('pbData', ctypes.POINTER(ctypes.c_char))]

    p = ctypes.create_string_buffer(encrypted, len(encrypted))
    blobin = DATA_BLOB(ctypes.sizeof(p), p)
    blobout = DATA_BLOB()
    retval = ctypes.windll.crypt32.CryptUnprotectData(
        ctypes.byref(blobin), None, None, None, None, 0, ctypes.byref(blobout))
    if not retval:
        raise ctypes.WinError()
    result = ctypes.string_at(blobout.pbData, blobout.cbData)
    ctypes.windll.kernel32.LocalFree(blobout.pbData)
    return result

def aes_decrypt(encrypted_txt):  # 新版本Chrome解密方法
    with open(os.path.join(os.environ['LOCALAPPDATA'],
                           r"Google\Chrome\User Data\Local State"), encoding='utf-8', mode="r") as f:
        jsn = json.loads(str(f.readline()))
    encoded_key = jsn["os_crypt"]["encrypted_key"]
    encrypted_key = base64.b64decode(encoded_key.encode())
    encrypted_key = encrypted_key[5:]
    key = dpapi_decrypt(encrypted_key)
    nonce = encrypted_txt[3:15]
    cipher = Cipher(algorithms.AES(key), None, backend=default_backend())
    cipher.mode = modes.GCM(nonce)
    decryptor = cipher.decryptor()
    return decryptor.update(encrypted_txt[15:])

def chrome_decrypt(encrypted_txt):  # 封装后Chrome解密函数
    if sys.platform == 'win32':
        try:
            if encrypted_txt[:4] == b'x01x00x00x00':
                decrypted_txt = dpapi_decrypt(encrypted_txt)
                return decrypted_txt.decode()
            elif encrypted_txt[:3] == b'v10':
                decrypted_txt = aes_decrypt(encrypted_txt)
                return decrypted_txt[:-16].decode()
        except WindowsError:
            return None
    else:
        raise WindowsError

def get_cookies_from_chrome(domain=None, sql=None):  # 给定域名，返回这个域名下的cookie信息
    if not sql:
        sql = f'SELECT name, encrypted_value as value FROM cookies where host_key like "%{domain}%"'
    else:
        sql = sql
    files=get_chrome_db_path()
    cookie_db =files.get('cookie_file')
    if os.environ.get('debug'):
        print("in get_cookies_from_chrome(),db=%s,sql=%s" % (cookie_db,sql))

    dsn = 'file:' + cookie_db + '?mode=ro&nolock=1'
    #con = sqlite3.connect(filename)
    con=sqlite3.connect(dsn,uri=True)
    con.row_factory = sqlite3.Row
    cur = con.cursor()
    cur.execute(sql)
    cookie = ''
    for row in cur:
        if row['value'] is not None:
            name = row['name']
            value = chrome_decrypt(row['value'])
            if value is not None:
                cookie += name + '=' + value + ';'
    return cookie

def get_url_from_chrome(url_query):  # 给定一个URL的SQL where条件查询语句，返回最后一条的url记录
    files=get_chrome_db_path()
    history_db = files.get('history_file')
    dsn = 'file:' + history_db + '?mode=ro&nolock=1'  # 设置只读方式打开，无锁方式，Chrome启动时会锁住History库，所以得nolock
    # print(dsn)
    con = sqlite3.connect(dsn, uri=True)
    con.row_factory = sqlite3.Row
    cur = con.cursor()
    sql = "SELECT * FROM urls where url like '%s' order by last_visit_time desc limit 1;" % url_query
    cur.execute(sql)
    for row in cur:
        return row['url']

def get_father_domain(
        full_domain):  # 给定类似testcase.software.ibm.com域名的字串，返回['testcase.software.ibm.com','.software.ibm.com','.ibm.com','.com']
    items = full_domain.split('.')
    N = len(items)
    res = []
    for i in range(N):
        one_item = []
        for j in range(i, N):
            one_item.append(items[j])
        one_domain = ".".join(one_item)
        if one_domain == full_domain:
            res.append(one_domain)
        else:
            res.append('.' + one_domain)
    return res

def gen_sql_clause(full_domain):
    domains = get_father_domain(full_domain)
    sql = 'SELECT name, encrypted_value as value FROM cookies'
    if not domains:
        return sql
    where_clause = ' where ' + " or ".join(["host_key='%s' or host_key='.%s'" % (domain,domain) for domain in domains])
    return sql + where_clause

def get_cookie_by_url(url):
    try:
        host = url.split('://', maxsplit=1)[1].split('/', maxsplit=1)[0]
        sql = gen_sql_clause(host)
        res = get_cookies_from_chrome(sql=sql)
    except Exception as e:
        print(e)
        return None
    return res

def get_profile_dir(): #通过后台进程，找到chrome.exe的进程，查看它带的--profile-directory=xx的参数，找到chrome的profile目录
    for process in psutil.process_iter():
        try:
            cmdline = process.cmdline()
            # cmdline类似如下数据：cmdline=['C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe', '--profile-directory=Profile 1', '--restart']
            if len(cmdline)<2: #cmd至少要带个参数才行
                continue
            cmd=os.path.basename(cmdline[0])
            #print(f"{cmdline=}")
            if 'chrome.exe' in cmd.lower():
                # 获取chrome进程的所有命令行参数
                #print(f"{cmdline=}")
                #查找cmdline数组元素中，含有--profile-directory的数据元素
                profile_cmdline=[x for x in cmdline if '--profile-directory' in x]
                #print(f"{profile_cmdline=}")
                #如果找到了，数据长度>0，然后返回等号=后部分的内容
                if len(profile_cmdline)>0:
                    #print(f"{profile_cmdline=}")
                    return profile_cmdline[0].split('=')[1]
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass

def search_file_in_path(fname,path):#在一个path下面遍历所有子目录，返回找到的第一个文件绝对路径
    search_path=os.path.join(path,'**')
    search_files = glob.glob(os.path.join(search_path, fname), recursive=True)
    # 遍历所有找到的文件，输出其所在的路径
    for search_file in search_files:
        file_path = os.path.abspath(search_file)
        return file_path

def get_chrome_db_path(): #通过去profile目录里面去找cookie和history文件，返回找到的绝对路径
    # 指定要查找的文件名
    file_name = 'cookies'
    data_path = os.path.join(os.environ['LOCALAPPDATA'], r"Google\Chrome\User Data")
    pdir=get_profile_dir()
    if pdir:
        profile_path=os.path.join(data_path,pdir)
    else:
        profile_path = os.path.join(data_path, 'Default')
    # 使用 glob.glob 函数遍历当前目录及其子目录中所有符合条件的文件
    cookie_file=search_file_in_path('cookies',profile_path)
    history_file = search_file_in_path('history', profile_path)
    return {'cookie_file':cookie_file,'history_file':history_file}

if __name__ == '__main__':
    # domain = 'exmail.qq.com'  # 目标网站域名
    # cookie = get_cookies_from_chrome(domain)
    # print(cookie)
    url = 'https://i.mi.com/contacts'
    res = get_cookie_by_url(url)
    print(res)


import pytest
from selenium import webdriver
import time
import math

from selenium.webdriver.common.by import By

link = "https://suninjuly.github.io/math.html"

def calc(x):
    return str(math.log(abs(12 * math.sin(x))))

@pytest.fixture(scope="function")
def browser():
    print("\nstart browser for test..")
    browser = webdriver.Chrome()
    yield browser
    print("\nquit browser..")
    browser.quit()

class TestLesson():

    def test_math_e2e_check(self, browser):
        browser.get(link)
        # проверяем ввод формулы в поле answer
        x_element = browser.find_element(By.ID, "input_value")
        x = int(x_element.text)
        y = calc(x)
        answer = browser.find_element(By.ID, "answer")
        answer.click()
        answer.send_keys(y)
        checkbox = browser.find_element(By.ID, "robotCheckbox")
        checkbox.click()
        assert checkbox.is_selected()
        radiobutton = browser.find_element(By.ID, "robotsRule")
        radiobutton.click()
        assert radiobutton.is_selected()
        button = browser.find_element(By.CSS_SELECTOR, ".btn")
        button.click()

    @pytest.mark.smoke
    def test_checked_radiobutton(self, browser):
        browser.get(link)
        # проверяем значение атрибута checked у people_radio
        people_radio = browser.find_element(By.ID, "peopleRule")
        people_checked = people_radio.get_attribute("checked")
        print("value of people radio: ", people_checked)
        assert people_checked is not None, "People radio is not selected by default"

    def test_checked_radiobutton2(self, browser):
        browser.get(link)
        # проверяем значение атрибута checked у robots_radio
        robots_radio = browser.find_element(By.ID, "robotsRule")
        robots_checked = robots_radio.get_attribute("checked")
        print("value of robots_radio: ", robots_checked)
        assert robots_checked is None

    def test_submit_disabled(self, browser):
        browser.get(link)
        # проверяем значение атрибута disabled у кнопки Submit
        button = browser.find_element(By.CSS_SELECTOR, '.btn')
        button_disabled = button.get_attribute("disabled")
        print("value of button Submit: ", button_disabled)
        assert button_disabled is None

    def test_submit_disabled_post_timeout(self, browser):
        browser.get(link)
        # проверяем значение атрибута disabled у кнопки Submit после таймаута
        time.sleep(15)
        button = browser.find_element(By.CSS_SELECTOR, '.btn')
        button_disabled = button.get_attribute("disabled")
        print("value of button Submit after 10sec: ", button_disabled)
        assert button_disabled is not None

# не забываем оставить пустую строку в конце файла


#-*-encoding:utf-8-*-
import os,sys
from PIL import Image
import subprocess
import cv2
import numpy as np
import threading

py_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.dirname(py_dir)
sys.path.append(project_dir)
import img_process.utils.resize as resize
import api_invoice.utils.data as data
import img_process.utils.change as change
import img_process.utils.check as check
import api_invoice.service.text as text
import api_common.utils.log as logger
import datetime
import threadpool
import time
import shutil
import uuid
import api_invoice.service.cardno as cardno_service
import api_invoice.service.vinno as vinno_service
import api_invoice.service.organno as organno_service
import api_invoice.service.engineno as engineno_service
import api_invoice.service.price as price_service
import api_invoice.service.img as img_service
import api_invoice.config.profile as profile_config
import api_invoice.dao.invoice_dao as invoice_dao
import api_invoice.dao.log_dao as log_dao
##入口初始化
## 设置profile
profile_config.profile.set_profile(sys.argv[0])

import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.3
set_session(tf.Session(config=config))

#公共参数
py_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.dirname(py_dir)

img_file= project_dir+"/data/invoice/img.txt"
result_file= project_dir+"/data/invoice/result.txt"

file_dir_img_root = project_dir+"/data/invoice/img/"
file_dir_img_error_root = project_dir+"/data/invoice/img_error/"
file_dir_img_tesseract_root = project_dir+"/data/invoice/img_tesseract/"

if not os.path.exists(file_dir_img_root):
    os.mkdir(file_dir_img_root)

if not os.path.exists(file_dir_img_error_root):
    os.mkdir(file_dir_img_error_root)
else :
    shutil.rmtree(file_dir_img_error_root)
    os.mkdir(file_dir_img_error_root)

if not os.path.exists(file_dir_img_tesseract_root):
    os.mkdir(file_dir_img_tesseract_root)
else :
    shutil.rmtree(file_dir_img_tesseract_root)
    os.mkdir(file_dir_img_tesseract_root)
test_name=[]

#test_name.append("111")
#test_name.append("autohomecar__wKjBwlgmeSaAJrTHAAG-BeYqJoM741")

#test_name.append("41586520-7668-chbe-1gce-n204abm4cf7d")
#test_name.append("04005040-7417-41d5-be2d-n203a829dmge")
#test_name.append("autohomecar__wKjBw1b7NFaAWJkvAAKAyzSQbeo282")
#test_name.append("autohomecar__wKjBw1g9EPOAEIgFAAHQVHt-NE8287")
#test_name.append("autohomecar__wKjBzFghLxiAN0CnADRxl7NnxJE096")
#test_name.append("01362710-8011-h74m-622d-n204afcc45mt")

muilti_thread = False

width =2048
exclude=[]
target_path = project_dir + "/tmp_predict/test3"
if os.path.exists(target_path):
    shutil.rmtree(target_path)
    os.mkdir(target_path)

target_path_error_char = project_dir + "/tmp_predict/error_char"
if os.path.exists(target_path_error_char):
    shutil.rmtree(target_path_error_char)
    os.mkdir(target_path_error_char)

target_path4 = project_dir + "/tmp_predict/test4"
if os.path.exists(target_path4):
    shutil.rmtree(target_path4)
    os.mkdir(target_path4)
else :
    #os.mkdir(target_path4)
    pass

mydateset_base_dir = "/home/autoai/mydataset/original/"

mydateset_base_dir2 = os.path.join(project_dir,"data/invoice/temp")
print("====mydateset_base_dir====",mydateset_base_dir)

def img_process(img):
    begin = datetime.datetime.now()
    id = img["id"]
    path = img["path"]
    file_name = img["name"]
    file_suffix = img["suffix"]
    #file_suffix='jpg'
    if (test_name == None or file_name in test_name or len(test_name) == 0):
        file_path_resize=None

        # 原图
        try:
            file_path_src = file_dir_img_root + file_name + '.' + file_suffix

            pil_img = Image.open(file_path_src)
            if pil_img.size[0]*pil_img.size[1]<480000:
                invoice_dao.update_error_status_by_id("图片分辨率太低,至少大于800*600",id)
                return
            file_path_src_copy = file_dir_img_root + file_name + '/000_src_copy' + file_name + '.' + file_suffix
            # 横竖调整图
            file_path_change_wh = file_dir_img_root + file_name + '/001_change_wh' + file_name + '.' + file_suffix
            # 检测发票表格图
            file_path_rect = file_dir_img_root + file_name + '/002_rect' + file_name + '.' + file_suffix
            # 大小调整
            file_path_resize = file_dir_img_root + file_name + '/003_resize' + file_name + '.' + file_suffix
            # 灰度二值图
            #file_path_gray = file_dir_img_root + file_name + '/gray' + file_name + '.' + file_suffix
            # 上下调整图
            file_path_change_tb = file_dir_img_root + file_name + '/004_change_tb' + file_name + '.' + file_suffix

            file_path_box_dir = file_dir_img_root + file_name + '/box'

            cv2.imwrite(file_path_src_copy, cv2.imread(file_path_src))
            # 横竖处理
            #change.change_wh(file_path_src, file_path_change_wh)

            # 上下重置
            # change.change_tb(file_path_resize,file_path_change_tb)
            # 矫正并抽出中心内容

            check_result = check.check_rect(file_path_src, file_path_rect)
            """
            if check_result:
                area = check_result["all_area"]
                check_result_new =check.check_rect(file_path_rect, file_path_rect+"_new.jpg",level='simple')
                if check_result_new:
                    area_new = check_result_new["area"]
                    if area_new > area*0.55:
                        file_path_rect = file_path_rect+"_new.jpg"
                        print("========= check_reck twice")
                    print("===========area",area,area_new)
            """
            # 大小重置
            resize.resize(file_path_rect, file_path_resize, width)

            change.change_tb(file_path_resize, file_path_change_tb)
        except Exception as e:
            print(e)
        finally:
            pass

        result = {}
        result["info"] = {}
        result["log"] = {}
        result["debug"] = {}
        info = result.get("info")

        try:
            if (file_path_resize):
                img_service.get_text(cardno_service,file_path_resize,file_path_change_tb,file_name,file_path_box_dir,result,file_dir_img_root)
        #except Exception as e:
        #    print(e)
        finally:
            pass

        cardno = info.get("cardno")
        #if cardno:
        #    invoice_dao.update_idcardno_by_id(cardno,id)
        #机构代码识别
        if not cardno:
            try:
                if (file_path_resize):
                    img_service.get_text(organno_service, file_path_resize, file_path_change_tb, file_name, file_path_box_dir,
                                 result,file_dir_img_root)
            except Exception as e:
                print(e)
            #if organno:
                #invoice_dao.update_roganno_by_id(organno, id)
        organno = info.get("organno");
        try:
            if (file_path_resize):
                img_service.get_text(vinno_service, file_path_resize, file_path_change_tb, file_name, file_path_box_dir,result,file_dir_img_root)
        except Exception as e:
            print(e)
        finally:
            pass
        vinno = info.get("vinno");
        #if vinno:
            #invoice_dao.update_vin_by_id(vinno, id)

        try:
            if (file_path_resize):
                img_service.get_text(engineno_service, file_path_resize, file_path_change_tb, file_name, file_path_box_dir, result,
                     file_dir_img_root)
        except Exception as e:
            print(e)
        engineno = info.get("engineno");
        if engineno:
            #invoice_dao.update_engineno_by_id(engineno, id)
            build_check_char_img("engineno", info, result)

        try:
            if (file_path_resize):
                img_service.get_text(price_service, file_path_resize, file_path_change_tb, file_name, file_path_box_dir, result,
                     file_dir_img_root)
        except Exception as e:
            print(e)

        price = info.get("price");
        if price:
            #invoice_dao.update_price_by_id(price, id)
            build_check_char_img("price", info, result)
        if not cardno:
            invoice_dao.update_unknow_status_by_id(id)
        end = datetime.datetime.now()
        #predict_time =(end - begin).microseconds//1000
        predict_time =(end - begin).seconds
        predict_type_idcard = result["debug"].get("predict_type_cardno")
        invoice_dao.update_invoice_by_id(id,idcard=cardno,vin=vinno,organ=organno,engine=engineno,price=price,predict_time=predict_time,predict_type_idcard=predict_type_idcard)

        out_result = {}
        out_result["time"] = predict_time
        invoice_dao.update_predict_time_by_id(out_result["time"], id)
        out_result["file_name"] = file_name
        out_result["info"] = result["info"]
        #out_result["cardno_machine_default"] = cardno_machine_default
        out_result["log"] = result["log"]

        out_result["page_top"] = result["debug"].get("page_top")
        log_dao.info(str(out_result),"invoice",img["id"])
        with open(result_file, 'a') as f:
            f.write(str(out_result)+"\n")
        thread = threading.current_thread()
        t=thread_list.get(thread.name)
        if t!=None:
            thread_list.pop(thread.name)

def build_check_char_img(field_name,info,result,server_char_dir="upper_201706",server_number_dir="number_201706",server_signal_dir="signal_201706",char_set="ABCDEFGHIJKLMNOPQRSTUVWXYZ",num_set="0123456789",signal_set=".¥-"):
    target_mydateset_base_dir=mydateset_base_dir
    if not os.path.exists(target_mydateset_base_dir):
        target_mydateset_base_dir =mydateset_base_dir2
    server_char_path = os.path.join(target_mydateset_base_dir,server_char_dir)
    if not os.path.exists(server_char_path):
        os.makedirs(server_char_path)
    server_number_path = os.path.join(target_mydateset_base_dir,server_number_dir)
    if not os.path.exists(server_number_path):
        os.makedirs(server_number_path)
    server_signal_path = os.path.join(target_mydateset_base_dir,server_signal_dir)
    if not os.path.exists(server_signal_path):
        os.makedirs(server_signal_path)
    field = info.get(field_name);
    field_file = result.get(field_name+"_file")
    logger.info("=============="+field_name+","+field_name+"_file", field, field_file)
    if field_file and field:
        text_type = result["log"][field_name+"_type"]
        if text_type.startswith("tesseract"):
            target_dir = file_dir_img_tesseract_root + file_name + "_"+field_name
            if os.path.exists(target_dir):
                shutil.rmtree(target_dir)
                # os.mkdir(target_dir)
            else:
                pass
                # os.mkdir(target_dir)
            shutil.copytree(file_dir_img_root + file_name, target_dir)
        img_src_name = os.path.basename(os.path.dirname(os.path.dirname(os.path.dirname(field_file))))
        box_dir = field_file + "_dir"
        box_src_dir = field_file + "_dir_src"
        logger.debug("=============="+field+",box_dir", field, box_dir)
        if os.path.exists(box_dir):
            files = os.listdir(box_dir)
            files_len = len(files)
            logger.info("=============", files_len, len(field), field, files)
            if files_len == len(field):
                logger.info("============= build box img :" + field)
                value = field
                for i in range(0, len(field)):
                    char_text = field[i]
                    try:
                        if char_text in num_set:
                            if os.path.exists(server_number_path):
                                shutil.copyfile(box_src_dir + "/" + str(i).zfill(3) + ".jpg",
                                                server_number_path + "/" + char_text + "_" + str(i).zfill(
                                                    3) + "_" + value + "_" + img_src_name + ".jpg")
                            pass
                        if char_text in char_set:
                            if os.path.exists(server_char_path):
                                shutil.copyfile(box_dir + "/" + str(i).zfill(3) + ".jpg",
                                                server_char_path + "/" + char_text + "_" + str(i).zfill(
                                                    3) + "_" + value + "_" + img_src_name + ".jpg")
                        if char_text in signal_set:
                            if os.path.exists(server_signal_path):
                                shutil.copyfile(box_dir + "/" + str(i).zfill(3) + ".jpg",
                                                server_signal_path + "/" + char_text + "_" + str(i).zfill(
                                                    3) + "_" + value + "_" + img_src_name + ".jpg")
                                pass
                    # except Exception as e:
                    #    logger.error("======:Exception:",e)
                    #    pass
                    finally:
                        pass
            else:
                logger.warn("================file and dada is diff================")
    else:
        target_dir = file_dir_img_error_root + file_name + "_"+field
        if os.path.exists(target_dir):
            shutil.rmtree(target_dir)
            # os.mkdir(target_dir)
        else:
            pass
            # os.mkdir(target_dir)
        shutil.copytree(file_dir_img_root + file_name, target_dir)
        pass
        # engineno

if os.path.exists(result_file):
    os.remove(result_file)
global thread_list
thread_list ={}

index = 0

def get_target_imgs():
    if len(test_name) == 0:
        # 读取图片列表
        imgs = data.getimgs(img_file, file_dir_img_root)
    else:
        muilti_thread = False
        if os.path.exists(file_dir_img_root):
            shutil.rmtree(file_dir_img_root)
            os.mkdir(file_dir_img_root)
        imgs = data.get_by_filenames(test_name, file_dir_img_root)
    target_imgs = []
    if imgs:
        for img in imgs:
            file_name = img["name"]
            # print(file_name)
            if file_name in exclude:
                # break
                pass
            else:
                file_path_src_dir = file_dir_img_root + file_name
                # print(file_path_src_dir)
                if os.path.exists(file_path_src_dir):
                    shutil.rmtree(file_path_src_dir)

                file_suffix = img["suffix"]
                if not test_name or len(test_name) == 0:
                    target_imgs.append(img)
                elif file_name in test_name:
                    target_imgs.append(img)
    return target_imgs

target_imgs = get_target_imgs()

if muilti_thread:
    pool = threadpool.ThreadPool(40)
    requests = threadpool.makeRequests(img_process, target_imgs)
    [pool.putRequest(req) for req in requests]
    pool.wait()
else :
    while target_imgs and len(target_imgs)>0:
        imgs_len = len(target_imgs)
        for img in target_imgs:
            index += 1
            path = img["path"]
            file_name = img["name"]
            print("=======begin=====================", index, "/", imgs_len,"[",img["id"],"]", file_name)
            img_process(img)
            print("=======end=====================", index, "/", imgs_len,"[",img["id"],"]")
        if len(test_name) == 0:
            target_imgs = get_target_imgs()
        else :
            break
text.write_param_config()


from secml.ml.features.tests import CPreProcessTestCases

from collections import OrderedDict

try:
    import torch
    import torchvision
except ImportError:
    CPreProcessTestCases.importskip("torch")
    CPreProcessTestCases.importskip("torchvision")
else:
    import torch
    from torch import nn, optim
    from torchvision import transforms
    torch.manual_seed(0)

from secml.array import CArray
from secml.ml.features.normalization import CNormalizerDNN
from secml.ml.classifiers import CClassifierPyTorch
from secml.data.loader import CDLRandom
from secml.optim.function import CFunction

def mlp(input_dims=100, hidden_dims=(50, 50), output_dims=10):
    """Multi-layer Perceptron"""
    if len(hidden_dims) < 1:
        raise ValueError("at least one hidden dim should be defined")
    if any(d <= 0 for d in hidden_dims):
        raise ValueError("each hidden layer must have at least one neuron")

    # Input layers
    layers = [
        ('linear1', torch.nn.Linear(input_dims, hidden_dims[0])),
        ('relu1', torch.nn.ReLU()),
    ]
    # Appending additional hidden layers
    for hl_i, hl_dims in enumerate(hidden_dims[1:]):
        prev_hl_dims = hidden_dims[hl_i]  # Dims of the previous hl
        i_str = str(hl_i + 2)
        layers += [
            ('linear' + i_str, torch.nn.Linear(prev_hl_dims, hl_dims)),
            ('relu' + i_str, torch.nn.ReLU())]
    # Output layers
    layers += [
        ('linear' + str(len(hidden_dims) + 1),
         torch.nn.Linear(hidden_dims[-1], output_dims))]

    # Creating the model with the list of layers
    return torch.nn.Sequential(OrderedDict(layers))

class TestCNormalizerPyTorch(CPreProcessTestCases):

    @classmethod
    def setUpClass(cls):
        cls.ds = CDLRandom(n_samples=40, n_classes=3,
                           n_features=20, n_informative=15,
                           random_state=0).load()

        model = mlp(input_dims=20, hidden_dims=(40,), output_dims=3)
        loss = nn.CrossEntropyLoss()
        optimizer = optim.SGD(model.parameters(), lr=1e-1)
        cls.net = CClassifierPyTorch(model=model, loss=loss,
                                     optimizer=optimizer, random_state=0,
                                     epochs=10, pretrained=True)
        cls.net.fit(cls.ds.X, cls.ds.Y)
        cls.norm = CNormalizerDNN(net=cls.net)

        CPreProcessTestCases.setUpClass()

    def test_normalization(self):
        """Testing normalization."""
        x = self.ds.X[0, :]

        self.logger.info("Testing normalization at last layer")

        self.norm.out_layer = None

        out_norm = self.norm.transform(x)
        out_net = self.net.get_layer_output(x, layer=None)

        self.logger.info("Output of normalize:\n{:}".format(out_norm))
        self.logger.info("Output of net:\n{:}".format(out_net))

        self.assert_allclose(out_norm, out_net)

        self.norm.out_layer = 'linear1'

        self.logger.info(
            "Testing normalization at layer {:}".format(self.norm.out_layer))

        out_norm = self.norm.transform(x)
        out_net = self.net.get_layer_output(x, layer=self.norm.out_layer)

        self.logger.info("Output of normalize:\n{:}".format(out_norm))
        self.logger.info("Output of net:\n{:}".format(out_net))

        self.assert_allclose(out_norm, out_net)

    def test_chain(self):
        """Test for preprocessors chain."""
        # Inner preprocessors should be passed to the pytorch clf
        with self.assertRaises(ValueError):
            CNormalizerDNN(net=self.net, preprocess='min-max')

    def test_gradient(self):
        """Test for gradient."""
        x = self.ds.X[0, :]

        layer = None
        self.norm.out_layer = layer
        self.logger.info("Returning gradient for layer: {:}".format(layer))
        shape = self.norm.transform(x).shape
        w = CArray.zeros(shape=shape)
        w[0] = 1
        grad = self.norm.gradient(x, w=w)

        self.logger.info("Output of gradient_f_x:\n{:}".format(grad))

        self.assertTrue(grad.is_vector_like)
        self.assertEqual(x.size, grad.size)

        layer = 'linear1'
        self.norm.out_layer = layer
        self.logger.info("Returning output for layer: {:}".format(layer))
        out = self.net.get_layer_output(x, layer=layer)
        self.logger.info("Returning gradient for layer: {:}".format(layer))
        grad = self.norm.gradient(x, w=out)

        self.logger.info("Output of grad_f_x:\n{:}".format(grad))

        self.assertTrue(grad.is_vector_like)
        self.assertEqual(x.size, grad.size)

    def test_aspreprocess(self):
        """Test for normalizer used as preprocess."""
        from secml.ml.classifiers import CClassifierSVM
        from secml.ml.classifiers.multiclass import CClassifierMulticlassOVA

        model = mlp(input_dims=20, hidden_dims=(40,), output_dims=3)
        loss = nn.CrossEntropyLoss()
        optimizer = optim.SGD(model.parameters(), lr=1e-1)
        net = CClassifierPyTorch(model=model, loss=loss,
                                 optimizer=optimizer, random_state=0,
                                 epochs=10, preprocess='min-max')
        net.fit(self.ds.X, self.ds.Y)

        norm = CNormalizerDNN(net=net)

        clf = CClassifierMulticlassOVA(
            classifier=CClassifierSVM, preprocess=norm)

        self.logger.info("Testing last layer")

        clf.fit(self.ds.X, self.ds.Y)

        y_pred, scores = clf.predict(
            self.ds.X, return_decision_function=True)
        self.logger.info("TRUE:\n{:}".format(self.ds.Y.tolist()))
        self.logger.info("Predictions:\n{:}".format(y_pred.tolist()))
        self.logger.info("Scores:\n{:}".format(scores))

        x = self.ds.X[0, :]

        self.logger.info("Testing last layer gradient")

        for c in self.ds.classes:
            self.logger.info("Gradient w.r.t. class {:}".format(c))

            grad = clf.grad_f_x(x, y=c)

            self.logger.info("Output of grad_f_x:\n{:}".format(grad))

            check_grad_val = CFunction(
                clf.decision_function, clf.grad_f_x).check_grad(
                    x, y=c, epsilon=1e-1)
            self.logger.info(
                "norm(grad - num_grad): %s", str(check_grad_val))
            self.assertLess(check_grad_val, 1e-3)

            self.assertTrue(grad.is_vector_like)
            self.assertEqual(x.size, grad.size)

        layer = 'linear1'
        norm.out_layer = layer

        self.logger.info("Testing layer {:}".format(norm.out_layer))

        clf.fit(self.ds.X, self.ds.Y)

        y_pred, scores = clf.predict(
            self.ds.X, return_decision_function=True)
        self.logger.info("TRUE:\n{:}".format(self.ds.Y.tolist()))
        self.logger.info("Predictions:\n{:}".format(y_pred.tolist()))
        self.logger.info("Scores:\n{:}".format(scores))

        self.logger.info("Testing 'linear1' layer gradient")
        grad = clf.grad_f_x(x, y=0)  # y is required for multiclassova
        self.logger.info("Output of grad_f_x:\n{:}".format(grad))

        self.assertTrue(grad.is_vector_like)
        self.assertEqual(x.size, grad.size)

    def test_get_set_state(self):
        """Test for `.get_state` and `.set_state` methods."""
        x = self.ds.X[0, :]
        self.norm.out_layer = None
        out_norm = self.norm.transform(x)

        self.logger.info(
            "Normalized sample before restoring state:\n{:}".format(out_norm))
        state = self.norm.get_state(return_optimizer=False)

        model = mlp(input_dims=20, hidden_dims=(40,), output_dims=3)
        net = CClassifierPyTorch(model=model, pretrained=True)
        norm_new = CNormalizerDNN(net)

        # Restore state
        norm_new.set_state(state)

        post_out_norm = self.norm.transform(x)
        self.logger.info(
            "Normalized sample after restoring state:\n{:}".format(post_out_norm))

        self.assert_array_equal(out_norm, post_out_norm)

if __name__ == '__main__':
    CPreProcessTestCases.main()


#!/usr/bin/env python

#import time as timemod
#from time import __orig__time
from epdblib import dbg
from epdblib import debug as log

__orig__time = time
def time():
    if dbg.is_dbg_callee():
        return __orig__time()
    if dbg.mode == 'normal':
        r = __orig__time()
        dbg.snapshottingcontrol.set_make_snapshot()
        return r


"""Script to calculate global IOU for segmentation tasks."""
import argparse
from pathlib import Path

import numpy as np
from joblib import Parallel, delayed
from tqdm import tqdm

from iglovikov_helper_functions.utils.image_utils import load_grayscale
from iglovikov_helper_functions.utils.metrics import (
    calculate_confusion_matrix_from_arrays,
    calculate_jaccard,
    calculate_tp_fp_fn,
)

def confusion_matrix_from_files(y_true_path: Path, y_pred_path: Path, num_classes: int) -> np.array:
    y_true = load_grayscale(y_true_path)
    y_pred = load_grayscale(y_pred_path)

    if not y_true.shape == y_pred.shape:
        raise ValueError(
            f"y_true and y_pred should have the same shape. "
            f"y_true shape = {y_true.shape} y_pred.shape = {y_pred.shape} "
            f"y_pred_path = {y_pred_path} "
            f"y_true_path = {y_true_path}"
        )

    return calculate_confusion_matrix_from_arrays(y_true, y_pred, num_classes=num_classes)

def calculate_ious_global(y_true_path: Path, y_pred_path: Path, num_classes: int, num_workers: int = 1) -> np.array:
    y_true_files = sorted(y_true_path.glob("*.png"))
    y_pred_files = sorted(y_pred_path.glob("*.png"))

    if not [x.name for x in y_pred_files] == [x.name for x in y_true_files]:
        raise AssertionError("Should have the same file names for y_true and y_pred.")

    def helper(file_name):
        return confusion_matrix_from_files(
            y_true_path.joinpath(file_name), y_pred_path.joinpath(file_name), num_classes
        )

    matrices = Parallel(n_jobs=num_workers)(delayed(helper)(file_path.name) for file_path in tqdm(y_pred_files))

    confusion_matrix = np.dstack(matrices).sum(axis=2)
    tp_fp_fn_dict = calculate_tp_fp_fn(confusion_matrix)

    return calculate_jaccard(tp_fp_fn_dict)

def get_args():
    parser = argparse.ArgumentParser("Calculate IOU for segmentation masks.")
    arg = parser.add_argument
    arg("-p", "--predictions_path", type=Path, help="Json with predictions.", required=True)
    arg("-g", "--ground_truth_path", type=Path, help="Json with ground truth.", required=True)
    arg("-n", "--num_classes", type=int, help="Number of classes to use.", required=True)
    arg("-j", "--num_workers", type=int, help="Number of workers to use.", default=12)
    return parser.parse_args()

if __name__ == "__main__":
    args = get_args()

    ious = calculate_ious_global(args.ground_truth_path, args.predictions_path, args.num_classes, args.num_workers)

    print(f"Mean IOU = {np.mean(ious)}")
    for class_id, iou in enumerate(ious):
        print(f"Class {class_id}: IOU = {iou}")


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def sumEvenGrandparent(self, root: TreeNode) -> int:
        self.s = 0

        def traverse(root,parent,grandparent):
            if root is None:
                return

            if grandparent is not None and grandparent.val % 2 == 0:
                self.s += root.val

            traverse(root.left,root,parent)
            traverse(root.right,root,parent)
        traverse(root,None,None)
        return self.s


import os
from PyPDF2 import PdfReader

class Files():

    def __init__(self, path):
        self.path = path

    def getFiles(self):
        files = []
        try:
            for diretorio, subpastas, arquivos in os.walk(self.path):
                for arquivo in arquivos:
                    if arquivo.endswith('.pdf') or arquivo.endswith('.PDF'):
                        files.append(os.path.join(diretorio, arquivo))
        except:
             print("Arquivos não encontrados.")
        finally:
            return files

    def readFile(self, pathFile):

        try:
            pages = []
            filter = []
            reader = PdfReader(pathFile)

            for i in range(len(reader.pages)):
                #print(i)
                current = reader.pages[i].extract_text().splitlines()
                for line in current:
                    pages.append(line)

            """ for line in pages:
                split = line.split(" ")
                if "/" in split[0]:
                    i=1
                    print(pages[pages.index(line)])
                    filter.append(pages[pages.index(line)])
                    while "CFE" not in pages[pages.index(line)+i]:
                        linha = pages[pages.index(line)+i]
                        splitted = linha.split(" ")
                        #print(len(splitted[0]))
                        if "UN" in splitted[-1]:
                            print(linha)
                            filter.append(linha)

                        if "CFE" in pages[pages.index(line)+i]:
                            break

                        i += 1 """

            for i in range(len(pages)):
                split = pages[i].split(" ")
                if "/" in split[0]:
                    j=1
                    #print(pages[pages.index(pages[i])])
                    filter.append(pages[pages.index(pages[i])])
                    while "CFE" not in pages[pages.index(pages[i])+j]:

                        linha = pages[pages.index(pages[i])+j]
                        splitted = linha.split(" ")
                        if "UN" in splitted[-1]:
                            #print(linha)
                            filter.append(linha)
                        elif "CFE" in pages[pages.index(pages[i])+j]:
                            j=1
                            break
                        print(pages.index(pages[pages.index(pages[i])+j]), len(pages), j)
                        j+=1

        except Exception as e:
            print('\n')
            print(f"   Erro ao ler arquivo {pathFile.split('/')[-1]}", '\n')
            print("  ", "Erro: ", e, '\n')
            print("  ", "Pulando arquivo...", '\n')


from unittest.mock import Mock, PropertyMock
import pytest

from sqlalchemy.orm.session import Session

from generator2.domain.object import ObjectService

class TestObjectService:
    def test_full_object(self, dbsession, object_):
        """
        Ensures that the shape of the dict returned matches expectations.

        The object being tested for looks like:
        {
            "kind": "bovine",
            "slug": "cowslug",
            "properties": [
                {
                    "label": "The cow says",
                    "instances": ["moo"]
                },
                {
                    "label": "name",
                    "instances": ["aloysius"]
                }
            ]
        }
        """
        object_under_test = ObjectService(dbsession)
        actual_object = object_under_test.full_object('cowslug')
        assert 'bovine' == actual_object.get('kind')
        assert 'cowslug' == actual_object.get('slug')
        assert 2 == len(actual_object['properties'])

        property_ = actual_object.get('properties')[0]
        assert 'The cow says' == property_.get('label')
        assert 1 == len(property_.get('instances'))

        name = actual_object.get('properties')[1]
        assert 'name' == name.get('label')
        assert ['aloysius'] == name.get('instances')

        instance = property_.get('instances')[0]
        assert 'moo' == instance

    @pytest.fixture
    def dbsession(self, object_, additional_properties):
        dbsession = Mock(Session)
        _lookup_object = Mock()
        _lookup_object.filter.return_value.one.return_value = object_
        _object_additional_properties = Mock()
        _object_additional_properties.filter.return_value.all.return_value = additional_properties
        dbsession.query.side_effect = (_lookup_object, _object_additional_properties)
        return dbsession

    @pytest.fixture
    def instance(self):
        instance = Mock()
        type(instance).value = PropertyMock(return_value='moo')
        return instance

    @pytest.fixture
    def additional_instance(self):
        instance = Mock()
        type(instance).value = PropertyMock(return_value='aloysius')
        return instance

    @pytest.fixture
    def object_(self, property_):
        object_ = Mock()
        type(object_).kind = PropertyMock(return_value='bovine')
        type(object_).slug = PropertyMock(return_value='cowslug')
        type(object_).properties = PropertyMock(return_value=[property_])
        return object_

    @pytest.fixture
    def property_(self, instance):
        property_ = Mock()
        type(property_).label = PropertyMock(return_value='The cow says')
        type(property_).instances = PropertyMock(return_value=[instance])
        return property_

    @pytest.fixture
    def additional_properties(self, additional_instance):
        property_ = Mock()
        type(property_).label = PropertyMock(return_value='name')
        type(property_).instances = PropertyMock(return_value=[additional_instance])
        return [property_]


from functools import cmp_to_key
import json
import networkx as nx
import os
import platform
import sys
import torch
import warnings
from collections import defaultdict
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
from termcolor import colored
from utils.logger import logger, shell_cmd
from utils.envs import enver, setup_envs_of_dit
from utils.calculator import (
    rect_area,
    rect_overlap,
    rect_contain,
    union_rects,
    expand_rect_bound,
)

warnings.filterwarnings("ignore")

try:
    from detectron2.config import CfgNode as CN
    from detectron2.config import get_cfg
    from detectron2.utils.visualizer import ColorMode, Visualizer
    from detectron2.data import MetadataCatalog
    from detectron2.data.detection_utils import read_image
    from detectron2.engine import DefaultPredictor

    from unilm.dit.object_detection.ditod import add_vit_config
except Exception as e:
    logger.err(e)
    raise e

# GPT-Paper repo path
repo_path = Path(__file__).parents[1]
# GPT-Paper repo parent path
repo_parent_path = repo_path.parent

detectron2_path = repo_parent_path / "detectron2"
unilm_path = repo_parent_path / "unilm"

class DITLayoutAnalyzer:
    """
    * app.py · nielsr/dit-document-layout-analysis
        * https://huggingface.co/spaces/nielsr/dit-document-layout-analysis/blob/main/app.py
    * unilm/dit/object_detection/inference.py
        * https://github.com/microsoft/unilm/blob/master/dit/object_detection/inference.py
    """

    def __init__(self, size="base"):
        self.model_size = size
        self.is_setup_model = False

    def setup_model(self, quiet=True):
        logger.note("> Setting up model of DITLayoutAnalyzer ...")
        logger.enter_quiet(quiet)
        enver.set_envs(set_proxy=True, cuda_device=True, huggingface=True)
        os.environ = enver.envs
        self.load_configs()
        self.load_weights()
        self.set_device()
        self.create_predictor()
        self.set_metadata()
        self.is_setup_model = True
        enver.restore_envs()
        os.environ = enver.envs
        logger.exit_quiet(quiet)

    def test_run(self):
        self.setup_model(quiet=False)
        # self.annotate_image(
        #     input_image_path=repo_path / "examples" / "example_pdf_4.png",
        #     output_image_path=repo_path / "examples" / "example_pdf_4_output.png",
        # )

    # Step 1: Instantiate config
    def load_configs(self):
        self.cfg = get_cfg()
        add_vit_config(self.cfg)

        if self.model_size == "large":
            size_str = "large"
        else:
            size_str = "base"

        yaml_name = f"cascade_dit_{size_str}.yaml"
        cascade_dit_yml = (
            unilm_path
            / "dit"
            / "object_detection"
            / "publaynet_configs"
            / "cascade"
            / yaml_name
        )

        logger.note(f"> Loading configs from `{yaml_name}`:")
        logger.file(f"  - {cascade_dit_yml}")

        self.cfg.merge_from_file(cascade_dit_yml)

    # Step 2: Load model weights to config
    def load_weights(self):
        if self.model_size == "large":
            size_str = "l"
        else:
            size_str = "b"

        pth_name = f"publaynet_dit-{size_str}_cascade.pth"
        publaynet_dit_cascade_pth = repo_path / "data" / pth_name
        if not publaynet_dit_cascade_pth.exists():
            raise FileNotFoundError(f"`{pth_name}` not found.")

        logger.note(f"> Loading weights from `{pth_name}`:")
        logger.file(f"  - {publaynet_dit_cascade_pth}")

        self.cfg.MODEL.WEIGHTS = str(publaynet_dit_cascade_pth)

    # Step 3: Set CUDA device
    def set_device(self):
        if torch.cuda.is_available():
            cuda_device = os.environ.get("CUDA_VISIBLE_DEVICES", 0)
            logger.note(f"> Using GPU:{cuda_device} ...")
            self.cfg.MODEL.DEVICE = "cuda"
        else:
            logger.note(f"> Using CPU ...")
            self.cfg.MODEL.DEVICE = "cpu"

    # Step 4: Create Predictor
    def create_predictor(self):
        logger.note("> Creating predictor ...")
        self.predictor = DefaultPredictor(self.cfg)

    # Step 5: Set meta data
    def set_metadata(self):
        logger.note("> Setting metadata ...")
        # logger.mesg(self.cfg.DATASETS)
        dataset_name = "publaynet_val"
        self.metadata = MetadataCatalog.get(dataset_name)
        # logger.mesg(self.metadata)
        self.thing_classes = ["text", "title", "list", "table", "figure"]
        self.metadata.set(thing_classes=self.thing_classes)

    def annotate_image(
        self,
        annotate_info_json_path,
        input_image_path=None,
        output_image_path=None,
        quiet=False,
    ):
        logger.enter_quiet(quiet)

        if input_image_path is None:
            raise ValueError("`input_image_path` is None!")

        logger.note(f"> Analyzing the layout of input image:")
        logger.file(f"  - {input_image_path}")

        image = read_image(str(input_image_path))
        pred_output = self.predictor(image)["instances"]
        pred_things = [self.thing_classes[c] for c in pred_output.pred_classes]

        logger.note("> Results:")
        image_height, image_width = pred_output.image_size
        logger.back(f"  - image_size: {image_width}(w) * {image_height}(h)")
        logger.mesg(f"  - num_instances: {len(pred_output)}")
        logger.back(f"  - pred_classes: {pred_output.pred_classes.tolist()}")
        logger.mesg(f"  - pred_things: {pred_things}")
        logger.back(f"  - pred_boxes: {pred_output.pred_boxes.tensor.tolist()}")
        logger.back(f"  - scores: {pred_output.scores.tolist()}")

        self.dump_annotate_info(
            annotate_info_json_path, input_image_path, output_image_path, pred_output
        )

        logger.exit_quiet(quiet)

        return pred_output

    def dump_annotate_info(
        self, annotate_info_json_path, input_image_path, output_image_path, output
    ):
        image_height, image_width = output.image_size
        self.annotate_infos = {
            "page": {
                "original_image_path": str(input_image_path),
                "current_image_path": str(output_image_path),
                "image_width": image_width,
                "image_height": image_height,
                "regions_num": len(output),
            },
            "regions": [],
        }
        pred_classes = output.pred_classes.tolist()
        pred_things = [self.thing_classes[c] for c in pred_classes]
        pred_boxes = [
            [round(x, 1) for x in box] for box in output.pred_boxes.tensor.tolist()
        ]
        pred_scores = [round(x * 100, 2) for x in output.scores.tolist()]
        for i in range(len(output)):
            region_info = {
                "idx": i + 1,
                "thing": pred_things[i],
                "score": pred_scores[i],
                "box": pred_boxes[i],
            }
            self.annotate_infos["regions"].append(region_info)

        logger.success(f"> Saving annotated info:")
        logger.file(f"  - {annotate_info_json_path}")
        with open(annotate_info_json_path, "w") as wf:
            wf.write(json.dumps(self.annotate_infos, indent=4))

        return annotate_info_json_path

class LayoutLMv3Analyzer:
    """
    * unilm/layoutlmv3 at master · microsoft/unilm
        * https://github.com/microsoft/unilm/tree/master/layoutlmv3#document-layout-analysis-on-publaynet
    """

    def __init__(self) -> None:
        pass

def draw_regions_on_page(
    regions_info_json_path,
    output_parent_path,
    spacing=2,
    show_region_idx=True,
    score_use_percent=True,
    overwrite=False,
    quiet=False,
):
    logger.enter_quiet(quiet)

    with open(regions_info_json_path, "r") as rf:
        regions_infos = json.load(rf)
    original_page_image_path = Path(regions_infos["page"]["original_image_path"])
    drawn_page_image_path = output_parent_path / original_page_image_path.name
    if not overwrite and drawn_page_image_path.exists():
        logger.exit_quiet(quiet)
        return

    region_colors = {
        "text": (0, 128, 0),
        "title": (128, 0, 0),
        "list": (0, 0, 128),
        "table": (128, 128, 0),
        "figure": (0, 0, 128),
    }
    page_num = int(original_page_image_path.stem.split("_")[-1])
    page_image = Image.open(original_page_image_path)
    page_image_width, page_image_height = page_image.size
    image_draw = ImageDraw.Draw(page_image, "RGBA")

    regions = regions_infos["regions"]
    logger.mesg(f"- Draw on Page {page_num} with {len(regions)} regions")
    logger.back(f"  - {drawn_page_image_path}")

    for region in regions:
        region_idx = region["idx"]
        region_box = region["box"]
        region_thing = region["thing"]
        region_score = region["score"]

        region_box = [
            region_box[0] - spacing,
            region_box[1] - spacing,
            region_box[2] + spacing,
            region_box[3] + spacing,
        ]

        text_font = ImageFont.truetype("times.ttf", 40)
        if show_region_idx:
            region_idx_str = f"{region_idx}."
        else:
            region_idx_str = ""

        if score_use_percent:
            region_score_str = f"{round(region_score)}%"
        else:
            region_score_str = f"{region_score}"

        text_str = f"{region_idx_str}{region_thing}({region_score_str})"
        text_bbox = image_draw.textbbox(
            region_box[:2], text_str, font=text_font, anchor="rt"
        )
        image_draw.text(
            region_box[:2], text_str, fill="black", font=text_font, anchor="rt"
        )
        region_rect_color = region_colors[region_thing]
        image_draw.rectangle(
            region_box,
            outline=region_rect_color,
            fill=(*region_rect_color, 64),
            width=2,
        )

        image_draw.rectangle(text_bbox, fill=(*region_rect_color, 80))
    page_image.save(drawn_page_image_path)
    logger.exit_quiet(quiet)

def calc_regions_overlaps(regions):
    """
    Return: List of list of tuples. List length is num of regions.
    In each level-1 list, there is level-2 list of tuples.

    For each tuple in i-th list:
    The 1st element is the idx of overlapped region with region i, and
    the 2nd element is the overlapped area ratio of overlapped region in region i.
    """
    regions_overlaps = []
    for i in range(len(regions)):
        region_i_overlaps = []
        for j in range(len(regions)):
            region_i = regions[i]
            region_j = regions[j]
            region_i_overlaps_j, region_i_j_overlapped_area_ratio = rect_overlap(
                region_i["box"], region_j["box"], t=-5
            )
            if i != j and region_i_overlaps_j:
                region_i_overlaps.append(
                    (regions[j]["idx"], region_i_j_overlapped_area_ratio)
                )
        regions_overlaps.append(region_i_overlaps)

    for i in range(len(regions)):
        region_i_overlaps = regions_overlaps[i]
        region_i = regions[i]

        if region_i_overlaps:
            region_i_overlaps_str = ", ".join(
                [f"{x[0]}({round(x[1],2)})" for x in region_i_overlaps]
            )
            region_i_overlaps_str = f"- {region_i_overlaps_str}"
            line_color = "light_red"
            overlap_region_num_str = f", overlaps with {len(region_i_overlaps)} regions"
        else:
            region_i_overlaps_str = ""
            line_color = "light_cyan"
            overlap_region_num_str = ""

        logger.store_indent()
        logger.indent(2)
        logger.back(
            colored(
                f"- {region_i['idx']}: {region_i['thing']} region ({region_i['score']})"
                + overlap_region_num_str,
                line_color,
            )
        )
        if region_i_overlaps_str:
            logger.indent(2)
            logger.back(region_i_overlaps_str)
        logger.restore_indent()

    return regions_overlaps

def remove_regions_overlaps(
    regions,
    regions_overlaps,
    overlap_area_ratio_threshold=0.9,
    ignore_score_threshold=0.15,
):
    """
    Rules of thumb:

    1. Keep the separated regions.
    2. For overlapped regions:
    (a) Ignore the regions with scores lower than `ignore_score_threshold`.
        (Reason: This would remove mis-categorized,
                 such as figures which contain texts that should be separated.)
    (b) Ignore the regions that are totally contained by other higher-score regions.
        (Reason: There are always other regions with higher score that overlaps with it.)
    (c) For the remained overlapped regions (after removing the cases in (a) and (b)),
        group the regions with connected components, and union them with bound rects.
    """
    filtered_regions = []
    for i in range(len(regions)):
        region_i = regions[i]
        region_i_overlaps = regions_overlaps[i]
        keep_region_i = True
        if region_i_overlaps:
            # Ignore the regions with scores lower than `ignore_score_threshold`.
            if region_i["score"] / 100 <= ignore_score_threshold:
                keep_region_i = False
                regions_overlaps[i] = []
                logger.back(
                    f"Ignore region {i+1} its score {(region_i['score'])} "
                    f"is lower than {round(ignore_score_threshold*100)}"
                )
            # Ignore the regions that are totally contained by other higher-score regions.
            else:
                for j in range(len(region_i_overlaps)):
                    region_j_idx, region_i_j_overlapped_area_ratio = region_i_overlaps[
                        j
                    ]
                    region_j = regions[region_j_idx - 1]
                    logger.back(
                        f"region {i+1} ({region_i['score']}) and region {region_j_idx} ({(region_j['score'])}) "
                        f"[{round(region_i_j_overlapped_area_ratio*100)}%]"
                    )
                    if (
                        region_i_j_overlapped_area_ratio >= overlap_area_ratio_threshold
                        and region_j["score"] >= region_i["score"]
                    ):
                        # Means this region is a totally child region, ignore it.
                        logger.back(
                            f"Ignore region {i+1} as region {region_j_idx} has score {(region_j['score'])} "
                            f"[{round(region_i_j_overlapped_area_ratio*100)}%]"
                        )
                        keep_region_i = False
                        regions_overlaps[i] = []
                        break
        # Keep separated regions
        if keep_region_i:
            filtered_regions.append(region_i)

    # Re-create the overlapped regions after removing the totally child regions
    # (totally child means: 100% contained by other higher-score regions)
    new_overlap_regions = []
    for i in range(len(regions_overlaps)):
        region_i_overlaps = regions_overlaps[i]
        region_i = regions[i]
        if region_i_overlaps:
            new_region_i_overlaps = []
            for j in range(len(region_i_overlaps)):
                region_j_idx, region_i_j_overlapped_area_ratio = region_i_overlaps[j]
                if regions_overlaps[region_j_idx - 1]:  # region_j is not removed
                    new_region_i_overlaps.append(
                        (region_j_idx, region_i_j_overlapped_area_ratio)
                    )

            if new_region_i_overlaps:
                new_overlap_regions.append(region_i)

    # Group the remained overlapped regions with connected components
    if new_overlap_regions:
        new_overlap_regions_idxs = [x["idx"] for x in new_overlap_regions]
        new_overlap_regions_str = ", ".join(map(str, new_overlap_regions_idxs))
        logger.warn(
            f"{len(new_overlap_regions)} regions remains overlapped: [{new_overlap_regions_str}]"
        )

        # Re-calculate the regions overlaps
        new_regions_overlaps = calc_regions_overlaps(new_overlap_regions)
        new_overlap_regions_edges = []
        for i, new_regions_overlaps in enumerate(new_regions_overlaps):
            region_idx = new_overlap_regions_idxs[i]
            for new_region_idx, new_region_overlap_area_ratio in new_regions_overlaps:
                new_overlap_regions_edges.append((region_idx, new_region_idx))

        # Union (combine) connected regions with Group Theory
        G = nx.Graph()
        G.add_nodes_from(new_overlap_regions_idxs)
        G.add_edges_from(new_overlap_regions_edges)
        connected_regions_components = list(nx.connected_components(G))
        logger.success(
            f"> Combine {connected_regions_components} to {len(connected_regions_components)} regions"
        )

        # Remove the remained overlapped regions from filtered regions
        # as they would be grouped (combined) to new large (and less) regions
        filtered_regions = [
            region
            for region in filtered_regions
            if region["idx"] not in new_overlap_regions_idxs
        ]

        # Create new combined regions from connected components
        for c_idx, connected_regions_component in enumerate(
            connected_regions_components
        ):
            connected_regions_idxs = list(connected_regions_component)
            connected_regions_boxes = [
                regions[region_idx - 1]["box"] for region_idx in connected_regions_idxs
            ]
            union_region_box = union_rects(connected_regions_boxes)
            union_region_score = 0
            union_region_thing = ""
            union_region_idx = len(regions) + c_idx + 1

            # Determine the thing and score of new combined region
            thing_scores = defaultdict(float)
            for i in connected_regions_idxs:
                region_i = regions[i - 1]
                region_i_thing = region_i["thing"]
                region_i_score = region_i["score"]
                region_i_area = rect_area(*region_i["box"])
                thing_scores[region_i_thing] += region_i_score * region_i_area

            connected_region_areas_sum = sum(
                rect_area(*x) for x in connected_regions_boxes
            )
            connected_region_scores_sum = sum(thing_scores.values())
            connected_region_score = (
                connected_region_scores_sum / connected_region_areas_sum
            )

            union_region_thing = max(thing_scores, key=thing_scores.get)
            union_region_score = round(connected_region_score, 2)

            union_region = {
                "idx": union_region_idx,
                "thing": union_region_thing,
                "score": union_region_score,
                "box": union_region_box,
            }

            logger.indent(2)
            logger.success(
                f"- New {union_region_thing} region {union_region_idx} ({union_region_score})"
            )
            logger.indent(-2)
            filtered_regions.append(union_region)

    return filtered_regions

class RegionsOrderer:
    def sort_two_regions_by_reading_order(self, region1, region2):
        """
        Return:
            -1: rect1 is before rect2
             1: rect1 is after rect2

        In this certain case, there is no need to consider the conditions
        that rects are intersected,
        as we have already split them to no-overlapped ones.

        1. Check Left and Right:
            r1 <= l2: rect1 is before rect2
            l1 >= r2: rect1 is after rect2
        2. Then check Top and Bottom:
            b1 <= t2: rect1 is before rect2
            t1 >= b2: rect1 is after rect2
        3. Others:
            raise Error

        Since rects have no intersections, the checks above is sufficient.

        """
        rect1 = region1["box"]
        rect2 = region2["box"]

        l1, t1, r1, b1 = expand_rect_bound(rect1, t=-5)
        l2, t2, r2, b2 = expand_rect_bound(rect2, t=-5)
        if r1 <= l2:
            return -1
        elif l1 >= r2:
            return 1
        else:
            if b1 <= t2:
                return -1
            elif t1 >= b2:
                return 1
            else:
                raise ValueError("Rects have intersections!")

    def sort_regions_by_reading_order(self, regions):
        for region in regions:
            region_box = region["box"]
            region_thing = region["thing"]
            region_score = region["score"]
            logger.back(
                f"- {region['idx']}: {region_thing} region ({region_score}) {region_box}"
            )
        sorted_regions = sorted(
            regions, key=cmp_to_key(self.sort_two_regions_by_reading_order)
        )
        for idx, region in enumerate(sorted_regions):
            sorted_regions[idx]["idx"] = idx + 1
            region_box = region["box"]
            region_thing = region["thing"]
            region_score = region["score"]
            logger.back(
                f"- {region['idx']}: {region_thing} region ({region_score}) {region_box}"
            )

        return sorted_regions

if __name__ == "__main__":
    dit_layout_analyzer = DITLayoutAnalyzer(size="large")
    dit_layout_analyzer.test_run()


import json
import os
from PIL import Image
import plotly.graph_objects as go
import polars as pl
from typing import Dict, List, Tuple, Union

class BenetechDataset:
    def __init__(self, CFG):
        self.CFG = CFG
        self.annotations = self.get_annotations()

    def get_annotations(self):
        self.annotations = []
        for annotation_path in os.listdir(self.CFG.train_ann_path):
            with open(f"{self.CFG.train_ann_path}/{annotation_path}") as annotation_f:
                dct = json.load(annotation_f)
                dct['id'] = annotation_path.split('.')[0]
                self.annotations.append(dct)
        return self.annotations

    def to_pl_df(self):
        def flatten_dict(d: Dict, parent_key: str = '', sep: str = '_') -> Dict:
            """Flatten a nested dictionary"""
            items = []
            for k, v in d.items():
                new_key = f"{parent_key}{sep}{k}" if parent_key else k
                if isinstance(v, dict):
                    items.extend(flatten_dict(v, new_key, sep=sep).items())
                else:
                    items.append((new_key, v))
            return dict(items)

        data = []
        for dataaaa in self.annotations:
            # Flatten the nested dictionaries
            flat_data = flatten_dict(dataaaa)
            data.append(flat_data)
        df = pl.DataFrame(data)
        return df

    def get_chart_type_counts(self):
        chart_type_counts = {
            "dot": 0,
            "line": 0,
            "scatter": 0,
            "vertical_bar": 0,
            "horizontal_bar": 0
        }

        for annotation in self.annotations:
            chart_type_counts[annotation["chart-type"]] += 1

        return chart_type_counts

    def load_annotation(self, name: str):
        with open(f"{self.CFG.train_ann_path}/{name}.json") as annotation_f:
            ann_example = json.load(annotation_f)
        return ann_example

    def get_coords(self, polygon, img_height):
        xs = [
            polygon["x0"],
            polygon["x1"],
            polygon["x2"],
            polygon["x3"],
            polygon["x0"]
        ]

        ys = [
            -polygon["y0"] + img_height,
            -polygon["y1"] + img_height,
            -polygon["y2"] + img_height,
            -polygon["y3"] + img_height,
            -polygon["y0"] + img_height
        ]

        return xs, ys

    def add_line_breaks(self, text: str, break_num: int = 7) -> str:
        words = text.split()
        new_text = ""
        for i, word in enumerate(words, start=1):
            new_text += word
            if i % break_num == 0:
                new_text += "
"
            else:
                new_text += " "
        return new_text

    def get_tick_value(self, name, data_series):
        for el in data_series:
            if el["x"] == name:
                return el["y"]
            elif el["y"] == name:
                return el["x"]

    def plot_annotated_image(self, name: str, scale_factor: int = 1.0) -> None:
        img_example = Image.open(f"{self.CFG.train_img_path}/{name}.jpg")
        ann_example = self.load_annotation(name)

        # create figure
        fig = go.Figure()

        # constants
        img_width = img_example.size[0]
        img_height = img_example.size[1]

        # add invisible scatter trace
        fig.add_trace(
            go.Scatter(
                x=[0, img_width],
                y=[0, img_height],
                mode="markers",
                marker_opacity=0
            )
        )

        # configure axes
        fig.update_xaxes(
            visible=False,
            range=[0, img_width]
        )

        fig.update_yaxes(
            visible=False,
            range=[0, img_height],
            # the scaleanchor attribute ensures that the aspect ratio stays constant
            scaleanchor="x"
        )

        # add image
        fig.add_layout_image(dict(
            x=0,
            sizex=img_width,
            y=img_height,
            sizey=img_height,
            xref="x", yref="y",
            opacity=1.0,
            layer="below",
            sizing="stretch",
            source=img_example
        ))

        # add bounding box
        fig.add_shape(
            type="rect",
            x0=ann_example["plot-bb"]["x0"],
            y0=-ann_example["plot-bb"]["y0"] + img_height,
            x1=ann_example["plot-bb"]["x0"] + ann_example["plot-bb"]["width"],
            y1=-(ann_example["plot-bb"]["y0"] + ann_example["plot-bb"]["height"]) + img_height,
            line=dict(color="RoyalBlue"),
        )

        # add polygons
        for text in ann_example["text"]:
            name = text["text"]

            if text["role"] == "tick_label":
                tick_value = self.get_tick_value(name, ann_example["data-series"])
                if tick_value:
                    name = f'Text: {name}
Value: {tick_value}'

            xs, ys = self.get_coords(text["polygon"], img_height)
            fig.add_trace(go.Scatter(
                x=xs, y=ys, fill="toself",
                name=self.add_line_breaks(name),
                hovertemplate="%{name}",
                mode='lines'
            ))

        # add x-axis dots
        xs = [dot["tick_pt"]["x"] for dot in ann_example["axes"]["x-axis"]["ticks"]]
        ys = [-dot["tick_pt"]["y"] + img_height for dot in ann_example["axes"]["x-axis"]["ticks"]]
        fig.add_trace(go.Scatter(
            x=xs, y=ys, mode='markers',
            name="x-axis"
        ))

        # add y-axis dots
        xs = [dot["tick_pt"]["x"] for dot in ann_example["axes"]["y-axis"]["ticks"]]
        ys = [-dot["tick_pt"]["y"] + img_height for dot in ann_example["axes"]["y-axis"]["ticks"]]
        fig.add_trace(go.Scatter(
            x=xs, y=ys, mode='markers',
            name="y-axis"
        ))

        # configure other layout
        fig.update_layout(
            width=img_width * scale_factor,
            height=img_height * scale_factor,
            margin={"l": 0, "r": 0, "t": 0, "b": 0},
            showlegend=False
        )

        # disable the autosize on double click because it adds unwanted margins around the image
        # and finally show figure
        fig.show(config={'doubleClick': 'reset'})


# _*_ coding:utf-8 _*_

class Quantity:
    ##描述符基于协议实现，无需创建子类（无需继承自别的类）

    def __init__(self,storage_name):

        self.storage_name=storage_name
        #Quantity实例有个storage_name属性,这是托管实例中存储值的属性的名称

    def __set__(self,instance,value):
        ##尝试为托管属性赋值时，会调用__set__方法。这里，self是描述符实例(即Lineitem.weight或LineItem.price)，instance是托管实例（LineItem实例），value是有设定的值

        if value>0:
            #这里必须直接处理托管实例的__dict__属性，如果使用内置的setattr函数，会在此触发__set__方法，导致无限递归
            instance.__dict__[self.storage_name]=value

        else:
            raise(u'value must > 0,but get %s'%(value))

class LineItem(object):

    weight=Quantity('weight')
    price=Quantity('price')

    def __init__(self,weight,price):

        self.weight=weight
        self.price=price

def test():

    lineitem=LineItem(1,2)
    print (lineitem.weight,lineitem.price)

    lineitem.weight=6
    #quantity描述符不需要实现__get__方法就能修改属性的值，这是与property不一样的地方,因为其默认调用了__getattribute__方法

    print (lineitem.weight,lineitem.price)

print (u'这个ItemLine还有一个缺点，在托管类的定义体中实例化描述符要重复输入属性的名称。如weight=Quantity("weight")')
print (u'如果能向下面一样声明就好了,weight=Quantity()')

if __name__ =='__main__':

    test()


import cv2
import numpy as np
import random
import pafy  # library to work with youtube videos
from recognition import process_number, recognize
from split_plate import split_plate, dumb_split_plate

# set url to grab video from youtube
url = 'https://www.youtube.com/watch?v=Li3FPoa3iYE'
url1 ='https://www.youtube.com/watch?v=DWo1ESaLg-Q'
num_set = set()  # to store read data
jobs = []
t_config = '--user-patterns pattern.txt --oem 3 --psm 13 -c tessedit_char_whitelist=-BDFGHJKLNPRSTUVXYZ0123456789  load_system_dawg=false load_freq_dawg=false'

# start parameters for masking
h_min = 8
h_max = 21
s_min = 112 #152
s_max = 205
v_min = 75 #195
v_max = 215 #244
confi = 0.8 # i need confidence no less than
pause = False
show_mask = False

# function runs as parameters changed

def set_mask(_):
    global lower, upper
    lower = np.array([cv2.getTrackbarPos("Hue_min", "params"), cv2.getTrackbarPos("Sat_min", "params"),
                      cv2.getTrackbarPos("Val_min", "params")])
    upper = np.array([cv2.getTrackbarPos("Hue_max", "params"), cv2.getTrackbarPos("Sat_max", "params"),
                      cv2.getTrackbarPos("Val_max", "params")])
    return ()

# define possible area size of a license plate
lp_max = 1000
lp_min = 400

counter = 0
tilt = 0  # if license plates are tilted, we need to fix it, set angle here (positive if the road turns right)

video = pafy.new(url)
best = video.getbest(preftype="mp4")

capture = cv2.VideoCapture()
capture.open(best.url)
s, img = capture.read()

# let's select roi
roi = cv2.selectROI('select', img)
cv2.destroyWindow('select')  # closing roi select window manually

# creating window to adjust parameters
cv2.namedWindow("params")
cv2.resizeWindow("params", 400, 200)
cv2.createTrackbar("Hue_min", "params", h_min, 178, set_mask)
cv2.createTrackbar("Hue_max", "params", h_max, 178, set_mask)
cv2.createTrackbar("Sat_min", "params", s_min, 255, set_mask)
cv2.createTrackbar("Sat_max", "params", s_max, 255, set_mask)
cv2.createTrackbar("Val_min", "params", v_min, 255, set_mask)
cv2.createTrackbar("Val_max", "params", v_max, 255, set_mask)

# building mask
lower = np.array([cv2.getTrackbarPos("Hue_min", "params"), cv2.getTrackbarPos("Sat_min", "params"),
                  cv2.getTrackbarPos("Val_min", "params")])
upper = np.array([cv2.getTrackbarPos("Hue_max", "params"), cv2.getTrackbarPos("Sat_max", "params"),
                  cv2.getTrackbarPos("Val_max", "params")])

# cropping image to roi
img = img[int(roi[1]):int(roi[1] + roi[3]), int(roi[0]):int(roi[0] + roi[2])]
min_diff = roi[2] * roi[3] / 5  # we skip the frame if it is changed in less pixels than min_diff

img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
old_img_gray = np.copy(img_gray)

def t_recognize(img,c=0.3):  # function to ocr images
    save_ll=False
    #print ('splitting...')
    chars = dumb_split_plate(img)

    if save_ll:  # save pic
        for char in chars:
            cv2.imwrite('./ll/'+str(random.randint(0,1000000)) + ".png", char)

    if not chars:
        #print ('[main]:no chars')
        return False,0,0
    #print('[main]: returned chars:', len(chars))
    if len(chars) > 1:
        #print('processing...')
        nums = process_number(chars)
    else:
        #print ('less than one char')
        return False,0,0
    if len(nums) > 1:
        #print('recognizing...')
        try:
            t,conf = recognize(nums,c)

        except:
            #print('something went wrong')
            return False,0,0
        return t, conf, chars
    return False,0

# start main loop
while True:
    if not pause:  # if pause not pressed ('p' button)
        old_img_gray = np.copy(img_gray)
        s, img = capture.read()
        img = img[int(roi[1]):int(roi[1] + roi[3]), int(roi[0]):int(roi[0] + roi[2])]
        img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        k = np.sum(img_gray != old_img_gray)
        if k < min_diff:
            # skip if no significant changes
            continue

    if not s:  # no image captured - continue to next frame
        continue

    # processing image before searching for contours
    img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    img_blur = cv2.blur(img_hsv, (10, 10), 2)
    img_eroded = cv2.erode(img_blur, (30, 30), iterations=2)
    mask = cv2.inRange(img_eroded, lower, upper)

    if show_mask: cv2.imshow('mask', mask)

    # detecting contours
    contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    img_copy = np.copy(img)  # we need a copy to draw on it and show
    for cont in contours:
        area = cv2.contourArea(cont)

        if lp_max > area > lp_min:  # checking if area of contour within defined range
            box = cv2.approxPolyDP(cont, 1, 1)
            x, y, w, h = cv2.boundingRect(box)
            if h * w < 100:  # if area is way too small
                continue
            cv2.rectangle(img_copy, (x, y), (x + w, y + h), (255, 0, 0), 3)  # draw a box on copy

            number = img[y :y + h, x - 2:x + w]  # saving cropped license plate

            if 3 < (w / h) < 6:  # typical proportions of LP

                try:
                    cv2.imshow('params', number)
                except:
                    #print("error in contours detection, size is : ",number.shape)  # sometimes we get 0 in contour dimensions
                    continue
                if tilt != 0:  # fixing proportions if needed
                    pts1 = np.float32([[0, 0 + tilt], [number.shape[0], 0], [0, number.shape[1]],
                                       [number.shape[0], number.shape[1] - tilt]])
                    pts2 = np.float32(
                        [[0, 0], [number.shape[0], 0], [0, number.shape[1]], [number.shape[0], number.shape[1]]])

                    M = cv2.getPerspectiveTransform(pts1, pts2)
                    number = cv2.warpPerspective(number, M, (number.shape[1], number.shape[0]))
                text, conf, chars = t_recognize(number)  # run recognition

                if text and 3 < len(text) and conf > 0.6:
                    print('Detected license plate: ', text, "conf=",conf)
                    dummy=np.zeros((40,100,3))
                    if conf > confi:
                        c=int(conf*100)
                        cv2.putText(dummy, text, (0, 30), cv2.FONT_HERSHEY_PLAIN,1.3,(0,255,255),1 )
                        cv2.putText(dummy, str(c)+'%', (15, 15), cv2.FONT_HERSHEY_PLAIN, 1, (0, 255, 0), 1)
                        cv2.imshow('detected',dummy)

    cv2.imshow("Contours", img_copy)

    key = cv2.waitKey(1) & 0xFF
    if key == ord('s'):  # save pic
        cv2.imwrite("car" + str(counter) + ".png", img)
    elif key == ord('q'):  # stop
        break
    elif key == ord('z') and chars:  # save chars
        for char in chars:
            cv2.imwrite ('./ll/'+str(random.randint(0,1000000))+'.png',char)
        print ('[main]: Saved...')
    elif key == ord('m'):  # turn mask window on/off
        if show_mask:
            cv2.destroyWindow('mask')
        show_mask = not show_mask
    elif key == ord('p'):  # set on pause
        pause = not pause

capture.release()
cv2.destroyAllWindows()


from scrapy import Spider
from scrapy.http import Request

from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader

import urllib.request, urllib.parse, urllib.error

class TrendnetSpider(Spider):
    name = "trendnet"
    allowed_domains = ["trendnet.com"]
    start_urls = ["http://www.trendnet.com/support/"]

    def parse(self, response):
        for entry in response.xpath("//select[@id='subtype_id']/option"):
            if entry.xpath(".//text()"):
                text = entry.xpath(".//text()").extract()[0]
                href = entry.xpath("./@value").extract()[0]
                yield Request(
                    url=urllib.parse.urljoin(response.url, href),
                    meta={"product": text},
                    headers={"Referer": response.url},
                    callback=self.parse_product)
    def parse_product(self, response):
        product = response.xpath("//div[@class='col-lg-12 col-md-6 col-sm-6']/h2/text()").extract()[0].strip().split(" ")[0].strip()
        device_class = response.xpath("//div[@class='g-px-10--lg g-pt-10']/div/h1[@class='g-font-weight-300 mb-0']/text()").get().strip()
        for tab in response.xpath("//div[@class='row  g-pt-20 g-pb-10 g-px-15']"):
            text = tab.xpath("./div/div/a/text()").extract()
            href = tab.xpath("./div/div/a/@onclick").extract()
            version = tab.xpath("./div/p/text()").extract()
            date = tab.xpath("./div[2]/text()").extract()
            if len(text) == 0:
                continue
            if "firmware" in text[0].lower():
                if "open" in href[0]:
                    link = href[0].split('\'')[1]
                    url = urllib.parse.urljoin("https://www.trendnet.com", link)
                    url = url.replace("inc_downloading.asp?", "inc_downloading.asp?button=Continue+with+Download&Continue=yes&")
                self.logger.debug(date[1].strip())
                self.logger.debug(text[0])
                self.logger.debug(url)
                self.logger.debug(version)
                self.logger.debug(device_class)
                self.logger.debug(product)
                item = FirmwareLoader(
                    item=FirmwareImage(), response=response, date_fmt=["%m/%d/%Y"])
                item.add_value("url", url)
                item.add_value("product", product)
                item.add_value("date", date[1].strip())
                if len(version) > 0:
                    item.add_value("version", version)
                item.add_value("vendor", self.name)
                yield item.load_item()


import matplotlib
import matplotlib.pyplot as plt
import scienceplots
import numpy as np
plt.style.use('ieee')

files = ['acc_dist_1.csv', 'acc_dist_2.csv', 'acc_dist_3.csv']
def compute(infile):
    f = open(infile, 'r')
    lines = f.readlines()
    f.close()
    gamma = list(map(float, lines[0].strip().split(',')[1:-1]))
    th_0_acc = list(map(float, lines[1].strip().split(',')[1:-1]))
    th_1_acc = list(map(float, lines[2].strip().split(',')[1:-1]))
    th_2_acc = list(map(float, lines[3].strip().split(',')[1:-1]))
    th_3_acc = list(map(float, lines[4].strip().split(',')[1:-1]))
    return gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc

if __name__ == '__main__':
    fig, ax = plt.subplots(figsize=[4, 4])
    gamma_pos = -5
    dist = [1, 2, 3]
    th_0_acc_list = [0, 0, 0]
    th_1_acc_list = [0, 0, 0]
    th_2_acc_list = [0, 0, 0]
    th_3_acc_list = [0, 0, 0]
    for i in range(1, 4):
        gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc = compute(files[i-1])
        th_0_acc_list[i-1] = th_0_acc[gamma_pos]
        th_1_acc_list[i-1] = th_1_acc[gamma_pos]
        th_2_acc_list[i-1] = th_2_acc[gamma_pos]
        th_3_acc_list[i-1] = th_3_acc[gamma_pos]
    ax.plot(dist, th_0_acc_list, '-*', label='DLS')
    ax.plot(dist, th_1_acc_list, '--*', label='DLSM $\\theta_s$=1')
    ax.plot(dist, th_2_acc_list, '--*', label='DLSM $\\theta_s$=2')
    ax.plot(dist, th_3_acc_list, '--*', label='DLSM $\\theta_s$=3')
    # fig.text(0.25, 0.1, '(a)', horizontalalignment='center')
    # fig.text(0.5, 0.1, '(b)', horizontalalignment='center')
    # fig.text(0.75, 0.1, '(c)', horizontalalignment='center')
    ax.set_xlabel('Minimum Levenshtein Distance of Indexes')
    ax.set_ylabel('Accuracy (γ=%.2f)'%gamma[gamma_pos])
    ax.set_xticks(dist)
    ax.legend()
    ax.grid(which='both')
    # plt.gcf().set_dpi(600)
    plt.savefig('dist_optimization.pdf')

    fig, ax = plt.subplots(figsize=[4, 4])
    gamma_pos = -5
    dist = [1, 2, 3]
    th_0_acc_list = [0, 0, 0]
    th_1_acc_list = [0, 0, 0]
    th_2_acc_list = [0, 0, 0]
    th_3_acc_list = [0, 0, 0]
    for i in range(1, 4):
        gamma, th_0_acc, th_1_acc, th_2_acc, th_3_acc = compute(files[i-1])
        th_0_acc_list[i-1] = th_0_acc[gamma_pos]
        th_1_acc_list[i-1] = th_1_acc[gamma_pos]
        th_2_acc_list[i-1] = th_2_acc[gamma_pos]
        th_3_acc_list[i-1] = th_3_acc[gamma_pos]

    arr = np.array([th_0_acc_list, th_1_acc_list, th_2_acc_list, th_3_acc_list])
    th = [0, 1, 2, 3]
    print(arr)
    ax.plot(th, arr[:, 0], '-*', label='Min. LD = 1')
    ax.plot(th, arr[:, 1], '--*', label='Min. LD = 2')
    ax.plot(th, arr[:, 2], '--*', label='Min. LD = 3')
    # fig.text(0.25, 0.1, '(a)', horizontalalignment='center')
    # fig.text(0.5, 0.1, '(b)', horizontalalignment='center')
    # fig.text(0.75, 0.1, '(c)', horizontalalignment='center')
    ax.set_xlabel('$\\theta_s$')
    ax.set_ylabel('Accuracy (γ=%.2f)'%gamma[gamma_pos])
    ax.set_xticks(th)
    ax.legend()
    ax.grid(which='both')
    # plt.gcf().set_dpi(600)
    plt.savefig('dist_optimization_1.pdf')
    plt.show()


import json

def writeJsonData(path, data_json):
    File_out = open(path, "w", encoding="utf-8")
    File_out.write(json.dumps(data_json, ensure_ascii=False))
    File_out.close()

def readJsonData(path):
    decode_file = open(path, "r", encoding="utf-8")
    tweets = json.load(decode_file)
    decode_file.close()
    return tweets