# -*- coding: utf-8 -*-
"""
    @Project     : wenshu_spider
    @File        : wenshu_case_detail_spider.py
    @Create_Date : 2019-05-21 15:39
    @Update_Date : 2019-05-21 15:39
    @Author      : liing
    @Email       : liing1221@163.com
    @Software    : PyCharm
"""
from __future__ import print_function
import os
import re
import time
import json
import random
import execjs
import requests
from urllib.parse import quote
from fake_useragent import UserAgent
from requests import adapters, Response
from concurrent.futures import ProcessPoolExecutor
from requests.exceptions import ChunkedEncodingError
from requests.exceptions import ReadTimeout, ProxyError, ConnectionError, ContentDecodingError
from middlewares.settings import RETRY, TIMEOUT, CONTRAST_TABLE, LEN_SQR, DATA_FILE, PROCESSES
from middlewares.proxymanage import ProxyManager
from middlewares.redismanage import RedisManager
from middlewares.mysqlmanage import SqlManager
from middlewares.js import js_wzwschallenge

requests.adapters.DEFAULT_RETRIES = 10

class CaseDetailSpider:
    '''根据传入的案件信息（docid， jstj）抓取解析存储案件详情'''

    def __init__(self, row):
        """

        """
        self.item = {}
        self.row = row
        self.params = quote(row['JSTJ'])
        self.retries = RETRY
        self.timeout = TIMEOUT
        self.len_sqr = LEN_SQR       # 申请人判断：解析时80以内判断是案件人信息，以上为文字描述
        self.guid = ''
        self.number = ''
        self.proxymanager = ProxyManager(pool=False)
        self.sqlmanager = SqlManager(2)
        # self.f = open('./middlewares/page_err.txt', 'a')
        # 保持回话
        self.s = requests.Session()
        pass

    def make_proxies(self):
        """"""
        # 　代理设置
        ip_port = self.proxymanager.get_proxy()
        ip = ip_port[0]
        port = ip_port[1].strip()
        proxies = {
            'http': 'http://{}:{}'.format(ip, port),
            'https': 'http://{}:{}'.format(ip, port)
        }
        return proxies

    @staticmethod
    def make_guid():
        guid1 = ''
        for _, i in enumerate(range(8)):
            num = int((1 + random.random()) * 65536) | 0
            guid = hex(num)[3:]
            if _ == 0 or _ == 3:
                guid1 += guid
            elif _ == 1 or _ == 2 or _ == 4:
                guid1 += (guid + '-')
            else:
                guid1 += guid
        return guid1

    def get_number(self):
        """"""
        ua = UserAgent().random
        headers = {
            'Host': 'wenshu.court.gov.cn',
            'Connection': 'keep-alive',
            'Accept': '*/*',
            'Origin': 'http://wenshu.court.gov.cn',
            'X-Requested-With': 'XMLHttpRequest',
            'Accept-Encoding': 'gzip,deflate',
            'Accept-Language': 'zh-CN,zh;q = 0.9',
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'Referer': 'http://wenshu.court.gov.cn/'
        }
        headers.setdefault('User-Agent', ua)
        url = 'http://wenshu.court.gov.cn/ValiCode/GetCode'
        data = {'guid': self.guid}
        try:
            proxies = self.make_proxies()
            response = self.s.post(url, data=data, proxies=proxies, headers=headers, timeout=TIMEOUT)
            # response = self.s.post(url, data=data, headers=headers)
            if isinstance(response, Response) and response.status_code == 200:
                number = response.text
                return number
            else:
                raise ValueError("获取number参数出错!")
        except (ReadTimeout, ProxyError, ConnectionError, ContentDecodingError, ChunkedEncodingError,
                        ValueError, KeyError):
            return self.get_number()

    def get_vjkl5(self):
        """"""
        ua = UserAgent().random
        url = 'http://wenshu.court.gov.cn/list/list/?sorttype=1&number={}&guid={}{}'
        url = url.format(self.number, self.guid, self.params, timeout=TIMEOUT)
        headers = {
            'Host': 'wenshu.court.gov.cn',
            'Connection': 'keep-alive',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,'
                      '*/*;q=0.8,application/signed-exchange;v=b3',
            'Upgrade-Insecure-Requests': '1',
            'Referer': 'http://wenshu.court.gov.cn/',
            'Accept-Encoding': 'gzip,deflate',
            'Accept-Language': 'zh-CN,zh;q = 0.9',
        }
        headers.setdefault('User-Agent', ua)
        try:
            proxies = self.make_proxies()
            print('进程：{} Proxies>>> {}'.format(os.getpid(), proxies))
            # print(self.s.cookies.get_dict())
            # self.s.cookies.pop('guid')
            response = self.s.get(url, proxies=proxies, headers=headers, timeout=TIMEOUT)
            # response = self.s.get(url, headers=headers)
            if isinstance(response, Response) and response.status_code == 200:
                response.encoding = 'utf-8'
                try:
                    dynamicurl = re.search('dynamicurl="(.*?)"', response.text).group(1)
                    wzwsquestion = re.search('wzwsquestion="(.*?)"', response.text).group(1)
                    wzwsfactor = re.search('wzwsfactor="(.*?)"', response.text).group(1)
                    wzwsmethod = re.search('wzwsmethod="(.*?)"', response.text).group(1)
                    wzwsparams = re.search('wzwsparams="(.*?)"', response.text).group(1)
                except AttributeError:
                    return None

                js_dynamicurl = '''
                var dynamicurl="{}";var wzwsquestion="{}";var wzwsfactor="{}";var wzwsmethod="{}";var wzwsparams="{}";
                '''.format(dynamicurl, wzwsquestion, wzwsfactor, wzwsmethod, wzwsparams)
                js_code = js_dynamicurl + js_wzwschallenge
                ctx = execjs.compile(js_code)
                wzwschallenge = ctx.call("wzwschallenge")
                print('进程：{} wzwschallenge>>> {}'.format(os.getpid(), wzwschallenge))
                next_url = 'http://wenshu.court.gov.cn' + dynamicurl + '?' + 'wzwschallenge=' + wzwschallenge

                self.s.get(next_url, proxies=proxies, headers=headers, timeout=TIMEOUT)
                # resp = self.s.get(next_url, headers=headers)
                vjkl5 = self.s.cookies.get_dict()['vjkl5']
                return vjkl5
            else:
                raise ValueError("获取number参数出错!")
        except (ReadTimeout, ProxyError, ConnectionError, ContentDecodingError, ChunkedEncodingError,
                ValueError, KeyError):
            return

    def get_page(self):
        """"""
        row = self.row
        retry = RETRY
        url1 = 'http://wenshu.court.gov.cn/content/content?DocID={}&KeyWord='.format(row['docid'])
        url = 'http://wenshu.court.gov.cn/CreateContentJS/CreateContentJS.aspx?DocID={}'.format(row['docid'])

        def fun():
            proxies = self.make_proxies()
            self.guid = self.make_guid()
            print('进程：{} Start get Number'.format(os.getpid()))
            self.number = self.get_number()
            print('进程：{} Start get vjkl5'.format(os.getpid()))
            self.vjkl5 = self.get_vjkl5()
            if not self.vjkl5:
                return
            nonlocal retry
            if retry <= 0:
                return None
            else:
                retry -= 1
            headers = {
                'Host': 'wenshu.court.gov.cn',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1',
                'Accept': 'text/javascript, application/javascript, */*',
                'X-Requested-With': 'XMLHttpRequest',
                'Referer': url1,
                'Accept-Encoding': 'gzip,deflate',
                'Accept-Language': 'zh-CN,zh;q=0.9'
            }

            try:
                print('进程：{} Start get Response'.format(os.getpid()))
                headers.setdefault('User-Agent', UserAgent().random)
                response = self.s.get(url, headers=headers, proxies=proxies, timeout=TIMEOUT)
                # response = self.s.get(url, headers=headers, timeout=TIMEOUT)
                if isinstance(response, Response) and response.status_code == 200:
                    response.encoding = 'utf-8'
                    try:
                        dynamicurl = re.search('dynamicurl="(.*?)"', response.text).group(1)
                        wzwsquestion = re.search('wzwsquestion="(.*?)"', response.text).group(1)
                        wzwsfactor = re.search('wzwsfactor="(.*?)"', response.text).group(1)
                        wzwsmethod = re.search('wzwsmethod="(.*?)"', response.text).group(1)
                        wzwsparams = re.search('wzwsparams="(.*?)"', response.text).group(1)
                    except AttributeError:
                        return None

                    js_dynamicurl = '''
                    var dynamicurl="{}";var wzwsquestion="{}";var wzwsfactor="{}";var wzwsmethod="{}";var wzwsparams
                    ="{}";'''.format(dynamicurl, wzwsquestion, wzwsfactor, wzwsmethod, wzwsparams)
                    js_code = js_dynamicurl + js_wzwschallenge
                    ctx = execjs.compile(js_code)
                    wzwschallenge = ctx.call("wzwschallenge")
                    print('进程：{} wzwschallenge>>> {}'.format(os.getpid(), wzwschallenge))
                    next_url = 'http://wenshu.court.gov.cn' + dynamicurl + '?' + 'wzwschallenge=' + wzwschallenge

                    resp = self.s.get(next_url, proxies=proxies, headers=headers, timeout=TIMEOUT)
                    # resp = self.s.get(next_url, headers=headers)
                    if isinstance(resp, Response) and resp.status_code == 200:
                        resp.encoding = 'utf-8'
                        if 'window.location.href=' in response.text:
                            raise ValueError("请求详情页出错!")
                        else:
                            return resp
                    else:
                        raise ValueError("请求详情页出错!")
                else:
                    raise ValueError("请求详情页出错!")
            except (ReadTimeout, ProxyError, ConnectionError, ContentDecodingError, ChunkedEncodingError,
                    ValueError, KeyError):
                if retry == 3 or retry == 1:
                    time.sleep(60)
                return fun()
        return fun()

    def parse_page(self, response):
        """"""
        item = {}
        item['page'] = response.content
        try:
            caseinfos = re.search(r'caseinfo=JSON.stringify\((.*?)\);\$\(', response.text, re.S).group(1)
        except AttributeError:
            return
        caseinfos = json.loads(caseinfos, encoding='utf-8')
        for caseinfo in caseinfos.items():
            if caseinfo[0] in CONTRAST_TABLE.keys():
                if caseinfo[0] not in item.keys():
                    item[CONTRAST_TABLE[caseinfo[0]]] = caseinfo[1]
                elif not item[CONTRAST_TABLE[caseinfo[0]]]:
                    item[CONTRAST_TABLE[caseinfo[0]]] = caseinfo[1]
            else:
                print('进程：{} 解析caseinfo:未存储字段：{}'.format(os.getpid(), caseinfo))

        item['WSFBRQ'] = re.search(r'\\"PubDate\\":\\"(.*?)\\",\\"Html', response.text, re.S).group(1)
        names = re.findall(r'>> {}'.format(os.getpid(), names))
        try:
            item["WBSB"] = re.search(r'(<a type=\'dir\' name=\'WBSB\'.*?)<a type', response.text, re.S).group(1)
        except AttributeError:
            item['WBSB'] = ''
        try:
            if "SSJL" in names:
                item['DSRXX'] = re.search(r'(<a type=\'dir\' name=\'DSRXX\'.*?)<a type', response.text, re.S).group(1)
            else:
                item['DSRXX'] = re.search(r'()', response.text, re.S).group(1)
        except AttributeError:
            item['DSRXX'] = ''
        try:
            item['SSJL'] = item['CPYZ'] = ''
            if "SSJL" in names and "CPYZ" in names:
                ssjl = re.search(r'(<a type=\'dir\' name=\'SSJL\'.*?)<a type', response.text, re.S)
                if ssjl:
                    item['SSJL'] = ssjl.group(1)
                cpyz = re.search(r'()', response.text, re.S)
                if cpyz:
                    item['CPYZ'] = cpyz.group(1)
            elif "SSJL" in names and "CPYZ" not in names:
                ssjl = re.search(r'()', response.text, re.S)
                if ssjl:
                    item['SSJL'] = ssjl.group(1)
            elif "SSJL" not in names and "CPYZ" in names:
                cpyz = re.search(r'()', response.text, re.S)
                if cpyz:
                    item['CPYZ'] = cpyz.group(1)
            else:
                pass
        except AttributeError:
            return
        try:
            item['PJJG'] = item['WBWB'] = ''
            if "PJJG" in names and 'WBWB' in names:
                pjjg = re.search(r'(<a type=\'dir\' name=\'PJJG\'.*?)<a type=', response.text, re.S)
                if pjjg:
                    item['PJJG'] = pjjg.group(1)
                div_wbwb = re.search(r'(<a type=\'dir\' name=\'WBWB\'.*?
)\\"}";', response.text, re.S)
                if div_wbwb:
                    item['WBWB'] = div_wbwb.group(1)
            elif "PJJG" in names and 'WBWB' not in names:
                pjjg = re.search(r'((.*?)<a type=', response.text, re.S)
                if pjjg:
                    item['PJJG'] = pjjg.group(1)
                div_wbwb = re.search(r'((.*?)
.*?(
审.*?
)\\"}";',
                                     response.text, re.S)
                if div_wbwb:
                    item['WBWB'] = div_wbwb.group(1)
            else:
                pass
        except AttributeError:
            pass

        if item['DSRXX']:
            elements = re.findall(r'LINE-HEIGHT.*?16pt;\'>(.*?)
', item['DSRXX'], re.S)
        else:
            elements = re.findall(r'LINE-HEIGHT.*?16pt;\'>(.*?)
', response.text, re.S)
        item['TCSQR'] = item['BSQR'] = item['YSBG'] = item['YSDSR'] = key = ''
        dic = {}
        try:
            for n, ele in enumerate(elements):
                if len(ele) > self.len_sqr:
                    break
                for j in ['法定代表人', '委托代理人']:
                    if j in ele:
                        dic[CONTRAST_TABLE[j]] += ele
                        continue
                for i in ['上诉人', '原告', "申请人", '申请再审人', "再审申请人", '被上诉人', '被告', '原审被告',
                          '被申请人', '原审第三人']:
                    el = ele[:len(i)]
                    if i == el:
                        if dic and key:
                            item[key] += json.dumps(dic, ensure_ascii=False)
                        key = CONTRAST_TABLE[i]
                        dic.clear()
                        dic[key] = ele
                        dic['WTDLR'] = dic['FDDBR'] = ''
                        break
            item[key] += json.dumps(dic, ensure_ascii=False)
        except KeyError:
            self.len_sqr = 2 * LEN_SQR
            for n, ele in enumerate(elements):
                if len(ele) > self.len_sqr:
                    break
                for j in ['法定代表人', '委托代理人']:
                    if j in ele:
                        dic[CONTRAST_TABLE[j]] += ele
                        continue
                for i in ['上诉人', '原告', "申请人", '申请再审人', "再审申请人", '被上诉人', '被告', '原审被告',
                          '被申请人', '原审第三人']:
                    el = ele[:len(i)]
                    if i == el:
                        if dic and key:
                            item[key] += json.dumps(dic, ensure_ascii=False)
                        key = CONTRAST_TABLE[i]
                        dic.clear()
                        dic[key] = ele
                        dic['WTDLR'] = dic['FDDBR'] = ''
                        break
            item[key] += json.dumps(dic, ensure_ascii=False)

        if item['WBWB']:
            wbwbs = re.findall(r'16pt;\'>(.*?)
', item['WBWB'], re.S)
        else:
            div_wbwb = re.search(r'(>审.*?
)\\"}";', response.text, re.S).group(1)
            wbwbs = re.findall(r'>(.*?)<', div_wbwb, re.S)
        item['SPZ'] = item['SPY'] = item['DLSPY'] = item['SJY'] = item['PSY'] = key = ''
        for wbwb in wbwbs:
            if len(wbwb) > 20:
                break
            for w in ["审判长", "审判员", "代理审判员", "书记员", "代书记员", "人民陪审员", "陪审员"]:
                spli = w[-1]
                part = wbwb.split(spli)
                part1 = part[0].replace('\u3000', '').replace(' ', '') + spli
                if len(part) < 2:
                    continue
                elif len(part) == 2:
                    part2 = part[1].replace('\u3000', '').replace(' ', '')
                else:
                    part2 = ';'.join(spli.join(part[1:]).replace('\u3000', '').replace(' ', '').split(part1))
                if w == part1:
                    key = CONTRAST_TABLE[w]
                    if w != "代书记员":
                        if not item[key]:
                            item[key] += part2
                        else:
                            item[key] += ';' + part2
                    else:
                        if not item[key]:
                            item[key] += part2 + '(代)'
                        else:
                            item[key] += ';' + part2 + '(代)'
                    break
        return item

    def save_item(self, item):
        """存储数据：入数据库，保存文件"""
        date = time.strftime('%Y%m%d', time.localtime(time.time()))
        filepath = DATA_FILE + date + '/'
        filename_page = filepath + '{}.txt'.format(item['docid'])
        if not os.path.exists(filepath):
            os.makedirs(filepath)
        with open(filename_page, 'wb') as f:
            f.write(item['page'])
        item['DOC'] = filename_page
        item.pop('page')
        self.sqlmanager.update_one_cases(item, self.row['id'])

    def __del__(self):
        self.s.close()

def put_caseTo_redis():
    ''''''
    dbmanager = SqlManager(4)
    redismanager = RedisManager()
    queue_key = 'queue_cases'
    for row in dbmanager.get_one_cases():
        if not row:
            break
        else:
            # if row['id'] < 1201:
            redismanager.put_queue(queue_key, row)

def get_details():
    """"""
    redismanager = RedisManager()
    queue_key = 'queue_cases'
    while True:
        row = redismanager.get_queue(queue_key)
        if row:
            print('进程：{} Start spider>> {}!'.format(os.getpid(), row))
            casedetailspider = CaseDetailSpider(row)
            print('进程：{} Start get page!'.format(os.getpid()))
            response = casedetailspider.get_page()
            if isinstance(response, Response):
                print('进程：{} Start parse page!'.format(os.getpid()))
                item = casedetailspider.parse_page(response)
                if item and item['SPZ']:
                    print('进程：{} Start save item!'.format(os.getpid()))
                    casedetailspider.save_item(item)
                else:
                    queue_key = 'queue_cases'
                    print('进程：{} '.format(os.getpid()), end='  ', flush=True)
                    redismanager.put_queue(queue_key, row)
            else:
                queue_key = 'queue_cases'
                print('进程：{} '.format(os.getpid()), end='  ', flush=True)
                redismanager.put_queue(queue_key, row)
        else:
            break

def cocurrent_run(processes):
    """建立进程池并发执行搜索"""
    pool = ProcessPoolExecutor()
    for i in range(processes):
        time.sleep(1)
        pool.submit(get_details)
    pool.shutdown()

if __name__ == "__main__":
    # put_caseTo_redis()
    cocurrent_run(PROCESSES)
    # get_details()


"""ResourceSync resource_list object

An resource_list is a set of resources with some metadata for
each resource. Comparison of resource lists from a source and
a destination allows understanding of whether the two are in
sync or whether some resources need to be updated at the
destination.

The resource_list object may also contain metadata and links.

Described in specification at:
http://www.openarchives.org/rs/resourcesync#DescResources

- mandatory <rs:md> element:
-- must have capability="resourcelist"
-- should have modified=".." attribute also, which should match the
   mtime of a file written.
- one <url> element for each resource

"""

import collections
import os
from datetime import datetime
import re
import sys
from urllib import URLopener

from list_base_with_index import ListBaseWithIndex
from sitemap import Sitemap
from mapper import Mapper, MapperError
from url_authority import UrlAuthority

class ResourceListDict(dict):
    """Default implementation of class to store resources in ResourceList

    Key properties of this class are:
    - has add(resource) method
    - is iterable and results given in alphanumeric order by resource.uri
    """

    def __iter__(self):
        """Iterator over all the resources in this resource_list"""
        self._iter_next_list = sorted(self.keys())
        self._iter_next_list.reverse()
        return(iter(self._iter_next, None))

    def _iter_next(self):
        if (len(self._iter_next_list)>0):
            return(self[self._iter_next_list.pop()])
        else:
            return(None)

    def uris(self):
        return(sorted(self.keys()))

    def add(self, resource, replace=False):
        """Add just a single resource"""
        uri = resource.uri
        if (uri in self and not replace):
            raise ResourceListDupeError("Attempt to add resource already in resource_list")
        self[uri]=resource

class ResourceListDupeError(Exception):
    pass

class ResourceList(ListBaseWithIndex):
    """Class representing an resource_list of resources

    This same class is used for both the source and the destination
    and is the central point of comparison the decide whether they
    are in sync or what needs to be copied to bring the destinaton
    into sync.

    An resource_list will admit only one resource with any given URI.

    Storage is unordered but the iterator imposes a canonical order
    which is currently alphabetical by URI.
    """

    def __init__(self, resources=None, md=None, ln=None, allow_multifile=None, mapper=None):
        if (resources is None):
            resources = ResourceListDict()
        super(ResourceList, self).__init__(resources=resources, md=md, ln=ln, mapper=mapper)
        self.capability_name = 'resourcelist'
        self.capability_md = 'resourcelist'
        self.resources_class = ResourceListDict

    def add(self, resource, replace=False):
        """Add a resource or an iterable collection of resources

        Will throw a ValueError if the resource (ie. same uri) already
        exists in the resource_list, unless replace=True.
        """
        if isinstance(resource, collections.Iterable):
            for r in resource:
                self.resources.add(r,replace)
        else:
            self.resources.add(resource,replace)

    def compare(self,src):
        """Compare the current resource_list object with the specified resource_list

        The parameter src must also be an resource_list object, it is assumed
        to be the source, and the current object is the destination. This
        written to work for any objects in self and sc, provided that the
        == operator can be used to compare them.

        The functioning of this method depends on the iterators for self and
        src providing access to the resource objects in URI order.
        """
        dst_iter = iter(self.resources)
        src_iter = iter(src.resources)
        same=ResourceList()
        updated=ResourceList()
        deleted=ResourceList()
        created=ResourceList()
        dst_cur=next(dst_iter,None)
        src_cur=next(src_iter,None)
        while ((dst_cur is not None) and (src_cur is not None)):
            #print 'dst='+dst_cur+'  src='+src_cur
            if (dst_cur.uri == src_cur.uri):
                if (dst_cur==src_cur):
                    same.add(dst_cur)
                else:
                    updated.add(src_cur)
                dst_cur=next(dst_iter,None)
                src_cur=next(src_iter,None)
            elif (not src_cur or dst_cur.uri < src_cur.uri):
                deleted.add(dst_cur)
                dst_cur=next(dst_iter,None)
            elif (not dst_cur or dst_cur.uri > src_cur.uri):
                created.add(src_cur)
                src_cur=next(src_iter,None)
            else:
                raise InternalError("this should not be possible")
        # what do we have leftover in src or dst lists?
        while (dst_cur is not None):
            deleted.add(dst_cur)
            dst_cur=next(dst_iter,None)
        while (src_cur is not None):
            created.add(src_cur)
            src_cur=next(src_iter,None)
        # have now gone through both lists
        return(same,updated,deleted,created)

    def has_md5(self):
        """Return true if at least one contained resource-like object has md5 data"""
        if (self.resources is None):
            return(False)
        for resource in self:
            if (resource.md5 is not None):
                return(True)
        return(False)


'''

    Make Two Figures Set as One Figures Using Shape Detect

    This is only for Toner. Take Contour of Figures and find rectangle which have smallest area.

    Because two figures(normal and defect) rotate with different angle, Make them match by this module

    Keypoint detection should spend too much time, so I use shape detection, make them match as fast.
'''

import cv2
import numpy as np
import math

PI = 3.141592

# 여백
whiteScreen = 10

def ShapeDetector(inputImage1, inputImage2):
    XPixelOfInputimage2 = 3840
    YPixelOfInputimage2 = 2160
    # Call Normal Image
    normal_image = cv2.imread(inputImage1)
    normal_image = cv2.resize(normal_image, dsize=(640, 480), interpolation=cv2.INTER_AREA)
    gray = cv2.cvtColor(normal_image, cv2.COLOR_BGR2GRAY)
    ret, blmage = cv2.threshold(gray,220, 255, cv2.THRESH_BINARY_INV)
    blmage = cv2.dilate(blmage, None)

    # Backup normal image for cut
    # This image has 4096 X 2160
    normalImageForCut = cv2.imread(inputImage1)

    # Find Contours of Figures
    mode = cv2.RETR_EXTERNAL
    method = cv2.CHAIN_APPROX_SIMPLE
    contours, hierarchy = cv2.findContours(blmage, mode, method)
    print('len(contours) = ', len(contours))

    maxLength = 0
    k = 0
    for i, cnt in enumerate(contours):
        perimeter = cv2.arcLength(cnt, closed = True)
        if perimeter > maxLength:
            maxLength = perimeter
            k = i
    print('maxLength = ', maxLength)
    cnt = contours[k]
    dst2 = normal_image.copy()
    cv2.drawContours(dst2, [cnt], 0, (255, 0, 0), 3)

    # After Find Contour, Find Smallest area BOX.
    # The pixel number of BOX is saved at box
    rect = cv2.minAreaRect(cnt)
    box = cv2.boxPoints(rect)
    box = np.int32(box)
    dst4 = dst2.copy()
    cv2.drawContours(dst4, [box], 0, (0,0,255), 2)
    cv2.imshow('normalImage', dst4)
    cv2.imwrite('./img/normalImage_dst4.png', dst4)

    # Call Defect Image
    defect = cv2.imread(inputImage2)
    defect = cv2.resize(defect, dsize=(640, 480), interpolation=cv2.INTER_AREA)
    gray_defect = cv2.cvtColor(defect, cv2.COLOR_BGR2GRAY)
    ret_defect, bImage_defect = cv2.threshold(gray_defect,220, 255, cv2.THRESH_BINARY_INV)
    bImage_defect = cv2.dilate(bImage_defect, None)

    # Backup Defect Image for Image Matching
    defectImageForMatching = cv2.imread(inputImage2)
    defectImageForMatching = cv2.resize(defectImageForMatching, dsize=(640, 480), interpolation=cv2.INTER_AREA)

    # Backup Defect Image for Cut
    defectImageForCut = cv2.imread(inputImage2)

    # Find Contours of Figures
    mode = cv2.RETR_EXTERNAL
    method = cv2.CHAIN_APPROX_SIMPLE
    contours_defect, hierarchy = cv2.findContours(bImage_defect, mode, method)
    print('len(contours_defect) = ', len(contours_defect))

    maxLength_defect = 0
    k_defect = 0
    for i, cnt in enumerate(contours_defect):
        perimeter = cv2.arcLength(cnt, closed = True)
        if perimeter > maxLength_defect:
            maxLength_defect = perimeter
            k_defect = i
    print('maxLength_defect = ', maxLength_defect)
    cnt_defect = contours_defect[k_defect]
    dst2_defect = defect.copy()
    cv2.drawContours(dst2_defect, [cnt], 0, (255, 0, 0), 3)

    # After Find Contour, Find Smallest area BOX.
    # The pixel number of BOX is saved at box_defect
    rect_defect = cv2.minAreaRect(cnt_defect)
    box_defect = cv2.boxPoints(rect_defect)
    box_defect = np.int32(box_defect)
    dst4_defect = dst2_defect.copy()
    cv2.drawContours(dst4_defect, [box_defect], 0, (0,0,255), 2)
    cv2.imshow('dst4_defect', dst4_defect)
    cv2.imwrite('./img/dst_defect.png', dst4_defect)

    print(' Box Coordinate for normal : ', box)
    print(' Box Coordinate for defect : ', box_defect)

    # Calculate With Mathematics
    # Calculate How many Rotate of Normal & Defect Image

    #
    # The order of components of box -> [Left Downside], [Left Upside], [Right Upside], [Right Downside]
    #

    slopeForHorizontalNormalImage = (box[2][1] - box[1][1]) / (box[2][0] - box[1][0])
    slopeForHorizontalDefectImage = (box_defect[2][1] - box_defect[1][1]) / (box_defect[2][0] - box_defect[1][0])

    if(slopeForHorizontalNormalImage != slopeForHorizontalDefectImage):
        print('Two Images are not matched. Start Matching...')
        print('Standard image is Normal Image.')

        AngleForNormalImage = math.atan(slopeForHorizontalNormalImage)
        AngleForDefectImage = math.atan(slopeForHorizontalDefectImage)

        print('Angle of Normal Image : ', AngleForNormalImage, ' Angle of Defect Image = ', AngleForDefectImage)

        # If normal image has bigger angle, below variable will be 0. if not, 1
        whichImageHasBiggerAngle = 0

        if(AngleForNormalImage > AngleForDefectImage):
            AngleDiscrepancy = AngleForNormalImage - AngleForDefectImage
        else :
            AngleDiscrepancy = AngleForDefectImage - AngleForNormalImage
            whichImageHasBiggerAngle = 1

        AngleDiscrepancyInDegree = AngleDiscrepancy*180/PI
        print('Difference between two angle : ', AngleDiscrepancy)
        print('Rotate defect Image as ', AngleDiscrepancyInDegree, 'degree')

        # Rotates Small Figures
        rows, cols, channels = defectImageForMatching.shape
        if whichImageHasBiggerAngle == 0:
            M1 = cv2.getRotationMatrix2D((rows/2, cols/2), -AngleDiscrepancyInDegree, 1.0)
        else:
            M1 = cv2.getRotationMatrix2D((rows/2, cols/2), AngleDiscrepancyInDegree, 1.0)
        rotatedDefectImage = cv2.warpAffine(defectImageForMatching, M1, (640, 480))

        # Rotates Image be cut
        rows, cols, channels = defectImageForCut.shape
        if whichImageHasBiggerAngle == 0:
            M2 = cv2.getRotationMatrix2D((rows/2, cols/2), -AngleDiscrepancyInDegree, 1.0)
        else:
            M2 = cv2.getRotationMatrix2D((rows/2, cols/2), AngleDiscrepancyInDegree, 1.0)
        rotatedDefectImageCut = cv2.warpAffine(defectImageForCut, M2, (XPixelOfInputimage2, YPixelOfInputimage2))

        # Find Smallest area box
        gray_rotatedDefect = cv2.cvtColor(rotatedDefectImage, cv2.COLOR_BGR2GRAY)
        ret_rotatedDefect, bImage_rotatedDefect = cv2.threshold(gray_rotatedDefect, 220, 255, cv2.THRESH_BINARY_INV)
        bImage_rotatedDefect = cv2.dilate(bImage_rotatedDefect, None)

        # Find Contours of Figures
        mode = cv2.RETR_EXTERNAL
        method = cv2.CHAIN_APPROX_SIMPLE
        contours_rotatedDefect, hierarchy_rotatedDefect = cv2.findContours(bImage_rotatedDefect, mode, method)
        print('len(contours_rotatedDefect) = ', len(contours_rotatedDefect))

        maxLength_rotatedDefect = 0
        k_rotatedDefect = 0
        for i, cnt in enumerate(contours_rotatedDefect):
            perimeter = cv2.arcLength(cnt, closed=True)
            if perimeter > maxLength_rotatedDefect:
                maxLength_rotatedDefect = perimeter
                k_rotatedDefect = i
        print('maxLength_defect = ', maxLength_rotatedDefect)
        cnt_rotatedDefect = contours_rotatedDefect[k_rotatedDefect]
        dst2_rotatedDefect = rotatedDefectImage.copy()
        cv2.drawContours(dst2_rotatedDefect, [cnt], 0, (255, 0, 0), 3)

        # After Find Contour, Find Smallest area BOX.
        # The pixel number of BOX is saved at box_defect
        rect_rotatedDefect = cv2.minAreaRect(cnt_rotatedDefect)
        box_rotatedDefect = cv2.boxPoints(rect_rotatedDefect)
        box_rotatedDefect = np.int32(box_rotatedDefect)
        dst4_rotatedDefect = dst2_rotatedDefect.copy()
        cv2.drawContours(dst4_rotatedDefect, [box_rotatedDefect], 0, (0, 0, 255), 2)
        print('Box point of rotated Image : ', box_rotatedDefect)
        cv2.imshow('Box point for rotated Image ', rotatedDefectImageCut)

        # This image has 4096 X 2160

        XDirectionPixelDefect = []
        YDirectionPixelDefect = []
        XDirectionPixelNormal = []
        YDirectionPixelNormal = []

        XDirectionPixelDefect.append((min([box_rotatedDefect[2][1] - whiteScreen , box_rotatedDefect[3][1] - whiteScreen])
                                     *YPixelOfInputimage2/480,
                                     max([box_rotatedDefect[3][1] + whiteScreen , box_rotatedDefect[0][1] + whiteScreen])
                                     *YPixelOfInputimage2/480))

        YDirectionPixelDefect.append((min([box_rotatedDefect[1][0] - whiteScreen , box_rotatedDefect[0][0] - whiteScreen])
                                      *XPixelOfInputimage2/640,
                                      max([box_rotatedDefect[2][0] + whiteScreen , box_rotatedDefect[3][0] + whiteScreen])
                                      *XPixelOfInputimage2/640))

        XDirectionPixelNormal.append((min([box[2][1] - whiteScreen , box[3][1] - whiteScreen])*YPixelOfInputimage2/480,
                                     max([box[3][1] + whiteScreen , box[0][1] + whiteScreen])*YPixelOfInputimage2/480))

        YDirectionPixelNormal.append((min([box[1][0] - whiteScreen , box[0][0] - whiteScreen])*XPixelOfInputimage2/640,
                                      max([box[2][0] + whiteScreen , box[3][0] + whiteScreen])*XPixelOfInputimage2/640))

        print('min and max value for Defect Image : ', XDirectionPixelDefect, YDirectionPixelDefect)
        print('min and max value for Normal Image : ', XDirectionPixelNormal, YDirectionPixelNormal)

        print('round number = ', int(round(XDirectionPixelDefect[0][0])),round(XDirectionPixelDefect[0][1]),
                                 round(YDirectionPixelDefect[0][0]),round(YDirectionPixelDefect[0][1]), 'For Defect')

        print('round number = ', int(round(XDirectionPixelNormal[0][0])),int(round(XDirectionPixelNormal[0][1])),
                                 int(round(YDirectionPixelNormal[0][0])),int(round(YDirectionPixelNormal[0][1])), 'For normal')

        CutRotatedDefectImage = rotatedDefectImageCut[int(round(XDirectionPixelDefect[0][0])):int(round(XDirectionPixelDefect[0][1])),
                                                      int(round(YDirectionPixelDefect[0][0])):int(round(YDirectionPixelDefect[0][1]))]

        CutNormalImage = normalImageForCut[int(round(XDirectionPixelNormal[0][0])):int(round(XDirectionPixelNormal[0][1])),
                                           int(round(YDirectionPixelNormal[0][0])):int(round(YDirectionPixelNormal[0][1]))]

        rowsDefect, colsDefect, channelsDefect = CutRotatedDefectImage.shape
        rowsNormal, colsNormal, channelsNormal = CutNormalImage.shape

        print('Pixel Size for images : ', rowsDefect , ' X', colsDefect, ' and channels : ', channelsDefect, 'For Defect')
        print('Pixel Size for images : ', rowsNormal , ' X', colsNormal, ' and channels : ', channelsNormal, 'For Normal')

        cv2.imwrite('./img/rotatedDefectImage.png', CutRotatedDefectImage)
        cv2.imwrite('./img/rotatedNormalImage.png', CutNormalImage)

        cv2.waitKey(0)
        cv2.destroyAllWindows()


#!/usr/bin/env python

import os

from setuptools import setup, find_packages

from minode import shared

README = open(os.path.join(
    os.path.abspath(os.path.dirname(__file__)), 'README.md')).read()

name, version = shared.user_agent.strip(b'/').split(b':')

setup(
    name=name.decode('utf-8'),
    version=version.decode('utf-8'),
    description='Python 3 implementation of the Bitmessage protocol.'
    ' Designed only to route objects inside the network.',
    long_description=README,
    license='MIT',
    author='Krzysztof Oziomek',
    url='https://github.com/g1itch/MiNode',
    packages=find_packages(),
    package_data={'': ['*.csv', 'tls/*.pem']},
    entry_points={'console_scripts': ['minode = minode.main:main']},
    classifiers=[
        "License :: OSI Approved :: MIT License"
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 3",
        "Topic :: Internet",
        "Topic :: Security :: Cryptography",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
)


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
import statsmodels.formula.api as sm
from sklearn.metrics import r2_score

b_high = 4609.9295
optimal_budget_high = (b_high/2)**2
b_low = 3054.9224
optimal_budget_low = (b_low/2)**2

low_train = pd.read_csv('low_train.csv').drop('index', 1).drop('Opening Weekend', 1).drop('Worldwide Gross', 1).drop('Domestic Gross', 1)
low_test = pd.read_csv('low_final_test.csv').drop('index', 1).drop('Opening Weekend', 1).drop('Worldwide Gross', 1).drop('Domestic Gross', 1)
high_train = pd.read_csv('high_train.csv').drop('index', 1).drop('Opening Weekend', 1).drop('Worldwide Gross', 1).drop('Domestic Gross', 1)
high_test = pd.read_csv('high_final_test.csv').drop('index', 1).drop('Opening Weekend', 1).drop('Worldwide Gross', 1).drop('Domestic Gross', 1)

low_train['Theatre Count exp'] = np.exp(low_train['Theater count']/1000)
low_test['Theatre Count exp'] = np.exp(low_test['Theater count']/1000)
high_train['Theatre Count exp'] = np.exp(high_train['Theater count']/1000)
high_test['Theatre Count exp'] = np.exp(high_test['Theater count']/1000)

# Split into gains and losses
low_train_neg = low_train[low_train['Previous Gross'] < 0]
low_train_pos = low_train[low_train['Previous Gross'] > 0]
low_test_neg = low_test[low_test['Previous Gross'] < 0]
low_test_pos = low_test[low_test['Previous Gross'] > 0]
high_train_neg = high_train[high_train['Previous Gross'] < 0]
high_train_pos = high_train[high_train['Previous Gross'] > 0]
high_test_neg = high_test[high_test['Previous Gross'] < 0]
high_test_pos = high_test[high_test['Previous Gross'] > 0]

X_high_train_pos = high_train_pos.drop('Budget', 1).drop('Theater count', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_high_train_neg = high_train_neg.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_high_train_neg['Previous Gross'] = abs(X_high_train_neg['Previous Gross'])
Y_high_train_pos = abs(high_train_pos['Budget'] - optimal_budget_high)
Y_high_train_neg = abs(high_train_neg['Budget'] - optimal_budget_high)
X_high_test_pos = high_test_pos.drop('Budget', 1).drop('Theater count', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_high_test_neg = high_test_neg.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_high_test_neg['Previous Gross'] = abs(X_high_test_neg['Previous Gross'])
Y_high_test_pos = abs(high_test_pos['Budget'] - optimal_budget_high)
Y_high_test_neg = abs(high_test_neg['Budget'] - optimal_budget_high)

X_low_train_pos = low_train_pos.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_low_train_neg = low_train_neg.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_low_train_neg['Previous Gross'] = abs(X_low_train_neg['Previous Gross'])
Y_low_train_pos = abs(low_train_pos['Budget'] - optimal_budget_low)
Y_low_train_neg = abs(low_train_neg['Budget'] - optimal_budget_low)
X_low_test_pos = low_test_pos.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_low_test_neg = low_test_neg.drop('Budget', 1).drop('Theater count', 1).drop('Spring', 1).drop('Unrated', 1).\
    drop('R', 1).drop('Short', 1)
X_low_test_neg['Previous Gross'] = abs(X_low_test_neg['Previous Gross'])
Y_low_test_pos = abs(low_test_pos['Budget'] - optimal_budget_low)
Y_low_test_neg = abs(low_test_neg['Budget'] - optimal_budget_low)

model = sm.OLS(Y_low_train_pos, X_low_train_pos)
results = model.fit()
y_predict = results.predict(X_low_test_pos)
r2 = r2_score(Y_low_test_pos, y_predict)
adj_r2 = 1-((1-r2)*((len(Y_low_test_pos)-1)/(len(Y_low_test_pos)-len(X_low_test_pos.columns)-1)))
print(results.summary())
print(r2, adj_r2)


# (C) Copyright 2015-2016 Hewlett Packard Enterprise Development Company LP

from monasca_agent.collector.checks import AgentCheck
from os.path import abspath
from os.path import exists
from os.path import join

from os import access
from os import stat
from os import walk

from os import X_OK

import logging

log = logging.getLogger(__name__)

class DirectoryCheck(AgentCheck):

    """This check is for monitoring and reporting metrics on the provided directory

    WARNING: the user/group that mon-agent runs as must have access to stat
    the files in the desired directory

    Config options:
        "directory" - string, the directory to gather stats for. required
    """

    def check(self, instance):
        if "directory" not in instance:
            error_message = 'DirectoryCheck: missing "directory" in config'
            log.error(error_message)
            raise Exception(error_message)

        directory = instance["directory"]
        abs_directory = abspath(directory)

        if not exists(abs_directory):
            error_message = "DirectoryCheck: the directory (%s) does not " \
                            "exist" % abs_directory
            log.error(error_message)
            raise Exception(error_message)

        dimensions = self._set_dimensions({"path": directory}, instance)
        self._get_stats(abs_directory, dimensions)

    def _get_stats(self, directory_name, dimensions):
        directory_bytes = 0
        directory_files = 0
        for root, dirs, files in walk(directory_name):
            for directory in dirs:
                directory_root = join(root, directory)
                if not access(directory_root, X_OK):
                    log.warn("DirectoryCheck: could not access directory {}".
                             format(directory_root))
            for filename in files:
                filename = join(root, filename)
                try:
                    file_stat = stat(filename)
                except OSError as ose:
                    log.warn("DirectoryCheck: could not stat file %s - %s" %
                             (filename, ose))
                else:
                    directory_files += 1
                    directory_bytes += file_stat.st_size

        # number of files
        self.gauge("directory.files_count", directory_files,
                   dimensions=dimensions)
        # total file size
        self.gauge("directory.size_bytes", directory_bytes,
                   dimensions=dimensions)
        log.debug("DirectoryCheck: Directory {0} size {1} bytes with {2} "
                  "files in it.".format(directory_name, directory_bytes,
                                        directory_files))


#
# @lc app=leetcode.cn id=494 lang=python3
#
# [494] 目标和
#

# @lc code=start
class Solution:
    def findTargetSumWays(self, nums: List[int], S: int) -> int:
        # --------- 回溯法
        # self.result = 0
        # if len(nums) == 0:
        #     return 0
        # self.backtrack(nums, 0, S)
        # return self.result

        # --------- 子集划分
        # sum(a) - sum(b) = target
        # sum(a) = target + sum(b)
        # sum(a) + sum(a) = target + sum(b) + sum(a)
        # sum(a) = (target + sum(nums))/2
        SUMS = sum(nums) + S
        if SUMS % 2 != 0 or sum(nums) < S:
            return 0
        sums = SUMS // 2

        return self.subsets(nums, sums)

    def subsets(self, nums, sums):
        n = len(nums)
        # dp[i][j] i个物品时，j的容量能否装下
        dp = [[0 for j in range(sums + 1)] for i in range(n + 1)]
        for i in range(len(dp)):
            dp[i][0] = 1

        for i in range(1, n + 1):
            for j in range(sums + 1):
                if j >= nums[i - 1]:
                    dp[i][j] = dp[i - 1][j] + dp[i - 1][j - nums[i - 1]]
                else:
                    dp[i][j] = dp[i - 1][j]
        return dp[n][sums]

    def backtrack(self, nums, i, rest):
        if i == len(nums):
            if rest == 0:
                self.result += 1
            return

        # 选择+
        rest -= nums[i]
        self.backtrack(nums, i + 1, rest)
        rest += nums[i]  # 撤销选择

        # 选择-
        rest += nums[i]
        self.backtrack(nums, i + 1, rest)
        rest -= nums[i]  # 撤销选择

# @lc code=end


import numpy as np
#import pandas as pd
import pickle
#import cv2
from sklearn.svm import SVC
from sklearn.cross_validation import train_test_split
import math

p = open('np_final_version.pickle', 'rb')
np_final_v = pickle.load(p)
p.close()

p = open('feature_vector.pickle', 'rb')
feature_vector = pickle.load(p)
p.close()

#p = open('/home/youssef/Downloads/Machine_Learning/Support_Vector_Machine/gender_classification/Feature_Vector_Final_CSV/image_path.pickle', 'rb')
#image_path = pickle.load(p)
#p.close()

#p = open('/home/youssef/Downloads/Machine_Learning/Support_Vector_Machine/gender_classification/Feature_Vector_Final_CSV/corrupted_image.pickle', 'rb')
#corrupted_image = pickle.load(p)
#p.close()

#p = open('SVM_3_Linear.pickle', 'rb')
#s = pickle.load(p)
#p.close()

#print(len(corrupted_image))
print(len(feature_vector))
feature_vector = np.array(feature_vector)
np_final_v = np_final_v[:, 3]

l = np_final_v.tolist()
# print(l)
for i in range(0,len(l)):
    if math.isnan(l[i]):
        l[i] = float(-1)

np_final_v = np.array(l)

X_train, X_test, y_train, y_test = train_test_split(feature_vector, np_final_v, test_size=0.2)
print(X_test.shape)
print(y_test.shape)
print(X_train.shape, y_train.shape)

clf = SVC(C=3, gamma='auto', kernel='linear')
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)
print(accuracy*100)


import numpy as np
import os

def read_datasets(subpaths):
    """
    Specify a path and read all subfolders
    read_dataset defines the individual order
    """
    Soldata_container = []
    Iexactdata_container = []
    Iapproxdata_container = []

    for i, subpath in enumerate(subpaths):
        print("Evaluating " + subpath + " data", end='\n\n')
        Iexactdata, Iapproxdata, Soldata = read_dataset(subpath)
        Iexactdata_container.append(Iexactdata)
        Iapproxdata_container.append(Iapproxdata)
        Soldata_container.append(Soldata)

    return np.array(Iexactdata_container), np.array(Iapproxdata_container),\
        np.array(Soldata_container)

def read_dataset(path):
    """
    Read the data from a specific folder;
    Emission exact
    [t, I_exact_E_dir, I_exact_ortho, freq/w, Iw_exact_E_dir, Iw_exact_ortho,
     Int_exact_E_dir, Int_exact_ortho]
    Emission approx
    [t, I_E_dir, I_ortho, freq/w, Iw_E_dir, Iw_ortho, Int_E_dir, Int_ortho]
    Solution
    [t, Solution, paths, electric_field, A_field]

    Parameters
    ----------
    path : String
        Path to the dataset

    Returns
    -------
    Iexactdata : np.ndarray
        Exact current and emission data
    Iapprox : np.ndarray
        Approx. current and emission data (kira & koch formula)
    Solution : np.ndarray
        Full solution density, paths, electric field and vector potential
    """
    filelist = os.listdir(path)
    Soldata = None
    Iexactdata = None
    Iapproxdata = None

    for filename in filelist:
        # Emissions I
        # [t, solution, paths, electric_field, A_field]
        if ('Sol_' in filename):
            print("Reading :", path, filename)
            Soldict = np.load(path + filename)
            Soldata = np.array([Soldict['t'], Soldict['solution'],
                                Soldict['paths'], Soldict['electric_field'],
                                Soldict['A_field']])

        # Emissions Iexact
        # [t, I_exact_E_dir, I_exact_ortho, freq/w, Iw_exact_E_dir,
        # Iw_exact_ortho, Int_exact_E_dir, Int_exact_ortho]
        if ('Iexact_' in filename):
            print("Reading :", path, filename)
            Iexactdata = np.array(np.load(path + filename))

        # Emission approximate
        if ('Iapprox_' in filename):
            print("Reading :", path, filename)
            Iapproxdata = np.array(np.load(path + filename))

    return Iexactdata, Iapproxdata, Soldata


"""Contains functions for combining csv output files of sonochiro and imagej for the Light on Nature project."""

import os

from collections import defaultdict
from glob import glob

def combine_sonochiro_files():
    """Return all combined sonochiro output files in directory with this name in a dict of lists with filename as key"""
    csv_files = defaultdict(list)
    for csv_file in glob(r"sonochiro_output_files\*.csv"):
        with open(csv_file, 'r') as sc_file:
            filename = os.path.split(csv_file)[1]
            for line in sc_file:
                csv_files[filename].append(line)
    return dict(csv_files)

def combine_imagej_files():
    """Return all combined csv imagej output files in directory with that name in a list.
    Rows consist of original filename + the value in the area column from imagej.
    """
    csv_files = []
    for csv_file in glob(r"imagej_output_files\*.csv"):
        with open(csv_file, 'r') as ij_file:
            filename = os.path.split(csv_file)[1]
            for line in ij_file:
                if line == ' ,Area,Mean,Min,Max\n':  # if it's a header
                    continue
                area = line.split(",")[1]  # take the area entry
                csv_files.append([filename, area])
    return csv_files


from encoder.params_model import model_embedding_size as speaker_embedding_size
from utils.argutils import print_args
from synthesizer.inference import Synthesizer
from encoder import inference as encoder
from vocoder import inference as vocoder
from pathlib import Path
import numpy as np
import librosa
import traceback

class VoiceCloner:
    def __init__(self):
        # Info & args
        enc_model_fpath = Path("encoder/saved_models/pretrained.pt")

        syn_model_dir = Path("synthesizer/saved_models/logs-pretrained/")
        voc_model_fpath = Path("vocoder/saved_models/pretrained/pretrained.pt")
        low_mem = False

        ## Load the models one by one.
        print("Preparing the encoder, the synthesizer and the vocoder...")
        encoder.load_model(enc_model_fpath)
        self.synthesizer = Synthesizer(syn_model_dir.joinpath("taco_pretrained"), low_mem=low_mem)
        vocoder.load_model(voc_model_fpath)

    def gen_audio(self, ref_audio, text):
        try:
            in_fpath = Path(ref_audio.replace("\"", "").replace("\'", ""))

            ## Computing the embedding
            # First, we load the wav using the function that the speaker encoder provides. This is
            # important: there is preprocessing that must be applied.

            # The following two methods are equivalent:
            # - Directly load from the filepath:
            # preprocessed_wav = encoder.preprocess_wav(in_fpath)
            # - If the wav is already loaded:
            original_wav, sampling_rate = librosa.load(in_fpath, sr=None)
            preprocessed_wav = encoder.preprocess_wav(original_wav, sampling_rate)
            print("Loaded file succesfully, rate=%s" % sampling_rate)

            # Then we derive the embedding. There are many functions and parameters that the
            # speaker encoder interfaces. These are mostly for in-depth research. You will typically
            # only use this function (with its default parameters):
            embed = encoder.embed_utterance(preprocessed_wav)
            print("Created the embedding")

            ## Generating the spectrogram
            # The synthesizer works in batch, so you need to put your data in a list or numpy array
            texts = [text]
            embeds = [embed]
            # If you know what the attention layer alignments are, you can retrieve them here by
            # passing return_alignments=True
            specs = self.synthesizer.synthesize_spectrograms(texts, embeds)
            spec = specs[0]
            print("Created the mel spectrogram")

            ## Generating the waveform
            print("Synthesizing the waveform:")
            # Synthesizing the waveform is fairly straightforward. Remember that the longer the
            # spectrogram, the more time-efficient the vocoder.
            generated_wav = vocoder.infer_waveform(spec)

            ## Post-generation
            # There's a bug with sounddevice that makes the audio cut one second earlier, so we
            # pad it.
            # generated_wav = np.pad(generated_wav, (0, self.synthesizer.sample_rate), mode="constant")
            print("\n samples = %s @ %s" % (len(generated_wav), self.synthesizer.sample_rate))

            return generated_wav

        except Exception as e:
            traceback.print_exc()
            print("Caught exception: %s" % repr(e))
            print("Restarting\n")


__author__ = 'tnorwood'

import logging
import zmq.green as zmq
import gevent
from feed import Feed
from zmqjsonrpc.api_proxy import APIProxy

log = logging.getLogger(__name__)
if not log.handlers:
    log.setLevel(logging.DEBUG)
    handler = logging.StreamHandler()
    handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s | %(name)s | %(levelname)s | %(message)s', '%H:%M:%S')
    handler.setFormatter(formatter)
    log.addHandler(handler)

SUBTYPE_LAST = 0
SUBTYPE_MID = 1

class FeedServerProxy:
    def __init__(self, zmq_context, feed_server_ip='tcp://127.0.0.1', feed_sever_data_port='6010', feed_server_api_port='6011'):
        self._context = zmq_context
        self._feed_server_ip = feed_server_ip
        self._feed_server_data_port = feed_sever_data_port
        self._feed_server_api_port = feed_server_api_port
        self._sub_filters = []
        self._feeds = {}
        self._create_sub_socket()
        self._inbound_response_greenlet = gevent.spawn(self._receive_inbound)

    def _create_sub_socket(self):
        self._sub_socket = self._context.socket(zmq.SUB)
        self._sub_socket.set_hwm(10000)
        self._sub_socket.connect(self._feed_server_ip + ":" + self._feed_server_data_port)

    def subscribe_to_market_data(self, asset_id, subscriber_id, callback, sub_type=SUBTYPE_MID):
        self._subscribe_to_feed(asset_id, subscriber_id, callback, sub_type)
        if asset_id not in self._sub_filters:
            self._request_feed_from_server(asset_id)

    def _subscribe_to_feed(self, asset_id, subscriber_id, callback, sub_type=SUBTYPE_MID):
        if asset_id in self._feeds:
            f = self._feeds[asset_id]
        else:
            f = Feed(asset_id, 5)
            self._feeds[asset_id] = f
        if sub_type == SUBTYPE_MID:
            f.subscribe_to_mid_updates(subscriber_id, callback)
        elif sub_type == SUBTYPE_LAST:
            f.subscribe_to_last_updates(subscriber_id, callback)

    def _request_feed_from_server(self, asset_id):
        self._sub_filters.append(asset_id)
        self._sub_socket.setsockopt(zmq.SUBSCRIBE, str(asset_id))
        proxy = APIProxy(self._context, self._feed_server_api_port, self._feed_server_ip)
        result = proxy.api_call("subscribe_to_market_data", {'asset_id': asset_id})
        proxy.close()

    def _receive_inbound(self):
        while True:
            asset_id = int(self._sub_socket.recv())
            update = self._sub_socket.recv_json()
            if asset_id in self._feeds:
                self._feeds[asset_id].process_update(update)
            gevent.sleep()

    def _shutdown(self):
        self._sub_socket.close()

    def join_child_greenlets(self):
        self._inbound_response_greenlet.join()


# -*- coding: utf-8 -*-
from odoo import api, fields, models
from datetime import date
from datetime import datetime
from datetime import *
import datetime
from odoo.tools.float_utils import float_round
from odoo.exceptions import UserError, ValidationError

class VacacionesOps(models.Model):
    _inherit = 'hr.contract'

    emple_perma = fields.Boolean(string="¿Empleado Permanente?")
    fecha_ingreso = fields.Date(string='Fecha Ingreso Real', help="Fecha de ingreso en la cual el empleado inicio con la empresa", required=True)
    #Actualizacion de la h a H en las horas
    hora_contractual = fields.Selection([('4', '4 Horas'),('5', '5 Horas'), ('6', '6 Horas'), ('8', '8 Horas'),('Permanente', 'Permanente')], string='Hora Contractual', required=True, default='4')

    #FUNCION QUE SE EJECUTARA UNA VEZ AL MES. A FINAL DE MES SE EJECUTARA
    @api.multi
    def validacion_vacaciones(self):
        estado = "open"
        perma = True
        ma = 0
        mos =  []
        zk_attendance = self.env['hr.leave.allocation']
        stage = self.env['hr.contract'].search([('state', '=', estado), ('emple_perma', '=', perma)])
        conf_val = self.env['res.config.settings'].search([], order='id desc', limit=1)
        leave_allocations = self.env['hr.leave.type'].search([('id', '=', conf_val.holiday_status_id.id)])
        #Fecha actual
        ha = datetime.datetime.now()
        fina = datetime.date(ha.year, ha.month, ha.day)
        for eta in stage:
            # Calculamos la diferencia de los días
            dias_totales = (fina - eta['fecha_ingreso']).days
            total = 365 / dias_totales
            to2 = float_round(total, precision_digits=2)
            if dias_totales > 365 and dias_totales < 730:
               #primer = 10 * to2
               for leave_allocation in leave_allocations:
                                zk_attendance.create({
                                        'name': leave_allocation.name,
                                        'holiday_status_id':leave_allocation.id,
                                        'number_of_days':0.83,
                                        'state': 'validate',
                                        'holiday_type': 'employee',
                                        'employee_id': eta['employee_id'].id})
            #Empleado Mayor a 2 años
            if dias_totales > 730 and dias_totales < 1095:
               for leave_allocation in leave_allocations:
                                zk_attendance.create({
                                        'name': leave_allocation.name,
                                        'holiday_status_id':leave_allocation.id,
                                        'number_of_days':1,
                                        'state': 'validate',
                                        'holiday_type': 'employee',
                                        'employee_id': eta['employee_id'].id})
            #Empleado Mayor a 3 años
            if dias_totales > 1095 and dias_totales < 1460:
               for leave_allocation in leave_allocations:
                                zk_attendance.create({
                                        'name': leave_allocation.name,
                                        'holiday_status_id':leave_allocation.id,
                                        'number_of_days':1.25,
                                        'state': 'validate',
                                        'holiday_type': 'employee',
                                        'employee_id': eta['employee_id'].id})
            #Empleado Mayor a 4 años
            if dias_totales > 1460:
               for leave_allocation in leave_allocations:
                                zk_attendance.create({
                                        'name': leave_allocation.name,
                                        'holiday_status_id':leave_allocation.id,
                                        'number_of_days':1.66,
                                        'state': 'validate',
                                        'holiday_type': 'employee',
                                        'employee_id': eta['employee_id'].id})

        #raise ValidationError(mos)

    @api.multi
    def validacion_vacaciones2(self):
        estado = "open"
        perma = True
        ma = 0
        mos =  []
        zk_attendance = self.env['hr.leave.allocation']
        stage = self.env['hr.contract'].search([('state', '=', estado), ('emple_perma', '=', perma)])
        conf_val = self.env['res.config.settings'].search([], order='id desc', limit=1)
        leave_allocations = self.env['hr.leave.type'].search([('id', '=', conf_val.holiday_status_id.id)])
        #for eta in stage:
        #Fecha actual  - fecha ingreso
        hasta = datetime.datetime.now()
        final = datetime.date(hasta.year, hasta.month, hasta.day)
        #to = float_round(porce_dias, precision_digits=2)

        raise ValidationError(final)


import requests
from bs4 import BeautifulSoup

c_url='https://new.syu.ac.kr/academic/scholarship-information/scholarship-notice/'

def crawl(c_url):
    r=requests.get(c_url)
    c=r.content
    soup=BeautifulSoup(c,'html.parser')

    nameList=soup.find('div',{'class':'md_notice_bx'}).findAll('span',{'class':'tit'})

    lee=''
    for name in nameList:
        print(name.get_text())
        lee=lee+name.get_text()
        lee= lee+'<br>'

    return lee

han=crawl(c_url)
print('end')
print(han)

크롤링/삼육대 크롤링.py


from __future__ import print_function
import sys
import time
from connection_utils import connect_to_central
from binascii import hexlify
from struct import unpack

central = connect_to_central()
if len(sys.argv) < 4:
    print("Error MUST specify destination file!")
    sys.exit()

file_handle = 1
start_time = time.time()
obj = central.bulk_transfer.get(file_handle)
duration = time.time() - start_time
print("downloaded %d bytes, in %.1f seconds" %(len(obj), duration))
central.bulk_transfer.erase(file_handle)
with open(sys.argv[3], 'wb') as f:
    f.write(obj)
    f.close()

central.disconnect()
time.sleep(.1)


### This app is for updating an order  list based on the uploaded image . The steps are
### 1- uploading an image , 2 - classifying it or predicting the image type as sunflower/pizza/dollar bill/dalmation/soccer ball.
###  3- If correctly predicted as the original uploaded image type , update the order  list .

from flask import Flask, render_template, request
import pickle as p
import numpy as np
from werkzeug.utils import secure_filename
import skimage
from sklearn.utils import Bunch
from skimage.io import imread
from skimage.transform import resize
from datetime import date
import sqlite3 as sql

app = Flask(__name__)

@app.route('/')
def features():
	return render_template('Upload_order.html')

@app.route('/classify_and_update' , methods =  ['GET' , 'POST'])
def classify_and_update():
	if request.method == 'POST':

## read in the image and format it for model prediction.
		images = []
		flat_data = []
		target = []
		f = request.files['file']
		f.save(secure_filename(f.filename))
		img = skimage.io.imread(f.filename)
		img_resized = resize(img, output_shape=(64, 64), anti_aliasing=True, mode='reflect')
		flat_data.append(img_resized.flatten())
		images.append(img_resized)
##
## get the image type from the form.
		form_data = request.form.to_dict()
		form_data = list(form_data.values())
		form_data = list(map(int, form_data))
		target.append(form_data[0])
## get the quantity
		quantity = form_data[1]
##
		image_dataset = Bunch(data = flat_data , target = target , target_names = 'sunflower' , images = images , DESCR = 'A pic for image classification')
		X_test = image_dataset.data
		y_test = image_dataset.target
##
## predict the image type
		predicted = model.predict(X_test)
		if predicted == 0:
			result = 'Dollar_bill'
		elif predicted == 1:
			result = 'Dalmation'
		elif predicted == 2:
			result = 'Soccer_ball'
		elif predicted == 3:
			result = 'Pizza'
		elif predicted ==4:
			result = 'Sunflower'
## check if the predicted is same as the original image type.
		if predicted == y_test:
			result_all = result + ' -Correctly Classified'
		else:
			result_all = result + ' -Incorrectly Classified'
## get the current date for inventory date
		order_date = date.today()
## insert a new record into inventory if correctly classified.
		if predicted == y_test:
			try:
				with sql.connect("database.db") as con:
					cur = con.cursor()
					cur.execute("INSERT into Order_list (name,quantity,date) VALUES(?,?,?)",(result, quantity, order_date))
					con.commit
					msg = "Record successfully added"
			except:
				con.rollback()
				msg = 'Error in insert '
			finally:
				con.close()
			result_all = result_all + "." + msg

		return render_template("image_classification_prediction.html", result = result_all)
## If the user want to list the inventory
@app.route('/list')
def get_list():
   con = sql.connect("database.db")
   con.row_factory = sql.Row

   cur = con.cursor()
   cur.execute("select * from Order_list")

   rows = cur.fetchall();
   return render_template("list_order.html",rows = rows)

if __name__ == '__main__':
	modelfile = 'Images_classification_model.sav'
	model = p.load(open(modelfile, 'rb'))
	print("model loaded")
	app.run(debug = True,host = '0.0.0.0' ,  port = 5000 )


# -*- coding: utf-8 -*-
from functools import reduce
from sklearn.metrics.pairwise import cosine_similarity
import os,copy
import sys,json
import string
from nltk.stem.lancaster import LancasterStemmer
from nltk.tokenize import word_tokenize
sys.path.append("..")
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from utils import preprocess_sen_new,load_json,split_Uppercase,remove_stopwords,dump_json
import numpy as np
def get_all_class_path(dir_parh):
    count_name_dict={}
    class_name_list={}
#    class_index_name={}
    class_count=0
    path_list=[]
    for root,j,k in os.walk(dir_parh):
        root=root.replace('\\',"/")
        for k_each in k:
#            print("k_each: ",root,root[-1],k_each)
            k_each=k_each.replace('\\',"/")
            if len(k_each.split("."))>2:
                continue
            class_name_list[k_each.split(".")[0]]=class_count
            count_name_dict[class_count]=k_each.split(".")[0]
#            class_index_name[]=
            class_count=class_count+1
            if root[-1]=="/":
                path_list.append(root+k_each)
#                print("path: ",root+k_each)
            else:
                path_list.append(root+"/"+k_each)
#                print("path: ",root+"/"+k_each)
    return  path_list,class_name_list,count_name_dict
'''
类信息：类名和类描述

每个方法API信息：方法描述，形参描述，返回值描述，形参名和方法名

is_signature  是否加入形参名，方法名
'''
def extrac_class_des(path,is_class_name_des=True,is_method_des=False,is_para_des=False,is_return_des=False,is_signature=False) :
#    print("path",path)

    json_data = load_json(path)
    class_all_des=""
    if is_class_name_des:
        class_name=split_Uppercase(json_data["class_name"])
        class_all_des=class_all_des+class_name+" "+json_data["class_description"]
    if is_method_des:#对每个方法提取相应的单元
        method_list=json_data["Methods"]
        for i,method in enumerate(method_list):
            class_all_des=class_all_des+" "+method["method_description"]+" "+split_Uppercase(method["method_name"])
            if is_return_des:
#                print("len: ",method['return_value']['return_description'])
                if len(method['return_value']['return_description'])>0:
                    class_all_des=class_all_des+" "+method['return_value']['return_description'][0]
            if is_para_des:
                if len(method['params'])>0:
                    for i,para in enumerate(method['params']):
                        class_all_des=class_all_des+" "+para["param_description"]
                        if is_signature:
                            class_all_des=class_all_des+" "+" ".join(para["param_name"])
#    print(class_all_des)
    return class_all_des
'''
对若干文档，TF-IDF计算
corpus是list，每一个元素是一个文档
'''
def Tfidf(corpus) :
#    print("corpus",corpus)
    vectorizer = CountVectorizer()
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))

    word = vectorizer.get_feature_names() #所有文本的关键字
    weight = tfidf.toarray()              #对应的tfidf矩阵
#    print("word: ",len(word))
#    print("weight: ",weight,weight.shape)
    return weight,word
#    sFilePath = './tfidffile'
#    if not os.path.exists(sFilePath) :
#        os.mkdir(sFilePath)
#
#    # 这里将每份文档词语的TF-IDF写入tfidffile文件夹中保存
#    for i in range(len(weight)) :
#　　　　 print(u"--------Writing all the tf-idf in the",i,u" file into ",sFilePath+'/'+string.zfill(i,5)+'.txt',"--------")
#        f = open(sFilePath+'/'+string.zfill(i,5)+'.txt','w+')
#        for j in range(len(word)) :
#            f.write(word[j]+"    "+str(weight[i][j])+"\n")
#        f.close()
def cos_sim(v1, v2):
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
'''
获得每篇文档的top_k个关键词
将所有文档的关键词进行合并
获得每篇文档的向量表示
生成相似度

'''
def get_all_key_words(weight,top_k):
    index_sort=copy.deepcopy(np.argsort(weight, axis=1)[:,::-1][:,:top_k])
    print(weight.shape)
    sigle_words_index=list(set(index_sort.flatten()))
#    sigle_words_index=list(set(reduce(lambda x,y: x+y, index_sort)))
#    for
    modify_weight=weight[:,sigle_words_index]
    print(modify_weight.shape)
    return modify_weight
#计算两个类的余弦相似矩阵
def cos_two_matrixs(matrix1,matrix2):
    import scipy.spatial as sp
    return 1 - sp.distance.cdist(matrix1, matrix2, 'cosine')
def modify_cos_by_keywords(cos_matrix,index_filter):
    return new_cos_matrix
#获得最相近的k个类
def get_topk_sim_class(top_k_class,sim_matrix):
    index_sort=copy.deepcopy(np.argsort(sim_matrix, axis=1)[:,::-1][:,:top_k_class])
    all_class_cos=[]
    for i,sim in enumerate(sim_matrix):
        cos_list=list(map(lambda x:sim[x] ,index_sort[i]))
        all_class_cos.append(cos_list)
    return index_sort,all_class_cos
'''
weight 每一行是一个文档   每一行的内容是每个词的tf-idf
index_sort 为每个类提取k个最大的tf-idf值
java_swift_words 存放重叠词的下标
java_swift_num 存放重叠的词的数目
java_swift_cos #存储向量的cos近似度

java_swift_map_index 相似度从大到小存储 每个java类对应的swift的类 下标
java_swift_map_words_index 按相似度从大到小存储 每个java类对应的swift的类的重叠词的下标 每个元素是一个列表 列表的每个元素

1.为每个文档提取k个关键词 index_sort(按tf-idf从大到小)
2.计算重叠的词的数目和重叠词的余弦值
这里的weight是java和swift的文档混合，java_num是分界点
for 每个java文档
    for 每个swift文档
        不同类的相交的关键词的数目
        不同类相交的关键词的向量列表
'''
def extract_keywords(top_k,java_weight,swift_weight):
#    index_sort=np.argsort(weight, axis=1)[::-1][:top_k]
#    print("java_weight.shape: ",java_weight.shape)
    swift_class_num=swift_weight.shape[0]
    java_class_num=java_weight.shape[0]
    index_java_sort=copy.deepcopy(np.argsort(java_weight, axis=1)[:,::-1][:,:top_k])
    index_swift_sort=copy.deepcopy(np.argsort(swift_weight, axis=1)[:,::-1][:,:top_k])
#    print("index_java_sort: ",index_java_sort.shape,index_swift_sort.shape)
    java_swift_num=np.zeros((java_class_num,swift_class_num))#存储重叠的词汇数目
    java_swift_cos=np.zeros((java_class_num,swift_class_num))#存储向量的cos近似度
    java_swift_words=[[] for i in range(len(java_weight))]#存储每个java类和swift类的重叠词汇的下标
    for i,java_class in enumerate(index_java_sort):
        for j,swift_class in enumerate(index_swift_sort):
            index_intersect=list(set(java_class)&set(swift_class))#存储相交的下标，即重叠的词汇
            if len(index_intersect)>0:
                java_swift_words[i].append(index_intersect)
                java_swift_num[i][j]=len(index_intersect)*-1
                java_vec=[java_weight[i][word_index] for word_index in index_intersect]
                swift_vec=[swift_weight[j][word_index] for word_index in index_intersect]
                java_swift_cos[i][j]=cos_sim(java_vec,swift_vec)*-1
            else:
                java_swift_words[i].append([])
                java_swift_num[i][j]=swift_class_num#这里设置成了最大值 ，实际应该是0
                java_swift_cos[i][j]=1#这里设置成了最大值1 ，实际应该是0
    java_swift_map_index=np.zeros((java_class_num,swift_class_num),dtype=int)#每一个元素存储swift的类下标
    java_swift_map_words_index=[]
    for row in range(java_swift_num.shape[0]):
        java_swift_map_index[row] = np.lexsort((java_swift_cos[row],java_swift_num[row]))
#        print("index: ",java_swift_words[row][0])
        java_swift_map_words_index.append( [java_swift_words[row][swift_index] for swift_index in java_swift_map_index[row]])
    return  index_java_sort,java_swift_map_index,java_swift_map_words_index
'''
1. 生成tf-idf的数据集，所有类文件路径
2. 训练tf-idf矩阵
3. 生成java到swift的类映射的排序
    确定每个类需要提取的关键词的数目
    按照关键词的重叠数目
    之后按照重叠关键词的余弦相似度
4. 对java类 提取前k个最相近的类写入文件
    映射的swift的类名+“------------”+关键词的数目 空格分开
'''
if __name__ == "__main__" :
#    (allfile,path) = getFilelist(sys.argv)

    all_path_list=[]
    java_class_name_list={}#存放tf-idf的每一行下标对应的类名  下标索引
    swift_class_name_list={}#存放tf-idf的每一行下标对应的类名  下标索引
    java_count_name_dict={}
    swift_count_name_dict={}
    java_dir_path="../android_api_doc_new/json_3_class_type/"#test
    swift_dir_path="../ios_api_doc/newest/all_2/"
    all_path_list,java_class_name_list,java_count_name_dict=get_all_class_path(java_dir_path)
    java_class_num=len(all_path_list)
    print("java_class_num: ",java_class_num)
    swift_path_list,swift_class_name_list,swift_count_name_dict=get_all_class_path(swift_dir_path)
    dump_json("../data/java_class_index.json",java_class_name_list)
    dump_json("../data/swift_class_index.json",swift_class_name_list)
    sim_matrixs=np.load("../data/tf_idf_class_map.npy")
    print("string map to string: ",sim_matrixs[java_class_name_list["String"]][swift_class_name_list["NSString"]])
    all_path_list.extend(swift_path_list)
    print("all_class_num: ",len(all_path_list))

    #制作文档集
    all_class_corpus=[]#每个元素是一个类
    for i,class_info_path in  enumerate(all_path_list):
        #def extrac_class_des(path,is_class_name_des=True,is_method_des=False,is_para_des=False,is_return_des=False,is_signature=False) :
        sen= extrac_class_des(class_info_path,True,True,True,True,True)
#        print("sen: ",sen)
        sen=preprocess_sen_new(sen,True)
#        print("sen_pre: ",sen)
        lancaster_stemmer = LancasterStemmer()
        input_str=word_tokenize(sen)
        sen=[lancaster_stemmer.stem(value) for value in input_str]
        sen=" ".join(remove_stopwords(sen))
#        print("sen: ",sen)
#        if i>2:
#            break
        all_class_corpus.append(sen)
#    all_class_corpus=["Swift makes it easy to create arrays in your code using an array literal: simply surround a comma-separated list of values with square brackets. Without any other information, Swift creates an array that includes the specified values, automatically inferring the array’s Element type."
 #                      ,"Arrays are one of the most commonly used data types in an app. You use arrays to organize your app’s data. Specifically, you use the Array type to hold elements of a single type, the array’s Element type. An array can store any kind of elements—from integers to strings to classes."]
#    for class_1 in all_class_corpus:
#        print("class: ",class_1)
#        break
    weight,word=Tfidf(all_class_corpus)

    top_k=10
    filter_weight=get_all_key_words(weight,top_k)
    sim_matrixs=cos_two_matrixs(np.array(filter_weight[:java_class_num]),np.array(filter_weight[java_class_num:]))
    np.save("../data/tf_idf_class_map.npy",sim_matrixs)
    map_java_list_name=["String","CharSequence","StringBuffer","StringBuilder","File","FileInputStream","FileOutputStream","ArrayList","LinkedList","Hashtable","HashSet"]

    top_k_class=10
    #获得与java最相近的10个swift 类
    index_sort,all_class_cos=get_topk_sim_class(top_k_class,sim_matrixs)
    #获得最相近的top_k_class的名字
    dir_class_map_path="../data/class_map_all_keywords/"
    java_index_list=list(map(lambda x: java_class_name_list[x],map_java_list_name ))
    swift_class_map_name_list=[]
    for i,each_java in enumerate(java_index_list):
        swift_class_map_name=list(map(lambda x: swift_count_name_dict[x],index_sort[each_java]))
        swift_class_map_name_list.append(swift_class_map_name)
        print("class: ",map_java_list_name[i],swift_class_map_name)
        print("sim_cos: ",all_class_cos[i])
        with open(dir_class_map_path+map_java_list_name[i]+".txt", 'w',encoding='utf-8', errors='ignore') as f_w:
            f_w.write(map_java_list_name[i]+"\n")
            for j in range(len(swift_class_map_name)):
                f_w.write(str(swift_class_map_name[j])+" "+str(all_class_cos[i][j])+"\n")

#            f_w.close()
#    map_java_list_name=java_class_name_list
##    map_java_list_name=["String","CharSequence","StringBuffer","StringBuilder","File","FileInputStream","FileOutputStream","ArrayList","LinkedList","Hashtable","HashSet"]
#    map_java_to_swift=[]
#    for i,value  in enumerate(map_java_list_name):
#        map_java_to_swift.append(list(weight[java_class_name_list[value]]))
#    java_key_words_index,java_swift_map_index,java_swift_map_words_index=extract_keywords(top_k,np.array(map_java_to_swift),np.array(weight[java_class_num:]))
##
#
#    dir_class_map_path="../data/class_map/"
#    like_class_num=20
#    for i,file_name in enumerate(map_java_list_name):
#        print("write file : ",file_name)
#        with open(dir_class_map_path+file_name+".txt", 'w',encoding='utf-8', errors='ignore') as f_w:
#            java_key_words=[]
#            for index in java_key_words_index[i]:
#                java_key_words.append(word[index])
#            f_w.write(" ".join(java_key_words)+"\n")
#            for j in range(like_class_num):
#                swift_index=java_swift_map_index[i][j]
#                key_words_list=[]
#                for key_word_index in java_swift_map_words_index[i][j]:
##                    print("java_swift_map_words_index[i][j]: ",java_swift_map_words_index[i][j],key_word_index)
#                    key_words_list.append(word[key_word_index])
#                f_w.write(swift_count_name_dict[swift_index]+": "+" ".join(key_words_list)+"\n")
##            f_w.close()
##    print("all_path_list: ",all_path_list)


import multiprocessing
import time
from tweepy import Stream
from tweepy import OAuthHandler
from tweepy.streaming import StreamListener

ckey=""
csecret=""
atoken=""
asecret=""

class listener(StreamListener):
    def on_status(self, status):

        try:
            textTweet= status.text+"\n"
            print(textTweet)
            saveFile = open("War.csv","a")
            saveFile.write(textTweet)
            saveFile.write("\n")
            saveFile.close()
            return True

        except BaseException as e:
            print("failed ondata",str(e))
            time.sleep(5)
    def on_error(self,status):
        print(status)

auth=OAuthHandler(ckey,csecret)
auth.set_access_token(atoken,asecret)
twitterStream = Stream(auth,listener())

twitterStream.filter(track=["war"],languages = ['en'])

TwitterProject/Find tweet word hashtag.py


# uncompyle6 version 3.7.4
# Python bytecode 2.5 (62131)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-i686/egg/sunflower/h5attr.py
# Compiled at: 2009-01-12 20:14:59
from __future__ import division, with_statement
__version__ = '$Revision: 192 $'
import sys
from .hdf5 import h5open
from ._utils import progress

def h5attr(h5filenames, set_attrs):
    for h5filename in h5filenames:
        progress(h5filename)
        with h5open(h5filename, 'r+') as (h5file):
            for leaf in h5file.walkNodes(classname='Leaf'):
                for (key, value) in set_attrs.iteritems():
                    setattr(leaf.attrs, key, value)

def parse_options(args):
    from ._optparse import OptionParser
    usage = '%prog [OPTION]... FILE...'
    version = '%%prog %s' % __version__
    parser = OptionParser(usage=usage, version=version)
    parser.add_option('-s', '--set-str', action='update', dest='set_attrs', default={}, metavar='KEY VALUE', help='set KEY=VALUE')
    parser.add_option('-i', '--set-int', action='update', type=int, dest='set_attrs', metavar='KEY VALUE', help='set KEY=VALUE')
    (options, args) = parser.parse_args(args)
    if not len(args) >= 1:
        parser.print_usage()
        sys.exit(1)
    return (options, args)

def main(args=sys.argv[1:]):
    (options, args) = parse_options(args)
    return h5attr(args, options.set_attrs)

if __name__ == '__main__':
    sys.exit(main())


#similarity between two docs
# http://stackoverflow.com/questions/8897593/similarity-between-two-text-documents

from sklearn.feature_extraction.text import TfidfVectorizer
#from numpy import dot, array
#from math import sqrt

##doc1f = open('testdoc1a.txt','r')
##doc1 = doc1f.read()
##doc1f.close()
##
##doc2f = open('testdoc1b_nondescripted.txt','r')
##doc2 = doc2f.read()
##doc2f.close()

#datadir = '/Users/robin/Documents/Thesis_local/corpora/blogs/train70_split/' #data directory
#infile = open('train70.txt','r')
#infile.close()

##for doc in train70docs[:2]:
##    infile = open(datadir + doc[:-1],'r') #[:-1] = no \n
##    text = infile.read().decode('utf-8', 'ignore').split()
##    infile.close()

##train70docs = ['/Users/robin/Documents/Thesis_local/corpora/blogs/train70_split/1752515.female.27.Science.Aries28.txt',\
##'/Users/robin/Documents/Thesis_local/corpora/blogs/train70_split/1752515.female.27.Science.Aries29.txt']
##
##documentsfiles = [open(f) for f in train70docs] #first two
##documents = [f.read() for f in documentsfiles] #first two

def main(doc1, doc2):
    documents = [doc1, doc2]
    tfidf = TfidfVectorizer().fit_transform(documents)
    pairwise_similarity = tfidf * tfidf.T
    return pairwise_similarity

#http://stackoverflow.com/questions/2380394/simple-implementation-of-n-gram-tf-idf-and-cosine-similarity-in-python

##inf = open('train70_freqs_2_smoothed.csv','r')
##f = inf.readlines()[202:204] #first two docs
##inf.close()
##
##uraw = f[0]
##ul = uraw.split(',')[:-1] #ignore line ending
##u = []
##for i in ul:
##    u.append(float(i)) #make strings into floats
##u = array(u) #make into numpy 1D array (9989 x 1)
##
##vraw = f[1]
##vl = vraw.split(',')[:-1]
##v = []
##for i in vl:
##    v.append(float(i))
##v = array(v)
##
##sim = dot(u,v) / (sqrt(dot(u,u))) * (sqrt(dot(v,v)))
##print sim
##


#!/usr/bin/env python3

from gi.repository import GLib

import subprocess
import threading
from datetime import datetime

from nwg_panel.tools import check_key

import gi

gi.require_version('Gtk', '3.0')
gi.require_version('Gdk', '3.0')

from gi.repository import Gtk, Gdk

class Clock(Gtk.EventBox):
    def __init__(self, settings):
        self.settings = settings
        Gtk.EventBox.__init__(self)
        self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL, spacing=0)
        self.add(self.box)
        self.label = Gtk.Label("")

        check_key(settings, "root-css-name", "root-clock")
        check_key(settings, "css-name", "clock")
        check_key(settings, "tooltip-text", "")
        check_key(settings, "on-left-click", "")
        check_key(settings, "on-right-click", "")
        check_key(settings, "on-middle-click", "")
        check_key(settings, "on-scroll-up", "")
        check_key(settings, "on-scroll-down", "")

        check_key(settings, "interval", 1)

        self.set_property("name", settings["root-css-name"])
        self.label.set_property("name", settings["css-name"])

        if settings["tooltip-text"]:
            self.set_tooltip_text(settings["tooltip-text"])

        if "format" not in settings:
            self.settings["format"] = "%a, %d. %b  %H:%M:%S"

        if settings["on-left-click"] or settings["on-right-click"] or settings["on-middle-click"] or settings[
            "on-scroll-up"] or settings["on-scroll-down"]:
            self.connect('button-press-event', self.on_button_press)
            self.add_events(Gdk.EventMask.SCROLL_MASK)
            self.connect('scroll-event', self.on_scroll)

            self.connect('enter-notify-event', self.on_enter_notify_event)
            self.connect('leave-notify-event', self.on_leave_notify_event)

        self.build_box()
        self.refresh()

        if settings["interval"] > 0:
            Gdk.threads_add_timeout_seconds(GLib.PRIORITY_LOW, settings["interval"], self.refresh)

    def update_widget(self, output):
        self.label.set_text(output)

        return False

    def get_output(self):
        now = datetime.now()
        try:
            time = now.strftime(self.settings["format"])
            GLib.idle_add(self.update_widget, time)
        except Exception as e:
            print(e)

    def refresh(self):
        thread = threading.Thread(target=self.get_output)
        thread.daemon = True
        thread.start()
        return True

    def build_box(self):
        self.box.pack_start(self.label, False, False, 4)
        self.label.show()

    def on_enter_notify_event(self, widget, event):
        self.get_style_context().set_state(Gtk.StateFlags.SELECTED)

    def on_leave_notify_event(self, widget, event):
        self.get_style_context().set_state(Gtk.StateFlags.NORMAL)

    def on_button_press(self, widget, event):
        if event.button == 1 and self.settings["on-left-click"]:
            self.launch(self.settings["on-left-click"])
        elif event.button == 2 and self.settings["on-middle-click"]:
            self.launch(self.settings["on-middle-click"])
        elif event.button == 3 and self.settings["on-right-click"]:
            self.launch(self.settings["on-right-click"])

    def on_scroll(self, widget, event):
        if event.direction == Gdk.ScrollDirection.UP and self.settings["on-scroll-up"]:
            self.launch(self.settings["on-scroll-up"])
        elif event.direction == Gdk.ScrollDirection.DOWN and self.settings["on-scroll-up"]:
            self.launch(self.settings["on-scroll-up"])
        else:
            print("No command assigned")

    def launch(self, cmd):
        print("Executing '{}'".format(cmd))
        subprocess.Popen('exec {}'.format(cmd), shell=True)


from typing import Tuple, Optional

from src.internal.adapters.entities.error import Error
from src.internal.biz.dao.base_dao import BaseDao
from src.internal.biz.deserializers.auth_code import AuthCodeDeserializer, DES_AUTH_CODE_FROM_DB_FULL, AUTH_CODE_ID, \
    AUTH_CODE_EDITED_AT
from src.internal.biz.entities.auth_code import AuthCode

class AuthCodeDao(BaseDao):

    async def get_by_account_main_id_code(self, auth_code: AuthCode) -> Tuple[Optional[AuthCode], Optional[Error]]:
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                """
                SELECT
                    id,
                    edited_at
                FROM
                    auth_code
                WHERE
                    account_main_id = $1 AND code = $2
                """, auth_code.account_main.id, auth_code.code
            )

            if not row:
                return None, None
            auth_code.id = row['id']
            auth_code.edited_at = row['edited_at']

            return auth_code, None

    async def remove_by_id(self, auth_code_id: int) -> Tuple[None, Optional[Error]]:
        async with self.pool.acquire() as conn:
            await conn.execute(
                """
                DELETE
                FROM auth_code
                WHERE id = $1
                """, auth_code_id
            )

            return None, None

    async def get_by_account_main_id(self, account_main_id: int) -> Tuple[Optional[AuthCode], Optional[Error]]:
        async with self.pool.acquire() as conn:
            row = await conn.fetchrow(
                f"""
                SELECT
                    id          {AUTH_CODE_ID},
                    edited_at   {AUTH_CODE_EDITED_AT}
                FROM
                    auth_code
                WHERE
                    account_main_id = $1
                """, account_main_id
            )

            if not row:
                return None, None
            return AuthCodeDeserializer.deserialize(row, DES_AUTH_CODE_FROM_DB_FULL), None

    async def update(self, auth_code_id: int, auth_code: AuthCode) -> Tuple[Optional[AuthCode], Optional[Error]]:
        async with self.pool.acquire() as conn:
            await conn.execute(
                """
                UPDATE
                    auth_code
                SET
                    edited_at = CURRENT_TIMESTAMP,
                    code = $1
                """, auth_code.code
            )

            return auth_code, None

    async def add(self, auth_code: AuthCode) -> Tuple[Optional[AuthCode], Optional[Error]]:
        async with self.pool.acquire() as conn:
            await conn.execute(
                """
                INSERT INTO auth_code(account_main_id, code) VALUES ($1, $2);
                """, auth_code.account_main.id, auth_code.code
            )

            return auth_code, None

    async def set_is_confirm(self, is_confirmed: bool) -> Tuple[None, Optional[Error]]:
        async with self.pool.acquire() as conn:
            await conn.execute(
                """
                UPDATE account_main
                SET is_confirmed = $1
                """, is_confirmed
            )
            return None, None


# -*- coding: utf-8 -*-

class BitVector(object):
    """
    Simple structure, that looks like a dict <index>:<0 or 1>, lets to store data as binary array
    without predefined size and load it back.
    """
    def __init__(self, data=None):
        if data is None:
            self._int_data = 0
        elif isinstance(data, int):
            self._int_data = data
        else:
            self._int_data = self._parse_data(data)

    @staticmethod
    def _parse_data(data):
        result = 0
        for i, c in enumerate(reversed(data)):
            result += ord(c) * 256 ** i

        return result

    def dump(self):
        n = self._int_data
        if n > 0:
            arr = []
            while n:
                n, rem = divmod(n, 256)
                arr.append(rem)
            return ''.join(map(chr, reversed(arr)))
        elif n == 0:
            return ''
        else:
            raise ValueError

    def __eq__(self, other):
        return (self.__class__ == other.__class__) and (self._int_data == other._int_data)

    def __ne__(self, other):
        return not self.__eq__(other)

    def __str__(self):
        return self.dump()

    def __getitem__(self, index):
        return (self._int_data >> index) & 1

    def __setitem__(self, index, value):
        if isinstance(value, bool):
            value = int(value)

        if value == 0:
            self._int_data &= ~(1 << index)
        elif value == 1:
            self._int_data |= 1 << index
        else:
            raise ValueError('Invalid value for BitVector')

class WeekDaysVector(BitVector):
    pass

EVERY_DAY_OF_WEEK = WeekDaysVector(127)  # 1111111
WEEK_END_DAYS = WeekDaysVector(65)       # 1000001
WORKING_DAYS = WeekDaysVector(62)        # 0111110


# -*- coding: utf-8 -*-
import importlib

def num2str(number, lang='pl'):
    module = importlib.import_module('.'.join(['langs', lang]))

    if number == 0:
        return module.numbers[0]

    result = ''
    digits = 0
    while number > 0:
        result = module.three_digits_str(number % 1000, digits) + result
        digits += 3
        number //= 1000

    return " ". join(result.split())


# coding: utf-8

#----------------------------------- import -----------------------------------

from argparse import ArgumentParser

#---------------------------------- fonction ----------------------------------

def get_args():
    """Renvoie les arguments passés en ligne de commande.

    Return:
        (tuple): Ensemble d'arguments.
    """
    parser = ArgumentParser(description="Convertit le fichier passé en "
        "argument en page HTML.")

    parser.add_argument('--file','-f',help="Chemin du fichier à traduire",required=True)
    parser.add_argument('--debug','-d',help="Affichage de debugs",required=False,action="store_true")
    # action='store_true' : De base, si aucune valeur n'est proposée, la
    # variable vaut None. Avec cet argument, on idndique qu'on attend pas de
    # valeur, seulement la présence de cette argument, et s'il est là, on
    # enregistre True.

    args = parser.parse_args()

    return args.__dict__

def debug(*args,**kwargs):
    """Fonction de debug."""
    if DEBUG:
        print("[DEBUG]",end=' ')
        print(*args,**kwargs)

def delete_space(txt):
    """Supprime les espaces au début et à la fin d'un string.

    Args:
        txt (str): Chaîne de caractères à simplifier.

    Return:
        (str): Chaîne de caractères.
    """
    res = txt
    c = res[0]

    while c == ' ':
        res = res[1:]
        c = res[0]

    c = res[-1]

    while c == ' ':
        res = res[:-1]
        c = res[-1]

    return res

#----------------------------------- classe -----------------------------------

#------------------------------------ main ------------------------------------

# initialisation
args      = get_args()
path_file = args['file']
DEBUG     = args['debug']


from datetime import datetime
from django.db import models
from rounds.exceptions import *
from rounds.models import Round, RoundTarget

SCORING_VALUE_CHOICES = (
    ('X', 'X'),
    ('10', '10'),
    ('9', '9'),
    ('8', '8'),
    ('7', '7'),
    ('6', '6'),
    ('5', '5'),
    ('4', '4'),
    ('3', '3'),
    ('2', '2'),
    ('1', '1'),
    ('M', 'M'),
)

class Scoresheet(models.Model):
    round = models.ForeignKey(Round, related_name='scoresheets')
    user = models.ForeignKey('auth.User', related_name='scoresheets')
    private = models.BooleanField(default=False)

    start_time = models.DateTimeField(auto_now_add=True)
    end_time = models.DateTimeField(null=True, editable=False)

    def is_live(self):
        return self.end_time is None

    def is_complete(self):
        for round_target in round.targets:
            arrow_count = ScoresheetArrow.objects.count(scoresheet=self, round_target=round_target)
            if arrow_count != round_target.total_arrows():
                return False

        return True

    def mark_complete(self):
        self.end_time = datetime.utcnow()
        self.save()

    def score_arrow(self, round_target, score):
        # can only score arrows if the round is live
        if not self.is_live():
            raise ScoresheetNotLiveError('cannot score arrows unless the scoresheet is live')

        # check the round_target is in the correct round
        if round_target.round != self.round:
            raise RoundTargetMismatchError('the given round_target is in the wrong round')

        arrow_count = ScoresheetArrow.objects.filter(scoresheet=self, round_target=round_target).count()
        if arrow_count >= round_target.total_arrows():
            raise ArrowCountOverflowError('all the arrows on this round_target have been scored')

        arrow = ScoresheetArrow()

        arrow.scoresheet = self
        arrow.round_target = round_target

        arrow.number = arrow_count + 1
        arrow.score = score

        arrow.save()

        if self.is_complete():
            self.mark_complete()

    def amend_arrow(self, round_target, number, score):
        # can only score arrows if the round is live
        if not self.is_live():
            raise ScoresheetNotLiveError('cannot score arrows unless the scoresheet is live')

        # check the round_target is in the correct round
        if round_target.round != self.round:
            raise RoundTargetMismatchError('the given round_target is in the wrong round')

        arrow = ScoresheetArrow.objects.get(scoresheet=self, round_target=round_target, number=number)

        #TODO other validation?

        arrow.score = score
        arrow.save()

    def unscore_arrow(self, round_target):
        # can only score arrows if the round is live
        if not self.is_live():
            raise ScoresheetNotLiveError('cannot score arrows unless the scoresheet is live')

        # check the round_target is in the correct round
        if round_target.round != self.round:
            raise RoundTargetMismatchError('the given round_target is in the wrong round')

        arrow = ScoresheetArrow.objects.filter(scoresheet=self, round_target=round_target).order_by('-number').first()

        #TODO other validation?

        arrow.delete()

    def __str__(self):
        return '%s - %s at %s' % (self.round, self.user, self.start_time)

class ScoresheetArrow(models.Model):
    scoresheet = models.ForeignKey(Scoresheet, related_name='arrows')
    round_target = models.ForeignKey(RoundTarget)

    number = models.IntegerField()
    score = models.CharField(max_length=2, choices=SCORING_VALUE_CHOICES)
    data = models.CharField(max_length=128, null=True, blank=True)
    time = models.DateTimeField(auto_now_add=True)

    def __str__(self):
        return '#%i - %s on %s' % (self.number, self.round_target, self.scoresheet)

    class Meta:
        unique_together = (('scoresheet', 'number'),)
        ordering = ('number',)


from setuptools import setup, find_packages

with open('README.md', 'r') as fi:
    long_description = fi.read()

setup(
    name='tdh-tcd',
    version='2.5.3',

    author='Dmitry Karikh',
    author_email='the.dr.hax@gmail.com',

    description='Twitch Chat Downloader',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/TheDrHax/Twitch-Chat-Downloader',

    install_requires=[
        'requests',
        'progressbar2',
        'iso8601'
    ],

    packages=find_packages(),
    package_data={'tcd': ['example.settings.json']},
    include_package_data=True,

    entry_points='''
    [console_scripts]
    tcd=tcd:main
    ''',

    classifiers=[
        'Environment :: Console',
        'Programming Language :: Python :: 3.6',
        'Intended Audience :: End Users/Desktop',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Topic :: Utilities'
    ]
)


from selenium import  webdriver # 调用selenium模块、webdriver模块
from selenium.webdriver.chrome.options import Options # 调用Option模块
from bs4 import BeautifulSoup
import time

chrome_options = Options() # 实例化Option对象
chrome_options.add_argument('--headless') # 把Chrome设置为静默模式
driver = webdriver.Chrome(options = chrome_options) # 声明浏览器对象
driver.get('https://y.qq.com/n/yqq/song/000xdZuV2LcQ19.html') # 访问页面
time.sleep(2)

button = driver.find_element_by_class_name('js_get_more_hot') # 根据类名找到【点击加载更多】
button.click() # 点击
time.sleep(2) # 等待两秒

comments = driver.find_element_by_class_name('js_hot_list').find_elements_by_class_name('js_cmt_li') # 使用
print(len(comments)) # 打印获取到的评论个数
for comment in comments: # 循环
    sweet = comment.find_element_by_tag_name('p') # 找到评论
    print ('评论：%s\n ---\n'%sweet.text) # 打印评论