from django.shortcuts import render ,redirect,HttpResponse,HttpResponseRedirect
# from django.views.generic import ListView,DetailView

from myapps.models import  Bloods_Group,Contact_us
from django.views.generic import ListView,DetailView, TemplateView
from django.core.mail import send_mail,BadHeaderError
from django.contrib import messages
from myapps.models import Donors_in_Site
from django.core.files.storage import FileSystemStorage
from depend.models import State,Country,District
# Create your views here.

def home(request):
   return render(request,'index.html')

def about_us(request):
   return render(request,'about_us.html')

# def Donor_Registration(request):

#     countryid = request.GET.get('country')

#     stateid = request.GET.get('state')
#     state = None
#     district = None
#     if countryid :
#         getcountry = Country.objects.get(name=countryid)

#         state = State.objects.filter(country=getcountry)
#     if stateid:
#         getstate = State.objects.get(name=stateid)
#         district = District.objects.filter(state=getstate)

#     country = Country.objects.all()
#     bloods= Bloods_Group.objects.all()

#     return render(request, 'Donor_Registration1.html',locals())

def contact_us(request):

    if request.method=='POST':
        name= request.POST.get('name')
        email= request.POST.get('email')
        subject = request.POST.get('subject', )
        message= request.POST.get('message')
        admin='khandokarsamad4@gmail.com'

        if subject and message and email:
           contact=Contact_us(sender_name=name,from_email=email,subject=subject,message=message)
           contact.save()
           send_mail(subject,message,email, [admin,email,name])
           messages.success(request,'Succesfully Send Message!')

        else:
           messages.error(request,'Make sure all fields are entered and valid!')

    return render(request, 'contact.html')

def Donor_Reg_Site(request):
  print("okk")

  if request.method=='POST' :
    country = request.POST.get('country')
    state = request.POST.get('state')
    district = request.POST.get('district')

    if not Country.objects.filter(name=country.title()).exists():
                c=''.join(country)
                c = Country(name=c.title())
                c.save()
                # c=c.capitalize()
                # print(1111111111111111111111)
                # c=Country.objects.create(name=c)

                # c.save()

    if not State.objects.filter(name=state.title()).exists():
                c = Country.objects.get(name=country.title())
                s = State(name=state.title(), country=c)

                s.save()
            #    s=s.capitalize()
            #    c=c.capitalize()
            #    print(22222222222222222)
            #    c = Country.objects.get(name=c)
            #    obj = State(country=c,name=s)

            #    obj.save()
    if not District.objects.filter(name=district.title()).exists():
                c = State.objects.get(name=state.title())
                s = District(name=district.title(), state=c )

                s.save()
    #            d=''.join(district)
    #            s=''.join(state)
    #            d=d.capitalize()
    #            s=s.capitalize()
    #            obj=District.objects.create(state=s,name=d)
    #            obj=obj.capitalize()
    #            obj.save()
    try:
        instance=Donors_in_Site.objects.get(user=request.user)

        if instance:
            instance.user=request.user
            phone = request.POST['phone']
            phone_str = ''.join(phone)
            instance.phone = phone_str

            blood = request.POST.get('blood'),
            blood_str = ''.join(blood)
            instance.blood=blood_str

            dob=request.POST['dob'],
            dob_str=''.join(dob)
            instance.Last_Donate = dob_str

            country = request.POST.get('country'),
            country_str=''.join(country)
            country_str=country_str.title()
            instance.country = country_str

            state = request.POST.get('state'),
            state_str=''.join(state)
            state_str=state_str.title()
            instance.state = state_str

            district = request.POST.get('district'),
            district_str=''.join(district)
            district_str=district_str.title()
            instance.district = district_str

            if len(request.FILES) == 0:
                print('no files')
            else:
                _, file = request.FILES.popitem()
                file = file[0]
                instance.image = file
                instance.save()

            instance.save()

            messages.success(request,'Data has been updated!')
            return render(request, 'index.html')
    except:
        en=Donors_in_Site.objects.create(
        user=request.user,
        phone = request.POST['phone'],
        blood = request.POST.get('blood' ),
        Last_Donate = request.POST['dob'],

        country = request.POST.get('country').title(),

        state = request.POST.get('state').title(),
        district = request.POST.get('district').title()
          )
        if len(request.FILES) == 0:
            print('no files')
        else:
            _, file = request.FILES.popitem()
            file = file[0]
            en.image = file
            en.save()

        print("okk")
        messages.success(request,'Data has been submitted!')
        return render(request, 'index.html')

  else:
        messages.error(request,'something mistake.please Reloade and try again!!')
        return render(request, 'dependantfield.html')

# Create your views here.

class donor(DetailView):
      model=Donors_in_Site
      template_name='loginapp/profile2.html'
      context_object_name='user'


import scrapy
from datetime import datetime
from scrapy.exceptions import CloseSpider
from scrapy.loader import ItemLoader

from tudo_celular.items.smartphone import SmartphoneItem

class FichasTecnicasSpider(scrapy.Spider):
    name = "fichas_tecnicas"
    allowed_domains = ["www.tudocelular.com"]
    url = "https://www.tudocelular.com"
    page = 1

    def start_requests(self):
        yield scrapy.Request(url=f"{self.url}/celulares/fichas-tecnicas.html", callback=self.parse)

    def parse(self, response):
        if response.status == 302:
            raise CloseSpider("No more pages scrapy will be closed...")

        for phone in response.xpath('//article[@class="phonelist_item"]'):
            item = ItemLoader(item=SmartphoneItem())
            item.add_value('Brand', phone.xpath('div[2]/a/h4/strong/text()').get())
            item.add_value('Model', phone.xpath('div[2]/a/h4/text()').get())
            item.add_value('Image', phone.css('img').attrib['style'])
            item.add_value('Score', phone.xpath('div[1]/div/strong/text()').get())
            item.add_value('Price', phone.xpath('div[2]/div/a[1]/span[2]/text()').get())
            item.add_value('Url', self.url + phone.xpath('a[@class="pic"]').attrib['href'])
            item.add_value('Extracted_at', datetime.now().isoformat())

            # Phone details
            details_url = self.url + phone.xpath('a[@class="pic"]').attrib['href']
            yield scrapy.Request(details_url, callback=self.parse_details, meta={'item': item})

        # Paginate
        self.page += 1
        next_page = f"{self.url}/celulares/fichas-tecnicas_{self.page}.html"
        yield response.follow(url=next_page, callback=self.parse)

    def parse_details(self, response):
        item = response.meta['item']
        details = {}
        sections = response.xpath('//div[@class="row_titles"]/h3/strong/text()').getall()
        sections.insert(0, "BÁSICO")
        titles = response.xpath('//div[@class="row_titles"]/ul[@class="phone_column_features"]')
        values = response.xpath('//div[@class="phone_column"]/ul[@class="phone_column_features"]')

        # Block to handle values
        for i in range(len(sections)):
            details[sections[i]] = {}
            # Block to handle values that have <i> and <b> tags
            for title, value in zip(titles[i].xpath('li'), values[i].xpath('li')):
                v = ''
                # Handle icon
                if value.xpath('i[@class]'):
                    v = value.xpath('i[@class]').attrib['class'] + ' '
                # Handle bold
                elif value.xpath('b/text()'):
                    v += value.xpath('b/text()').get() + ' '
                # Handle price
                elif value.xpath('a/b/text()'):
                    v += value.xpath('a/b/text()').get() + ' '

                # Handle text
                if value.xpath('text()'):
                    v += value.xpath('text()').get()

                details[sections[i]].update({
                    title.xpath('text()').get(): v
                })

        item.add_value('Details', details)
        yield item.load_item()


'''
    Escreva uma função que receba dois parâmetros: uma tupla com dois números e uma string. De acordo com a string
    recebida, e com a Tabela, a função deve realizar diferentes operações com os números e retornar o resultado.
    Obs. No caso de a string de entrada ser diferente das strings da Tabela,
    a função deve retornar uma string com a seguinte mensagem: A operação informada não foi reconhecida.

    SOMA = soma de dois números
    MULT = multiplicação de dois números
    DIV = divisão de dois números
    SUB = subtração de dois números
    MOD = retorna o resto da divisão do primeiro pelo segundo
    POT = retorna o primeiro elevado pelo segundo
'''

def operacao(t):
    ''' Recebe uma tupla com dois números (x,y) e uma operação, retorna o resultado da operação
    '''
    x = t[0]
    y = t[1]
    op = t[2]
    #print(x,y, op.lower())

    operacao = op.lower()

    if (operacao == 'soma'):
        result = x + y
    elif (operacao == 'mult'):
        result = x * y
    elif (operacao == 'div'):
        result = x / y
    elif (operacao == 'sub'):
        result = x - y
    elif (operacao == 'mod'):
        result = x % y
    elif (operacao == 'pot'):
        result = x ** y
    else:
        result = 'A operação informada não foi reconhecida!'

    print(result)

t = (1, 2, 'POTAGE')
operacao(t)


import io
import os
import sys
import mmap
import time
import math

def getbit(value,position):
    return (value >> position)&1

def setbit(value,position,bitvaule):
    if bitvaule == 0:
        mask = 0xFFFFFFFF ^ (1 << position)
        return value & mask
    else:
        mask = 1 << position
        return value | mask

def __get_phy_buf_addr():
    fd = open('/sys/class/u-dma-buf/phy_buf/phys_addr','r')
    phy_addr = int(fd.read().rstrip(),16)
    fd.close()
    return phy_addr

def __get_phy_buf_size():
    fd = open('/sys/class/u-dma-buf/phy_buf/size','r')
    size = int(fd.read().rstrip())
    fd.close()
    return size

def addr_cal(addr,length):
    mem_span = 2**math.ceil(math.log(length,2))
    mem_start = addr-(addr%mem_span)
    if addr%mem_span == 0:
        mem_end = mem_start+mem_span
    else:
        mem_end = mem_start+2*mem_span
    page_start = math.floor(addr/mmap.PAGESIZE)*mmap.PAGESIZE
    page_end = math.ceil((addr+length)/mmap.PAGESIZE)*mmap.PAGESIZE
    mem_start = max(mem_start,page_start)
    mem_end = min(mem_end,page_end)
    mem_span = mem_end-mem_start
    trans_start = addr-mem_start
    trans_end = trans_start + length
    return [mem_start, mem_span, trans_start, trans_end]

system_buf_addr=__get_phy_buf_addr()
system_buf_size=__get_phy_buf_size()

class axilite:
    size = -1
    mem = None
    fd = -1
    def __init__(self,addr=0,size=0x1000):
        if addr%mmap.PAGESIZE != 0:
            raise Exception('The address you want to map is not page aligned, exiting!')
        if size%mmap.PAGESIZE != 0:
            raise Exception('Minimal allocating unit is one PAGE, exiting!')
        try:
            self.fd=os.open('/dev/mem',os.O_RDWR | os.O_SYNC)
        except IOError:
            raise Exception('Unable to open /dev/mem, maybe try with sudo?')
        self.size = size
        self.mem = mmap.mmap(self.fd,size,flags=mmap.MAP_SHARED,prot=mmap.PROT_READ|mmap.PROT_WRITE,offset=addr)
    def __read(self, length, offset):
        if offset+length > self.size:
            print('Trying to read beyond the buffer edge')
            return 1
        mem_start, mem_span, trans_start, trans_end = addr_cal(offset, length)
        self.mem.seek(mem_start)
        raw_bytes=self.mem.read(mem_span)
        return raw_bytes[trans_start:trans_end]
    def __write(self, raw_bytes, offset):
        length=len(raw_bytes)
        if offset+length > self.size:
            print('Trying to write beyond the buffer edge')
            return 1
        mem_start, mem_span, trans_start, trans_end = addr_cal(offset, length)
        if mem_span > mmap.PAGESIZE:
            tmp_buf = bytearray(mem_span)
            tmp_buf[0:mmap.PAGESIZE] = self.__read(mmap.PAGESIZE,mem_start)
            tmp_buf[mem_span-mmap.PAGESIZE:mem_span] = self.__read(mmap.PAGESIZE,mem_start+mem_span-mmap.PAGESIZE)
        else:
            tmp_buf = bytearray(self.__read(mem_span,mem_start))
        tmp_buf[trans_start:trans_end] = raw_bytes
        self.mem.seek(mem_start)
        self.mem.write(tmp_buf)
        return 0
    def read32(self, offset=0):
        return int.from_bytes(self.__read(4,offset),sys.byteorder)
    def read64(self, offset=0):
        return int.from_bytes(self.__read(8,offset),sys.byteorder)
    def read(self, length, offset=0):
        return self.__read(length,offset)
    def write32(self, value, offset=0):
        raw_bytes = value.to_bytes(4,sys.byteorder)
        return self.__write(raw_bytes,offset)
    def write64(self, value, offset=0):
        raw_bytes = value.to_bytes(8,sys.byteorder)
        return self.__write(raw_bytes,offset)
    def write(self, raw_bytes, offset=0):
        return self.__write(raw_bytes, offset)
    def clean(self):
        if self.mem is not None:
            self.mem.close()
            self.mem = None
        if self.fd > 0:
            os.close(self.fd)
            self.fd = -1

class phy_buf:
    size = system_buf_size
    mem = None
    mem_fd = -1
    phy_addr = system_buf_addr
    offset = 0
    def __init__(self,offset=0,size=system_buf_size):
        if size+offset > system_buf_size:
            raise Exception('System has in total '+ str(system_buf_size) +' bytes reserve memory, requsting buffer is beyond that edge, exiting!')
        if offset%mmap.PAGESIZE != 0:
            raise Exception('The address you want to allocate is not page aligned, exiting!')
        if size%mmap.PAGESIZE != 0:
            raise Exception('Minimal allocating unit is one PAGE, exiting!')
        if not os.path.exists('/dev/phy_buf'):
            raise Exception('Physical buffer not exist, exiting!')
        try:
            self.mem_fd = os.open('/dev/phy_buf', os.O_RDWR | os.O_SYNC)
        except IOError:
            raise Exception('Unable to open /dev/phy_buf!')
        self.phy_addr = system_buf_addr+offset
        self.size = size
        self.mem = mmap.mmap(self.mem_fd,size,flags=mmap.MAP_SHARED,prot=mmap.PROT_READ|mmap.PROT_WRITE,offset=offset)
    def read(self, length, offset=0):
        if offset+length > self.size:
            print('Trying to read beyond the buffer edge')
            return 1
        mem_start, mem_span, trans_start, trans_end = addr_cal(offset, length)
        self.mem.seek(mem_start)
        raw_bytes=self.mem.read(mem_span)
        return raw_bytes[trans_start:trans_end]
    def write(self, raw_bytes, offset=0):
        length=len(raw_bytes)
        if offset+length > self.size:
            print('Trying to write beyond the buffer edge')
            return 1
        mem_start, mem_span, trans_start, trans_end = addr_cal(offset, length)
        if mem_span > mmap.PAGESIZE:
            tmp_buf = bytearray(mem_span)
            tmp_buf[0:mmap.PAGESIZE] = self.read(mmap.PAGESIZE,mem_start)
            tmp_buf[mem_span-mmap.PAGESIZE:mem_span] = self.read(mmap.PAGESIZE,mem_start+mem_span-mmap.PAGESIZE)
        else:
            tmp_buf = bytearray(self.read(mem_span,mem_start))
        tmp_buf[trans_start:trans_end] = raw_bytes
        self.mem.seek(mem_start)
        self.mem.write(tmp_buf)
        return 0
    def clean(self):
        if self.mem is not None:
            self.mem.close()
            self.mem = None
        if self.mem_fd > 0:
            os.close(self.mem_fd)
            self.mem_fd = -1

class axidma:
    axil = None
    phy_buf = None
    bufsize = system_buf_size
    dma_bufsize = 2**26
    ps_phy_addr = system_buf_addr
    def __init__(self,axil_offset=0,buf_size=system_buf_size,buf_offset=0,dma_buflen=26):
        self.axil = axilite(axil_offset)
        self.phy_buf = phy_buf(buf_offset,buf_size)
        self.bufsize = buf_size
        self.dma_bufsize = 2**dma_buflen
        self.ps_phy_addr = self.phy_buf.phy_addr
    def buf_write(self,raw_bytes,offset=0):
        return self.phy_buf.write(raw_bytes,offset)
    def buf_read(self,size,offset=0):
        return self.phy_buf.read(size,offset)
    def __mm2s(self,addr,size,timeout,sync):
        CR_ADDR = 0x0
        SR_ADDR = 0x4
        SA_ADDR = 0x18
        LEN_ADDR = 0x28
        for i in range(timeout):
            isIdle = getbit(self.axil.read32(SR_ADDR),0) or getbit(self.axil.read32(SR_ADDR),1) #halt or idle
            if isIdle == 1:
                break
            else:
                time.sleep(0.001)
            if i == timeout-1:
                print('mm2s time out!')
                return 1
        oldCtrlVal=self.axil.read32(CR_ADDR)
        newCtrlVal=setbit(oldCtrlVal,0,1)
        self.axil.write32(newCtrlVal,CR_ADDR) #start
        self.axil.write64(addr,SA_ADDR) #Address
        self.axil.write32(size,LEN_ADDR) #Length
        newCtrlVal=setbit(newCtrlVal,0,0)
        self.axil.write32(newCtrlVal,CR_ADDR) #set start to 0
        if sync:
            for i in range(timeout):
                isIdle = getbit(self.axil.read32(SR_ADDR),0) or getbit(self.axil.read32(SR_ADDR),1) #halt or idle
                if isIdle == 1:
                    break
                else:
                    time.sleep(0.001)
                if i == timeout-1:
                    print('mm2s time out!')
                    return 1
        return 0
    def __s2mm(self,addr,size,timeout,sync):
        CR_ADDR = 0x30
        SR_ADDR = 0x34
        DA_ADDR = 0x48
        LEN_ADDR = 0x58
        for i in range(timeout):
            isIdle = getbit(self.axil.read32(SR_ADDR),0) or getbit(self.axil.read32(SR_ADDR),1) #halt or idle
            if isIdle == 1:
                break
            else:
                time.sleep(0.001)
            if i == timeout-1:
                print('s2mm time out!')
                return 1
        oldCtrlVal=self.axil.read32(CR_ADDR)
        newCtrlVal=setbit(oldCtrlVal,0,1)
        self.axil.write32(newCtrlVal,CR_ADDR) #start
        self.axil.write64(addr,DA_ADDR) #Address
        self.axil.write32(size,LEN_ADDR)
        newCtrlVal=setbit(newCtrlVal,0,0)
        self.axil.write32(newCtrlVal,CR_ADDR) #set start to 0
        if sync:
            for i in range(timeout):
                isIdle = getbit(self.axil.read32(SR_ADDR),0) or getbit(self.axil.read32(SR_ADDR),1) #halt or idle
                if isIdle == 1:
                    break
                else:
                    time.sleep(0.001)
                if i == timeout-1:
                    print('s2mm time out!')
                    return 1
        return 0
    def mm2s(self,size,offset=0,timeout=2000,sync=False):
        if size+offset > self.bufsize:
            print("Trying to access beyond the allocated buffer area, exiting!")
            return 1
        #1.Data->Phy_mem in PS DDR
        #2.AXIDMA in PL <- Phy_mem in PS DDR (can be multiple)
        #    2 happens when 1 is completely done
        #This implementation works, but it's not the best performance,
        #    because 2 and 1 can be pipelined
        #If PS PL bandwidth is the bottleneck for the app,
        #    pingpong ram and pipeline can be used to optimize in the future
        for loc in range(0,size,self.dma_bufsize):
            curr_loc = self.ps_phy_addr + offset + loc
            size_bytes_to_go = min(self.dma_bufsize,size-loc)
            if self.__mm2s(curr_loc,size_bytes_to_go,timeout,sync):
                return 1
        return 0
    def s2mm(self,size,offset=0,timeout=2000,sync=False):
        if size+offset > self.bufsize:
            print("Trying to access beyond the allocated buffer area, exiting!")
            return 1
        for loc in range(0,size,self.dma_bufsize):
            curr_loc = self.ps_phy_addr + offset + loc
            size_bytes_to_come = min(self.dma_bufsize,size-loc)
            if self.__s2mm(curr_loc,size_bytes_to_come,timeout,sync):
                return 1
        return 0
    def clean(self):
        if self.phy_buf is not None:
            self.phy_buf.clean()
            self.phy_buf = None
        if self.axil is not None:
            self.axil.clean()
            self.axil = None

class axicdma:
    axil = None
    dma_bufsize = 2**23
    def __init__(self,axil_offset=0):
        self.axil = axilite(axil_offset)
    def __movedata(self,src_addr,dst_addr,size,timeout,sync):
        CR_ADDR = 0x0
        SR_ADDR = 0x4
        SA_ADDR = 0x18
        DA_ADDR = 0x20
        LEN_ADDR = 0x28
        for i in range(timeout):
            isIdle = getbit(self.axil.read32(SR_ADDR),1) #idle
            if isIdle == 1:
                break
            else:
                time.sleep(0.001)
            if i == timeout-1:
                print('CDMA time out!')
                return 1
        cr_value = self.axil.read32(CR_ADDR)
        new_cr_value = setbit(cr_value,12,0)
        self.axil.write32(new_cr_value,CR_ADDR) #control signal
        self.axil.write64(src_addr,SA_ADDR) #source address
        self.axil.write64(dst_addr,DA_ADDR) #destination address
        self.axil.write32(size,LEN_ADDR) #transfer size
        if sync:
            for i in range(timeout):
                isIdle = getbit(self.axil.read32(SR_ADDR),1) #idle
                if isIdle == 1:
                    break
                else:
                    time.sleep(0.001)
                if i == timeout-1:
                    print('CDMA time out!')
                    return 1
        sr_value = self.axil.read32(SR_ADDR)
        new_sr_value = setbit(sr_value,12,1)
        self.axil.write32(new_sr_value,SR_ADDR)
        return 0
    def movedata(self,src_addr,dst_addr,size,timeout=2000,sync=False):
        for loc in range(0,size,self.dma_bufsize):
            curr_loc_src = src_addr + loc
            curr_loc_dst = dst_addr + loc
            size_transfer = min(self.dma_bufsize,size-loc)
            if self.__movedata(curr_loc_src,curr_loc_dst,size_transfer,timeout,sync):
                return 1
        return 0
    def clean(self):
        if self.axil is not None:
            self.axil.clean()
            self.axil = None

class fpgamem:
    cdma = None
    phy_buf = None
    ps_base_addr = system_buf_addr
    pl_base_addr = 0
    def __init__(self,cdma_offset=0,ps_buf_offset=0,ps_buf_size=system_buf_size,fpgamem_mapbase=0):
        self.cdma = axicdma(cdma_offset)
        self.phy_buf = phy_buf(ps_buf_offset,ps_buf_size)
        self.ps_base_addr = system_buf_addr + ps_buf_offset
        self.pl_base_addr = fpgamem_mapbase
    def buf_write(self,raw_bytes,offset=0):
        return self.phy_buf.write(raw_bytes,offset)
    def buf_read(self,size,offset=0):
        return self.phy_buf.read(size,offset)
    def get(self,pl_offset=0,size=4096,sync=False):
        return self.cdma.movedata(self.pl_base_addr+pl_offset,self.ps_base_addr,size,sync)
    def put(self,pl_offset=0,size=4096,sync=False):
        return self.cdma.movedata(self.ps_base_addr,self.pl_base_addr+pl_offset,size,sync)
    def file2mem(self,f,pl_offset=0,size=-1):
        if isinstance(f,io.BufferedIOBase):
            fd=f
        elif isinstance(f,str):
            fd=open(f,'rb')
        else:
            print('ERROR: input file should be in binary format')
            return 1
        if size == -1:
            remain_size = 0x7ffff000 #size limit
        else:
            remain_size = size
        pl_addr=pl_offset
        while remain_size > 0:
            bytes_to_read = min(self.phy_buf.size,remain_size)
            remain_size -= bytes_to_read
            raw_bytes = fd.read(bytes_to_read)
            if len(raw_bytes) == 0:
                break
            self.buf_write(raw_bytes)
            self.put(pl_addr,len(raw_bytes),True)
            pl_addr+=bytes_to_read
        if isinstance(f,str):
            fd.close()
    def mem2file(self,f,pl_offset=0,size=4096):
        if isinstance(f,io.BufferedIOBase):
            fd=f
        elif isinstance(f,str):
            fd=open(f,'wb')
        else:
            print('ERROR: output file should be in binary format')
            return 1
        remain_size = size
        pl_addr=pl_offset
        while remain_size > 0:
            bytes_to_read = min(self.phy_buf.size,remain_size)
            remain_size -= bytes_to_read
            self.get(pl_addr,bytes_to_read,True)
            raw_bytes = self.buf_read(bytes_to_read)
            fd.write(raw_bytes)
            pl_addr+=bytes_to_read
            if len(raw_bytes) != bytes_to_read:
                break
        if isinstance(f,str):
            fd.close()
    def clean(self):
        if self.cdma is not None:
            self.cdma.clean()
            self.cdma = None
        if self.phy_buf is not None:
            self.phy_buf.clean()
            self.phy_buf = None


import matplotlib.pyplot as plt
import numpy as np

def linked_circles(radii,speeds,phases,time=1,n=1001,draw=True):

    t_vals = np.linspace(0,time*2*np.pi,n)

    X,Y = [],[]

    for t in t_vals:
        x = 0
        y = 0
        for R,S,P in zip(radii,speeds,phases):
            x += R*np.sin(t*S+(P*2*np.pi))
            y += R*np.cos(t*S+(P*2*np.pi))
        X.append(x)
        Y.append(y)

    if draw == True:

        fig = plt.figure()
        fig.set_size_inches(12,12)
        ax = plt.axes()
        ax.set_aspect("equal","datalim")
        ax.axis('off')
        ax.set_xticks([])
        ax.set_yticks([])

        plt.plot(X,Y)

    return X,Y

# Doesn't work at all really
def fourier_approxiation(C,num_circles,time=1,draw=True):

    t_vals = np.linspace(0,2*np.pi,len(C))

    def calc_coef(n,C):
        cn = C*np.exp(-1j*2*n*np.pi*t_vals)
        return sum(cn)/len(cn)

    coefs = [calc_coef(N,C) for N in range(1,num_circles+1)]

    R = np.absolute(coefs)
    S = np.array(range(1,num_circles+1))
    P = np.angle(coefs)

    print(R)
    print(S)
    print(P)

    linked_circles(R,S,P,time,len(C),draw=draw)

if __name__ == '__main__':

#    from Conversions import xy_to_complex
#    from ParametricPolygon import regular_polygon

    num_cos = 5
    R = [n*2-1 for n in range(num_cos,0,-1)]
    S = [(n+1)*2-1 for n in range(0,num_cos,1)]
    P = [0]*num_cos
    x,y = linked_circles(R,S,P,1,n=2001)

    num_cos = 4
    R = [(n*2)-1 for n in range(num_cos,0,-1)]
    S = [(n*2)-1 for n in range(0,num_cos,1)]
    P = [0]*num_cos
    x,y = linked_circles(R,S,P,1,n=2001)

    num_cos = 6
    R = [5**n for n in range(0,num_cos,1)]
    S = [6**n for n in range(num_cos,0,-1)]
    P = [0]*num_cos
    x,y = linked_circles(R,S,P,.5,n=4001)


from skmultiflow.evaluation import EvaluatePrequential
from skmultiflow.data import UnsupervisedStream

from models.ae import AE
from models.model import Model
from monitoring.monitoring import BaseMonitor
from data.dataset import Dataset
from config import Configuration

if __name__ == '__main__':

    config = Configuration()
    config.parse_commandline()
    config.new_experiment()

    X = Dataset()[config.CHANNEL]  # already segmented and overlapped!
    stream = UnsupervisedStream(X)
    stream.prepare_for_use()

    # ae = self.clf  (issue #5: Make these tests as classes)
    ae = Model(
        AE(
            # architecture hyperparameters
            n_input=1,
            n_hidden=config.n_hidden,
            n_output=1,
            n_layers=config.n_layers,
        )
    )

    monitor = BaseMonitor(
        ae,
        # learning hyperparameters
        # temperature,
        # learning_rate,
        # weight_decay
    )

    evaluator = EvaluatePrequential(
        n_wait=50,
        pretrain_size=2000,
        batch_size=1,
        metrics=[
            'true_vs_predicted',
            'mean_square_error',
            'mean_absolute_error'
        ],
        output_file=config.EXPERIMENT_PERSISTENCE + '/prequential.npy',
        show_plot=True
    )

    evaluator.evaluate(stream=stream, model=monitor)


from datetime import datetime
import json
import pandas as pd
import sys

OUT_DIR = "metadata/"

def datify(u):
    return datetime.fromtimestamp(u).strftime('%Y-%m-%d') if u else "Indefinite"

def mapper(data):

    round_mapping = {
        "0x1b165fe4da6bc58ab8370ddc763d367d29f50ef0": {
            "name": "Climate Solutions",
            "color": "0AAD72"
        },
        "0xd95a1969c41112cee9a2c931e849bcef36a16f4c": {
            "name" : "Open Source Software",
            "color": "00A9B7"
        },
        "0xe575282b376e3c9886779a841a2510f1dd8c2ce4": {
            "name": "Ethereum Infrastructure",
            "color": "FCB53B"
        }
    }

    work_start_date  = 1663819200
    impact_end_date  = 0
    collection       = "Gitcoin Alpha Round"
    allowlist        = "ipfs://bafkreiaxdog4clqiitnarc4rrzpgdlcjsg6k2nr2n2t4thwklccza34ubi"
    default_image    = "ipfs://bafkreicchjbpbb2hfcg5mtmlz3zktf2wt5dnux2rzx33ta7b6bhrozlbgi"

    app_data         = data['application']
    project_data     = app_data['project']
    answer_data      = app_data['answers']

    project_address  = app_data['recipient']
    project_name     = project_data['title']
    project_descr    = project_data['description']
    project_url      = project_data['website']

    project_date     = int(str(project_data.get('createdAt', '1673829248'))[:10])
    project_icon     = "ipfs://" + project_data.get('logoImg', 'bafkreiejljnf6xf6kwcvh3wjef5xa3n7gscdumrmurmt4otkozbx5524r4')
    project_banner   = "ipfs://" + project_data.get('bannerImg', 'bafkreigkmcufguhakp4nbucca6d2rt7nw7ourdnkqfbs2gvsue4j4ohsly')
    project_github   = project_data.get('projectGithub')
    project_twitter  = project_data.get('projectTwitter')

    project_funding  = answer_data[1].get('answer')
    project_teamsize = answer_data[2].get('answer')

    round_contract   = app_data['round']
    round_name       = round_mapping[round_contract]['name']
    #bg_color         = round_mapping[round_contract]['color']

    return {
        "name": project_name,
        "description": project_descr,
        "external_url": project_url,
        "image": default_image,
        #"background_color": bg_color,
        "properties": {
            "impact_scope": {
                "name": "Impact Scope",
                "value": [round_name],
                "display_value": round_name
            },
            "work_scope": {
                "name": "Work Scope",
                "value": [project_name],
                "display_value": project_name
            },
            "work_timeframe": {
                "name": "Work Timeframe",
                "value": [work_start_date, project_date],
                "display_value": f"{datify(work_start_date)} → {datify(project_date)}"
            },
            "impact_timeframe": {
                "name": "Impact Timeframe",
                "value": [project_date, impact_end_date],
                "display_value": f"{datify(project_date)} → {datify(impact_end_date)}"
            },
            "contributors": {
                "name": "Contributors",
                "value": project_address,
                "display_value": project_address
            },
            "rights": {
                "name": "Rights",
                "value": ["public-display", "-transfers"],
                "display_value": "Public display"
            },
        },
        "hidden_properties": {
            "version": "1.0.0",
            "collection": collection,
            "icon": project_icon,
            "banner": project_banner,
            "allowlist": allowlist
        }
    }

def get_metadata(data):
    try:
        return mapper(data)
    except Exception as e:
        print(data)
        print(e)
        return None

def parse_csv(csv_path, out_dir):
    df = pd.read_csv(csv_path)

    for _, row in df.iterrows():
        project_name = row['title']
        metadata = get_metadata(eval(row['ipfs_data']))

        filename = project_name.replace("/","-")
        filename = filename if filename[0] != '.' else filename[1:]
        out_path = f"{out_dir}/{filename}.json"
        out_file = open(out_path, "w")
        json.dump(metadata, out_file, indent=4)

        out_file.close()

if __name__ == "__main__":

    csv_path = sys.argv[1]
    out_dir = sys.argv[2] if len(sys.argv) == 3 else OUT_DIR
    parse_csv(csv_path, out_dir)


from __future__ import absolute_import

import os

from nose.tools import eq_

from aws.s3 import s3file
from aws.s3.s3test_utils import S3TestBase

QUOTE_EN = 'a journey of a thousand miles begins with a single step'
QUOTE_CH = u'千里之行，始於足下'

class S3FileTest(S3TestBase):
  def test_basic_read(self):
    path = self.get_test_path('test_basic_read.txt')
    key = self.get_key(path)
    with self.cleaning(path):
      key.set_contents_from_string(QUOTE_EN)
      eq_(QUOTE_EN, s3file.open(key, 'r').read())
      eq_(QUOTE_EN[:4], s3file.open(key, 'r').read(length=4))

  def test_unicode_read(self):
    path = self.get_test_path('test_unicode_read.txt')
    key = self.get_key(path)
    with self.cleaning(path):
      key.set_contents_from_string(QUOTE_CH)
      eq_(QUOTE_CH.encode('utf-8'), s3file.open(key, 'r').read())
      eq_(QUOTE_CH.encode('utf-8')[:4], s3file.open(key, 'r').read(length=4))

  def test_seek(self):
    path = self.get_test_path('test_seek.txt')
    key = self.get_key(path)
    with self.cleaning(path):
      key.set_contents_from_string(QUOTE_EN)
      f = s3file.open(key, 'r')
      f.seek(0, os.SEEK_SET)
      eq_(QUOTE_EN[:2], f.read(2))
      f.seek(1, os.SEEK_SET)
      eq_(QUOTE_EN[1:][:2], f.read(2))
      f.seek(-1, os.SEEK_END)
      eq_(QUOTE_EN[-1:], f.read())
      f.seek(0, os.SEEK_SET)
      f.seek(2, os.SEEK_CUR)
      eq_(QUOTE_EN[2:][:2], f.read(2))


# -*- coding: utf-8 -*-

import requests
from lxml import etree

CN_NUM = {u'〇': 0,
          u'一': 1,
          u'二': 2,
          u'三': 3,
          u'四': 4,
          u'五': 5,
          u'六': 6,
          u'七': 7,
          u'八': 8,
          u'九': 9,
          u'零': 0,
          u'壹': 1,
          u'贰': 2,
          u'叁': 3,
          u'肆': 4,
          u'伍': 5,
          u'陆': 6,
          u'柒': 7,
          u'捌': 8,
          u'玖': 9,
          u'貮': 2,
          u'两': 2,
}

CN_UNIT = {
    u'十': 10,
    u'拾': 10,
    u'百': 100,
    u'佰': 100,
    u'千': 1000,
    u'仟': 1000,
    u'万': 10000,
    u'萬': 10000,
    u'亿': 100000000,
    u'億': 100000000,
    u'兆': 1000000000000,
}

def cn2digTransformer(cn):
    lcn = list(cn)
    unit = 0  # 当前的单位
    ldig = []  # 临时数组

    ret = 0
    tmp = 0

    if lcn[-2:] == ['零', '十']:    # “一千零十”
        ret = 10
        lcn = lcn[:-2]

    while lcn:
        cndig = lcn.pop()
        if cndig in CN_UNIT:          # python2: CN_UNIT.has_key(cndig)
            unit = CN_UNIT.get(cndig)
            if unit == 10000:
                ldig.append('w')  # 标示万位
                unit = 1
            elif unit == 100000000:
                ldig.append('y')  # 标示亿位
                unit = 1
            elif unit == 1000000000000:  # 标示兆位
                ldig.append('z')
                unit = 1
            continue
        else:
            dig = CN_NUM.get(cndig)
            if dig==None:
                return False
            else:
                if unit:
                    dig *= unit
                    if len(ldig) == 1 and isinstance(ldig[0], int) and ldig[0] < 10:   # “七百五”“七千五”的“五”不是个位
                        ldig[0] = ldig[0]*unit//10
                    if len(ldig) == 2 and isinstance(ldig[0], int) and ldig[0] < 10:   # “七亿五”“七万五”的“五”不是个位
                        if ldig[-1] == 'w':
                            ldig[0] *= 1000
                        elif ldig[-1] == 'y':
                            ldig[0] *= 10000000
                        elif ldig[-1] == 'z':
                            ldig[0] *= 100000000000
                    unit = 0

                ldig.append(dig)

    if unit == 10:  # 处理10-19的数字
        ldig.append(10)

    while ldig:
        x = ldig.pop()
        if x == 'w':
            tmp *= 10000
            ret += tmp
            tmp = 0
        elif x == 'y':
            tmp *= 100000000
            ret += tmp
            tmp = 0
        elif x == 'z':
            tmp *= 1000000000000
            ret += tmp
            tmp = 0
        else:
            tmp += x
    ret += tmp
    return str(ret)

class Book(object):

    def start(self, name):
        """根据关键词获取搜索页"""
        url = "http://www.xbiquge.la/modules/article/waps.php"
        response = requests.post(url, data={"searchkey": name})
        if response.status_code == 200:
            content = response.content
            html = etree.HTML(content)
            book_list = html.xpath('//table[@class="grid"]/tr')
            book_list2 = []
            print("搜索结果如下")
            for book in book_list:
                b_name = book.xpath('./td[@class="even"]/a/text()')
                if b_name:
                    b_name = b_name[0]
                    b_id = book.xpath('./td[@class="even"]/a/@href')[0]
                    b_author = book .xpath('./td[@class="even"]/text()')[0]
                    book_list2.append((b_name, b_id))
                    print(b_name, b_id, b_author)
                    print("")
            name = input("请从输入以上列表中的书籍名或输入NO退出:")
            if name == "NO" or name == "No" or name == "no" or name == "nO":
                return
            else:
                for b in book_list2:
                    if name == b[0]:
                        b_url = b[1]
                        self.get_page(b_url)

    def get_page(self, link):
        """定位需要开始的章节"""
        s_list = self.get_section(link)

        start_s = int(input("开始章节:"))
        now_s = 1
        for s in s_list:
            t, u = s
            if int(now_s) >= int(start_s):
                self.get_info(t, u, now_s)
                input("回车继续")
            now_s += 1

    def get_section(self, link):
        """获取章节列表"""
        response = requests.get(link)
        if response.status_code == 200:
            content = response.content
            html = etree.HTML(content)
            section_list = html.xpath('//div[@id ="list"]/dl/dd/a/@href')
            title_list = html.xpath('//div[@id="list"]/dl/dd/a/text()')
            a = 1
            for t in title_list:
                print(a, t)
                a += 1
            return zip(title_list, section_list)
        else:
            return []

    def get_info(self, title, link, now_s):
        """获取并输出章节内容"""
        link = "http://www.xbiquge.la" + link
        print(link)
        print(now_s, title)
        response = requests.get(link)
        if response. status_code == 200:
            content = response.content
            html = etree.HTML(content)
            body = html.xpath('//div[@id="content"]/text()')
            for b in body:
                 print(b)

if __name__ == '__main__':
    # book_name = input("书籍名:")
    book_name = "圣光"
    book = Book()
    book.start(book_name)


import random
import time

from PySide2.QtWidgets import *
from functools import partial
from qt_material import apply_stylesheet, list_themes, QtStyleTools
from JasonUI.buttons import DarkerButton

class NineBoxDraw(QWidget):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.order = [(0, 0), (0, 1), (0, 2), (1, 2), (2, 2), (2, 1), (2, 0), (1, 0)]
        self.current_position = 0
        self.buttons = [[QPushButton("奖品") for j in range(3)] for i in range(9)]
        self.buttons[1][1] = DarkerButton()
        self.buttons[1][1].setStyleSheet(self.buttons[1][1].styleSheet() + "\nQPushButton{color: gray;}")
        self.buttons[1][1].setText("抽奖！")
        self.buttons[1][1].setFixedSize(100, 100)
        self.buttons[1][1].clicked.connect(self.draw)
        self.grid = QGridLayout(self)
        for line in range(3):
            for col in range(3):
                self.grid.addWidget(self.buttons[line][col], line, col)
        for (row, col) in self.order:
            self.buttons[row][col].clicked.connect(partial(self.on_text_edit, row, col))
            self.buttons[row][col].setFixedSize(100, 100)

    def on_text_edit(self, row, col):
        d = EditDialog(row, col, self)
        d.show()

    def draw(self):
        index = self.current_position
        row, col = self.order[index]
        self.buttons[row][col].setStyleSheet(self.buttons[row][col].styleSheet().rstrip("\nQPushButton{"
                                                                                "background-color: pink;}"))
        stop_point = random.randint(1, 8)
        for i in range(self.current_position, 50+stop_point):
            index = i % 8;
            row, col = self.order[index]
            button_to_shine = self.buttons[row][col]
            button_to_shine.setStyleSheet(button_to_shine.styleSheet() + "\nQPushButton{background-color: pink;}")
            QApplication.processEvents()
            if 0.2 - i*0.01 >= 0.03:
                time.sleep(0.2 - i*0.01)
            elif 50+stop_point - i <= 12:
                time.sleep(0.03+(12 - 50+stop_point + i)*0.01)
            else:
                time.sleep(0.03)
            if i != 50+stop_point - 1:
                button_to_shine.setStyleSheet(button_to_shine.styleSheet().rstrip("\nQPushButton{background-color: pink;}"))
        self.current_position = i % 8

class EditDialog(QDialog):
    def __init__(self, row, col, parent=None):
        super().__init__(parent)
        self.coordinates = (row, col)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("请输入奖品名称：")
        self.line_edit = QLineEdit(self)
        self.btn = QPushButton("确定")
        self.btn.clicked.connect(self.quit)
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.line_edit)
        self.vbox.addWidget(self.btn)

    def quit(self):
        self.parentWidget().buttons[self.coordinates[0]][self.coordinates[1]].setText(self.line_edit.text())
        self.close()

class SelectBatchDialog(QDialog):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("在这里批量添加奖品名称，一行一个奖品，共八个，从左上角开始顺时针顺序")
        self.text_edit = QTextEdit(self)
        self.submit = QPushButton("确定")
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.text_edit)
        self.vbox.addWidget(self.submit)
        self.submit.clicked.connect(self.submit_names)

    def submit_names(self):
        text = self.text_edit.toPlainText().strip().split('\n')
        for index in range(8):
            w = self.parentWidget().w
            row, col = w.order[index]
            w.buttons[row][col].setText(text[index].strip())
        self.close()

class AboutDialog(QDialog):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.vbox = QVBoxLayout(self)
        self.label = QLabel("这是关于作者、本程序，以及本程序的使用条款")
        self.text_edit = QTextEdit(self)
        self.text_edit.setHtml("
本程序的作者是河北省保定市的一名中学生，"
                               "欢迎通过jasoncoder@qq.com与作者联系，或进行关于编程问题的交流探讨。
"
                               "这个程序受版权保护，不得随意进行修改。未经作者同意，不得商用。
"
                               "这个程序是完全免费且开源的，可以"
                               "在"
                               "https://github.com/Jason-XII/Nine-Box-Draw上查看"
                               "此项目的源代码，以及更为详细的使用说明。
")
        self.text_edit.setReadOnly(True)
        self.submit = QPushButton("我已阅读")
        self.vbox.addWidget(self.label)
        self.vbox.addWidget(self.text_edit)
        self.vbox.addWidget(self.submit)

class MainWindow(QMainWindow, QtStyleTools):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("九宫格抽奖程序")
        self.w = NineBoxDraw(self)
        self.setCentralWidget(self.w)
        self.menu = self.menuBar()
        self.more_menu = self.menu.addMenu("更多……")
        self.add_more = self.more_menu.addAction("批量添加奖品名称")
        self.add_more.triggered.connect(self.select_names)
        self.themes_menu = self.more_menu.addMenu("设置程序外观")
        self.about = self.more_menu.addAction("关于")
        self.about.triggered.connect(self.about_toggled)
        self.ac = []
        t = list_themes() + ['default']
        for theme_name in t:
            act = self.themes_menu.addAction(theme_name)
            act.triggered.connect(partial(self.set_theme, theme_name, t.index(theme_name)))
            act.setCheckable(True)
            self.ac.append(act)
        self.ac[-1].setChecked(True)
        self.checked_index = len(t) - 1

    def select_names(self):
        d = SelectBatchDialog(self)
        d.show()

    def set_theme(self, theme_name, index):
        self.apply_stylesheet(self, theme_name)
        self.ac[self.checked_index].setChecked(False)
        self.checked_index = index

    def about_toggled(self):
        d = AboutDialog(self)
        d.show()

app = QApplication([])
window = MainWindow()

# setup stylesheet
extra = {

    # Button colors
    'danger': '#dc3545',
    'warning': '#ffc107',
    'success': '#17a2b8',
}
apply_stylesheet(app, theme='default', extra=extra)

# run
window.show()
app.exec_()


import re
from collections import defaultdict

data = [i.strip() for i in open("inputs/7.txt").readlines()]

def parse(data):
  bags = defaultdict(dict)
  for line in data:
    bag = re.match(r'(.*) bags contain', line).groups()[0]
    for c, b in re.findall(r'(\d+) (\w+ \w+) bag', line):
      bags[bag][b] = int(c)
  return bags

def solve_p1(bags):
  ret = set()
  def search(colour):
    for i in bags:
      if colour in bags[i]:
        ret.add(i)
        search(i)
  search('shiny gold')
  return len(ret)

def solve_p2(bags):
  def search(bag):
    count = 1
    for x in bags[bag]:
      m = bags[bag][x]
      count += m * search(x)
    return count
  return search('shiny gold') - 1

print(solve_p1(parse(data)))
print(solve_p2(parse(data)))


from read_inputs.ListNode import ListNode

# Solved
def detectCycle(head: ListNode) -> ListNode:
    if head is None:
        return None

    if head.next is None:
        return None

    mapNodes = set()

    while head:
        if head in mapNodes:
            return head

        mapNodes.add(head)
        head = head.next

    return None

# Initialize a linked list with values [3, 2, 0, -4]
node1 = ListNode(3)
node2 = ListNode(2)
node3 = ListNode(0)
node4 = ListNode(-4)

node1.next = node2
node2.next = node3
node3.next = node4
node4.next = node2

isCycle = detectCycle(node1)
if isCycle:
    print("List has cycle in it at value => ", isCycle.val)
else:
    print("No cycle in list")


import FWCore.ParameterSet.Config as cms
from  PhysicsTools.NanoAOD.common_cff import *
### NanoAOD v5 (for 2016,2017,2018), for different recipe please modify accordingly
from Configuration.Eras.Modifier_run2_nanoAOD_94X2016_cff import run2_nanoAOD_94X2016
from Configuration.Eras.Modifier_run2_nanoAOD_94XMiniAODv2_cff import run2_nanoAOD_94XMiniAODv2
from Configuration.Eras.Modifier_run2_nanoAOD_102Xv1_cff import run2_nanoAOD_102Xv1
from Configuration.Eras.Modifier_run2_nanoAOD_106Xv1_cff import run2_nanoAOD_106Xv1
from Configuration.Eras.Modifier_run2_nanoAOD_106Xv2_cff import run2_nanoAOD_106Xv2
from JMEAnalysis.JetToolbox.jetToolbox_cff import jetToolbox
from JMEAnalysis.JetToolbox.jetToolbox_cff import _addProcessAndTask

from  PhysicsTools.PatAlgos.recoLayer0.jetCorrFactors_cfi import patJetCorrFactors #Added 3/20/23
from  PhysicsTools.PatAlgos.producersLayer1.jetUpdater_cfi import updatedPatJets #Added 3/20/23

# ---------------------------------------------------------
# This is the part the user should modify
def setupCustomizedJetToolbox(process):

    runOnMC = False	#This directory is for Data reprocessing

    #### AK4 PUPPI jets

    ak4btagdiscriminators = [
            	'pfDeepCSVJetTags:probb',
            	'pfDeepCSVJetTags:probbb',
            	'pfDeepCSVJetTags:probc',
            	'pfDeepCSVJetTags:probudsg',
#            	'pfDeepFlavourJetTags:probb',
#            	'pfDeepFlavourJetTags:probbb',
#            	'pfDeepFlavourJetTags:problepb',
#            	'pfDeepFlavourJetTags:probc',
#            	'pfDeepFlavourJetTags:probuds',
#            	'pfDeepFlavourJetTags:probg',
#		'pfMassDecorrelatedParticleNetJetTags:probXbb', #3/1/23 Added ParticleNet Discriminators
#		'pfMassDecorrelatedParticleNetJetTags:probXcc',
#		'pfMassDecorrelatedParticleNetJetTags:probXqq',
#		'pfMassDecorrelatedParticleNetJetTags:probQCDc',
#		'pfMassDecorrelatedParticleNetJetTags:probQCDcc',
#		'pfMassDecorrelatedParticleNetJetTags:probQCDb',
#		'pfMassDecorrelatedParticleNetJetTags:probQCDbb',
#		'pfMassDecorrelatedParticleNetJetTags:probQCDothers'
    ]
    ak4btaginfos = [ 'pfDeepCSVTagInfos' ] #'pfDeepFlavourTagInfos'

    jetToolbox(process, 'ak4', 'dummyseq', 'noOutput',
               dataTier='nanoAOD',
               PUMethod='Puppi', JETCorrPayload='AK4PFPuppi',
               #addQGTagger=True,
               runOnMC=False,
               Cut='pt > 15.0 && abs(eta) < 2.4',
               bTagDiscriminators=ak4btagdiscriminators,
               bTagInfos=ak4btaginfos,
               verbosity=4
               )

    #Exclusive to Data version
    if hasattr(process, "patJetsPuppi"):
        process.patJetsPuppi.addGenPartonMatch = cms.bool(False)
        process.patJetsPuppi.addGenJetMatch = cms.bool(False)
        process.patJetsPuppi.JetPartonMapSource = cms.InputTag("")
        process.patJetsPuppi.JetFlavourInfoSource = cms.InputTag("")

    #### AK8 PUPPI jets
    ak8btagdiscriminators = [
                        'pfBoostedDoubleSecondaryVertexAK8BJetTags',
#                        'pfMassIndependentDeepDoubleBvLJetTags:probQCD',
#                        'pfMassIndependentDeepDoubleBvLJetTags:probHbb',
#                        'pfMassIndependentDeepDoubleCvLJetTags:probQCD',
#                        'pfMassIndependentDeepDoubleCvLJetTags:probHcc',
#                        'pfMassIndependentDeepDoubleCvBJetTags:probHbb',
#                        'pfMassIndependentDeepDoubleCvBJetTags:probHcc',
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:bbvsLight",
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:ccvsLight",
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:TvsQCD",
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:ZHccvsQCD",
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:WvsQCD",
#                        "pfMassDecorrelatedDeepBoostedDiscriminatorsJetTags:ZHbbvsQCD"
#			'pfMassDecorrelatedParticleNetJetTags:probXbb', #2/22/23 Added ParticleNet Discriminators
#			'pfMassDecorrelatedParticleNetJetTags:probXcc',
#			'pfMassDecorrelatedParticleNetJetTags:probXqq',
#			'pfMassDecorrelatedParticleNetJetTags:probQCDc',
#			'pfMassDecorrelatedParticleNetJetTags:probQCDcc',
#			'pfMassDecorrelatedParticleNetJetTags:probQCDb',
#			'pfMassDecorrelatedParticleNetJetTags:probQCDbb',
#			'pfMassDecorrelatedParticleNetJetTags:probQCDothers'
            ]

    jetToolbox(process, 'ak8', 'adummyseq', 'noOutput',
               dataTier='nanoAOD',
               PUMethod='Puppi', JETCorrPayload='AK8PFPuppi',
	       subJETCorrPayload='AK4PFPuppi',
               runOnMC=runOnMC,
               Cut='pt > 100.0',
               bTagDiscriminators=ak8btagdiscriminators,
               verbosity=4,
	       addSoftDrop=True,
               addSoftDropSubjets=True,
               addPruning=True,
               addNsub=True,
               addEnergyCorrFunc=True,
	       addEnergyCorrFuncSubjets=True,	# Added 7/13
#	       runParticleNetMD=True	#Added 3/7/23
               )
#    particlenet_stuff = '''

#    process.SelectJetCorrFactorsAK8 = patJetCorrFactors.clone(src='selectedPatJetsAK8PFPuppiSoftDrop',
#        levels = cms.vstring('L1FastJet',
#            'L2Relative',
#            'L3Absolute',
#        'L2L3Residual'),
#        payload = cms.string('AK8PFPuppi'),
#        primaryVertices = cms.InputTag("offlineSlimmedPrimaryVertices"),
#    )
    process.SelectJetCorrFactorsAK8 = patJetCorrFactors.clone(src='packedPatJetsAK8PFPuppiSoftDrop',
        levels = cms.vstring('L1FastJet',
            'L2Relative',
            'L3Absolute',
        'L2L3Residual'),
        payload = cms.string('AK8PFPuppi'),
        primaryVertices = cms.InputTag("offlineSlimmedPrimaryVertices"),
    )

#    process.updatedSelectJetsAK8 = updatedPatJets.clone(
#        addBTagInfo=False,
#        jetSource='selectedPatJetsAK8PFPuppiSoftDrop',
#	jetCorrFactorsSource=cms.VInputTag(cms.InputTag("SelectJetCorrFactorsAK8")),
#    )
    process.updatedSelectJetsAK8 = updatedPatJets.clone(
        addBTagInfo=False,
        jetSource='packedPatJetsAK8PFPuppiSoftDrop',
	jetCorrFactorsSource=cms.VInputTag(cms.InputTag("SelectJetCorrFactorsAK8")),
    )

#    print("asdf")
    print(process.patAlgosToolsTask)

    from PhysicsTools.PatAlgos.tools.jetTools import updateJetCollection
    from RecoBTag.ONNXRuntime.pfParticleNet_cff import pfMassDecorrelatedParticleNetJetTags
    flav_names = ["probQCDothers", "probQCDb", "probQCDbb", "probQCDc", "probQCDcc", "probXqq", "probXbb", "probXcc"]
    pfMassDecorrelatedParticleNetJetTagsProbs = ['pfMassDecorrelatedParticleNetJetTags:' + n for n in flav_names]
    ak8btagdiscriminators += pfMassDecorrelatedParticleNetJetTagsProbs

    print("DEBUG: ParticleNetMD Tagging")

    JETCorrLevels = ['L2Relative', 'L3Absolute', 'L2L3Residual']

    updateJetCollection(
    	process,
    	jetSource=cms.InputTag('updatedSelectJetsAK8'),
    	rParam=0.8,
    	jetCorrections=('AK8PFPuppi', cms.vstring(JETCorrLevels), 'None'),
    	btagDiscriminators=ak8btagdiscriminators,
        postfix='AK8'
    )

    _addProcessAndTask(process, 'SelectJetCorrFactorsAK8', process.SelectJetCorrFactorsAK8)
    _addProcessAndTask(process, 'updatedSelectJetsAK8', process.updatedSelectJetsAK8)

    print(process.patAlgosToolsTask)

    from RecoBTag.ONNXRuntime.pfParticleNet_cff import pfMassDecorrelatedParticleNetJetTags
    process.pfParticleNetTagInfosAK8.jet_radius = 0.8
    process.pfMassDecorrelatedParticleNetJetTagsAK8ParticleNet = pfMassDecorrelatedParticleNetJetTags.clone(
        src = 'pfParticleNetTagInfos',
#        preprocess_json = '/home/akobert/CMSSW_10_6_29/src/ParticleNetMD/preprocess.json',
#        model_path = '/home/akobert/CMSSW_10_6_29/src/ParticleNetMD/ParticleNetMD.onnx',
        preprocess_json = 'PhysicsTools/data/ParticleNet-MD/ak8/preprocess.json',
        model_path = 'PhysicsTools/data/ParticleNet-MD/ak8/ParticleNet-MD.onnx',
    )
    print("DEBUG: Checkpoint 1")

    srcJets = cms.InputTag('selectedUpdatedPatJetsAK8')

    process.ak8WithUserData = cms.EDProducer("PATJetUserDataEmbedder",
        src=srcJets,
        userFloats=cms.PSet(),
        userInts=cms.PSet(),
    )

    process.finalSelectedJetsAK8 = cms.EDFilter("PATJetRefSelector",
        src = cms.InputTag("ak8WithUserData"),
        cut = cms.string("")
    )
    print("DEBUG: Checkpoint 2")

    process.selectedPatJetsAK8PFPuppiJTBTable = cms.EDProducer("SimpleCandidateFlatTableProducer", src=cms.InputTag('finalSelectedJetsAK8'), name=cms.string("selectedPatJetsAK8PFPuppi"), cut=cms.string(""), doc=cms.string("Selected AK8 PUPPI Jets with ParticleNet Taggers"), singleton=cms.bool(False), extension=cms.bool(False), variables=cms.PSet(P4Vars,
#	msoftdrop = Var("groomedMass()", float, doc="Corrected soft drop mass with PUPPI", precision=10),
	msoftdrop = Var("userFloat('ak8PFJetsPuppiSoftDropMass')", float, doc='Softdrop mass', precision=10),
        area=Var("jetArea()", float, doc="jet catchment area, for JECs", precision=10),
        nMuons = Var("?hasOverlaps('muons')?overlaps('muons').size():0", int, doc="number of muons in the jet"),
        muonIdx1 = Var("?overlaps('muons').size()>0?overlaps('muons')[0].key():-1", int, doc="index of first matching muon"),
        muonIdx2 = Var("?overlaps('muons').size()>1?overlaps('muons')[1].key():-1", int, doc="index of second matching muon"),
        electronIdx1 = Var("?overlaps('electrons').size()>0?overlaps('electrons')[0].key():-1", int, doc="index of first matching electron"),
        electronIdx2 = Var("?overlaps('electrons').size()>1?overlaps('electrons')[1].key():-1", int, doc="index of second matching electron"),
         nElectrons = Var("?hasOverlaps('electrons')?overlaps('electrons').size():0", int, doc="number of electrons in the jet"),
        jetId = Var("userInt('looseId')+userInt('tightId')*2+4*userInt('tightIdLepVeto')",int,doc="Jet ID flags bit1 is loose, bit2 is tight, bit3 is tightLepVeto"),
        nConstituents = Var("numberOfDaughters()",int,doc="Number of particles in the jet"),
        rawFactor = Var("1.-jecFactor('Uncorrected')",float,doc="1 - Factor to get back to raw pT",precision=6),
        tau1 = Var("userFloat('NjettinessAK8Puppi:tau1')", float, doc="Nsubjettiness (1 axis)", precision=10),
        tau2 = Var("userFloat('NjettinessAK8Puppi:tau2')", float, doc="Nsubjettiness (2 axis)", precision=10),
        tau3 = Var("userFloat('NjettinessAK8Puppi:tau3')", float, doc="Nsubjettiness (3 axis)", precision=10),
	tau4 = Var("userFloat('NjettinessAK8Puppi:tau4')", float, doc="Nsubjettiness (4 axis)", precision=10),
        n2b1 = Var("?hasUserFloat('nb1AK8PuppiSoftDrop:ecfN2')?userFloat('nb1AK8PuppiSoftDrop:ecfN2'):-99999.", float, doc="N2 with beta=1 (for jets with raw pT>250 GeV)", precision=10),
	n3b1 = Var("?hasUserFloat('nb1AK8PuppiSoftDrop:ecfN3')?userFloat('nb1AK8PuppiSoftDrop:ecfN3'):-99999.", float, doc="N3 with beta=1 (for jets with raw pT>250 GeV)", precision=10),
	chHEF = Var("chargedHadronEnergyFraction()", float, doc="charged Hadron Energy Fraction", precision= 6),
        neHEF = Var("neutralHadronEnergyFraction()", float, doc="neutral Hadron Energy Fraction", precision= 6),
        chEmEF = Var("chargedEmEnergyFraction()", float, doc="charged Electromagnetic Energy Fraction", precision= 6),
        neEmEF = Var("neutralEmEnergyFraction()", float, doc="neutral Electromagnetic Energy Fraction", precision= 6),
	muEF = Var("muonEnergyFraction()", float, doc="muon Energy Fraction", precision= 6),
	)
    )
    print("DEBUG: Checkpoint 3")

    for prob in pfMassDecorrelatedParticleNetJetTagsProbs:
        name = 'ParticleNetMD_' + prob.split(':')[1]
        name = name.replace('QCDothers', 'QCD')
        setattr(process.selectedPatJetsAK8PFPuppiJTBTable.variables, name, Var("bDiscriminator('%s')" % prob, float, doc=prob, precision=-1))

#    process.selectedPatSubJetsAK8Table = cms.EDProducer("SimpleCandidateFlatTableProducer", src=cms.InputTag("selectedPatJetsAK8PFPuppiSoftDropPacked", "SubJets"), name=cms.string("SelectedSubJetAK8"), cut=cms.string(""), doc=cms.string("Selected AK8 PUPPI SubJets"), singleton=cms.bool(False), extension=cms.bool(False), variables=cms.PSet(P4Vars,
#	)
#    )
#    process.selectedPatSubJetsAK8Table.variables.pt.precision = 10
    print("DEBUG: Checkpoint 4")

    _addProcessAndTask(process, 'ak8WithUserData', process.ak8WithUserData)
    _addProcessAndTask(process, 'finalSelectedJetsAK8', process.finalSelectedJetsAK8)
    _addProcessAndTask(process, 'selectedPatJetsAK8PFPuppiJTBTable', process.selectedPatJetsAK8PFPuppiJTBTable)
#    _addProcessAndTask(process, 'selectedPatSubJetsAK8Table', process.selectedPatSubJetsAK8Table)

    #Avoid Circular Dependency
#    process.jetMC.remove(process.patJetPartons)
#    _addProcessAndTask(process, 'patJetPartons', process.patJetPartons)
#    '''
    print("test10")
    return process

# ---------------------------------------------------------

def nanoJTB_customizeMC(process):
    print("test9")
    run2_nanoAOD_94X2016.toModify(process, setupCustomizedJetToolbox)
    run2_nanoAOD_94XMiniAODv2.toModify(process, setupCustomizedJetToolbox)
    run2_nanoAOD_102Xv1.toModify(process, setupCustomizedJetToolbox)
    run2_nanoAOD_106Xv1.toModify(process, setupCustomizedJetToolbox)
    run2_nanoAOD_106Xv2.toModify(process, setupCustomizedJetToolbox)
    #process.NANOAODSIMoutput.fakeNameForCrab = cms.untracked.bool(True)  # needed for crab publication
    return process


# -*-coding:utf-8-*-
# @auth ivan
# @time 2018-01-16 14:29
# @goal config
from logging import NOTSET

__all__ = [
    'main_path_data',
    'main_path_out',
    'main_path_test',
    'data_Name',
    'data_Tame',
    'data_type',
    'out_Name',
    'out_type',
    'size',
    'times',
    'deletions',
    'data',
    'data_T',
    'pro_gra_dot',
    'min_cpu',
    'log_mode',
    'log_level',
    'log_format',
    'log_encode'
]

main_path_data = u'G:\\OUT\\07.AI\\MLTools\\data\\'

main_path_out = u'G:\\OUT\\07.AI\\MLTools\\out\\'

main_path_test = u'G:\\OUT\\07.AI\\MLTools\\out\\test\\'

data_Name = u'm1m2_sample'

data_Tame = u'm1m2_sample_T200(MISS)'

data_type = u'.csv'

out_Name = u'fields'

out_type = u'.log'

size = 0.05

times = 5

deletions = {
    'R_POS_CNT_16_Pct_Avg_POS_CNT_1N': [-99000792.0, 85],
    'R_Con_Incs_in_INC_Pay_P_BAL': []
}

special = [-99000792, -99000784, -99000776]

data = main_path_data + data_Name + data_type

data_T = main_path_data + data_Tame + data_type

pro_gra_dot = 'C:\\Program Files (x86)\\Graphviz2.38\\bin\\dot.exe'

min_cpu = 2

log_mode = 'w'

log_level = NOTSET

log_format = '%(levelname)s| %(message)s'

log_encode = 'utf-8'


# @author - Vikesh Khanna
# This file takes as input the data from FootballData.co.uk in csv format and extracts the following different csv files-
# teams - csv file with each team name. The row number of the team is important as it will be used as pointer in other matrices
# train - The training matrix csv. There are 2*(n+1) columns. Each n+1 columns is teamID and the n features we are capturing. (see legend)

import csv
import sys
import numpy as np
import scipy.io as io
from utils import *

def main(args):
	if len(args)<1:
		print("Usage: python parse.py <csv_file>")
		sys.exit(1)

	csvfile = args[0]

	f = open(csvfile)
	reader = csv.reader(f)

	# Should match the Feature class elements
	header = reader.next()

	# team list
	teams = []

	# train matrix - 2*(n+1) rows. Each row is a match. First row is the team ID, n+1 features of home team, (1, n+1) features of the away team
	train = []

	for row in reader:
		features = {Constants.HOME : [], Constants.AWAY: []}
		outcome = None

		for i in range(len(row)):
			feature = row[i]
			field = header[i]

			if field == Features.HomeTeam or field == Features.AwayTeam:
				if feature not in teams:
					teams.append(feature)

			# Header contains the feature - Useful feature. Must.Take
			if field in Features.FMAP:
				# Find home or away
				location = Features.FMAP[field]

				if field==Features.HomeTeam or field==Features.AwayTeam:
					feature = teams.index(feature) + 1
				else:
					feature = float(feature)

				# Add feature to the appropriate list
				features[location].append(feature)

			elif field==Features.FTR:
				outcome = Features.FTR_MAP[feature]

		# Horizontally join the two lists
		train.append(features[Constants.HOME] + features[Constants.AWAY] + [outcome])

	OUTFILE = "%s.mat"%csvfile

	#Write to mat file
	np_teams = np.array(teams)
	np_train = np.array(train)
	np_dict = {'teams':np_teams, 'train':np_train}

	io.savemat(OUTFILE, np_dict)

if __name__=="__main__":
	main(sys.argv[1:])


def extractYoraikun(item):
	"""
	# Yoraikun
	"""
	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or 'preview' in item['title'].lower():
		return None

	snames = [
			'The Rise of the Shield Hero',
			'Konjiki no Wordmaster',
			'Sevens',
			'Dragoon',
			'The Lazy King',
			'Me, Her, and the Ballistic Weaponry [Antique]',
		]

	tlut = {tmp.lower(): tmp for tmp in snames}

	tlut['ikctawbwtwh']                      = "I Kinda Came to Another World, But Where's the Way Home"
	tlut['okitegami kyouko']                 = "The Memorandum of Okitegami Kyouko"
	tlut['battlefield masurawo']             = "Sentou Jousai Masurawo"
	tlut['the christmas of the shield hero'] = "The Rise of the Shield Hero"
	tlut['alchemuls']                        = "Waga Hero no Tame no Alchemuls"
	tlut['aura']                             = "AURA: Koga Maryuin\'s Last Battle"

	ltags = [tmp.lower() for tmp in item['tags']]
	for key, value in tlut.items():
		if key in ltags:
			tl_type = 'translated'

			return buildReleaseMessageWithType(item, value, vol, chp, frag=frag, postfix=postfix, tl_type=tl_type)

	titlemap = [
		('Mary-san Comes on Foot',  'Mary Comes on Foot',      'translated'),
		('Mary Comes on Foot',      'Mary Comes on Foot',      'translated'),
	]

	for titlecomponent, name, tl_type in titlemap:
		if titlecomponent.lower() in item['title'].lower():
			return buildReleaseMessageWithType(item, name, vol, chp, frag=frag, postfix=postfix, tl_type=tl_type)

	return False


from __future__ import absolute_import
from __future__ import unicode_literals
import os

import mindspore.ops
import mindspore.nn as nn
import numpy as np
import sklearn.neighbors as sn
from skimage import color

class NNEncLayer:
    """
    Layer which encodes ab map into Q colors

    Args:
        args (namespace): Network parameters.

    Inputs:
        - **img_ab** (tensor) - AB channel of image.

    Outputs:
         - **encode** (ndarray) - Coded image AB channel.
         - **max_encode** (ndarray) - Coding maximum.
    """

    def __init__(self, args):
        self.nne = 32
        self.sigma = 0.5
        self.enc_dir = args.resource
        self.nnenc = NNEncode(self.nne, self.sigma, km_filepath=os.path.join(self.enc_dir, 'pts_in_hull.npy'))

        self.x = 224
        self.y = 224
        self.q = self.nnenc.k

    def forward(self, img_ab):
        encode = self.nnenc.encode_points_mtx_nd(img_ab)
        max_encode = np.argmax(encode, axis=1).astype(np.int32)
        return encode, max_encode

    def reshape(self, top):
        top[0].reshape(self.N, self.Q, self.X, self.Y)

class PriorBoostLayer:
    """
    Layer boosts ab values based on their rarity

    Args:
        args (namespace): Network parameters.

    Inputs:
        - **bottom** (ndarray) - Calculate boost.

    Outputs:
        - **boost** (ndarray) - Layer boosts ab values
    """

    def __init__(self, args):
        self.enc_dir = args.resource
        self.gamma = .5
        self.alpha = 1.
        self.pc = PriorFactor(self.alpha, gamma=self.gamma, priorFile=os.path.join(self.enc_dir, 'prior_probs.npy'))

        self.x = 224
        self.y = 224

    def forward(self, bottom):
        boost = self.pc.forward(bottom, axis=1)
        return boost

class NonGrayMaskLayer:
    """
    Layer outputs a mask based on if the image is grayscale or not

    Inputs:
        - **bottom** (ndarray) - Image ab channel values.

    Outputs:
        - **boost** (ndarray) - Layer outputs a mask
    """

    def forward(self, bottom):
        bottom = bottom.asnumpy()
        mask = (np.sum(np.sum(np.sum((np.abs(bottom) > 5).astype('float'), axis=1),
                              axis=1), axis=1) > 0)[:, na(), na(), na()].astype('float')
        return mask

class PriorFactor:
    """
    Handles prior factor

    Args:
        alpha (int): Prior correction factor.
        gamma (int): Percentage to mix in uniform prior with empirical prior.
        verbose (namespace): Network parameters.
        priorFile(str)： File which contains prior probabilities across classes

    Inputs:
        - **bottom** (ndarray) - Image ab channel values.

    Outputs:
        - **boost** (ndarray) - A priori factor after handles
    """

    def __init__(self, alpha, gamma=0, priorFile=''):
        self.alpha = alpha
        self.gamma = gamma
        # self.verbose = verbose
        self.prior_probs = np.load(priorFile)
        self.uni_probs = np.zeros_like(self.prior_probs)
        self.uni_probs[self.prior_probs != 0] = 1.
        self.uni_probs = self.uni_probs / np.sum(self.uni_probs)

        # convex combination of empirical prior and uniform distribution
        self.prior_mix = (1 - self.gamma) * self.prior_probs + self.gamma * self.uni_probs

        # set prior factor
        self.prior_factor = self.prior_mix ** -self.alpha
        self.prior_factor = self.prior_factor / np.sum(self.prior_probs * self.prior_factor)  # re-normalize

        # implied empirical prior
        self.implied_prior = self.prior_probs * self.prior_factor
        self.implied_prior = self.implied_prior / np.sum(self.implied_prior)  # re-normalize

    def forward(self, data_ab_quant, axis=1):
        """Handles prior factor"""
        data_ab_maxind = np.argmax(data_ab_quant, axis=axis)
        corr_factor = self.prior_factor[data_ab_maxind]
        if axis == 0:
            boost = corr_factor[na(), :]
        elif axis == 1:
            boost = corr_factor[:, na(), :]
        elif axis == 2:
            boost = corr_factor[:, :, na(), :]
        elif axis == 3:
            boost = corr_factor[:, :, :, na()]
        return boost

class NNEncode:
    """
    Encode points using NN search and Gaussian kernel

    Args:
        NN (int): Number of nearest neighbors to be selected.
        sigma (int): A uniform distribution with weight.
        km_filepath (str): A prior file path.
        cc(int)：Correction factor

    Inputs:
        - **pts_nd** (ndarray) - Image ab channel values.

    Outputs:
        - **pts_enc_nd** (ndarray) - Coded image AB channel.
    """

    def __init__(self, nne, sigma, km_filepath='', cc=-1):
        self.p_index = None
        self.pts_enc_flt = None
        if check_value(cc, -1):
            self.cc = np.load(km_filepath)
        else:
            self.cc = cc
        self.k = self.cc.shape[0]
        self.nne = int(nne)
        self.sigma = sigma
        self.nbr = sn.NearestNeighbors(n_neighbors=nne, algorithm='ball_tree').fit(self.cc)
        self.used = False

    def encode_points_mtx_nd(self, pts_nd, axis=1, same_block=True):
        """Encode points using NN search and Gaussian kernel"""
        pts_flt = flatten_nd_array(pts_nd, axis=axis)
        p = pts_flt.shape[0]
        if same_block and self.used:
            self.pts_enc_flt[...] = 0  # already pre-allocated
        else:
            self.used = True
            self.pts_enc_flt = np.zeros((p, self.k))
            self.p_index = np.arange(0, p, dtype='int')[:, na()]
        pts_flt = pts_flt.asnumpy()
        (dists, index) = self.nbr.kneighbors(pts_flt)

        wts = np.exp(-dists ** 2 / (2 * self.sigma ** 2))
        wts = wts / np.sum(wts, axis=1)[:, na()]
        self.pts_enc_flt[self.p_index, index] = wts

        pts_enc_nd = unflatten_2d_array(self.pts_enc_flt, pts_nd, axis=axis)
        return pts_enc_nd

# *****************************
# ***** Utility functions *****
# *****************************
def check_value(cc, val):
    """
    Check to see if an array is a single element equaling a particular value for pre-processing inputs in a function
    """
    if np.array(cc).size == 1:
        if cc == val:
            return True
    return False

def na():
    """shorthand for new axis"""
    return np.newaxis

def flatten_nd_array(pts_nd, axis=1):
    """
    Flatten an nd array into a 2d array with a certain axis

    Inputs:
        - **pts_nd** (ndarray) - Image ab channel values.
        - **axis** (int) - Dimension

    Outputs:
        - **pts_flt**(ndarray)    2d array with a certain axis
    """

    ndim = pts_nd.dim()
    shp = np.array(pts_nd.shape)
    nax = np.setdiff1d(np.arange(0, ndim), np.array((axis)))  # non axis indices
    npts = np.prod(shp[nax])
    axorder = tuple(np.concatenate((nax, np.array(axis).flatten()), axis=0).tolist())
    transpose = mindspore.ops.Transpose()
    pts_flt = transpose(pts_nd, axorder)
    pts_flt = pts_flt.view(npts.item(), shp[axis].item())
    return pts_flt

def unflatten_2d_array(pts_flt, pts_nd, axis=1, squeeze=False):
    """
    Unflatten a 2d array with a certain axis

    Inputs:
        - **pts_flt** (ndarray) - 2d array with a certain axis
        - **pts_nd** (ndarray) - Image ab channel values.
        - **axis** (int) - Dimension
        - **squeeze** (bool) - whether squeeze it out

    Outputs:
        - **pts_out** (ndarray) - Unflatten a 2d array
    """
    ndim = pts_nd.dim()
    shp = np.array(pts_nd.shape)
    nax = np.setdiff1d(np.arange(0, ndim), np.array(axis))  # non axis indices

    if squeeze:
        axorder = nax
        axorder_rev = np.argsort(axorder)
        new_shp = shp[nax].tolist()

        pts_out = pts_flt.reshape(new_shp)
        pts_out = pts_out.transpose(axorder_rev)
    else:
        axorder = np.concatenate((nax, np.array(axis).flatten()), axis=0)
        axorder_rev = tuple(np.argsort(axorder).tolist())
        m = pts_flt.shape[1]
        new_shp = shp[nax].tolist()
        new_shp.append(m)
        pts_out = pts_flt.reshape(new_shp)
        pts_out = pts_out.transpose(axorder_rev)
    return pts_out

def decode(data_l, conv8_313, r_path, rebalance=1):
    """
    Merge gray channel and ab channel to output color images

    Inputs:
        - **data_l** (tensor) - Gray scale channel of image
        - **conv8_313** (tensor) - AB channel of image
        - **rebalance** (int) - Rebalance factor

    Outputs:
        - **img_rgb** (ndarray) - RGB image
    """
    data_l = data_l[0] + 50
    data_l = data_l.asnumpy().transpose((1, 2, 0))
    conv8_313 = conv8_313[0]
    enc_dir = r_path
    conv8_313_rh = conv8_313 * rebalance
    softmax = nn.Softmax(axis=0)
    class8_313_rh = softmax(conv8_313_rh).asnumpy().transpose((1, 2, 0))
    class8 = np.argmax(class8_313_rh, axis=-1)
    cc = np.load(os.path.join(enc_dir, 'pts_in_hull.npy'))
    data_ab = cc[class8[:][:]]
    data_ab = data_ab.repeat(4, axis=0).repeat(4, axis=1)
    img_lab = np.concatenate((data_l, data_ab), axis=-1)
    img_rgb = color.lab2rgb(img_lab)
    return img_rgb


from ibm_cloud_sdk_core import ApiException, read_external_sources
import os
import pytest
from ibm_dpx_services.data_product_exchange_api_service_v1 import *

#
# This file provides an example of how to use the Data Product Exchange API Service service.
#
# The following configuration properties are assumed to be defined:
# DATA_PRODUCT_EXCHANGE_API_SERVICE_URL=<service base url>
# DATA_PRODUCT_EXCHANGE_API_SERVICE_AUTH_TYPE=iam
# DATA_PRODUCT_EXCHANGE_API_SERVICE_APIKEY=<IAM apikey>
# DATA_PRODUCT_EXCHANGE_API_SERVICE_AUTH_URL=<IAM token service base URL - omit this if using the production environment>
#
# These configuration properties can be exported as environment variables, or stored
# in a configuration file and then:
# export IBM_CREDENTIALS_FILE=<name of configuration file>
#
config_file = 'data_product_exchange_api_service_v1.env'

data_product_exchange_api_service_service = None

config = None

# Variables to hold link values
create_data_product_version_by_catalog_id_link = None
delete_data_product_version_by_user_id_link = None
deliver_data_product_version_by_user_id_link = None
get_data_product_by_user_id_link = None
get_data_product_version_by_user_id_link = None
get_list_of_data_product_by_catalog_id_link = None
get_status_by_catalog_id_link = None
update_data_product_version_by_user_id_link = None

##############################################################################
# Start of Examples for Service: DataProductExchangeApiServiceV1
##############################################################################
# region
class TestDataProductExchangeApiServiceV1Examples:
    """
    Example Test Class for DataProductExchangeApiServiceV1
    """

    @classmethod
    def setup_class(cls):
        global data_product_exchange_api_service_service
        if os.path.exists(config_file):
            os.environ['IBM_CREDENTIALS_FILE'] = config_file

            # begin-common

            data_product_exchange_api_service_service = DataProductExchangeApiServiceV1.new_instance()

            # end-common
            assert data_product_exchange_api_service_service is not None

            # Load the configuration
            global config
            config = read_external_sources(DataProductExchangeApiServiceV1.DEFAULT_SERVICE_NAME)

        print('Setup complete.')

    needscredentials = pytest.mark.skipif(
        not os.path.exists(config_file), reason="External configuration not available, skipping..."
    )

    @needscredentials
    def test_initialize_example(self):
        """
        initialize request example
        """
        try:
            print('\ninitialize() result:')
            # begin-initialize

            response = data_product_exchange_api_service_service.initialize(
                include=['delivery_methods', 'data_product_samples', 'domains_multi_industry'],
            )
            initialize_resource = response.get_result()

            print(json.dumps(initialize_resource, indent=2))

            # end-initialize

            global create_data_product_version_by_catalog_id_link
            create_data_product_version_by_catalog_id_link = initialize_resource['container']['id']
            global get_status_by_catalog_id_link
            get_status_by_catalog_id_link = initialize_resource['container']['id']
            global get_list_of_data_product_by_catalog_id_link
            get_list_of_data_product_by_catalog_id_link = initialize_resource['container']['id']
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_create_data_product_version_example(self):
        """
        create_data_product_version request example
        """
        try:
            print('\ncreate_data_product_version() result:')
            # begin-create_data_product_version

            container_reference_model = {
                # 'id': 'd29c42eb-7100-4b7a-8257-c196dbcca1cd',
                'id': create_data_product_version_by_catalog_id_link,
                'type': 'catalog',
            }

            response = data_product_exchange_api_service_service.create_data_product_version(
                container=container_reference_model,
                name='My New Data Product',
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-create_data_product_version

            global get_data_product_version_by_user_id_link
            get_data_product_version_by_user_id_link = data_product_version['id']
            global update_data_product_version_by_user_id_link
            update_data_product_version_by_user_id_link = data_product_version['id']
            global delete_data_product_version_by_user_id_link
            delete_data_product_version_by_user_id_link = data_product_version['id']
            global get_data_product_by_user_id_link
            get_data_product_by_user_id_link = data_product_version['data_product']['id']
            global deliver_data_product_version_by_user_id_link
            deliver_data_product_version_by_user_id_link = data_product_version['id']
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_initialize_status_example(self):
        """
        get_initialize_status request example
        """
        try:
            print('\nget_initialize_status() result:')
            # begin-get_initialize_status

            response = data_product_exchange_api_service_service.get_initialize_status()
            initialize_resource = response.get_result()

            print(json.dumps(initialize_resource, indent=2))

            # end-get_initialize_status

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_data_product_example(self):
        """
        get_data_product request example
        """
        try:
            print('\nget_data_product() result:')
            # begin-get_data_product

            response = data_product_exchange_api_service_service.get_data_product(
                id=get_data_product_by_user_id_link,
            )
            data_product = response.get_result()

            print(json.dumps(data_product, indent=2))

            # end-get_data_product

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_list_data_products_example(self):
        """
        list_data_products request example
        """
        try:
            print('\nlist_data_products() result:')
            # begin-list_data_products

            all_results = []
            pager = DataProductsPager(
                client=data_product_exchange_api_service_service,
                limit=10,
            )
            while pager.has_next():
                next_page = pager.get_next()
                assert next_page is not None
                all_results.extend(next_page)

            print(json.dumps(all_results, indent=2))

            # end-list_data_products
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_list_data_product_versions_example(self):
        """
        list_data_product_versions request example
        """
        try:
            print('\nlist_data_product_versions() result:')
            # begin-list_data_product_versions

            all_results = []
            pager = DataProductVersionsPager(
                client=data_product_exchange_api_service_service,
                asset_container_id=get_list_of_data_product_by_catalog_id_link,
                # data_product='testString',
                # state='draft',
                # version='testString',
                limit=10,
            )
            while pager.has_next():
                next_page = pager.get_next()
                assert next_page is not None
                all_results.extend(next_page)

            print(json.dumps(all_results, indent=2))

            # end-list_data_product_versions
        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_get_data_product_version_example(self):
        """
        get_data_product_version request example
        """
        try:
            print('\nget_data_product_version() result:')
            # begin-get_data_product_version

            response = data_product_exchange_api_service_service.get_data_product_version(
                id=get_data_product_version_by_user_id_link,
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-get_data_product_version

        except ApiException as e:
            pytest.fail(str(e))

    @needscredentials
    def test_update_data_product_version_example(self):
        """
        update_data_product_version request example
        """
        try:
            print('\nupdate_data_product_version() result:')
            # begin-update_data_product_version

            json_patch_operation_model = {
                'op': 'replace',
                'path': '/description',
            }

            response = data_product_exchange_api_service_service.update_data_product_version(
                id=update_data_product_version_by_user_id_link,
                json_patch_instructions=[json_patch_operation_model],
            )
            data_product_version = response.get_result()

            print(json.dumps(data_product_version, indent=2))

            # end-update_data_product_version

        except ApiException as e:
            pytest.fail(str(e))

    # @needscredentials
    # def test_deliver_data_product_version_example(self):
    #     """
    #     deliver_data_product_version request example
    #     """
    #     try:
    #         print('\ndeliver_data_product_version() result:')
    #         # begin-deliver_data_product_version

    #         item_reference_model = {
    #             'id': 'd29c42eb-7100-4b7a-8257-c196dbcca1cd',
    #         }

    #         order_reference_model = {
    #             'id': '4705e047-1808-459a-805f-d5d13c947637',
    #             'items': [item_reference_model],
    #         }

    #         response = data_product_exchange_api_service_service.deliver_data_product_version(
    #             id=deliver_data_product_version_by_user_id_link,
    #             order=order_reference_model,
    #         )
    #         delivery_resource = response.get_result()

    #         print(json.dumps(delivery_resource, indent=2))

    #         # end-deliver_data_product_version

    #     except ApiException as e:
    #         pytest.fail(str(e))

    @needscredentials
    def test_delete_data_product_version_example(self):
        """
        delete_data_product_version request example
        """
        try:
            # begin-delete_data_product_version

            response = data_product_exchange_api_service_service.delete_data_product_version(
                id=delete_data_product_version_by_user_id_link,
            )

            # end-delete_data_product_version
            print('\ndelete_data_product_version() response status code: ', response.get_status_code())

        except ApiException as e:
            pytest.fail(str(e))

# endregion
##############################################################################
# End of Examples for Service: DataProductExchangeApiServiceV1
##############################################################################


import xml.etree.ElementTree as ET
import os
import sqlite3
import thread
import unicodedata
import threading
import json
import re

from multiprocessing.dummy import Pool as ThreadPool

########################################################################################################
############################### extract topics from raw files 	##############################
########################################################################################################
def extract_tag(in_path):
	"""
	return a set of tags the raw file has: (indexing_service, online_producer)
	"""
	tag_indexing = []
	tag_online = []
	tree = ET.parse(in_path)
	head = tree.getroot().find("head")
	for descr in head.findall('docdata/identified-content/classifier[@type="descriptor"]'):
		if descr.get('class') == 'indexing_service':
			tag_indexing.append(descr.text)
		if descr.get('class') == 'online_producer':
			tag_online.append(descr.text)
	return set(tag_indexing), set(tag_online)

def extract_tag_all(root_dir):
	"""
	save an index of all the tags and files
	"""
	index = {}
	online_prod = {}
	for root, dirs, files in os.walk(root_dir):
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			try:
				tag_indexing, tag_online= extract_tag(p)
			except:
				tag_indexing, tag_online = set([]),set([])
				print("Both empty: "+p)
			#name = ("/").join(p.split("/")[-3:]).split(".")[0]
			name = p.split("/")[-1].split(".")[0]
			if tag_indexing == set([]):
				tag_indexing.update(["NO TAG"])
			if tag_online == set([]):
				tag_online.update(["NO TAG"])

			for tag in tag_indexing:
				item_index = index.get(tag)
				if item_index is None:
					index[tag] = [name]
				else:
					item_index.append(name)
					index.update([(tag,item_index)])
			for tag in tag_online:
				item_index = online_prod.get(tag)
				if item_index is None:
					online_prod[tag] = [name]
				else:
					item_index.append(name)
					online_prod.update([(tag,item_index)])
	f = open(root_dir+'topics_indexing_services.json','wb')
	json.dump(index,f)
	f.close()
	f = open(root_dir + 'topics_online_producer.json','wb')
	json.dump(online_prod,f)
	f.close()

def file_to_topic(root_path):
	"""
	for all the files under root_path, map file id to its topics.
	Save result: "file_to_topics.json"
	file_id:([Topics_by_indexing_service], [topics_by_online_service])
	"""
	processed = set()
	out = {}
	try:
		with open(root_path+"file_to_topics.json") as json_data:
			d = json.load(json_data)
			processed = set(d.keys())
	except:
		print("No previous work available!")

	for root, dirs, files in os.walk(root_path):
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			tag_indexing, tag_online= extract_tag(p)
			file_id = p.split("/")[-1].split(".")[0]
			if file_id in processed:
				continue

			if tag_indexing == set([]):
				tag_indexing.update(["NO TAG"])
			if tag_online == set([]):
				tag_online.update(["NO TAG"])

			# tmp = {file_id:(list(tag_indexing), list(tag_online))}
			out[file_id] = (list(tag_indexing), list(tag_online))

	with open(root_path+'file_to_topics.json','w') as f_json:
		json.dump(out, f_json)
	print(">>Done for "+root_path)

########################################################################################################
################## extract summary, full-article, headlines from raw files 	#########################
########################################################################################################

def extract(in_path):
	tree = ET.parse(in_path)
	summary=""
	body = tree.getroot().find("body")
	for head in body.findall('body.head/abstract/p'):
		summary += head.text
	article=""
	body = tree.getroot().find("body")
	contents = body.find('body.content')

	for content in contents.iter('block'):
		para = content.get('class')
		if para!="full_text":
			continue
		for txt in content.findall('p'):
			article+=txt.text

	h1 = tree.getroot().find("body/body.head/hedline/hl1")
	meta = h1.text

	return summary,article, meta

# extract and parse plain text into root/summary, root/meta, root/content
def extractall(root_dir):
	paths = [] # list of all files under root
	new_storage = ["summary","content","meta"]
	process_path = [root_dir+ns for ns in new_storage]

	# make folders under root if they don't already exist
	for ps in process_path:
		if not os.path.exists(ps):
			os.makedirs(ps)

	for root, dirs, files in os.walk(root_dir):
		# get all .xml file path under year/XX/XX
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			try:
				extraction = extract(p)
			except Exception:
				print("Parsing error: "+p)
				continue
			name = p.split("/")[-1].split(".")[0]

			# skip if contain non-ascii or some parts missing
			if extraction[0]=="" or extraction[1]=="" or extraction[2]=="":
				print("missing part on "+name)
				continue
			for i in range(3):
				f = open(process_path[i]+"/"+name+".txt","wb")
				try:
					f.write(extraction[i])
				except UnicodeEncodeError:
					print("Unicode encoding: "+p)
					f.write(extraction[i].encode("utf8"))
				f.close()

class myThreads(threading.Thread):
	def __init__(self,in_dir,number):
		threading.Thread.__init__(self)
		self._dir = in_dir
		self._number = number
	def run(self):
		print("Starting: "+str(self._number))
		extract_tag_all(self._dir)
		print("Existing: "+str(self._number))

def extract_multi(in_dir,start,finish):
	threads = []
	for i in range(start,finish+1):
		path= in_dir+str(i)+"/"
		thread = myThreads(path,i)
		threads.append(thread)

	for thread in threads:
		thread.start()

	for t in threads:
		t.join()
	print(str(start)+"-"+str(finish)+" Done")

########################################################################################################
################################ preprocess extracted files ###################################
########################################################################################################
def annotate(content, in_dir,start,end):
    for i in range(start,end):
        in_path = in_dir+content+'/'+str(i)+content
        out_path = in_dir+content+"_annotated/"+str(i)+content+"_annotated"
        annotate_all(in_path,out_path)

        print("Annotating "+content+"for "+str(i)+"Done!")

def annotate_all(in_dir, out_dirs):
	import corenlpy
	if not os.path.exists(out_dirs):
		os.makedirs(out_dirs)

	corenlpy.corenlp(in_dirs=in_dir, out_dir=out_dirs,
		annotators=['tokenize','ssplit','pos','lemma','ner','parse','depparse','dcoref','relation'],
		threads=10,output_format='xml')

########################################################################################################
############################### store paths of all files     ################################
########################################################################################################
# store all file paths under root into db
def store(root_dir):
	conn = sqlite3.connect("nyt.db")
	c = conn.cursor()
	c.execute("""CREATE TABLE IF NOT EXISTS nyt(file_id integer, orinigal_path text NOT NULL);""")

	# entry:[(id,path)]
	entry = []
	for root, dirs, files in os.walk(root_dir):
		# get all .xml file path under year/XX/XX
		path = [os.path.join(root,name) for name in files if len(name.split("."))==2 and name.split(".")[1]=="xml"]
		for p in path:
			name = p.split("/")[-1].split(".")[0]
			entry.append((name,p))

	c.executemany("INSERT INTO nyt VALUES (?,?)",entry)

	conn.commit()
	conn.close()

# given file id, return path to its original raw xml file
def get_path(file_id):
	conn = sqlite3.connect("nyt.db")
	c = conn.cursor()
	query = "SELECT * FROM nyt WHERE file_id=="+str(file_id)
	c.execute(query)
	out = c.fetchone()
	conn.close()
	return out[1]

#################################################################################################
################################ preprocess summary files ###################################
#################################################################################################
def substi(text):
	"""
	Find all ; and replace with '.' get rid of (M), photo(s)/drawing(s)
	"""

	text = re.sub(r'photo|drawing|map|chart|table|graph|diagram',"", text)
	text = re.sub(r'\([A-Z]\)','',text)

	text_sep = text.split(".")
	text_sep = [s.strip() for s in text_sep if len(s)>=3]
	text_new = ". ".join(text_sep)

	return text_new

def substi_all(root_dir):
	"""
	For all files under root_dir, preprocess and write back
	"""
	l = os.listdir(root_dir)
	for p in l:
		_dir= root_dir+p
		try:
			with open(_dir) as f:
				text = f.read()
				text = substi(text)
		except:
			print("Cannot modify "+ _dir)
			continue
		try:
			with open(_dir,'wb') as f:
				f.write(text)
		except:
			print("Cannot write into "+ _dir)
			continue

	print(root_dir+" Done!")

#######################################################################################################
################################################# Testing	###########################################
#######################################################################################################

# testing
def test_old():
	# test extract
	in_path ="/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/02/04/1823747.xml"

	summary, content, meta = extract(in_path)
	# print(len(summary),len(content),len(meta))
	# f = open("test1","wb")
	# try:
	# 	f.write(content)
	# except UnicodeEncodeError:
	# 	print("Encode in Unicode!")
	# 	f.write(content.encode("utf8"))
	# f.close()

	#test extractall
	# in_dir =s
	in_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2006'
	in_dir_200701 = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/01'
	in_dir_200702 = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007/02'
	tag_path = "/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2006/01/02/1729112.xml"
	#extractall(in_dir_200701)
	# thread1 = myThreads(in_dir_200701,"0701")
	# thread2 = myThreads(in_dir_200702,"0702")
	# thread1.start()
	# thread2.start()
	#extract_multi(in_dir,2006,2007)
	extract_tag_all(in_dir)

	#test annotate_all
	in_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/2007content'
	out_dir = '/Users/liujingyun/Desktop/NLP/nyt_corpus/summary_annotated'
	#annotate_all(in_dir, out_dir)

	for i in range(1987,2008):
		in_path = '/Users/liujingyun/Desktop/NLP/nyt_corpus/data/'+str(i)+'content'
		out_path = '/Users/liujingyun/Desktop/NLP/nyt_corpus/content_annotated/'+str(i)
		#annotate_all(in_path, out_path)

def test_substi():
	# _dir = "/home/ml/jliu164/corpus/nyt_corpus/summary/2007summary/"
	# substi_all(_dir)

	pool = ThreadPool(6)
	root_dir = '/home/ml/jliu164/corpus/nyt_corpus/summary/'
	dirs = [root_dir+str(i)+"summary/" for i in range(2002,2007)]
	results = pool.map(substi_all, dirs)
	pool.close()
	pool.join()

if __name__ == '__main__':
	# test()
	test_substi()


def first_second(my_list):
    max1, max2 = my_list[0], my_list[0]

    for n in my_list:
        if n > max1:
            max2 = max1
            max1 = n
        elif n > max2 and n != max1:
            max2 = n

    return max1, max2

if __name__ == '__main__':
    myList = [84, 85, 86, 87, 85, 90, 85, 83, 23, 45, 84, 1, 2, 0]
    print(first_second(myList))