"""

>>> This is an implementation of DE360, the flow estimator for 360-degree imagery with Latitude-aware convolution.

>>> This is modified from official PWC-Net code. We thank the authors of PWC-Net for sharing codes for their great works.

"""

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

from correlation_package import correlation
from LatConv import LatConv

import pdb
import cv2
import sys
import numpy as np

def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
    return nn.Sequential(
            nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
                        padding=padding, dilation=dilation, bias=True),
            nn.LeakyReLU(0.1))

def predict_flow(in_planes):
    return nn.Conv2d(in_planes,2,kernel_size=3,stride=1,padding=1,bias=True)

def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
    return nn.ConvTranspose2d(in_planes, out_planes, kernel_size, stride, padding, bias=True)

class DE360(nn.Module):

    def __init__(self, md=4):
        """
        input: md --- maximum displacement (for correlation. default: 4), after warpping
        """
        super(DE360,self).__init__()

        self.conv1a  = conv(3,   16, kernel_size=3, stride=2)
        self.conv1aa = conv(16,  16, kernel_size=3, stride=1)
        self.conv1b  = conv(16,  16, kernel_size=3, stride=1)
        self.conv2a  = conv(16,  32, kernel_size=3, stride=2)
        self.conv2aa = conv(32,  32, kernel_size=3, stride=1)
        self.conv2b  = conv(32,  32, kernel_size=3, stride=1)
        self.conv3a  = conv(32,  64, kernel_size=3, stride=2)
        self.conv3aa = conv(64,  64, kernel_size=3, stride=1)
        self.conv3b  = conv(64,  64, kernel_size=3, stride=1)
        self.conv4a  = conv(64,  96, kernel_size=3, stride=2)
        self.conv4aa = conv(96,  96, kernel_size=3, stride=1)
        self.conv4b  = conv(96,  96, kernel_size=3, stride=1)
        self.conv5a  = conv(96, 128, kernel_size=3, stride=2)
        self.conv5aa = conv(128,128, kernel_size=3, stride=1)
        self.conv5b  = conv(128,128, kernel_size=3, stride=1)
        self.conv6aa = conv(128,196, kernel_size=3, stride=2)
        self.conv6a  = conv(196,196, kernel_size=3, stride=1)
        self.conv6b  = conv(196,196, kernel_size=3, stride=1)

        self.LatConv2 = LatConv(32, 3)
        self.LatConv3 = LatConv(64, 3)
        self.LatConv4 = LatConv(96, 3)
        self.LatConv5 = LatConv(128, 3)
        self.LatConv6 = LatConv(196, 3)

        self.corr = correlation.Correlation(pad_size = md, kernel_size=1, max_displacement=md, stride1=1, stride2=1, corr_multiply=1)
        self.leakyRELU = nn.LeakyReLU(0.1)

        nd = (2*md+1)**2
        dd = np.cumsum([128,128,96,64,32])

        od = nd
        self.conv6_0 = conv(od,      128, kernel_size=3, stride=1)
        self.conv6_1 = conv(od+dd[0],128, kernel_size=3, stride=1)
        self.conv6_2 = conv(od+dd[1],96,  kernel_size=3, stride=1)
        self.conv6_3 = conv(od+dd[2],64,  kernel_size=3, stride=1)
        self.conv6_4 = conv(od+dd[3],32,  kernel_size=3, stride=1)
        self.predict_flow6 = predict_flow(od+dd[4])
        self.deconv6 = deconv(2, 2, kernel_size=4, stride=2, padding=1)
        self.upfeat6 = deconv(od+dd[4], 2, kernel_size=4, stride=2, padding=1)

        od = nd+128+4
        self.conv5_0 = conv(od,      128, kernel_size=3, stride=1)
        self.conv5_1 = conv(od+dd[0],128, kernel_size=3, stride=1)
        self.conv5_2 = conv(od+dd[1],96,  kernel_size=3, stride=1)
        self.conv5_3 = conv(od+dd[2],64,  kernel_size=3, stride=1)
        self.conv5_4 = conv(od+dd[3],32,  kernel_size=3, stride=1)
        self.predict_flow5 = predict_flow(od+dd[4])
        self.deconv5 = deconv(2, 2, kernel_size=4, stride=2, padding=1)
        self.upfeat5 = deconv(od+dd[4], 2, kernel_size=4, stride=2, padding=1)

        od = nd+96+4
        self.conv4_0 = conv(od,      128, kernel_size=3, stride=1)
        self.conv4_1 = conv(od+dd[0],128, kernel_size=3, stride=1)
        self.conv4_2 = conv(od+dd[1],96,  kernel_size=3, stride=1)
        self.conv4_3 = conv(od+dd[2],64,  kernel_size=3, stride=1)
        self.conv4_4 = conv(od+dd[3],32,  kernel_size=3, stride=1)
        self.predict_flow4 = predict_flow(od+dd[4])
        self.deconv4 = deconv(2, 2, kernel_size=4, stride=2, padding=1)
        self.upfeat4 = deconv(od+dd[4], 2, kernel_size=4, stride=2, padding=1)

        od = nd+64+4
        self.conv3_0 = conv(od,      128, kernel_size=3, stride=1)
        self.conv3_1 = conv(od+dd[0],128, kernel_size=3, stride=1)
        self.conv3_2 = conv(od+dd[1],96,  kernel_size=3, stride=1)
        self.conv3_3 = conv(od+dd[2],64,  kernel_size=3, stride=1)
        self.conv3_4 = conv(od+dd[3],32,  kernel_size=3, stride=1)
        self.predict_flow3 = predict_flow(od+dd[4])
        self.deconv3 = deconv(2, 2, kernel_size=4, stride=2, padding=1)
        self.upfeat3 = deconv(od+dd[4], 2, kernel_size=4, stride=2, padding=1)

        od = nd+32+4
        self.conv2_0 = conv(od,      128, kernel_size=3, stride=1)
        self.conv2_1 = conv(od+dd[0],128, kernel_size=3, stride=1)
        self.conv2_2 = conv(od+dd[1],96,  kernel_size=3, stride=1)
        self.conv2_3 = conv(od+dd[2],64,  kernel_size=3, stride=1)
        self.conv2_4 = conv(od+dd[3],32,  kernel_size=3, stride=1)
        self.predict_flow2 = predict_flow(od+dd[4])
        self.deconv2 = deconv(2, 2, kernel_size=4, stride=2, padding=1)

        self.dc_conv1 = conv(od+dd[4], 128, kernel_size=3, stride=1, padding=1,  dilation=1)
        self.dc_conv2 = conv(128,      128, kernel_size=3, stride=1, padding=2,  dilation=2)
        self.dc_conv3 = conv(128,      128, kernel_size=3, stride=1, padding=4,  dilation=4)
        self.dc_conv4 = conv(128,      96,  kernel_size=3, stride=1, padding=8,  dilation=8)
        self.dc_conv5 = conv(96,       64,  kernel_size=3, stride=1, padding=16, dilation=16)
        self.dc_conv6 = conv(64,       32,  kernel_size=3, stride=1, padding=1,  dilation=1)
        self.dc_conv7 = predict_flow(32)

        for m in self.modules():
            if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
                nn.init.kaiming_normal(m.weight.data, mode='fan_in')
                if m.bias is not None:
                    m.bias.data.zero_()

    def warp(self, x, flo):
        """
        warp an image/tensor (im2) back to im1, according to the optical flow
        x: [B, C, H, W] (im2)
        flo: [B, 2, H, W] flow
        """
        B, C, H, W = x.size()
        # mesh grid
        xx = torch.arange(0, W).view(1,-1).repeat(H,1)
        yy = torch.arange(0, H).view(-1,1).repeat(1,W)
        xx = xx.view(1,1,H,W).repeat(B,1,1,1)
        yy = yy.view(1,1,H,W).repeat(B,1,1,1)
        grid = torch.cat((xx,yy),1).float()

        if x.is_cuda:
            grid = grid.cuda()
        vgrid = Variable(grid) + flo

        # scale grid to [-1,1]
        vgrid[:,0,:,:] = 2.0*vgrid[:,0,:,:].clone() / max(W-1,1)-1.0
        vgrid[:,1,:,:] = 2.0*vgrid[:,1,:,:].clone() / max(H-1,1)-1.0

        vgrid = vgrid.permute(0,2,3,1)
        output = nn.functional.grid_sample(x, vgrid)
        mask = torch.autograd.Variable(torch.ones(x.size())).cuda()
        mask = nn.functional.grid_sample(mask, vgrid)

        mask[mask<0.9999] = 0
        mask[mask>0] = 1

        return output*mask

    def forward(self, im1, im2):
        """ Estimate flow from img2 to img1 """

        c11 = self.conv1b(self.conv1aa(self.conv1a(im1)))
        c21 = self.conv1b(self.conv1aa(self.conv1a(im2)))

        c12 = self.conv2b(self.conv2aa(self.conv2a(c11)))
        c22 = self.conv2b(self.conv2aa(self.conv2a(c21)))

        c13 = self.conv3b(self.conv3aa(self.conv3a(c12)))
        c23 = self.conv3b(self.conv3aa(self.conv3a(c22)))

        c14 = self.conv4b(self.conv4aa(self.conv4a(c13)))
        c24 = self.conv4b(self.conv4aa(self.conv4a(c23)))

        c15 = self.conv5b(self.conv5aa(self.conv5a(c14)))
        c25 = self.conv5b(self.conv5aa(self.conv5a(c24)))

        c16 = self.conv6b(self.conv6a(self.conv6aa(c15)))
        c26 = self.conv6b(self.conv6a(self.conv6aa(c25)))

        # Latitude-aware convolution
        dc16 = self.LatConv6(c16)
        dc26 = self.LatConv6(c26)

        corr6 = self.corr(dc16, dc26)
        corr6 = self.leakyRELU(corr6)

        x = torch.cat((self.conv6_0(corr6), corr6),1)
        x = torch.cat((self.conv6_1(x), x),1)
        x = torch.cat((self.conv6_2(x), x),1)
        x = torch.cat((self.conv6_3(x), x),1)
        x = torch.cat((self.conv6_4(x), x),1)
        flow6 = self.predict_flow6(x)
        up_flow6 = self.deconv6(flow6)
        up_feat6 = self.upfeat6(x)

        warp5 = self.warp(c25, up_flow6*0.625)
        # Latitude-aware convolution
        dc15 = self.LatConv5(c15)
        dwarp5 = self.LatConv5(warp5)

        corr5 = self.corr(dc15, dwarp5)
        corr5 = self.leakyRELU(corr5)

        x = torch.cat((corr5, c15, up_flow6, up_feat6), 1)
        x = torch.cat((self.conv5_0(x), x),1)
        x = torch.cat((self.conv5_1(x), x),1)
        x = torch.cat((self.conv5_2(x), x),1)
        x = torch.cat((self.conv5_3(x), x),1)
        x = torch.cat((self.conv5_4(x), x),1)
        flow5 = self.predict_flow5(x)

        #up_flow5 = self.deconv5(flow5) + 2 * F.interpolate(up_flow6*0.625, (2*flow5.size(2), 2*flow5.size(3)), mode = 'bicubic')
        up_flow5 = self.deconv5(flow5)
        up_feat5 = self.upfeat5(x)

        warp4 = self.warp(c24, up_flow5*1.25)
        # Latitude-aware convolution
        dc14 = self.LatConv4(c14)
        dwarp4 = self.LatConv4(warp4)

        corr4 = self.corr(dc14, dwarp4)
        corr4 = self.leakyRELU(corr4)

        x = torch.cat((corr4, c14, up_flow5, up_feat5), 1)
        x = torch.cat((self.conv4_0(x), x),1)
        x = torch.cat((self.conv4_1(x), x),1)
        x = torch.cat((self.conv4_2(x), x),1)
        x = torch.cat((self.conv4_3(x), x),1)
        x = torch.cat((self.conv4_4(x), x),1)
        flow4 = self.predict_flow4(x)
        #up_flow4 = self.deconv4(flow4) + 2 * F.interpolate(up_flow5*1.25, (2*flow4.size(2), 2*flow4.size(3)), mode = 'bicubic')
        up_flow4 = self.deconv4(flow4)
        up_feat4 = self.upfeat4(x)

        warp3 = self.warp(c23, up_flow4*2.5)
        # Latitude-aware convolution
        dc13 = self.LatConv3(c13)
        dwarp3 = self.LatConv3(warp3)

        corr3 = self.corr(dc13, dwarp3)
        corr3 = self.leakyRELU(corr3)

        x = torch.cat((corr3, c13, up_flow4, up_feat4), 1)
        x = torch.cat((self.conv3_0(x), x),1)
        x = torch.cat((self.conv3_1(x), x),1)
        x = torch.cat((self.conv3_2(x), x),1)
        x = torch.cat((self.conv3_3(x), x),1)
        x = torch.cat((self.conv3_4(x), x),1)
        flow3 = self.predict_flow3(x)
        #up_flow3 = self.deconv3(flow3) + 2 * F.interpolate(up_flow4*2.5, (2*flow3.size(2), 2*flow3.size(3)), mode = 'bicubic')
        up_flow3 = self.deconv3(flow3)
        up_feat3 = self.upfeat3(x)

        warp2 = self.warp(c22, up_flow3*5.0)
        # Latitude-aware convolution
        dc12 = self.LatConv2(c12)
        dwarp2 = self.LatConv2(warp2)

        corr2 = self.corr(dc12, dwarp2)
        corr2 = self.leakyRELU(corr2)

        x = torch.cat((corr2, c12, up_flow3, up_feat3), 1)
        x = torch.cat((self.conv2_0(x), x),1)
        x = torch.cat((self.conv2_1(x), x),1)
        x = torch.cat((self.conv2_2(x), x),1)
        x = torch.cat((self.conv2_3(x), x),1)
        x = torch.cat((self.conv2_4(x), x),1)
        flow2 = self.predict_flow2(x)
        flow2 = flow2 + up_flow3 * 5.0

        x = self.dc_conv4(self.dc_conv3(self.dc_conv2(self.dc_conv1(x))))
        flow2 = flow2 + self.dc_conv7(self.dc_conv6(self.dc_conv5(x)))

        f_output = {'flow2' : flow2, 'flow3' : flow3, 'flow4' : flow4, 'flow5' : flow5, 'flow6' : flow6, 'corr2' : corr2}

        return f_output


from zope.interface.verify import verifyObject

from twisted.internet.interfaces import (ITransport, IPushProducer, IConsumer,
    IReactorTCP, IReactorSSL, IReactorUNIX, IAddress, IListeningPort,
    IConnector)
from twisted.internet.address import IPv4Address
from twisted.trial.unittest import TestCase
from twisted.test.proto_helpers import (StringTransport, MemoryReactor,
    RaisingMemoryReactor)
from twisted.internet.protocol import ClientFactory, Factory

class StringTransportTests(TestCase):
    """
    Tests for L{twisted.test.proto_helpers.StringTransport}.
    """
    def setUp(self):
        self.transport = StringTransport()

    def test_interfaces(self):
        """
        L{StringTransport} instances provide L{ITransport}, L{IPushProducer},
        and L{IConsumer}.
        """
        self.assertTrue(verifyObject(ITransport, self.transport))
        self.assertTrue(verifyObject(IPushProducer, self.transport))
        self.assertTrue(verifyObject(IConsumer, self.transport))

    def test_registerProducer(self):
        """
        L{StringTransport.registerProducer} records the arguments supplied to
        it as instance attributes.
        """
        producer = object()
        streaming = object()
        self.transport.registerProducer(producer, streaming)
        self.assertIdentical(self.transport.producer, producer)
        self.assertIdentical(self.transport.streaming, streaming)

    def test_disallowedRegisterProducer(self):
        """
        L{StringTransport.registerProducer} raises L{RuntimeError} if a
        producer is already registered.
        """
        producer = object()
        self.transport.registerProducer(producer, True)
        self.assertRaises(
            RuntimeError, self.transport.registerProducer, object(), False)
        self.assertIdentical(self.transport.producer, producer)
        self.assertTrue(self.transport.streaming)

    def test_unregisterProducer(self):
        """
        L{StringTransport.unregisterProducer} causes the transport to forget
        about the registered producer and makes it possible to register a new
        one.
        """
        oldProducer = object()
        newProducer = object()
        self.transport.registerProducer(oldProducer, False)
        self.transport.unregisterProducer()
        self.assertIdentical(self.transport.producer, None)
        self.transport.registerProducer(newProducer, True)
        self.assertIdentical(self.transport.producer, newProducer)
        self.assertTrue(self.transport.streaming)

    def test_invalidUnregisterProducer(self):
        """
        L{StringTransport.unregisterProducer} raises L{RuntimeError} if called
        when no producer is registered.
        """
        self.assertRaises(RuntimeError, self.transport.unregisterProducer)

    def test_initialProducerState(self):
        """
        L{StringTransport.producerState} is initially C{'producing'}.
        """
        self.assertEqual(self.transport.producerState, 'producing')

    def test_pauseProducing(self):
        """
        L{StringTransport.pauseProducing} changes the C{producerState} of the
        transport to C{'paused'}.
        """
        self.transport.pauseProducing()
        self.assertEqual(self.transport.producerState, 'paused')

    def test_resumeProducing(self):
        """
        L{StringTransport.resumeProducing} changes the C{producerState} of the
        transport to C{'producing'}.
        """
        self.transport.pauseProducing()
        self.transport.resumeProducing()
        self.assertEqual(self.transport.producerState, 'producing')

    def test_stopProducing(self):
        """
        L{StringTransport.stopProducing} changes the C{'producerState'} of the
        transport to C{'stopped'}.
        """
        self.transport.stopProducing()
        self.assertEqual(self.transport.producerState, 'stopped')

    def test_stoppedTransportCannotPause(self):
        """
        L{StringTransport.pauseProducing} raises L{RuntimeError} if the
        transport has been stopped.
        """
        self.transport.stopProducing()
        self.assertRaises(RuntimeError, self.transport.pauseProducing)

    def test_stoppedTransportCannotResume(self):
        """
        L{StringTransport.resumeProducing} raises L{RuntimeError} if the
        transport has been stopped.
        """
        self.transport.stopProducing()
        self.assertRaises(RuntimeError, self.transport.resumeProducing)

    def test_disconnectingTransportCannotPause(self):
        """
        L{StringTransport.pauseProducing} raises L{RuntimeError} if the
        transport is being disconnected.
        """
        self.transport.loseConnection()
        self.assertRaises(RuntimeError, self.transport.pauseProducing)

    def test_disconnectingTransportCannotResume(self):
        """
        L{StringTransport.resumeProducing} raises L{RuntimeError} if the
        transport is being disconnected.
        """
        self.transport.loseConnection()
        self.assertRaises(RuntimeError, self.transport.resumeProducing)

    def test_loseConnectionSetsDisconnecting(self):
        """
        L{StringTransport.loseConnection} toggles the C{disconnecting} instance
        variable to C{True}.
        """
        self.assertFalse(self.transport.disconnecting)
        self.transport.loseConnection()
        self.assertTrue(self.transport.disconnecting)

    def test_specifiedHostAddress(self):
        """
        If a host address is passed to L{StringTransport.__init__}, that
        value is returned from L{StringTransport.getHost}.
        """
        address = object()
        self.assertIdentical(StringTransport(address).getHost(), address)

    def test_specifiedPeerAddress(self):
        """
        If a peer address is passed to L{StringTransport.__init__}, that
        value is returned from L{StringTransport.getPeer}.
        """
        address = object()
        self.assertIdentical(
            StringTransport(peerAddress=address).getPeer(), address)

    def test_defaultHostAddress(self):
        """
        If no host address is passed to L{StringTransport.__init__}, an
        L{IPv4Address} is returned from L{StringTransport.getHost}.
        """
        address = StringTransport().getHost()
        self.assertIsInstance(address, IPv4Address)

    def test_defaultPeerAddress(self):
        """
        If no peer address is passed to L{StringTransport.__init__}, an
        L{IPv4Address} is returned from L{StringTransport.getPeer}.
        """
        address = StringTransport().getPeer()
        self.assertIsInstance(address, IPv4Address)

class ReactorTests(TestCase):
    """
    Tests for L{MemoryReactor} and L{RaisingMemoryReactor}.
    """

    def test_memoryReactorProvides(self):
        """
        L{MemoryReactor} provides all of the attributes described by the
        interfaces it advertises.
        """
        memoryReactor = MemoryReactor()
        verifyObject(IReactorTCP, memoryReactor)
        verifyObject(IReactorSSL, memoryReactor)
        verifyObject(IReactorUNIX, memoryReactor)

    def test_raisingReactorProvides(self):
        """
        L{RaisingMemoryReactor} provides all of the attributes described by the
        interfaces it advertises.
        """
        raisingReactor = RaisingMemoryReactor()
        verifyObject(IReactorTCP, raisingReactor)
        verifyObject(IReactorSSL, raisingReactor)
        verifyObject(IReactorUNIX, raisingReactor)

    def test_connectDestination(self):
        """
        L{MemoryReactor.connectTCP}, L{MemoryReactor.connectSSL}, and
        L{MemoryReactor.connectUNIX} will return an L{IConnector} whose
        C{getDestination} method returns an L{IAddress} with attributes which
        reflect the values passed.
        """
        memoryReactor = MemoryReactor()
        for connector in [memoryReactor.connectTCP(
                              "test.example.com", 8321, ClientFactory()),
                          memoryReactor.connectSSL(
                              "test.example.com", 8321, ClientFactory(),
                              None)]:
            verifyObject(IConnector, connector)
            address = connector.getDestination()
            verifyObject(IAddress, address)
            self.assertEqual(address.host, "test.example.com")
            self.assertEqual(address.port, 8321)
        connector = memoryReactor.connectUNIX("/fake/path", ClientFactory())
        verifyObject(IConnector, connector)
        address = connector.getDestination()
        verifyObject(IAddress, address)
        self.assertEqual(address.name, "/fake/path")

    def test_listenDefaultHost(self):
        """
        L{MemoryReactor.listenTCP}, L{MemoryReactor.listenSSL} and
        L{MemoryReactor.listenUNIX} will return an L{IListeningPort} whose
        C{getHost} method returns an L{IAddress}; C{listenTCP} and C{listenSSL}
        will have a default host of C{'0.0.0.0'}, and a port that reflects the
        value passed, and C{listenUNIX} will have a name that reflects the path
        passed.
        """
        memoryReactor = MemoryReactor()
        for port in [memoryReactor.listenTCP(8242, Factory()),
                     memoryReactor.listenSSL(8242, Factory(), None)]:
            verifyObject(IListeningPort, port)
            address = port.getHost()
            verifyObject(IAddress, address)
            self.assertEqual(address.host, '0.0.0.0')
            self.assertEqual(address.port, 8242)
        port = memoryReactor.listenUNIX("/path/to/socket", Factory())
        verifyObject(IListeningPort, port)
        address = port.getHost()
        verifyObject(IAddress, address)
        self.assertEqual(address.name, "/path/to/socket")

    def test_readers(self):
        """
        Adding, removing, and listing readers works.
        """
        reader = object()
        reactor = MemoryReactor()

        reactor.addReader(reader)
        reactor.addReader(reader)

        self.assertEqual(reactor.getReaders(), [reader])

        reactor.removeReader(reader)

        self.assertEqual(reactor.getReaders(), [])

    def test_writers(self):
        """
        Adding, removing, and listing writers works.
        """
        writer = object()
        reactor = MemoryReactor()

        reactor.addWriter(writer)
        reactor.addWriter(writer)

        self.assertEqual(reactor.getWriters(), [writer])

        reactor.removeWriter(writer)

        self.assertEqual(reactor.getWriters(), [])


import configparser
import datetime
import optparse
import tempfile
import sys
import os
import traceback
import shutil
import time
import multiprocessing as mp
import subprocess
import threading
import random
import math
import gzip
from inspect import currentframe, getframeinfo
from operator import itemgetter
from Bio.Seq import Seq
from Bio.Alphabet import generic_dna
from Bio import SeqIO
from Bio.SeqRecord import SeqRecord
import utilsSR.utilsSR as utils
sys.stdout.write("module loaded\n")
sys.stdout.flush()

def run_analysis(LIB_DIC, ACC_ID, PLOIDY, OPTIONS, LOCA_PROGRAMS, CONFIG, DICO_CHR, QUEUE, PATHNAME, PARSEUNMAPPED):

	TMP = tempfile.NamedTemporaryFile().name.split('/')[-1]

	REF = CONFIG.get('Reference', 'genome')
	SAMTOOLS = LOCA_PROGRAMS.get('Programs','samtools')
	BWA = LOCA_PROGRAMS.get('Programs','bwa')
	JAVA = LOCA_PROGRAMS.get('Programs','java')
	PICARD = LOCA_PROGRAMS.get('Programs','picard')
	GATK = LOCA_PROGRAMS.get('Programs','gatk')
	UseUnifiedGenotyperForBaseRecal =  'no'
	PYTHON = LOCA_PROGRAMS.get('Programs','python')
	PLOTBAMSTAT = LOCA_PROGRAMS.get('Programs','plotbamstats')
	PREFIX = OPTIONS.prefix
	if CONFIG.has_section('Variant'):
		if CONFIG.has_option('Variant', 'UseUnifiedGenotyperForBaseRecal'):
			UseUnifiedGenotyperForBaseRecal =  CONFIG.get('Variant', 'UseUnifiedGenotyperForBaseRecal')

	if 'a' in OPTIONS.steps:
		#1 Mapping
		#2 Merging
		to_return = utils.run_step_A(ACC_ID, LIB_DIC, BWA, REF, TMP, JAVA, PICARD, SAMTOOLS, PREFIX, QUEUE, PARSEUNMAPPED, PLOTBAMSTAT)
		if to_return == 0:
			pass
		else:
			return to_return

	if 'b' in OPTIONS.steps:
		#3 removing duplicates
		to_return = utils.run_step_B(JAVA, PICARD, ACC_ID, TMP, PREFIX, QUEUE)
		if to_return == 0:
			pass
		else:
			return to_return

	if 'c' in OPTIONS.steps:
		#4 indel realignment
		to_return = utils.run_step_C(ACC_ID, JAVA, GATK, REF, CONFIG, PREFIX, QUEUE)
		if to_return == 0:
			pass
		else:
			return to_return

	if 'd' in OPTIONS.steps:
		#5 Base recalibration
		to_return = utils.run_step_D(CONFIG, ACC_ID, UseUnifiedGenotyperForBaseRecal, JAVA, GATK, REF, PLOIDY, PREFIX, QUEUE)
		if to_return == 0:
			pass
		else:
			return to_return

	if 'e' in OPTIONS.steps:
		#6 GVCF generation
		to_return = utils.run_step_E(ACC_ID, PYTHON, REF, DICO_CHR, PREFIX, QUEUE, PATHNAME)
		if to_return == 0:
			pass
		else:
			return to_return
	return to_return

def main_run_analysis(job):

	try:
		rslt = run_analysis(job[0],job[1],job[2],job[3],job[4],job[5], job[6], job[7], job[8], job[9])
	except Exception as e:
		print (e)
		rslt = 1
	finally:
		return rslt

def main_combine(job):

	try:
		rslt = utils.create_pseudo_VCF(job[0],job[1],job[2],job[3],job[4],job[5],job[6],job[7])
	except Exception as e:
		print (e)
		rslt = 1
	finally:
		return rslt

def main_prepa_combine(job):

	try:
		rslt = utils.createGVCF(job[0],job[1],job[2])
	except Exception as e:
		print (e)
		rslt = 1
	finally:
		return rslt

def main_combine_Large(job):

	try:
		rslt = utils.create_pseudo_VCF_Large(job[0],job[1],job[2],job[3],job[4],job[5],job[6],job[7])
	except Exception as e:
		print (e)
		rslt = 1
	finally:
		return rslt

def main_merging_sub_vcf(job):

	try:
		rslt = utils.merge_sub_vcf(job[0],job[1],job[2],job[3])
	except Exception as e:
		print (e)
		rslt = 1
	finally:
		return rslt

def __main__():
	#Parse Command Line
	parser = optparse.OptionParser(
		description="This program go through all steps needed to call SNP from filtered fastq file (From mapping to SNP calling).",
		epilog="Program designed by Guillaume MARTIN (guillaume.martin@cirad.fr)")
	# Wrapper options.
	parser.add_option( '-c', '--conf',		dest='conf',		default=None,		help='A configuration file containing path to references fastq files.'
	'The conf file should contain 2 sections ([Libraries] and [Reference]) and 2 additional ones ([Mapping], [Variant]).\t\t\t\t\t[Libraries] '
	'section should look like as follows :\t[Libraries]\t\t\t\t\t\tlib1 = genome_name path_to_mate1 path_to_mate2 ploidy\t\tlib2 = genome_name '
	'path_to_single ploidy\t\t\t...\t\t\t\t\t\t[Reference] section should look like as follows :\t[Reference]\t\t\t\t\t\tgenome = path_to_the_reference_sequence\t\t'
	'[Variant] section may contain 4 options and should look like :\t\t\t\t\t\t[Variant]\t\t\t\t\t\tindel = path to vcf_of_known_indels\t\t\t\t\tsnp = '
	'path to vcf_of_known_SNPs\t\t\t\tHCopt = additional options to pass to HaplotypeCaller\t\tUseUnifiedGenotyperForBaseRecal = yes or no (if not filled default = no)')
	parser.add_option( '-t', '--thread',	dest='thread',		default='1',		help='Max number of accessions treated at the same time (integer), [default: %default]')
	parser.add_option( '-q', '--queue',		dest='queue',		default=None,		help='Queue to use if SGE is installed on your machine. Do not fill otherwise, [default: %default]')
	parser.add_option( '-p', '--prefix',	dest='prefix',		default='All_lib',	help='Prefix for output bam and vcf containing all libraries. [default: %default]')
	parser.add_option( '-g', '--outgzip',	dest='outgzip',		default='n',		help='Output files in gzip format. [Default: %default]')
	parser.add_option( '-k', '--keepUn',	dest='keepUn',		default='n',		help='Keep unmapped reads in a file. [Default: %default]')
	parser.add_option( '-C', '--chrom',	dest='chrom',		default='all',		help='Chromosomes to work with (only for step f). If "all", all chromosomes will be used for calling. Either: a list of chromosome names separated by ":" [Default: %default]')
	parser.add_option( '-s', '--steps',		dest='steps',		default=None,		help='A string containing steps to perform:\t\t\t\t'
	'a: Aligning libraries\t\t\t\t\t'
	'b: Removing duplicates\t\t\t\t\t\t\t'
	'c: Indel realignment\t\t\t\t\t\t'
	'd: Bases recalibration\t\t\t\t\t\t'
	'e: Allele counting\t\t\t\t\t\t'
	'f: Genotype calling\t\t\t\t\t'
	'g: Merging genotype calling\t\t\t\t\t'
	'h: Mapping statistics calculation\t\t\t\t\t')
	(options, args) = parser.parse_args()

	if options.conf == None:
		sys.exit('--conf argument is missing')
	if options.steps == None:
		sys.exit('--steps argument is missing')

	nbProcs = int(options.thread)

	#Loading the file locating programs
	pathname = os.path.dirname(sys.argv[0])
	loca_programs = configparser.RawConfigParser()
	loca_programs.read(pathname+'/loca_programs.conf')

	#Loading the configuration file
	config = configparser.RawConfigParser()
	config.read(options.conf)

	ref = config.get('Reference', 'genome')
	samtools = loca_programs.get('Programs','samtools')
	bwa = loca_programs.get('Programs','bwa')
	java = loca_programs.get('Programs','java')
	picard = loca_programs.get('Programs','picard')
	gatk = loca_programs.get('Programs','gatk')
	python = loca_programs.get('Programs','python')

	# Checking options
	if options.keepUn != 'y' and options.keepUn != 'n':
		sys.exit('Please enter either "y" or "n" to --keepUn options\n')

	# Getting accessions to work with
	dico_lib = {}
	dico_ploidy = {}
	for n in config.options('Libraries'):
		data =  config.get('Libraries', n).split()
		if not(data[0] in dico_lib):
			dico_lib[data[0]] = {}
		dico_lib[data[0]][n] = data[1:-1]
		if data[0] in dico_ploidy:
			if dico_ploidy[data[0]] != data[-1]:
				sys.exit('This is embarrassing... The accession '+data[0]+' has two different ploidy level??')
		else:
			dico_ploidy[data[0]] = data[-1]

	# Checking files
	if 'a' in options.steps:
		BugInFile = 0
		for n in dico_lib:
			for lib in dico_lib[n]:
				for file in dico_lib[n][lib]:
					if not(os.path.isfile(file)):
						BugInFile = 1
						sys.stdout.write('This is embarrassing: the file '+file+' for library '+lib+' has not been found\n')
		if BugInFile:
			sys.exit('The program exited because some files were not found...')

	# Reference indexation
	sys.stdout.write('Checking for reference sequence index for picard ...\n')
	if not(os.path.isfile(ref+'.fai')):
		create_index = '%s faidx %s' % (samtools, ref)
		utils.run_job(getframeinfo(currentframe()), create_index, 'Error in creating .fai:\n')
		sys.stdout.write('Done\n')
	else:
		sys.stdout.write('Index found for the reference sequence.\n')

	sys.stdout.write('Checking for reference sequence index ...\n')
	if not(os.path.isfile(ref+'.amb')):
		sys.stdout.write('No index found for the reference sequence. Indexing ...\n')
		index = '%s index -a bwtsw %s 2>/dev/null' % (bwa, ref)
		utils.run_job(getframeinfo(currentframe()), index, 'Error in creating index for bwa:\n')
		sys.stdout.write('Done\n')
	else:
		sys.stdout.write('Index found for the reference sequence.\n')

	sys.stdout.write('Checking for reference sequence .dict for picard ...\n')
	if not(os.path.isfile('.'.join(ref.split('.')[:-1])+'.dict')):
		create_dict = '%s -XX:ParallelGCThreads=1 -Xmx8G -jar %s CreateSequenceDictionary REFERENCE=%s OUTPUT=%s' % (java, picard, ref, '.'.join(ref.split('.')[:-1])+'.dict')
		utils.run_job(getframeinfo(currentframe()), create_dict, 'Error in creating .dict:\n')
		sys.stdout.write('Done\n')
	else:
		sys.stdout.write('.dict found for the reference sequence.\n')
	sys.stdout.flush()

	# Obtaining chromosome order in multifasta
	dico_chr = {}
	if 'e' in options.steps or 'f' in options.steps or 'g' in options.steps or 'E' in options.steps or 'F' in options.steps:
		list_chr_order = []
		sequence_dict = list(SeqIO.parse(ref, "fasta"))
		for i in range(len(sequence_dict)):
			list_chr_order.append([sequence_dict[i].id, len(str(sequence_dict[i].seq))])
			dico_chr[sequence_dict[i].id] = len(str(sequence_dict[i].seq))
		del sequence_dict

	# Running analysis
	if 'a' in options.steps or 'b' in options.steps or 'c' in options.steps or 'd' in options.steps or 'e' in options.steps:
		# threads = []
		listJobs = []
		for n in dico_lib:
			sys.stdout.flush()
			listJobs.append([dico_lib[n], n, dico_ploidy[n], options, loca_programs, config, dico_chr, options.queue, pathname, options.keepUn])

		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_run_analysis, listJobs)

		for n in results:
			if n != 0:
				sys.stderr.write('*******************************\n\n')
				sys.stderr.write(str(n[1])+'\n')
				sys.stderr.write('*******************************\n')

		for n in results:
			if n != 0:
				sys.stdout.write('A problem occured in '+n[0]+' accession. See error log for more information.\n')

	# Creating pseudo VCF
	if 'f' in options.steps:

		# getting accession list
		liste_accessions = sorted(list(dico_lib.keys()))

		# Selecting chromosomes
		if options.chrom == 'all':
			ChrList = list(dico_chr.keys())
		else:
			ChrList = options.chrom.split(':')

		# Optimizing window size
		totalSize = 0
		for chr in ChrList:
			if not chr in dico_chr:
				sys.exit('This is embarrassing... chromosome: '+chr+' has not been found in the reference fasta file...\n')
			totalSize += dico_chr[chr]

		# window = int(totalSize/float(nbProcs))
		window = 1000000
		dicoWindow = {}
		for chr in ChrList:
			dicoWindow[chr] = []
			start = 0
			while start < dico_chr[chr]:
				if start + window <= dico_chr[chr]:
					dicoWindow[chr].append([start, start + window])
				else:
					dicoWindow[chr].append([start, dico_chr[chr]])
				start += window

		listJobs = []
		for chr in dicoWindow:
			for pos in dicoWindow[chr]:
				listJobs.append([liste_accessions, ref, options.prefix, dico_ploidy, list_chr_order, chr, pos[0], pos[1]])

		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_combine, listJobs)

		for n in results:
			if n != 0:
				sys.stdout.write(str(n)+'\n')

		# Merging files by chromosomes
		listJobs = []
		for chr in dicoWindow:
			listJobs.append([options.prefix, chr, dicoWindow[chr], options.outgzip])
		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_merging_sub_vcf, listJobs)

		for n in results:
			if n != 0:
				sys.stdout.write(str(n)+'\n')

		sys.stdout.write('Setp f finished\n')

	# Merging pseudo VCF
	if 'g' in options.steps:
		utils.merge_vcf(options.prefix, dico_chr)

	# Mapping statistics calculation
	if 'h' in options.steps:
		utils.Calc_stats(options.prefix, dico_lib)

	# Merging GVCF into one large GVCF (For very large data set: greater than the limit allowed by unix)
	if 'E' in options.steps:

		# getting accession list
		liste_accessions = sorted(list(dico_lib.keys()))

		# Selecting chromosomes
		if options.chrom == 'all':
			ChrList = list(dico_chr.keys())
		else:
			ChrList = options.chrom.split(':')

		# Preparing chromosomal launching
		listJobs = []
		for chr in ChrList:
			listJobs.append([chr, liste_accessions, options.prefix])

		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_prepa_combine, listJobs)

		for n in results:
			if n != 0:
				sys.stdout.write(str(n)+'\n')

	# Creating the vcf
	if 'F' in options.steps:

		# getting accession list
		liste_accessions = sorted(list(dico_lib.keys()))

		# Selecting chromosomes
		if options.chrom == 'all':
			ChrList = list(dico_chr.keys())
		else:
			ChrList = options.chrom.split(':')

		# Optimizing window size
		totalSize = 0
		for chr in ChrList:
			if not chr in dico_chr:
				sys.exit('This is embarrassing... chromosome: '+chr+' has not been found in the reference fasta file...\n')
			totalSize += dico_chr[chr]

		# window = int(totalSize/float(nbProcs))
		window = 1000000
		dicoWindow = {}
		for chr in ChrList:
			dicoWindow[chr] = []
			start = 0
			while start < dico_chr[chr]:
				if start + window <= dico_chr[chr]:
					dicoWindow[chr].append([start, start + window])
				else:
					dicoWindow[chr].append([start, dico_chr[chr]])
				start += window

		listJobs = []
		for chr in dicoWindow:
			for pos in dicoWindow[chr]:
				listJobs.append([liste_accessions, ref, options.prefix, dico_ploidy, list_chr_order, chr, pos[0], pos[1]])

		# for n in listJobs:
			# utils.create_pseudo_VCF_Large(n[0],n[1],n[2],n[3],n[4],n[5],n[6],n[7])

		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_combine_Large, listJobs)

		for n in results:
			if n != 0:
				sys.stdout.write(str(n)+'\n')

		# Merging files by chromosomes
		listJobs = []
		for chr in dicoWindow:
			listJobs.append([options.prefix, chr, dicoWindow[chr], options.outgzip])
		pool = mp.Pool(processes=nbProcs)
		results = pool.map(main_merging_sub_vcf, listJobs)

		for n in results:
			if n != 0:
				sys.stdout.write(str(n)+'\n')

		sys.stdout.write('Setp F finished\n')

if __name__ == "__main__": __main__()


from unittest.mock import Mock

import pytest

from pytools.libpythonpro.libpythonpro import github_api

@pytest.fixture
def avatar_url(mocker):
    resp_mock = Mock()
    url = 'https://avatars.githubusercontent.com/u/61382982?v=4'
    resp_mock.json.return_value = {
        'login': 'lu1zibrahim',
        'id': 61382982,
        'node_id': 'MDQ6VXNlcjYxMzgyOTgy',
        'avatar_url': url,
    }
    get_mock = mocker.patch('pytools.libpythonpro.libpythonpro.github_api.requests.get')
    get_mock.return_value = resp_mock
    return url

def test_buscar_avatar(avatar_url):
    url = github_api.buscar_avatar('lu1zibrahim')
    assert avatar_url == url

def test_buscar_avatar_integração():
    url = github_api.buscar_avatar('lu1zibrahim')
    assert 'https://avatars.githubusercontent.com/u/61382982?v=4' == url


"""
给定 S 和 T 两个字符串，当它们分别被输入到空白的文本编辑器后，
判断二者是否相等，并返回结果。 # 代表退格字符。
输入：S = "ab##", T = "c#d#"
输出：true
解释：S 和 T 都会变成 “”。
注意：如果对空文本输入退格字符，文本继续为空。

"""

class Solution:
    def backspaceCompare(self, S: str, T: str) -> bool:
        # 48ms 26.26%
        S_list = []
        for ch in S:
            if ch != '#':
                S_list.append(ch)
            elif len(S_list) > 0:
                S_list.pop()
        T_list = []
        for ch in T:
            if ch != '#':
                T_list.append(ch)
            elif len(T_list) > 0:
                T_list.pop()
        if S_list == T_list:
            return True
        else:
            return False

    def backspaceCompare2(self, S: str, T: str) -> bool:
        # 48ms
        # 反向遍历 双指针
        import itertools
        def F(S):
            skip = 0
            for x in reversed(S):
                if x == '#':
                    skip += 1
                elif skip:
                    skip -= 1
                else:
                    yield x

        return all(x == y for x, y in itertools.zip_longest(F(S), F(T)))

if __name__ == "__main__":
    S, T = "ab##", "c#d#"
    s = Solution()
    print(s.backspaceCompare2(S, T))

844_!生成器_比较含退格的字符串.py


# -*- coding: utf-8 -*-
"""This module contains auxiliary functions for the creation of tables in the main notebook."""

import pandas as pd
import numpy as np
from statsmodels.formula.api import ols,wls

from auxiliary.auxiliary_datasets import *
from auxiliary.auxiliary_functions import *
from auxiliary.auxiliary_subdatasets import *

def create_table1():
    """
      Creates Table 1.

    """
    data=pd.read_stata('data/census_30cities.dta')
    df=data[['age','citizen','metaread','year','educ','perwt']]

    edu_filter = ['N/A or no schooling','Nursery school to grade 4',
                  'Grade 5, 6, 7, or 8','Grade 9','Grade 10','Grade 11']
    citizen_filter = ['Not a citizen', 'Naturalized citizen']

    #Filter total-workers and lowskilled-worker
    df=df[df['age'].astype(float).between(16,64)]
    df1=df[(df['citizen'].isin(citizen_filter))&(df['educ'].isin(edu_filter))]

    #Create statistic table
    total=pd.crosstab(df['metaread'],df['year'],df['perwt'],colnames=" ",aggfunc='sum')
    lowskilled=pd.crosstab(df1['metaread'],df1['year'],values=df1['perwt'],colnames=" ",aggfunc='sum')

    new_index=(['Atlanta', 'Baltimore', 'Boston','Buffalo', 'Chicago','Cincinnati',
                'Cleveland', 'Columbus','Dallas-Fort Worth', 'Denver-Boulder','Detroit',
                'Honolulu','Houston','Kansas City', 'Los Angeles','Miami','Milwaukee',
                'Minneapolis', 'New Orleans','New York', 'Philadelphia','Phoenix',
                'Pittsburgh','Portland', 'San Diego','San Francisco', 'Seattle',
                'St. Louis', 'Tampa','Washington'])

    #Edit table1
    table1 = pd.DataFrame()
    table1=lowskilled.div(total, axis = 0)
    table1.index=new_index
    table1 = table1.style.format("{:.2%}")
    table1.index.rename('City', inplace=True)

    return table1

def create_table2(dt):

    tbls = ['p025','p2550', 'p5075', 'p75100','p90100']

    """ Looping over dataframe percentile"""
    index=["Usual hrs.per week|H>0","Work at leat 50hrs.(percent)",
           "Work at least 60hrs.(percent)","Age","Married","Children under 18","Child under 6"]
    for i, tbl in enumerate(tbls):
        df = dt[dt[tbl]==1]
        g=df.groupby(["year"])
        table=g.apply(lambda x: pd.Series(np.average(x[["chrswork","work50","work60","age",'married','children',"child5"]],
                                                     weights=x["perwt"], axis=0),index))
        table=table.transpose()
        tbls[i]=table

    """ Merge sub-table """
    table2 = pd.concat([tbls[0].iloc[:,0:3],tbls[1].iloc[:,0:3],tbls[2].iloc[:,0:3],
                        tbls[3].iloc[:,0:3],tbls[4].iloc[:,0:3]], axis=1)
    table2.columns = pd.MultiIndex.from_product([['0 -25 percentile','25 -50 percentile','50 -75 percentile',
                                                  '75 -100 percentile','90 -100 percentile'],['1980','1990','2000']])
    table2=table2.style.format("{:.2f}")

    return table2

def create_table3(dt):

    #Create education filter
    dt=educ_group(dt)
    #Datalist
    edus= ['Highschool','SomeCollege', 'College', 'Master','Advanced']
    tbl = ['tb1','tb2', 'tb3', 'tb4','tb5']

    """Looping over dataframe percentile"""
    index=["Usual hrs.per week|H>0","Labor force participation","Work at leat 50hrs.(percent)",
           "Work at least 60hrs.(percent)","Married","Children under 18","Child under 6"]
    for i, edu in enumerate(edus):
        df=dt[dt['edulv']==edu]
        g=df.groupby(["year"])
        table0=g.apply(lambda x: pd.Series(np.average(x[["lflw","work50","work60",'married','children',"child5"]],
                                                      weights=x["perwt"], axis=0)))
        idx = ~np.isnan(df["chrswork"])
        table1=g.apply(lambda x: pd.Series(np.average(x.chrswork[idx],weights=x.perwt[idx],axis=0)))
        table =pd.concat([table1,table0],axis=1)
        tbl[i] =table.transpose()
        tbl[i].index=index

    #merge sub-table
    table3 = pd.concat([tbl[0].iloc[:,0:3],tbl[1].iloc[:,0:3],
                        tbl[2].iloc[:,0:3],tbl[3].iloc[:,0:3],tbl[4].iloc[:,0:3]], axis=1)
    table3.columns = pd.MultiIndex.from_product([['At most high school grad','Some College','College grad',
                                              "Master's degree",'Professional degree or PhD'],
                                             ['1980','1990','2000']])
    table3=table3.style.format("{:.2f}")
    return table3

def create_table4(dt):
    #Create wage_percentile variables
    cols=['p025','p2550','p5075','p75100','p90100']
    for col in cols:
        dt.loc[:,col]=0
    dt.loc[dt['hwage']>dt['hwagep90r'],'p90100']=1
    dt.loc[dt['hwage']>dt['hwagep75r'],'p75100']=1
    dt.loc[(dt['hwage']<=dt['hwagep75r'])&(dt['hwage']>dt['hwagep50r']),'p5075']=1
    dt.loc[(dt['hwage']<=dt['hwagep50r'])&(dt['hwage']>dt['hwagep25r']),'p2550']=1
    dt.loc[dt['hwage']<=dt['hwagep25r'],'p025']=1

    dt["sex"]='Female'
    dt.loc[dt['male']==1,"sex"]='Male'
    #Datalist
    plist =['p025', 'p2550','p5075','p75100','p90100']
    tbl = ['tbl25','tbl50', 'tbl75', 'tbl75100','tbl90100']

    #Split data for female only
    df=dt[(dt['male']==0)]

    "weekhswork",
    """Looping over dataframe percentile"""
    index=["Hrs./week market work (condional on reporting wage)",
           "Age","Married","Children under 18","Child under 6"]
    for i, p in enumerate(plist):
        f=df[df[p]==1]
        fm=dt[dt[p]==1]
        gf=f.groupby(["year"])
        gfm=fm.groupby(["year","sex"])
        tf=gf.apply(lambda x: pd.Series(np.nanmean(x[["uhrsworkcon","age",'married','children',"child5"]],axis=0),index))
        tfm=gfm.apply(lambda x: pd.Series(np.nanmean(x['weekhswork']))).unstack()
        tfm.columns=['Hrs./week on household chores','Hrs./week on household chores (by men in same wage bracket)']
        table=pd.concat([tfm,tf],axis=1)
        tbl[i]=table.transpose()

    #merge sub-table
    table4 = pd.concat([tbl[0].iloc[:,0:3],
                        tbl[1].iloc[:,0:3],
                        tbl[2].iloc[:,0:3],
                        tbl[3].iloc[:,0:3],
                        tbl[4].iloc[:,0:3]],axis=1)
    table4.columns = pd.MultiIndex.from_product([["0 - 25th","25 - 50th","50 - 75th","75 - 100th",
                                                  "90 - 100th"],['1980','2000']])
    table4=table4.style.format("{:.2f}")

    return table4

def create_table5(df):

    #Create group
    p025 = df.query('hwage<=hwagep25r')
    p2550 = df.query('hwage>hwagep25r and hwage<= hwagep50r')
    p5075 = df.query('hwage>hwagep50r and hwage<=hwagep75r')
    p75100 = df.query('hwage>hwagep75r')
    p90100 = df.query('hwage>hwagep90r')

    #Datalist
    dflist =[p025, p2550,p5075,p75100,p90100]
    tbl = ['tbl25','tbl50', 'tbl75', 'tbl75100','tbl90100']

    """Looping over dataframe percentile"""
    for i, dt in enumerate(dflist):
        """ Create female table %percentage"""
        key=["Dummy for positive exp. in housekeeping","Housekeeping exp. |E>0 (1990 dollars)","(std)"]
        g=dt.groupby(["year"])
        table = pd.concat([g.apply(lambda x: np.average(x['dum340310c1'],weights=x['finlwt21'])).round(3),
                           g.apply(lambda x: np.average(x['avcost340310'],weights=x['finlwt'])).round(1),
                           g.apply(lambda x: aweighted_std(x['avcost340310'],x['finlwt']))],
                          axis=0, keys=key).unstack()
        tbl[i]=table

    #merge sub-table
    table5 = pd.concat([tbl[0].iloc[:,0:3],
                        tbl[1].iloc[:,0:3],
                        tbl[2].iloc[:,0:3],
                        tbl[3].iloc[:,0:3],
                        tbl[4].iloc[:,0:3]], axis=1)
    table5.columns = pd.MultiIndex.from_product([["0 - 25th percentile","25 - 50th percentile","50 - 75th percentile",
                                              "75 - 100th percentile","90 - 100th percentile"],['1980','1990','2000']])

    return table5

def create_table6(wls1,wls2,wls3,wls4):

    table6 =pd.DataFrame(np.full((6, 4), np.nan))

    table6.iloc[0,:]=["{:.3f}".format(wls1.params['ins']),"{:.3f}".format(wls2.params['ins']),
                      "{:.3f}".format(wls3.params['ins']),"{:.3f}".format(wls4.params['ins'])]    #coeff
    table6.iloc[1,:]=["({:.3f})".format(wls1.bse['ins']),"({:.3f})".format(wls2.bse['ins']),
                      "({:.3f})".format(wls3.bse['ins']),"({:.3f})".format(wls4.bse['ins'])]      #std
    table6.iloc[2,:]=["Basic","Basic","Basic","Basic"]
    table6.iloc[3,:]=["No","Yes","No","No"]
    table6.iloc[4,:]=["No","No","Yes","No"]
    table6.iloc[5,:]=[wls1.nobs/3,wls2.nobs/3,wls3.nobs/3,wls4.nobs/3]                                          #observation

    table6.columns= pd.MultiIndex.from_product([["ln[(LS Imm + LS Nat)/Labor Force]"],["(1)","(2)","(3)","(4)"]])
    table6.index=['Log(∑share i,j,1970 × LS Imm jt)',"",'Controls','Excludes California',
                  'Excludes Miami, New York City, and Los Angeles ','Number of cities ']

    return table6


import pytest

import numpy as np
from os.path import join
from warnings import catch_warnings

from hera_sim.config import CONFIG_PATH
from hera_sim.defaults import defaults
from hera_sim.interpolators import Beam, Tsky
from hera_sim.sigchain import gen_bandpass

def test_config_swap():
    defaults.set("h1c")
    config1 = defaults().copy()
    defaults.set("h2c", refresh=True)
    assert config1 != defaults()

def test_direct_config_path():
    config = join(CONFIG_PATH, "H2C.yaml")
    defaults.set(config, refresh=True)
    # check some of the parameters
    assert defaults()["integration_time"] == 8.59
    assert isinstance(defaults()["Tsky_mdl"], Tsky)
    assert isinstance(defaults()["omega_p"], Beam)

def test_null_config():
    defaults.set(None, refresh=True)
    assert defaults() == {}
    defaults.deactivate()

def test_multiple_param_specification():
    config = {0: {"Nfreqs": 100}, 1: {"Nfreqs": 200}}
    with catch_warnings(record=True) as w:
        defaults.set(config, refresh=True)
        # make sure that there's an error message
        assert w[0].message != ""
    defaults.deactivate()

def test_bandpass_changes():
    defaults.set("h1c", refresh=True)
    fqs = np.linspace(0.1, 0.2, 100)
    np.random.seed(0)
    bp = gen_bandpass(fqs, [0])[0]
    defaults.set("h2c", refresh=True)
    np.random.seed(0)
    assert not np.all(bp == gen_bandpass(fqs, [0])[0])
    defaults.deactivate()

def test_activate_and_deactivate():
    defaults.activate()
    assert defaults._override_defaults
    defaults.deactivate()
    assert not defaults._override_defaults

def test_dict_unpacking():
    config = {
        "setup": {
            "frequency_array": {"Nfreqs": 100, "start_freq": 100e6},
            "time_array": {"Ntimes": 50, "start_time": 2e6},
        },
        "telescope": {"omega_p": np.ones(100)},
    }
    defaults.set(config, refresh=True)
    for value in defaults().values():
        assert not isinstance(value, dict)
    defaults.deactivate()

def test_refresh():
    # choose some defaults to start with
    defaults.set("h1c")
    # now use new, simple defaults
    config = {"Nfreqs": 100}
    defaults.set(config, refresh=True)
    # check that refresh is working
    assert "Ntimes" not in defaults()
    assert "Nfreqs" in defaults()
    # default behavior is to not refresh, just update
    config = {"Nfreqs": 200, "Ntimes": 50}
    defaults.set(config)
    assert "Ntimes" in defaults()
    assert defaults("Nfreqs") == 200
    defaults.deactivate()

def test_call_with_bad_component():
    defaults.set("h1c")
    with pytest.raises(KeyError) as err:
        defaults("not_a_valid_key")
    assert "not_a_valid_key not found in configuration." == err.value.args[0]
    defaults.deactivate()

def test_bad_config_type():
    not_a_string = 1
    with pytest.raises(ValueError) as err:
        defaults.set(not_a_string)
    assert "The configuration must be a" in err.value.args[0]
    defaults.deactivate()

def test_bad_config_file():
    with pytest.raises(FileNotFoundError):
        defaults.set("not_a_file")
    defaults.deactivate()


import orange
import random, math
from orngABCN2 import ABCN2

def ruleToString(rule, showDistribution = True):
    def selectSign(oper):
        if oper == orange.ValueFilter_continuous.Less:
            return "<"
        elif oper == orange.ValueFilter_continuous.LessEqual:
            return "<="
        elif oper == orange.ValueFilter_continuous.Greater:
            return ">"
        elif oper == orange.ValueFilter_continuous.GreaterEqual:
            return ">="
        else: return "="

    if not rule:
        return "None"
    conds = rule.filter.conditions
    domain = rule.filter.domain

    ret = "IF "
    if len(conds)==0:
        ret = ret + "TRUE"

    for i,c in enumerate(conds):
        if i > 0:
            ret += " AND "
        if type(c) == orange.ValueFilter_discrete:
            ret += domain[c.position].name + "=" + str([domain[c.position].values[int(v)] for v in c.values])
        elif type(c) == orange.ValueFilter_continuous:
            ret += domain[c.position].name + selectSign(c.oper) + str(c.ref)
    if rule.classifier and type(rule.classifier) == orange.DefaultClassifier and rule.classifier.defaultVal:
        ret = ret + " THEN "+domain.classVar.name+"="+\
        str(rule.classifier.defaultValue)
        if showDistribution:
            ret += str(rule.classDistribution)
    elif rule.classifier and type(rule.classifier) == orange.DefaultClassifier and type(domain.classVar) == orange.EnumVariable:
        ret = ret + " THEN "+domain.classVar.name+"="+\
        str(rule.classDistribution.modus())
        if showDistribution:
            ret += str(rule.classDistribution)
    return ret

class LaplaceEvaluator(orange.RuleEvaluator):
    def __call__(self, rule, data, weightID, targetClass, apriori):
        if not rule.classDistribution:
            return 0.
        sumDist = rule.classDistribution.cases
        if not sumDist or (targetClass>-1 and not rule.classDistribution[targetClass]):
            return 0.
        # get distribution
        if targetClass>-1:
            return (rule.classDistribution[targetClass]+1)/(sumDist+2)
        else:
            return (max(rule.classDistribution)+1)/(sumDist+len(data.domain.classVar.values))

class WRACCEvaluator(orange.RuleEvaluator):
    def __call__(self, rule, data, weightID, targetClass, apriori):
        if not rule.classDistribution:
            return 0.
        sumDist = rule.classDistribution.cases
        if not sumDist or (targetClass>-1 and not rule.classDistribution[targetClass]):
            return 0.
        # get distribution
        if targetClass>-1:
            pRule = rule.classDistribution[targetClass]/apriori[targetClass]
            pTruePositive = rule.classDistribution[targetClass]/sumDist
            pClass = apriori[targetClass]/apriori.cases
        else:
            pRule = sumDist/apriori.cases
            pTruePositive = max(rule.classDistribution)/sumDist
            pClass = apriori[rule.classDistribution.modus()]/sum(apriori)
        if pTruePositive>pClass:
            return pRule*(pTruePositive-pClass)
        else: return (pTruePositive-pClass)/max(pRule,1e-6)

class mEstimate(orange.RuleEvaluator):
    def __init__(self, m=2):
        self.m = m
    def __call__(self, rule, data, weightID, targetClass, apriori):
        if not rule.classDistribution:
            return 0.
        sumDist = rule.classDistribution.abs
        if self.m == 0 and not sumDist:
            return 0.
        # get distribution
        if targetClass>-1:
            p = rule.classDistribution[targetClass]+self.m*apriori[targetClass]/apriori.abs
            p = p / (rule.classDistribution.abs + self.m)
        else:
            p = max(rule.classDistribution)+self.m*apriori[rule.classDistribution.modus()]/apriori.abs
            p = p / (rule.classDistribution.abs + self.m)
        return p

class RuleStopping_apriori(orange.RuleStoppingCriteria):
    def __init__(self, apriori=None):
        self.apriori =  None

    def __call__(self,rules,rule,examples,data):
        if not self.apriori:
            return False
        if not type(rule.classifier) == orange.DefaultClassifier:
            return False
        ruleAcc = rule.classDistribution[rule.classifier.defaultVal]/rule.classDistribution.abs
        aprioriAcc = self.apriori[rule.classifier.defaultVal]/self.apriori.abs
        if ruleAcc>aprioriAcc:
            return False
        return True

class LengthValidator(orange.RuleValidator):
    """ prune rules with more conditions than self.length. """
    def __init__(self, length=-1):
        self.length = length

    def __call__(self, rule, data, weightID, targetClass, apriori):
        if self.length >= 0:
            return len(rule.filter.conditions) <= self.length
        return True

def supervisedClassCheck(examples):
    if not examples.domain.classVar:
        raise Exception("Class variable is required!")
    if examples.domain.classVar.varType == orange.VarTypes.Continuous:
        raise Exception("CN2 requires a discrete class!")

class CN2Learner(orange.RuleLearner):
    def __new__(cls, examples=None, weightID=0, **kwargs):
        self = orange.RuleLearner.__new__(cls, **kwargs)
        if examples is not None:
            self.__init__(**kwargs)
            return self.__call__(examples, weightID)
        else:
            return self

    def __init__(self, evaluator = orange.RuleEvaluator_Entropy(), beamWidth = 5, alpha = 1.0, **kwds):
        self.__dict__.update(kwds)
        self.ruleFinder = orange.RuleBeamFinder()
        self.ruleFinder.ruleFilter = orange.RuleBeamFilter_Width(width = beamWidth)
        self.ruleFinder.evaluator = evaluator
        self.ruleFinder.validator = orange.RuleValidator_LRS(alpha = alpha)

    def __call__(self, examples, weight=0):
        supervisedClassCheck(examples)

        cl = orange.RuleLearner.__call__(self,examples,weight)
        rules = cl.rules
        return CN2Classifier(rules, examples, weight)

class CN2Classifier(orange.RuleClassifier):
    def __init__(self, rules=None, examples=None, weightID = 0, **argkw):
        self.rules = rules
        self.examples = examples
        self.weightID = weightID
        self.classVar = None if examples is None else examples.domain.classVar
        self.__dict__.update(argkw)
        if examples is not None:
            self.prior = orange.Distribution(examples.domain.classVar, examples)

    def __call__(self, example, result_type=orange.GetValue):
        classifier = None
        for r in self.rules:
         #   r.filter.domain = example.domain
            if r(example) and r.classifier:
                classifier = r.classifier
                classifier.defaultDistribution = r.classDistribution
                break
        if not classifier:
            classifier = orange.DefaultClassifier(example.domain.classVar, self.prior.modus())
            classifier.defaultDistribution = self.prior

        if result_type == orange.GetValue:
          return classifier(example)
        if result_type == orange.GetProbabilities:
          return classifier.defaultDistribution
        return (classifier(example),classifier.defaultDistribution)

    def __str__(self):
        retStr = ruleToString(self.rules[0])+" "+str(self.rules[0].classDistribution)+"\n"
        for r in self.rules[1:]:
            retStr += "ELSE "+ruleToString(r)+" "+str(r.classDistribution)+"\n"
        return retStr

# Kako nastavim v c++, da mi ni potrebno dodati imena
class CN2UnorderedLearner(orange.RuleLearner):
    def __new__(cls, examples=None, weightID=0, **kwargs):
        self = orange.RuleLearner.__new__(cls, **kwargs)
        if examples is not None:
            self.__init__(**kwargs)
            return self.__call__(examples, weightID)
        else:
            return self

    def __init__(self, evaluator = orange.RuleEvaluator_Laplace(), beamWidth = 5, alpha = 1.0, **kwds):
        self.__dict__.update(kwds)
        self.ruleFinder = orange.RuleBeamFinder()
        self.ruleFinder.ruleFilter = orange.RuleBeamFilter_Width(width = beamWidth)
        self.ruleFinder.evaluator = evaluator
        self.ruleFinder.validator = orange.RuleValidator_LRS(alpha = alpha)
        self.ruleFinder.ruleStoppingValidator = orange.RuleValidator_LRS(alpha = 1.0)
        self.ruleStopping = RuleStopping_apriori()
        self.dataStopping = orange.RuleDataStoppingCriteria_NoPositives()

    def __call__(self, examples, weight=0):
        supervisedClassCheck(examples)

        rules = orange.RuleList()
        self.ruleStopping.apriori = orange.Distribution(examples.domain.classVar,examples)
        progress=getattr(self,"progressCallback",None)
        if progress:
            progress.start = 0.0
            progress.end = 0.0
            distrib = orange.Distribution(examples.domain.classVar, examples, weight)
            distrib.normalize()
        for targetClass in examples.domain.classVar:
            if progress:
                progress.start = progress.end
                progress.end += distrib[targetClass]
            self.targetClass = targetClass
            cl = orange.RuleLearner.__call__(self,examples,weight)
            for r in cl.rules:
                rules.append(r)
        if progress:
            progress(1.0,None)
        return CN2UnorderedClassifier(rules, examples, weight)

class CN2UnorderedClassifier(orange.RuleClassifier):
    def __init__(self, rules = None, examples = None, weightID = 0, **argkw):
        self.rules = rules
        self.examples = examples
        self.weightID = weightID
        self.classVar = examples.domain.classVar if examples is not None else None
        self.__dict__.update(argkw)
        if examples is not None:
            self.prior = orange.Distribution(examples.domain.classVar, examples)

    def __call__(self, example, result_type=orange.GetValue, retRules = False):
        def add(disc1, disc2, sumd):
            disc = orange.DiscDistribution(disc1)
            sumdisc = sumd
            for i,d in enumerate(disc):
                disc[i]+=disc2[i]
                sumdisc += disc2[i]
            return disc, sumdisc

        # create empty distribution
        retDist = orange.DiscDistribution(self.examples.domain.classVar)
        covRules = orange.RuleList()
        # iterate through examples - add distributions
        sumdisc = 0.
        for r in self.rules:
            if r(example) and r.classDistribution:
                retDist, sumdisc = add(retDist, r.classDistribution, sumdisc)
                covRules.append(r)
        if not sumdisc:
            retDist = self.prior
            sumdisc = self.prior.abs

        if sumdisc > 0.0:
            for c in self.examples.domain.classVar:
                retDist[c] /= sumdisc
        else:
            retDist.normalize()

        if retRules:
            if result_type == orange.GetValue:
              return (retDist.modus(), covRules)
            if result_type == orange.GetProbabilities:
              return (retDist, covRules)
            return (retDist.modus(),retDist,covRules)
        if result_type == orange.GetValue:
          return retDist.modus()
        if result_type == orange.GetProbabilities:
          return retDist
        return (retDist.modus(),retDist)

    def __str__(self):
        retStr = ""
        for r in self.rules:
            retStr += ruleToString(r)+" "+str(r.classDistribution)+"\n"
        return retStr

class RuleClassifier_bestRule(orange.RuleClassifier):
    def __init__(self, rules, examples, weightID = 0, **argkw):
        self.rules = rules
        self.examples = examples
        self.classVar = examples.domain.classVar
        self.__dict__.update(argkw)
        self.prior = orange.Distribution(examples.domain.classVar, examples)

    def __call__(self, example, result_type=orange.GetValue):
        retDist = orange.Distribution(example.domain.classVar)
        bestRule = None
        for r in self.rules:
            if r(example) and (not bestRule or r.quality>bestRule.quality):
                for v_i,v in enumerate(example.domain.classVar):
                    retDist[v_i] = r.classDistribution[v_i]
                bestRule = r
        if not bestRule:
            retDist = self.prior
        else:
            bestRule.used += 1
        sumdist = sum(retDist)
        if sumdist > 0.0:
            for c in self.examples.domain.classVar:
                retDist[c] /= sumdisc
        else:
            retDist.normalize()
        # return classifier(example, result_type=result_type)
        if result_type == orange.GetValue:
          return retDist.modus()
        if result_type == orange.GetProbabilities:
          return retDist
        return (retDist.modus(),retDist)

    def __str__(self):
        retStr = ""
        for r in self.rules:
            retStr += ruleToString(r)+" "+str(r.classDistribution)+"\n"
        return retStr

class CovererAndRemover_multWeights(orange.RuleCovererAndRemover):
    def __init__(self, mult = 0.7):
        self.mult = mult
    def __call__(self, rule, examples, weights, targetClass):
        if not weights:
            weights = orange.newmetaid()
            examples.addMetaAttribute(weights,1.)
            examples.domain.addmeta(weights, orange.FloatVariable("weights-"+str(weights)), True)
        newWeightsID = orange.newmetaid()
        examples.addMetaAttribute(newWeightsID,1.)
        examples.domain.addmeta(newWeightsID, orange.FloatVariable("weights-"+str(newWeightsID)), True)
        for example in examples:
            if rule(example) and example.getclass() == rule.classifier(example,orange.GetValue):
                example[newWeightsID]=example[weights]*self.mult
            else:
                example[newWeightsID]=example[weights]
        return (examples,newWeightsID)

class CovererAndRemover_addWeights(orange.RuleCovererAndRemover):
    def __call__(self, rule, examples, weights, targetClass):
        if not weights:
            weights = orange.newmetaid()
            examples.addMetaAttribute(weights,1.)
            examples.domain.addmeta(weights, orange.FloatVariable("weights-"+str(weights)), True)
        try:
            coverage = examples.domain.getmeta("Coverage")
        except:
            coverage = orange.FloatVariable("Coverage")
            examples.domain.addmeta(orange.newmetaid(),coverage, True)
            examples.addMetaAttribute(coverage,0.0)
        newWeightsID = orange.newmetaid()
        examples.addMetaAttribute(newWeightsID,1.)
        examples.domain.addmeta(newWeightsID, orange.FloatVariable("weights-"+str(newWeightsID)), True)
        for example in examples:
            if rule(example) and example.getclass() == rule.classifier(example,orange.GetValue):
                try:
                    example[coverage]+=1.0
                except:
                    example[coverage]=1.0
                example[newWeightsID]=1.0/(example[coverage]+1)
            else:
                example[newWeightsID]=example[weights]
        return (examples,newWeightsID)

def rule_in_set(rule,rules):
    for r in rules:
        if rules_equal(rule,r):
            return True
    return False

def rules_equal(rule1,rule2):
    if not len(rule1.filter.conditions)==len(rule2.filter.conditions):
        return False
    for c1 in rule1.filter.conditions:
        found=False # find the same condition in the other rule
        for c2 in rule2.filter.conditions:
            try:
                if not c1.position == c2.position: continue # same attribute?
                if not type(c1) == type(c2): continue # same type of condition
                if type(c1) == orange.ValueFilter_discrete:
                    if not type(c1.values[0]) == type(c2.values[0]): continue
                    if not c1.values[0] == c2.values[0]: continue # same value?
                if type(c1) == orange.ValueFilter_continuous:
                    if not c1.oper == c2.oper: continue # same operator?
                    if not c1.ref == c2.ref: continue #same threshold?
                found=True
                break
            except:
                pass
        if not found:
            return False
    return True

class noDuplicates_validator(orange.RuleValidator):
    def __init__(self,alpha=.05,min_coverage=0,max_rule_length=0,rules=orange.RuleList()):
        self.rules = rules
        self.validator = orange.RuleValidator_LRS(alpha=alpha,min_coverage=min_coverage,max_rule_length=max_rule_length)

    def __call__(self, rule, data, weightID, targetClass, apriori):
        if rule_in_set(rule,self.rules):
            return False
        return bool(self.validator(rule,data,weightID,targetClass,apriori))

class ruleSt_setRules(orange.RuleStoppingCriteria):
    def __init__(self,validator):
        self.ruleStopping = orange.RuleStoppingCriteria_NegativeDistribution()
        self.validator = validator

    def __call__(self,rules,rule,examples,data):
        ru_st = self.ruleStopping(rules,rule,examples,data)
        if not ru_st:
            self.validator.rules.append(rule)
        return bool(ru_st)
#
#def CN2SDUnorderedLearner(examples = None, weightID=0, **kwds):
#    cn2 = CN2SDUnorderedLearnerClass(**kwds)
#    if examples:
#        return cn2(examples, weightID)
#    else:
#        return cn2

class CN2SDUnorderedLearner(CN2UnorderedLearner):
    def __new__(cls, examples=None, weightID=0, **kwargs):
        self = CN2UnorderedLearner.__new__(cls, **kwargs)
        if examples is not None:
            self.__init__(**kwargs)
            return self.__call__(examples, weightID)
        else:
            return self

    def __init__(self, evaluator = WRACCEvaluator(), beamWidth = 5, alpha = 0.05, mult=0.7, **kwds):
        CN2UnorderedLearnerClass.__init__(self, evaluator = evaluator,
                                          beamWidth = beamWidth, alpha = alpha, **kwds)
        self.coverAndRemove = CovererAndRemover_multWeights(mult=mult)

    def __call__(self, examples, weight=0):
        supervisedClassCheck(examples)

        oldExamples = orange.ExampleTable(examples)
        classifier = CN2UnorderedLearnerClass.__call__(self,examples,weight)
        for r in classifier.rules:
            r.filterAndStore(oldExamples,weight,r.classifier.defaultVal)
        return classifier

# Miscellaneous - utility functions
def avg(l):
    if len(l)==0:
        return 0.
    return sum(l)/len(l)

def var(l):
    if len(l)<2:
        return 0.
    av = avg(l)
    vars=[math.pow(li-av,2) for li in l]
    return sum(vars)/(len(l)-1)

def median(l):
    if len(l)==0:
        return 0.
    l.sort()
    le = len(l)
    if le%2 == 1:
        return l[(le-1)/2]
    else:
        return (l[le/2-1]+l[le/2])/2

def perc(l,p):
    l.sort()
    return l[int(math.floor(p*len(l)))]

def createRandomDataSet(data):
    newData = orange.ExampleTable(data)
    # shuffle data
    cl_num = newData.toNumeric("C")
    random.shuffle(cl_num[0][:,0])
    clData = orange.ExampleTable(orange.Domain([newData.domain.classVar]),cl_num[0])
    for d_i,d in enumerate(newData):
        d[newData.domain.classVar] = clData[d_i][newData.domain.classVar]
    return newData

# estimated fisher tippett parameters for a set of values given in vals list (+ deciles)
def compParameters(vals,oldMi=0.5,oldBeta=1.1):
    # compute percentiles
    vals.sort()
    N = len(vals)
    percs = [avg(vals[int(float(N)*i/10):int(float(N)*(i+1)/10)]) for i in range(10)]
    if N<10:
        return oldMi, oldBeta, percs
    beta = math.sqrt(6*var(vals)/math.pow(math.pi,2))
    beta = min(2.0,max(oldBeta, beta))
    mi = max(oldMi, avg(vals) - 0.57721*beta)
    return mi, beta, percs

def computeDists(data, weight=0, targetClass=0, N=100, learner=None):
    """ Compute distributions of likelihood ratio statistics of extreme (best) rules.  """
    if not learner:
        learner = createLearner()

    #########################
    ## Learner preparation ##
    #########################
    oldStopper = learner.ruleFinder.ruleStoppingValidator
    evaluator = learner.ruleFinder.evaluator
    learner.ruleFinder.evaluator = orange.RuleEvaluator_LRS()
    learner.ruleFinder.evaluator.storeRules = True
    learner.ruleFinder.ruleStoppingValidator = orange.RuleValidator_LRS(alpha=1.0)
    learner.ruleFinder.ruleStoppingValidator.max_rule_complexity = 0

    # loop through N (sampling repetitions)
    maxVals = []
    for d_i in range(N):
        # create data set (remove and randomize)
        tempData = createRandomDataSet(data)
        learner.ruleFinder.evaluator.rules = orange.RuleList()
        # Next, learn a rule
        bestRule = learner.ruleFinder(tempData,weight,targetClass,orange.RuleList())
        maxVals.append(bestRule.quality)
    extremeDists=[compParameters(maxVals,1.0,1.0)]

    #####################
    ## Restore learner ##
    #####################
    learner.ruleFinder.evaluator = evaluator
    learner.ruleFinder.ruleStoppingValidator = oldStopper
    return extremeDists

def createEVDistList(evdList):
    l = orange.EVDistList()
    for el in evdList:
        l.append(orange.EVDist(mu=el[0],beta=el[1],percentiles=el[2]))
    return l

class CovererAndRemover_Prob(orange.RuleCovererAndRemover):
    """ This class impements probabilistic covering. """
    def __init__(self, probAttribute=None, sigAttribute=None):
        self.indices = None
        self.probAttribute = probAttribute
        self.bestRule = []

    def initialize(self, examples, weightID, targetClass, apriori):
        self.bestRule = [None]*len(examples)
        self.probAttribute = orange.newmetaid()
        examples.addMetaAttribute(self.probAttribute,-1.e-6)
        examples.domain.addmeta(self.probAttribute, orange.FloatVariable("Probs"))
        for example in examples:
##            if targetClass<0 or (example.getclass() == targetClass):
            example[self.probAttribute] = apriori[targetClass]/apriori.abs
        return examples

    def getBestRules(self, currentRules, examples, weightID):
        bestRules = orange.RuleList()
        for r in currentRules:
            if hasattr(r.learner, "argumentRule") and not orngCN2.rule_in_set(r,bestRules):
                bestRules.append(r)
        for r_i,r in enumerate(self.bestRule):
            if r and not rule_in_set(r,bestRules) and examples[r_i].getclass()==r.classifier.defaultValue:
                bestRules.append(r)
        return bestRules

    def remainingExamplesP(self, examples, targetClass):
        pSum, pAll = 0.0, 0.0
        for ex in examples:
            if ex.getclass() == targetClass:
                pSum += ex[self.probAttribute]
                pAll += 1.0
        return pSum/pAll

    def __call__(self, rule, examples, weights, targetClass):
        if targetClass<0:
            for example_i, example in enumerate(examples):
                if rule(example) and rule.quality>example[self.probAttribute]-0.01:
                    example[self.probAttribute] = rule.quality+0.01
                    self.bestRule[example_i]=rule
        else:
            for example_i, example in enumerate(examples): #rule.classifier.defaultVal == example.getclass() and
                if rule(example) and rule.quality>example[self.probAttribute]:
                    example[self.probAttribute] = rule.quality+0.001
                    self.bestRule[example_i]=rule
##                if rule.classifier.defaultVal == example.getclass():
##                    print example[self.probAttribute]
        # compute factor
        return (examples,weights)

def add_sub_rules(rules, examples, weight, learner, dists):
    apriori = orange.Distribution(examples.domain.classVar,examples,weight)
    newRules = orange.RuleList()
    for r in rules:
        newRules.append(r)

    # loop through rules
    for r in rules:
        tmpList = orange.RuleList()
        tmpRle = r.clone()
        tmpRle.filter.conditions = []
        tmpRle.parentRule = None
        tmpRle.filterAndStore(examples,weight,r.classifier.defaultVal)
        tmpList.append(tmpRle)
        while tmpList and len(tmpList[0].filter.conditions) <= len(r.filter.conditions):
            tmpList2 = orange.RuleList()
            for tmpRule in tmpList:
                # evaluate tmpRule
                oldREP = learner.ruleFinder.evaluator.returnExpectedProb
                learner.ruleFinder.evaluator.returnExpectedProb = False
                learner.ruleFinder.evaluator.evDistGetter.dists = createEVDistList(dists[int(r.classifier.defaultVal)])
                tmpRule.quality = learner.ruleFinder.evaluator(tmpRule,examples,weight,r.classifier.defaultVal,apriori)
                learner.ruleFinder.evaluator.returnExpectedProb = oldREP
                # if rule not in rules already, add it to the list
                if not True in [rules_equal(ri,tmpRule) for ri in newRules] and len(tmpRule.filter.conditions)>0 and tmpRule.quality > apriori[r.classifier.defaultVal]/apriori.abs:
                    newRules.append(tmpRule)
                # create new tmpRules, set parent Rule, append them to tmpList2
                if not True in [rules_equal(ri,tmpRule) for ri in newRules]:
                    for c in r.filter.conditions:
                        tmpRule2 = tmpRule.clone()
                        tmpRule2.parentRule = tmpRule
                        tmpRule2.filter.conditions.append(c)
                        tmpRule2.filterAndStore(examples,weight,r.classifier.defaultVal)
                        if tmpRule2.classDistribution.abs < tmpRule.classDistribution.abs:
                            tmpList2.append(tmpRule2)
            tmpList = tmpList2
    for cl in examples.domain.classVar:
        tmpRle = orange.Rule()
        tmpRle.filter = orange.Filter_values(domain = examples.domain)
        tmpRle.parentRule = None
        tmpRle.filterAndStore(examples,weight,int(cl))
        tmpRle.quality = tmpRle.classDistribution[int(cl)]/tmpRle.classDistribution.abs
        newRules.append(tmpRle)
    return newRules

#def CN2EVCUnorderedLearner(examples = None, weightID=0, **kwds):
#    cn2 = CN2EVCUnorderedLearnerClass(**kwds)
#    if examples:
#        return cn2(examples, weightID)
#    else:
#        return cn2

class CN2EVCUnorderedLearner(ABCN2):
    """This is implementation of CN2 + EVC as evaluation + LRC classification.
        Main parameters:
          -- ...
    """
    def __init__(self, width=5, nsampling=100, rule_sig=1.0, att_sig=1.0, min_coverage = 1., max_rule_complexity = 5.):
        ABCN2.__init__(self, width=width, nsampling=nsampling, rule_sig=rule_sig, att_sig=att_sig,
                       min_coverage=int(min_coverage), max_rule_complexity = int(max_rule_complexity))


from bookstore.books.models import Book
import cloudinary.uploader
from django.db.models.signals import pre_save, pre_delete
from django.dispatch import receiver

@receiver(pre_save, sender=Book)
def delete_old_book_image_on_change(sender, instance, **kwargs):
    try:
        old_image = sender.objects.get(pk=instance.pk).image
        if old_image:
            new_image = instance.image
            if not old_image == new_image:
                cloudinary.uploader.destroy(old_image.public_id)
    except instance.DoesNotExist:
        return

@receiver(pre_delete, sender=Book)
def image_delete_on_book_delete(sender, instance, **kwargs):
    if instance.image:
        cloudinary.uploader.destroy(instance.image.public_id)


from concurrent import futures

import grpc
import time
import uuid
import argparse
import threading

import helper_funcs

from google.protobuf.timestamp_pb2 import Timestamp

import interfaces.globalmessage_pb2 as global_msg

import interfaces.chatserver_pb2_grpc as rpc

import interfaces.nameservice_pb2 as ns_msg
import interfaces.nameservice_pb2_grpc as ns_rpc
import interfaces.databasefacade_pb2_grpc as db_rpc

from src.client import MAX_RETRIES, MAX_TIMEOUT

class ChatServer(rpc.ChatServerServicer):  # inheriting here from the protobuf rpc file which is generated

    def __init__(self, ns, reg_msg):
        # List with all the chat history

        self.new_chat_event = threading.Event()
        self.chat_lock = threading.Lock()
        self.chats = {}
        self.chat_last_time = {}
        self.database_send = {}

        self.client_lock = threading.Lock()
        self.clients = {}

        self.database = None

        self.ns = ns
        self.reg_msg = reg_msg

        # initilize event to set so first message will be written
        self.peer_receive_event = threading.Event()
        self.peer_receive_event.set()

        threading.Thread(name="Database writeback", target=self.__database_write_back, daemon=True).start()

    def __database_write_back(self):
        print("Starting database write thread")
        if self.database == None:
            print("No database write object getting one from the name service")
            database_struct = self.ns.getDatabase(self.reg_msg)
            database_chan = grpc.insecure_channel(database_struct.ipAddress + ":" + str(database_struct.port))
            self.database = db_rpc.DatabaseServerStub(database_chan)

        while True:
            self.new_chat_event.wait(MAX_TIMEOUT)
            if self.new_chat_event.is_set():
                with self.chat_lock:
                    for g, n in self.chats.items():
                        if g in self.database_send and not self.database_send[g]:
                            self.database.UploadMessage(n)
                            self.database_send[g] = True

    # The stream which will be used to send new messages to clients
    def ChatStream(self, request: global_msg.Group, context):
        """
        This is a response-stream type call. This means the server can keep sending messages
        Every client opens this connection and waits for server to send new messages

        :param request_iterator:
        :param context:
        :return:
        """

        if not request.IsInitialized() and len(request.name) == 0:
            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Must specify a group to stream messages")

        peer = context.peer()
        print(f"Request for group {request.name} from peer {peer}")

        with self.chat_lock:
            print("Checking chat history")
            if self.chats.get(request.name, None) == None:
                print("Adding dictionary entry")
                self.chats[request.name] = None
                self.chat_last_time[request.name] = None

            if self.database_send.get(request.name, None) == None:
                self.database_send[request.name] = False

            with self.client_lock:
                print("Setting up peer information")
                if self.clients.get(request.name, None) == None:
                    print("Add dictionary for peer")
                    self.clients[request.name] = {}
                    self.clients[request.name]["received"] = 0
                else:
                    print("Add peer connection info and increment ")

                    self.clients[request.name][peer] = 0
                    self.clients[request.name]["total"] = self.clients[request.name].get("total", 0) + 1

                print(self.clients[request.name])

        # lastindex = 0 if len(self.chats[request.name]) == 0 else len(self.chats[request.name])

        # For every client a infinite loop starts (in gRPC's own managed thread)
        print("Testing active context before streaming")
        while True:
            # Check if there are any new messages
            print("Waiting for new message event")
            self.new_chat_event.wait(MAX_TIMEOUT)
            if self.new_chat_event.is_set():
                with self.chat_lock:
                    print("Checking for new messages")
                    if self.chat_last_time == None or self.chat_last_time.get(request.name, None) == None:
                        # no new message
                        continue

                    # if self.chats[request.name].timestamp.ToDatetime() > self.chat_last_time[request.name].ToDatetime():
                    #     # chat message is newer, send to client

                    with self.client_lock:

                        print("Checking if all peers have received message")
                        print(self.clients[request.name])
                        # if all clients received signal (may cause issue may signal numerous times, will return instantly if event is set)
                        if self.clients[request.name]["received"] == self.clients[request.name]["total"] and self.database_send[request.name]:
                            print("Clearing chat event setting all peers received event")
                            self.new_chat_event.clear()
                            self.peer_receive_event.set()
                            continue

                        print("Checking if this peer has received this message already")
                        # if this client already received this message move on
                        if self.clients[request.name][peer] > 0:
                            continue

                        try:
                            print("Passing message to client")
                            yield self.chats[request.name]

                            print("Update peer informaiton since this peer will get info unless rpc excpetion occured")
                            self.clients[request.name][peer] = self.clients[request.name][peer] + 1
                            self.clients[request.name]["received"] = self.clients[request.name]["received"] + 1
                            print(self.clients[request.name])

                        except grpc.RpcError as e:
                            status = e.code()

                            print(f"Received rpc error when yielding message to client: {status}")
                            context.cancel()
                            # decrement total clients since 1 fell off
                            self.clients[request.name]["total"] = self.clients[request.name]["total"] - 1

                            break

                            # need to cleanup clients dict to remove peer information

        print("Context closed returning from call")

            # while len(self.chats[request.name]) > lastindex:
            #     n = self.chats[request.name][lastindex]
            #     lastindex += 1
            #     yield n

    def SendNote(self, request: global_msg.Note, context):
        """
        This method is called when a clients sends a Note to the server.

        :param request:
        :param context:
        :return:
        """
        # this is only for the server console
        if not request.IsInitialized() and not request.user.IsInitialized() and not request.group.IsInitialized():
            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Must initilize user and group information")

        if len(request.user.userName) == 0 or len(request.group.name) == 0:
            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "Must include a username and group name")

        if not (request.timestamp.IsInitialized() and len( request.timestamp.ListFields() ) > 0):
            context.abort(grpc.StatusCode.INVALID_ARGUMENT, "All messages require timestamp")
        print(request)
        print("[{} @ {}] {} in group {}".format(request.user.displayName, request.timestamp, request.message, request.group.name))
        # Add it to the chat history
        num_waits = 0
        print("Entering loop")
        while context.is_active():
            print("Waiting for peer event")
            self.peer_receive_event.wait(MAX_TIMEOUT)
            if self.peer_receive_event.is_set():
                print("Peer event set checking current message against chat queue")
                with self.chat_lock:
                    # if no chats exist then add to chat list
                    if self.chats.get(request.group.name, None) == None:
                        print("No new messages in queue, setting this message")
                        self.chats[request.group.name] = request
                        self.chat_last_time[request.group.name] = request.timestamp
                        print("Clear peer event set new chat event")
                        self.peer_receive_event.clear()
                        self.new_chat_event.set()
                        return global_msg.Empty()

                    # a chat message already exists
                    else:
                        print("Validating time of current message in queue with incoming message")
                        # if new message in past then inform client it needs to resend
                        if self.chat_last_time[request.group.name].ToDatetime() > request.timestamp.ToDatetime():
                            context.abort(grpc.StatusCode.FAILED_PRECONDITION, "Message is too late in past resend")
                        # need to check peer receive information
                        else:
                            with self.client_lock:
                                print("Resetting peer information")
                                self.clients[request.group.name]["received"] = 0
                                # reset peer values
                                for peer, _ in self.clients[request.group.name].items():
                                    if peer != "received" and peer != "total":
                                        self.clients[request.group.name][peer] = 0
                                print(self.clients[request.group.name])

                                print("Setting new chat message")
                                self.chats[request.group.name] = request
                                self.chat_last_time[request.group.name] = request.timestamp

                                print("Clear peer receive and set new chat event")
                                self.peer_receive_event.clear()
                                self.new_chat_event.set()
                                return global_msg.Empty()

            num_waits = num_waits + 1

            if num_waits > MAX_RETRIES:
                context.abort(grpc.StatusCode.RESOURCE_EXHAUSTED, "Clients took to long to respond to previous message")
        print("Context fell off")

        # if self.chats.get(request.group.name, None) == None:
        #     self.chats[request.group.name] = []

        # if self.clients.get(request.user.userName, None) == None:
        #     group_index = len(self.chats[request.user.userName])
        #     self.clients[request.user.userName] = {request.group.name : group_index }
        # else:
        #     self.clients[request.user.userName][request.group.name] = len(self.chats[request.group.name])

        # self.chats[request.group.name].append(request)
        return global_msg.Empty()  # something needs to be returned required by protobuf language, we just return empty msg

def parse_args():
    parser = argparse.ArgumentParser(description="Chat Server for the distributed chat system")

    parser.add_argument("--port", action="store", type=int)
    parser.add_argument("--id", action="store", type=str)

    return parser.parse_args()

if __name__ == '__main__':
    port = 11915  # a random port for the server to run on
    id = str(uuid.uuid4())

    args = parse_args()

    if args.port != None:
        port = args.port

    if args.id != None:
        id = args.id

    # After starting server need to register with nameservice
    port_ns = 3535
    address_ns = '127.0.0.1'
    ns = grpc.insecure_channel(address_ns + ":" + str(port_ns))
    conn_ns = ns_rpc.NameServerStub(ns)
    registerMessage = ns_msg.RegisterServer()
    registerMessage.ipAddress = helper_funcs.get_ip()
    registerMessage.id = id
    registerMessage.port = port
    registerMessage.timestamp.GetCurrentTime()

    # the workers is like the amount of threads that can be opened at the same time, when there are 10 clients connected
    # then no more clients able to connect to the server.
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=100))  # create a gRPC server
    rpc.add_ChatServerServicer_to_server(ChatServer(conn_ns, registerMessage), server)  # register the server to gRPC
    # gRPC basically manages all the threading and server responding logic, which is perfect!
    print('Starting server. Listening...')
    server.add_insecure_port('127.0.0.1:' + str(port))

    conn_ns.registerChatServer(registerMessage)

    server.start()

    # Server starts in background (in another thread) so keep waiting
    # if we don't wait here the main thread will end, which will end all the child threads, and thus the threads
    # from the server won't continue to work and stop the server

    server.wait_for_termination()


from flask import Flask, render_template, request, jsonify
import cv2
import numpy as np
import base64
import io
from PIL import Image
from selfie_line_drawing import face_detection, canny_edge, canny_edge_auto, laplacian_of_gaussian

app = Flask(__name__)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/api/line-drawing', methods=['POST'])
def line_drawing():
    if 'file' not in request.files:
        return jsonify({'error': 'No file uploaded'}), 400

    file = request.files['file']
    if file.filename == '':
        return jsonify({'error': 'No file selected'}), 400

    method = request.form.get('method', 'canny')
    blur_kernel_size = int(request.form.get('blur_kernel_size', 5))
    low_threshold = int(request.form.get('low_threshold', 100))
    high_threshold = int(request.form.get('high_threshold', 200))

    input_image = Image.open(file.stream)
    input_image = cv2.cvtColor(np.array(input_image), cv2.COLOR_RGB2BGR)

    face_image = face_detection(input_image)

    if method == 'canny':
        edge_image = canny_edge(face_image, blur_kernel_size, low_threshold, high_threshold)
    elif method == 'auto_canny':
        edge_image = canny_edge_auto(face_image, blur_kernel_size)

    elif method == 'log':
        edge_image = laplacian_of_gaussian(face_image, blur_kernel_size)

    _, buffer = cv2.imencode('.jpg', edge_image)
    output_image_base64 = base64.b64encode(buffer).decode('utf-8')

    return jsonify({'image': output_image_base64})

if __name__ == '__main__':
    app.run(debug=True, port=5005)


import pandas as pd
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

def main():
  CAMINHO_DATASET_NORMAL= 'hepatitisNormal.data'

  NAMES = ['CLASS','AGE','SEX','STEROID','ANTIVIRALS','FATIGUE','MALAISE','ANOREXIA',
          'LIVER BIG','LIVER FIRM','SPLEEN PALPABLE','SPIDERS','ASCITES','VARICES',
          'BILIRUBIN','ALK PHOSPHATE','SGOT','ALBUMIN','PROTIME','HISTOLOGY']

  TARGET = 'CLASS'

  # Faz a leitura do arquivo
  df = pd.read_csv(CAMINHO_DATASET_NORMAL, # Nome do arquivo com dados
                      names = NAMES)  # Nome das colunas

  def VisualizePcaProjection(finalDf, targetColumn):
      fig = plt.figure(figsize = (8,8))
      ax = fig.add_subplot(1,1,1)
      ax.set_xlabel('Principal Component 1', fontsize = 15)
      ax.set_ylabel('Principal Component 2', fontsize = 15)
      ax.set_title('PCA Hepatits Dataset', fontsize = 20)
      targets = [0, 1, ]
      colors = ['r', 'g']
      for target, color in zip(targets,colors):
          indicesToKeep = finalDf[targetColumn] == target
          ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1'],
                    finalDf.loc[indicesToKeep, 'principal component 2'],
                    c = color, s = 50)
      ax.legend(targets)
      ax.grid()
      plt.show()

  # Separating out the features
  x = df.loc[:, FEATURES].values

  # Separating out the target
  y = df.loc[:,[TARGET]].values

  # Standardizing the features
  x = StandardScaler().fit_transform(x)
  normalizedDf = pd.DataFrame(data = x, columns = FEATURES)
  normalizedDf = pd.concat([normalizedDf, df[[TARGET]]], axis = 1)

  # PCA projection
  pca = PCA()
  principalComponents = pca.fit_transform(x)

  principalDf = pd.DataFrame(data = principalComponents[:,0:2],
                                columns = ['principal component 1',
                                            'principal component 2'])
  finalDf = pd.concat([principalDf, df[[TARGET]]], axis = 1)

  VisualizePcaProjection(finalDf, TARGET)

if __name__ == "__main__":
    main()


import sys

def input():
    return sys.stdin.readline().rstrip()

def travel(current, visited, cost, i):
    global min_cost
    if cost >= min_cost:
        return None
    if i == N - 1:
        if trip_map[current][0]:
            cost += trip_map[current][0]
            min_cost = min(cost, min_cost)
        return None
    for k in range(1, N):
        if not visited[k] and trip_map[current][k]:
            tmp = visited[:]
            tmp[k] = True
            travel(k, tmp, cost + trip_map[current][k], i + 1)

min_cost = 1e10
N = int(input())
trip_map = [list(map(int, input().split())) for _ in range(N)]

travel(0, [False] * (N + 1), 0, 0)
print(min_cost)