from datetime import datetime

from flask import current_app
from flask_login import login_user, current_user, logout_user

from flaskbb.forum.models import Category, Forum, Topic, Post, ForumsRead, \
    TopicsRead, Report
from flaskbb.user.models import User
from flaskbb.utils.settings import flaskbb_config

def test_category_save(database):
    """Test the save category method."""
    category = Category(title="Test Category")
    category.save()

    assert category.title == "Test Category"

def test_category_delete(category):
    """Test the delete category method."""
    category.delete()

    category = Category.query.filter_by(id=category.id).first()

    assert category is None

def test_category_delete_with_user(topic):
    """Test the delete category method with recounting the users post
    counts.
    """
    user = topic.user
    forum = topic.forum
    category = topic.forum.category

    assert user.post_count == 1
    assert forum.post_count == 1
    assert forum.topic_count == 1

    category.delete([user])

    assert user.post_count == 0

    category = Category.query.filter_by(id=category.id).first()
    topic = Topic.query.filter_by(id=topic.id).first()

    assert category is None
    # The topic should also be deleted
    assert topic is None

def test_category_delete_with_forum(forum):
    """When deleting a category, all of his forums should also be deleted."""
    forum.category.delete()

    assert forum is not None
    assert forum.category is not None

    category = Category.query.filter_by(id=forum.category.id).first()
    forum = Forum.query.filter_by(id=forum.id).first()

    assert forum is None
    assert category is None

def test_category_get_forums(forum, user):
    category = forum.category

    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)
        assert current_user.is_authenticated
        cat, forums = Category.get_forums(category.id, current_user)

        # Check if it is a list because in a category there are normally more
        # than one forum in it (not in these tests)
        assert isinstance(forums, list) is True

        assert forums == [(forum, None)]
        assert cat == category

        # Test the same thing with a logged out user
        logout_user()
        assert not current_user.is_authenticated
        cat, forums = Category.get_forums(category.id, current_user)

        # Check if it is a list because in a category there are normally more
        # than one forum in it (not in these tests)
        assert isinstance(forums, list) is True

        assert forums == [(forum, None)]
        assert cat == category

def test_category_get_all(forum, user):
    category = forum.category

    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)
        assert current_user.is_authenticated
        categories = Category.get_all(current_user)

        # All categories are stored in a list
        assert isinstance(categories, list)
        # The forums for a category are also stored in a list
        assert isinstance(categories[0][1], list)

        assert categories == [(category, [(forum, None)])]

        # Test with logged out user
        logout_user()
        assert not current_user.is_authenticated
        categories = Category.get_all(current_user)

        # All categories are stored in a list
        assert isinstance(categories, list)
        # The forums for a category are also stored in a list
        assert isinstance(categories[0][1], list)

        assert categories == [(category, [(forum, None)])]

def test_forum_save(category, moderator_user):
    """Test the save forum method"""
    forum = Forum(title="Test Forum", category_id=category.id)
    forum.moderators = [moderator_user]
    forum.save()

    assert forum.title == "Test Forum"
    assert forum.moderators == [moderator_user]

def test_forum_delete(forum):
    """Test the delete forum method."""
    forum.delete()

    forum = Forum.query.filter_by(id=forum.id).first()

    assert forum is None

def test_forum_delete_with_user_and_topic(topic, user):
    """Now test the delete forum method with a topic inside."""
    assert user.post_count == 1

    topic.forum.delete([user])

    forum = Forum.query.filter_by(id=topic.forum_id).first()

    assert forum is None

    assert user.post_count == 0

def test_forum_update_last_post(topic, user):
    """Test the update last post method."""
    post = Post(content="Test Content 2")
    post.save(topic=topic, user=user)

    assert topic.forum.last_post == post

    post.delete()

    topic.forum.update_last_post()

    assert topic.forum.last_post == topic.first_post

def test_forum_update_read(database, user, topic):
    """Test the update read method."""
    forumsread = ForumsRead.query.\
        filter(ForumsRead.user_id == user.id,
               ForumsRead.forum_id == topic.forum_id).first()

    topicsread = TopicsRead.query.\
        filter(TopicsRead.user_id == user.id,
               TopicsRead.topic_id == topic.id).first()

    forum = topic.forum

    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)

        # Should return False because topicsread is None
        assert not forum.update_read(current_user, forumsread, topicsread)

        # This is the first time the user visits the topic
        topicsread = TopicsRead()
        topicsread.user_id = user.id
        topicsread.topic_id = topic.id
        topicsread.forum_id = topic.forum_id
        topicsread.last_read = datetime.utcnow()
        topicsread.save()

        # hence, we also need to create a new forumsread entry
        assert forum.update_read(current_user, forumsread, topicsread)

        forumsread = ForumsRead.query.\
            filter(ForumsRead.user_id == user.id,
                   ForumsRead.forum_id == topic.forum_id).first()

        # everything should be up-to-date now
        assert not forum.update_read(current_user, forumsread, topicsread)

        post = Post(content="Test Content")
        post.save(user=user, topic=topic)

        # Updating the topicsread tracker
        topicsread.last_read = datetime.utcnow()
        topicsread.save()

        # now the forumsread tracker should also need a update
        assert forum.update_read(current_user, forumsread, topicsread)

        logout_user()
        # should fail because the user is logged out
        assert not forum.update_read(current_user, forumsread, topicsread)

def test_forum_update_read_two_topics(database, user, topic, topic_moderator):
    """Test if the ForumsRead tracker will be updated if there are two topics
    and where one is unread and the other is read.
    """
    forumsread = ForumsRead.query.\
        filter(ForumsRead.user_id == user.id,
               ForumsRead.forum_id == topic.forum_id).first()

    forum = topic.forum

    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)

        # This is the first time the user visits the topic
        topicsread = TopicsRead()
        topicsread.user_id = user.id
        topicsread.topic_id = topic.id
        topicsread.forum_id = topic.forum_id
        topicsread.last_read = datetime.utcnow()
        topicsread.save()

        # will not create a entry because there is still one unread topic
        assert not forum.update_read(current_user, forumsread, topicsread)

def test_forum_url(forum):
    assert forum.url == "http://localhost:5000/forum/1-test-forum"

def test_forum_slugify(forum):
    assert forum.slug == "test-forum"

def test_forum_get_forum(forum, user):
    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)

        forum_with_forumsread = \
            Forum.get_forum(forum_id=forum.id, user=current_user)

        assert forum_with_forumsread == (forum, None)

        # Test with logged out user
        logout_user()

        forum_with_forumsread = \
            Forum.get_forum(forum_id=forum.id, user=current_user)

        assert forum_with_forumsread == (forum, None)

def test_forum_get_topics(topic, user):
    forum = topic.forum
    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)

        topics = Forum.get_topics(forum_id=forum.id, user=current_user)

        assert topics.items == [(topic, topic.last_post, None)]

        # Test with logged out user
        logout_user()

        topics = Forum.get_topics(forum_id=forum.id, user=current_user)

        assert topics.items == [(topic, topic.last_post, None)]

def test_topic_save(forum, user):
    """Test the save topic method with creating and editing a topic."""
    post = Post(content="Test Content")
    topic = Topic(title="Test Title")

    assert forum.last_post_id is None
    assert forum.post_count == 0
    assert forum.topic_count == 0

    topic.save(forum=forum, post=post, user=user)

    assert topic.title == "Test Title"

    topic.title = "Test Edit Title"
    topic.save()

    assert topic.title == "Test Edit Title"

    # The first post in the topic is also the last post
    assert topic.first_post_id == post.id
    assert topic.last_post_id == post.id

    assert forum.last_post_id == post.id
    assert forum.post_count == 1
    assert forum.topic_count == 1

def test_topic_delete(topic):
    """Test the delete topic method"""
    assert topic.user.post_count == 1
    assert topic.post_count == 1
    assert topic.forum.topic_count == 1
    assert topic.forum.post_count == 1

    topic.delete()

    forum = Forum.query.filter_by(id=topic.forum_id).first()
    user = User.query.filter_by(id=topic.user_id).first()
    topic = Topic.query.filter_by(id=topic.id).first()

    assert topic is None
    assert user.post_count == 0
    assert forum.topic_count == 0
    assert forum.post_count == 0
    assert forum.last_post_id is None

def test_topic_move(topic):
    """Tests the topic move method."""
    forum_other = Forum(title="Test Forum 2", category_id=1)
    forum_other.save()

    forum_old = Forum.query.filter_by(id=topic.forum_id).first()

    assert topic.move(forum_other)

    assert forum_old.topics.all() == []
    assert forum_old.last_post_id is None

    assert forum_old.topic_count == 0
    assert forum_old.post_count == 0

    assert forum_other.last_post_id == topic.last_post_id
    assert forum_other.topic_count == 1
    assert forum_other.post_count == 1

def test_topic_move_same_forum(topic):
    """You cannot move a topic within the same forum."""
    assert not topic.move(topic.forum)

def test_topic_tracker_needs_update(database, user, topic):
    """Tests if the topicsread tracker needs an update if a new post has been
    submitted.
    """
    forumsread = ForumsRead.query.\
        filter(ForumsRead.user_id == user.id,
               ForumsRead.forum_id == topic.forum_id).first()

    topicsread = TopicsRead.query.\
        filter(TopicsRead.user_id == user.id,
               TopicsRead.topic_id == topic.id).first()

    with current_app.test_request_context():
        assert topic.tracker_needs_update(forumsread, topicsread)

        # Update the tracker
        topicsread = TopicsRead()
        topicsread.user_id = user.id
        topicsread.topic_id = topic.id
        topicsread.forum_id = topic.forum_id
        topicsread.last_read = datetime.utcnow()
        topicsread.save()

        forumsread = ForumsRead()
        forumsread.user_id = user.id
        forumsread.forum_id = topic.forum_id
        forumsread.last_read = datetime.utcnow()
        forumsread.save()

        # Now the topic should be read
        assert not topic.tracker_needs_update(forumsread, topicsread)

        post = Post(content="Test Content")
        post.save(topic=topic, user=user)

        assert topic.tracker_needs_update(forumsread, topicsread)

def test_untracking_topic_does_not_delete_it(database, user, topic):
    user.track_topic(topic)
    user.save()
    user.untrack_topic(topic)

    # Note that instead of returning None from the query below, the
    # unpatched verson will actually raise a DetachedInstanceError here,
    # due to the fact that some relationships don't get configured in
    # tests correctly and deleting would break them or something.  The
    # test still fails for the same reason, however: the topic gets
    # (albeit unsuccessfully) deleted when it is removed from
    # topictracker, when it clearly shouldn't be.
    user.save()

    assert Topic.query.filter(Topic.id == topic.id).first()

def test_topic_tracker_needs_update_cleared(database, user, topic):
    """Tests if the topicsread needs an update if the forum has been marked
    as cleared.
    """
    forumsread = ForumsRead.query.\
        filter(ForumsRead.user_id == user.id,
               ForumsRead.forum_id == topic.forum_id).first()

    topicsread = TopicsRead.query.\
        filter(TopicsRead.user_id == user.id,
               TopicsRead.topic_id == topic.id).first()

    with current_app.test_request_context():
        assert topic.tracker_needs_update(forumsread, topicsread)

        # Update the tracker
        forumsread = ForumsRead()
        forumsread.user_id = user.id
        forumsread.forum_id = topic.forum_id
        forumsread.last_read = datetime.utcnow()
        forumsread.cleared = datetime.utcnow()
        forumsread.save()

        # Now the topic should be read
        assert not topic.tracker_needs_update(forumsread, topicsread)

def test_topic_update_read(database, user, topic):
    """Tests the update read method if the topic is unread/read."""
    forumsread = ForumsRead.query.\
        filter(ForumsRead.user_id == user.id,
               ForumsRead.forum_id == topic.forum_id).first()

    with current_app.test_request_context():
        # Test with logged in user
        login_user(user)
        assert current_user.is_authenticated

        # Update the tracker
        assert topic.update_read(current_user, topic.forum, forumsread)
        # Because the tracker is already up-to-date, it shouldn't update it
        # again.
        assert not topic.update_read(current_user, topic.forum, forumsread)

        # Adding a new post - now the tracker shouldn't be up-to-date anymore.
        post = Post(content="Test Content")
        post.save(topic=topic, user=user)

        forumsread = ForumsRead.query.\
            filter(ForumsRead.user_id == user.id,
                   ForumsRead.forum_id == topic.forum_id).first()

        # Test tracker length
        flaskbb_config["TRACKER_LENGTH"] = 0
        assert not topic.update_read(current_user, topic.forum, forumsread)
        flaskbb_config["TRACKER_LENGTH"] = 1
        assert topic.update_read(current_user, topic.forum, forumsread)

        # Test with logged out user
        logout_user()
        assert not current_user.is_authenticated
        assert not topic.update_read(current_user, topic.forum, forumsread)

def test_topic_url(topic):
    assert topic.url == "http://localhost:5000/topic/1-test-topic-normal"

def test_topic_slug(topic):
    assert topic.slug == "test-topic-normal"

def test_post_save(topic, user):
    """Tests the save post method."""
    post = Post(content="Test Content")
    post.save(topic=topic, user=user)

    assert post.content == "Test Content"

    post.content = "Test Edit Content"
    post.save()

    assert post.content == "Test Edit Content"

    assert topic.user.post_count == 2
    assert topic.post_count == 2
    assert topic.last_post == post
    assert topic.forum.post_count == 2

def test_post_delete(topic):
    """Tests the delete post method with three different post types.
    The three types are:
        * First Post
        * A post between the first and last post (middle)
        * Last Post
    """
    post_middle = Post(content="Test Content Middle")
    post_middle.save(topic=topic, user=topic.user)
    assert topic.post_count == 2  # post_middle + first_post

    post_last = Post(content="Test Content Last")
    post_last.save(topic=topic, user=topic.user)

    # first post + post_middle + post_last
    assert topic.post_count == 3
    assert topic.forum.post_count == 3
    assert topic.user.post_count == 3

    post_middle.delete()

    # Check the last posts
    assert topic.last_post == post_last
    assert topic.forum.last_post == post_last
    assert topic.post_count == 2

    post_last.delete()

    # only the first_post remains
    assert topic.post_count == 1
    assert topic.forum.post_count == 1
    assert topic.user.post_count == 1
    assert topic.first_post_id == topic.last_post_id

    assert topic.forum.last_post_id == topic.last_post_id

def test_report(topic, user):
    """Tests if the reports can be saved/edited and deleted with the
    implemented save and delete methods."""
    report = Report(reason="Test Report")
    report.save(user=user, post=topic.first_post)
    assert report.reason == "Test Report"

    report.reason = "Test Report Edited"
    report.save()
    assert report.reason == "Test Report Edited"

    report.delete()
    report = Report.query.filter_by(id=report.id).first()
    assert report is None

def test_forumsread(topic, user):
    """Tests if the forumsread tracker can be saved/edited and deleted with the
    implemented save and delete methods."""
    forumsread = ForumsRead()
    forumsread.user_id = user.id
    forumsread.forum_id = topic.forum_id
    forumsread.last_read = datetime.utcnow()
    forumsread.save()
    assert forumsread is not None

    forumsread.delete()
    forumsread = ForumsRead.query.\
        filter_by(forum_id=forumsread.forum_id).\
        first()
    assert forumsread is None

def test_topicsread(topic, user):
    """Tests if the topicsread tracker can be saved/edited and deleted with the
    implemented save and delete methods."""
    topicsread = TopicsRead()
    topicsread.user_id = user.id
    topicsread.topic_id = topic.id
    topicsread.forum_id = topic.forum_id
    topicsread.last_read = datetime.utcnow()
    topicsread.save()
    assert topicsread is not None

    topicsread.delete()
    topicsread = TopicsRead.query.\
        filter_by(topic_id=topicsread.topic_id).\
        first()
    assert topicsread is None

def test_hiding_post_updates_counts(forum, topic, user):
    new_post = Post(content='spam')
    new_post.save(user=user, topic=topic)
    new_post.hide(user)
    assert user.post_count == 1
    assert topic.post_count == 1
    assert forum.post_count == 1
    assert topic.last_post != new_post
    assert forum.last_post != new_post
    assert new_post.hidden_by == user
    new_post.unhide()
    assert topic.post_count == 2
    assert user.post_count == 2
    assert forum.post_count == 2
    assert topic.last_post == new_post
    assert forum.last_post == new_post
    assert new_post.hidden_by is None

def test_hiding_topic_updates_counts(forum, topic, user):
    assert forum.post_count == 1
    topic.hide(user)
    assert forum.post_count == 0
    assert topic.hidden_by == user
    assert forum.last_post is None
    topic.unhide()
    assert forum.post_count == 1
    assert topic.hidden_by is None
    assert forum.last_post == topic.last_post

def test_hiding_first_post_hides_topic(forum, topic, user):
    assert forum.post_count == 1
    topic.first_post.hide(user)
    assert forum.post_count == 0
    assert topic.hidden_by == user
    assert forum.last_post is None
    topic.first_post.unhide()
    assert forum.post_count == 1
    assert topic.hidden_by is None
    assert forum.last_post == topic.last_post

def test_retrieving_hidden_posts(topic, user):
    new_post = Post(content='stuff')
    new_post.save(user, topic)
    new_post.hide(user)

    assert Post.query.get(new_post.id) is None
    assert Post.query.with_hidden().get(new_post.id) == new_post
    assert Post.query.filter(Post.id == new_post.id).first() is None
    hidden_post = Post.query\
        .with_hidden()\
        .filter(Post.id == new_post.id)\
        .first()
    assert hidden_post == new_post

def test_retrieving_hidden_topics(topic, user):
    topic.hide(user)

    assert Topic.query.get(topic.id) is None
    assert Topic.query.with_hidden().get(topic.id) == topic
    assert Topic.query.filter(Topic.id == topic.id).first() is None
    hidden_topic = Topic.query\
        .with_hidden()\
        .filter(Topic.id == topic.id)\
        .first()
    assert hidden_topic == topic


'''
Module to test the memory on the devise. This is done to check tht a memory error is less or more likely to occur based on an input size.
Created on Apr 18, 2019

@author:
'''
#! /usr/bin/python
import ctypes
import sys

#the size to evaluate
size = int(sys.argv[1])

'''
Class to carry out the memory test
'''
class MemoryTest(ctypes.Structure):
    _fields_ = [  ('chars' , ctypes.c_char*size * 1024*1024 ) ]

    try:
        test = MemoryTest()
        print('success => {0:>4}MB was allocated'.format(size) )
    except:
        print('failure => {0:>4}MB can not be allocated'.format(size) )


#Implementing Priority Queue from scratch using Lists

#Class for creating priority queues with the priority being their value
class PriorityQueue:

#Constructor
	def __init__(self):
		#Initialise the queue
		self.queue = []

#Return the string version of the object
	def __str__(self):
		#A string of all elements of the queue
		return ' '.join([str(i) for i in self.queue])

#Function to check if the queue is empty
	def isEmpty(self):
		#If length of queue is empty, return True else False
		return len(self.queue) == []

#Function to insert/append an element at the rear of a queue
	def insert(self, data):
		#Append the data to the queue
		self.queue.append(data)

#Funtion to remove an element from the queue based on priority
	def delete(self):
		try:
			#Set the index of the maximum element to 0
			max= 0
			#Iterate the whole queue
			for i in range(len(self.queue)):
				#Check for the max element in the whole queue
				if (self.queue[i] > self.queue[max]):
					#Update the value of max
					max = i
			#Get the maximum element which has to be removed
			item = self.queue[max]
			#Delete the maximum element
			del self.queue[max]
			#Return its value
			return item
		except IndexError:
			print()
			exit()

if __name__=='__main__':

	#Create a priority queue
	pQueue= PriorityQueue()

	#Insert values in the priority queue
	pQueue.insert(2)
	pQueue.insert(4)
	pQueue.insert(6)
	pQueue.insert(0)

	#Print the priority queue
	print (pQueue)

	#Remove all the elements from the priority queue
	while not pQueue.isEmpty():
		print(pQueue.delete())


import json
from Util import common
negativeFilePath = common.get_configuration("training", "NEGATIVE_DIC")
negativeDoc  = open(negativeFilePath).readlines()
neg = []
for line in negativeDoc:
    neg.append(line.strip())

positiveFilePath = common.get_configuration("training", "POSITIVE_DIC")
positiveDoc  = open(positiveFilePath).readlines()
pog = []
for line in positiveDoc:
    pog.append(line.strip())

with open("c:/negative.json","w") as out_q:
    out_q.write(json.dumps(neg))

with open("c:/positive.json","w") as out_q:
    out_q.write(json.dumps(pog))


from pox.core import core
import pox.openflow.libopenflow_01 as of
import pox.lib.packet as pkt
from pox.lib.addresses import EthAddr

log = core.getLogger()

def _handle_PacketIn (event):
  packet = event.parsed

  if packet.find("arp"):
    # Reply to ARP
    a = packet.find("arp")
    if a.opcode == a.REQUEST:
      r = pkt.arp()
      r.hwtype = a.hwtype
      r.prototype = a.prototype
      r.hwlen = a.hwlen
      r.protolen = a.protolen
      r.opcode = r.REPLY
      r.hwdst = a.hwsrc
      r.protodst = a.protosrc
      r.protosrc = a.protodst
      r.hwsrc = EthAddr("02:00:DE:AD:BE:EF")
      e = pkt.ethernet(type=packet.type, src=r.hwsrc, dst=a.hwsrc)
      e.payload = r

      msg = of.ofp_packet_out()
      msg.data = e.pack()
      msg.actions.append(of.ofp_action_output(port = of.OFPP_IN_PORT))
      msg.in_port = event.port
      event.connection.send(msg)

      log.info("%s ARPed for %s", r.protodst, r.protosrc)

  elif packet.find("icmp"):
    # Reply to pings

    # Make the ping reply
    icmp = pkt.icmp()
    icmp.type = pkt.TYPE_ECHO_REPLY
    icmp.payload = packet.find("icmp").payload

    # Make the IP packet around it
    ipp = pkt.ipv4()
    ipp.protocol = ipp.ICMP_PROTOCOL
    ipp.srcip = packet.find("ipv4").dstip
    ipp.dstip = packet.find("ipv4").srcip

    # Ethernet around that...
    e = pkt.ethernet()
    e.src = packet.dst
    e.dst = packet.src
    e.type = e.IP_TYPE

    # Hook them up...
    ipp.payload = icmp
    e.payload = ipp

    # Send it back to the input port
    msg = of.ofp_packet_out()
    msg.actions.append(of.ofp_action_output(port = of.OFPP_IN_PORT))
    msg.data = e.pack()
    msg.in_port = event.port
    event.connection.send(msg)

    log.debug("%s pinged %s", ipp.dstip, ipp.srcip)

def launch ():
  core.openflow.addListenerByName("PacketIn", _handle_PacketIn)

  log.info("Pong component running.")


from typing import TypedDict

class FooDict(TypedDict):
    foo: int
    bar: int

foo = FooDict(foo=3, bar=3)

print(foo["foo"]) # Works
print(foo["bar"]) # Works
reveal_type(("foo", "bar")) # Revealed type is 'Tuple[Literal['foo']?, Literal['bar']?]'

for key in ("foo", "bar"):
    reveal_type(key) # Revealed type is 'builtins.str'
    print(foo[key]) # TypedDict key must be a string literal; expected one of ('foo', 'bar')


#!/usr/bin/env python3
# mapper for X-MN matrix function
# matrix input should be in form of 'matrix_identifier,row_number,[row values]', 1 is matrix M, 2 is matrix N, 3 is matrix X
# parameter input should be rows of matrix M, and columns in matrix N

import sys

m = int(sys.argv[1])  # rows in M
n = int(sys.argv[2])  # cols in N

# input comes from stdin
for line in sys.stdin:

    line = line.strip()
    field = line.split(",")
    field = [f.strip() for f in field]  # if spaces between entries

    matrix_num, row = int(field[0]), int(field[1])  # which matrix; row val is index 1

    for col, value in enumerate(field[2:]):
        # matrix M
        if matrix_num == 1:
            # generate number of key value pairs
            for k in range(n):
                print(f'{row},{k}\tM,{col},{value}')

        # matrix N
        elif matrix_num == 2:
            # generate number of key value pairs
            for k in range(m):
                print(f'{k},{col}\tN,{row},{value}')

        # matrix X
        else:
            print(f'{row},{col}\tX,{-1},{value}')


"""Main module"""
import concurrent.futures
import time
import requests

def chunks(l, n):
    """Yield successive n-sized chunks from l. (https://stackoverflow.com/a/312464/157880)"""
    for i in range(0, len(l), n):
        yield l[i:i + n]

class VirusTotal:
    """Main class"""

    def __init__(self, configfile):
        import configparser
        import os
        config = configparser.ConfigParser()
        config.read(configfile)
        self.apikey = config["VIRUSTOTAL"]["apikey"]
        self.num_async_workers = config["client"].get("num_workers", "0")
        if not int(self.num_async_workers):
            # We're i/o bound. Let's load this up.
            self.num_async_workers = os.cpu_count() * 5
        else:
            self.num_async_workers = int(self.num_async_workers)

        self.filereportapi = "https://www.virustotal.com/vtapi/v2/file/report"

    def batch_get_report(self, hashlist, allinfo=False):
        hashes = ",".join(hashlist)
        params = {'apikey': self.apikey, 'resource': hashes}
        if allinfo:
            params['allinfo'] = 1
        for retry in range(5):
            response = requests.get(self.filereportapi, params=params)
            if response.status_code == 200:
                return response.json()
            else:
                sys.stderr.write(hashes + "\n")
                sys.stderr.write(str(response.status_code) + '\n')
                time.sleep(1)

    def batch_get_report_async(self, hashlist, allinfo=False):
        finaldata = []
        with concurrent.futures.ThreadPoolExecutor(max_workers=self.num_async_workers) as executor:
            future_to_list = {executor.submit(
                self.batch_get_report, chunk, allinfo): chunk for chunk in chunks(hashlist, 25)}
            for future in concurrent.futures.as_completed(future_to_list):
                hashsublist = future_to_list[future]
                try:
                    finaldata += future.result()
                except Exception as exc:
                    print('%r generated an exception: %s' % (md5sublist, exc))
                else:
                    print('GOOD')
        return finaldata

if __name__ == "__main__":
    import sys
    vt = VirusTotal(sys.argv[1])
    finaldata = []

    print(vt.batch_get_report_async(
        ['411475F231585EEF5C8F00055F422FB1', 'F5BA2982D2BD2241E4A4AB24B37514D8']))


from sequana.rnadiff import RNADiffResults, RNADiffAnalysis, RNADesign
from . import test_dir
import pytest

def test_design():
    d = RNADesign(f"{test_dir}/data/rnadiff/design.csv")
    assert d.comparisons == [('Complemented_csrA', 'Mut_csrA'),  ('Complemented_csrA', 'WT'),  ('Mut_csrA', 'WT')]
    assert d.conditions ==  ['Complemented_csrA', 'Mut_csrA', 'WT']

    d = RNADesign(f"{test_dir}/data/rnadiff/design.csv", reference="WT")
    assert d.comparisons == [('Complemented_csrA', 'WT'),  ('Mut_csrA', 'WT')]
    assert d.conditions ==  ['Complemented_csrA', 'Mut_csrA', 'WT']

@pytest.mark.xfail(reason="too slow or service may be down")
def test_rnadiff_onefolder():

    # Featurecounts are saved in sequana/resources/testing/rnadiff/rnadiff_onecond_ex1
    # generated from Featurecount of the file to be found in
    # sequana/resources/testing/featurecounts/featurecounts_ex1

    counts = f"{test_dir}/data/rnadiff/rnadiff_onecond_ex1/counts.csv"
    design = f"{test_dir}/data/rnadiff/rnadiff_onecond_ex1/design.csv"
    gff = f"{test_dir}/data/rnadiff/rnadiff_onecond_ex1/Lepto.gff"

    an = RNADiffAnalysis(counts, design,
            condition="condition", comparisons=[("Complemented_csrA", "WT")],
            fc_feature="gene", fc_attribute="ID", gff=gff)
    an

    r = an.run()

    r.plot_count_per_sample()
    r.plot_percentage_null_read_counts()
    #r.plot_volcano()
    r.plot_pca()
    r.plot_mds()
    r.plot_isomap()
    r.plot_density()
    r.plot_boxplot_normeddata()
    r.plot_boxplot_rawdata()
    r.plot_dendogram()
    r.plot_dispersion()
    r.plot_feature_most_present()

    r.comparisons['Complemented_csrA_vs_WT'].plot_volcano()
    r.comparisons['Complemented_csrA_vs_WT'].plot_padj_hist()
    r.comparisons['Complemented_csrA_vs_WT'].plot_pvalue_hist()

    r.summary()
    r.alpha = 1
    r.log2_fc = 1


from __future__ import division
from __future__ import print_function
import os, time, scipy.io, shutil, importlib
import numpy as np
import argparse
import torch
import torch.nn as nn
from skimage import io

from utils import AverageMeter, chw_to_hwc
from dataset.loader import RealNoise, SynNoise

parser = argparse.ArgumentParser(description = 'Train')
parser.add_argument('--model', default='resnet', type=str, help='model name')
parser.add_argument('--ps', default=512, type=int, help='patch size')
parser.add_argument('--lr', default=1e-4, type=float, help='learning rate')
parser.add_argument('--epochs', default=1000, type=int, help='sum of epochs')
parser.add_argument('--freq', default=500, type=int, help='learning rate update frequency')
parser.add_argument('--save_freq', default=100, type=int, help='save result frequency')
parser.add_argument('--syn', action='store_true', help='use synthetic noisy images')
args = parser.parse_args()

def adjust_learning_rate(optimizer, epoch):
	if not epoch % args.freq and epoch:
		for param_group in optimizer.param_groups:
			param_group['lr'] = param_group['lr'] * 0.1
	return optimizer

def train(train_loader, model, criterion, optimizer, epoch, result_dir):
	losses = AverageMeter()
	model.train()

	for ind, (noise_img, origin_img) in enumerate(train_loader):
		st = time.time()

		input_var = noise_img.cuda()
		target_var = origin_img.cuda()

		output = model(input_var)
		loss = criterion(output, target_var)

		losses.update(loss.item())

		optimizer.zero_grad()
		loss.backward()
		optimizer.step()

		print('[{0}][{1}]\t'
			'lr: {lr:.5f}\t'
			'Loss: {loss.val:.4f} ({loss.avg:.4f})\t'
			'Time: {time:.3f}'.format(
			epoch, ind,
			lr=optimizer.param_groups[-1]['lr'],
			loss=losses,
			time=time.time()-st))

		if epoch % args.save_freq == 0:
			if not os.path.isdir(os.path.join(result_dir, '%04d'%epoch)):
				os.makedirs(os.path.join(result_dir, '%04d'%epoch))

			origin_np = origin_img.numpy()
			noise_np = noise_img.numpy()
			output_np = output.cpu().detach().numpy()

			origin_np_img = chw_to_hwc(origin_np[0])
			noise_np_img = chw_to_hwc(noise_np[0])
			output_img = chw_to_hwc(np.clip(output_np[0], 0, 1))

			temp = np.concatenate((origin_np_img, noise_np_img, output_img), axis=1)
			io.imsave(os.path.join(result_dir, '%04d/train_%d.jpg'%(epoch, ind)), np.uint8(temp * 255))

if __name__ == '__main__':
	save_dir = os.path.join('./save_model/', args.model)
	result_dir = os.path.join('./result/', args.model)

	model = importlib.import_module('.' + args.model, package='model').Network()
	print(model)
	model.cuda()

	if os.path.exists(os.path.join(save_dir, 'checkpoint.pth.tar')):
		# load existing model
		model_info = torch.load(os.path.join(save_dir, 'checkpoint.pth.tar'))
		print('==> loading existing model:', os.path.join(save_dir, 'checkpoint.pth.tar'))
		model.load_state_dict(model_info['state_dict'])
		optimizer = torch.optim.Adam(model.parameters())
		optimizer.load_state_dict(model_info['optimizer'])
		cur_epoch = model_info['epoch']
	else:
		if not os.path.isdir(save_dir):
			os.makedirs(save_dir)
		# create model
		optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
		cur_epoch = 0

	criterion = nn.MSELoss()
	criterion.cuda()

	if args.syn:
		train_dataset = SynNoise('./data/syn/', patch_size=args.ps)
	else:
		train_dataset = RealNoise('./data/real/', patch_size=args.ps)

	train_loader = torch.utils.data.DataLoader(
		train_dataset, batch_size=1, shuffle=True, pin_memory=True)

	for epoch in range(cur_epoch, args.epochs + 1):

		optimizer = adjust_learning_rate(optimizer, epoch)
		train(train_loader, model, criterion, optimizer, epoch, result_dir)

		torch.save({
			'epoch': epoch + 1,
			'state_dict': model.state_dict(),
			'optimizer' : optimizer.state_dict()},
			os.path.join(save_dir, 'checkpoint.pth.tar'))


import xlrd
from datetime import date,datetime
import os
import xlwt

def xlrdApi():
    data = xlrd.open_workbook('/home/luohong/Downloads/111.xlsx')
    sheetNames = data.sheet_names()
    sheetCounts = data.nsheets
    print(sheetNames)
    print(sheetCounts)

    sheetObj = data.sheet_by_name('1')
    one_sheet_names = sheetObj.name
    one_sheet_nrows = sheetObj.nrows
    one_sheet_ncols = sheetObj.ncols
    one_sheet_rows_data = sheetObj.row_values(0)
    one_sheet_col_data = sheetObj.col_values(0)
    print(one_sheet_names)
    print(one_sheet_nrows)
    print(one_sheet_ncols)
    print(one_sheet_rows_data)
    print(one_sheet_col_data)

def writeDateByXlrd():
    book = xlwt.Workbook() # 新建工作簿
    table = book.add_sheet('Over',cell_overwrite_ok=True) # 如果对同一单元格重复操作会发生overwrite Exception，cell_overwrite_ok为可覆盖
    sheet = book.add_sheet('Test') # 添加工作页
    #sheet = book.add_sheet('Test') # 添加工作页
    sheet.write(1,1,'A')
    style = xlwt.XFStyle() # 新建样式
    font = xlwt.Font() #新建字体
    font.name = 'Times New Roman'
    font.bold = True
    style.font = font # 将style的字体设置为font
    table.write(0,0,'Test',style)
    book.save(filename_or_stream='/home/luohong/Downloads/111.xlsx') # 一定要保存

if __name__ == '__main__':
    writeDateByXlrd()


# Import the libraries needed to parse XML
import urllib.request, urllib.parse, urllib.error
import xml.etree.ElementTree as ET
import ssl

# Ignore SSL certificate errors
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

# Enter either one of the following URLS:
# url = 'http://py4e-data.dr-chuck.net/comments_42.xml'
# url = 'http://py4e-data.dr-chuck.net/comments_1554150.xml'
url = input('Enter - ')

# Retrieve data using urllib request
data = urllib.request.urlopen(url, context=ctx).read()
tree = ET.fromstring(data)

# Use the finall method to retrieve all the comments in the XML
lst = tree.findall('comments/comment')

# Create a list to create a running balance for count.
total = 0

# Loop through the tags and retieve the count information
for tag in lst :

    # find and grab the text from the tag 'count', convert it into integer and save to variable.
    count = int(tag.find('count').text)

    # Get a running balance
    total = total + count

# Print the total of the data you fetched.
print(total)


import setuptools

with open("../meta/version") as fh:
    version = fh.read().strip()

setuptools.setup(
    name="anki",
    version=version,
    author="Ankitects Pty Ltd",
    description="Anki's library code",
    long_description="Anki's library code",
    long_description_content_type="text/markdown",
    url="https://apps.ankiweb.net",
    packages=setuptools.find_packages(".", exclude=["tests"]),
    license="License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
    package_data={"anki": ["py.typed"]},
    classifiers=[],
    python_requires=">=3.7",
    install_requires=[
        "beautifulsoup4",
        "requests",
        "decorator",
        "protobuf",
        'psutil; sys_platform == "win32"',
        'distro; sys_platform != "darwin" and sys_platform != "win32"',
    ],
)


import os

import pandas as pd

# from graphsimviz_backend.tasks.task_hook import TaskHook
# import graphsimviz_backend.graphsimqt as graphsimqt

def get_data_dir():
    return "/usr/src/data/"

def get_network_comp_dir(networkType, network1, network2, id_space):
    id_space_suffix = '_' + id_space if id_space == 'ICD10' or id_space == 'UMLS' else ''
    dir1 = os.path.join(get_data_dir(), f'{network1}_vs_{network2}{id_space_suffix}')
    if networkType == 'drug-disease':
        return os.path.join(get_data_dir(), f'GED_drug_indication_distances_vs_DrPD{id_space_suffix}')
    if os.path.exists(dir1):
        return dir1
    dir2 = os.path.join(get_data_dir(), f'{network2}_vs_{network1}{id_space_suffix}')
    if os.path.exists(dir2):
        return dir2
    return None

def get_local_dist_file(dir):
    return os.path.join(dir, 'local_distances.csv')

def get_local_p_value_file(dir):
    return os.path.join(dir, 'local_empirical_p_values.csv')

def get_global_score_files(networkType, dir):
    if networkType == 'drug-disease':
        return [os.path.join(dir, 'global_empirical_p_values.csv'),
                os.path.join(dir, 'global_mwu_p_values_together.csv')]
    return [os.path.join(dir, 'global_empirical_p_values.csv'),
            os.path.join(dir, 'global_mwu_p_values.csv')]

def get_global_scores(global_files):
    global_p_values = pd.read_csv(str(global_files[0]))
    global_mwu_values = pd.read_csv(str(global_files[1]))
    return {'empirical_p_values': global_p_values.to_dict(), 'mwu_p_values': global_mwu_values.to_dict()}

def get_local_scores(path_to_local_p_values, node_ids):
    import pandas as pd
    local_p_values = pd.read_csv(str(path_to_local_p_values))
    local_p_values = local_p_values[local_p_values['node'].isin(node_ids)]
    local_values = {}
    for distance_type in set(local_p_values['distance_type']):
        p_values = local_p_values[['node', 'p_value','distance_type']]
        p_values = p_values[p_values['distance_type'] == distance_type]
        p_values = p_values.rename(columns={'p_value': 'local_p_value'})
        local_values[distance_type] = p_values.to_dict()
    return local_values

def get_cluster_scores(networkType, path_to_local_distances, node_ids, mwu):
    import pandas as pd
    import scipy.stats as sps

    p_values = {'distance_type': [], 'p_value': []}

    local_distances = pd.read_csv(str(path_to_local_distances))
    distance_types = list(set(local_distances['distance_type']))
    all_nodes = set(local_distances['node'])

    if mwu:
        for distance_type in distance_types:
            selection = (local_distances['distance_type'] == distance_type) & (local_distances['node'].isin(node_ids))
            p_values['distance_type'].append(distance_type)
            x = local_distances[selection & ~local_distances['permuted']]['distance'].to_numpy()
            y = local_distances[selection & local_distances['permuted']]['distance'].to_numpy()
            try:
                if len(node_ids) < 9:
                    _, mwu_p_value = sps.mannwhitneyu(x=x, y=y, alternative='less', method='asymptotic')
                else:
                    _, mwu_p_value = sps.mannwhitneyu(x=x, y=y, alternative='less')
            except ValueError:
                mwu_p_value = 1.0
            p_values['p_value'].append(mwu_p_value)

    else:
        batch_len = len(all_nodes) * len(distance_types)

        sum_local_distance_true = {'distance_type': [], 'sum_distance': []}
        for distance_type in distance_types:
            sum_true = 0.0
            for node_id in node_ids:
                try:
                    true_distance = local_distances[
                        (local_distances['node'] == node_id) &
                        ~local_distances['permuted'] & (
                                local_distances['distance_type'] == distance_type)]['distance'].to_list()[0]
                    sum_true += true_distance
                except:
                    # print(f'Node {node_id} not found in {distance_type} distance type')
                    pass
            sum_local_distance_true['distance_type'].append(distance_type)
            sum_local_distance_true['sum_distance'].append(sum_true)

        sum_local_distance_random_list = []
        for i in range(1, 1001):
            sum_local_distance_random = {'distance_type': [], 'sum_distance': []}
            ld_slice = local_distances.loc[i * batch_len:(i + 1) * batch_len]
            for distance_type in distance_types:
                sum_random = 0.0
                for node_id in node_ids:
                    try:
                        rand_distance = ld_slice[(ld_slice['node'] == node_id) & ld_slice['permuted'] &
                                                 (ld_slice['distance_type'] == distance_type)]['distance'].to_list()[0]
                        sum_random += rand_distance
                    except:
                        # print(f'Node {node_id} not found in {distance_type} distance type')
                        pass
                sum_local_distance_random['distance_type'].append(distance_type)
                sum_local_distance_random['sum_distance'].append(sum_random)
            sum_local_distance_random_list.append(sum_local_distance_random)

        if sum_local_distance_random_list[0].get('distance_type') == sum_local_distance_true.get('distance_type'):
            for dt in (range(3) if networkType != 'drug-disease' else range(1)):
                distance_type = sum_local_distance_random_list[0].get('distance_type')[dt]
                numleq = 0
                p_values['distance_type'].append(distance_type)
                for rd in sum_local_distance_random_list:
                    if rd.get('sum_distance')[dt] <= sum_local_distance_true.get('sum_distance')[dt]:
                        numleq += 1
                p_values['p_value'].append(float(numleq + 1) / float(1000 + 1))

    return pd.DataFrame.from_dict(p_values).to_dict()

def calculate_global_scores(networkType, network1, network2, id_space):
    dir = get_network_comp_dir(networkType, network1, network2, id_space)
    print(dir)
    global_file = get_global_score_files(networkType, dir)
    print(global_file)
    return get_global_scores(global_file)

def calculate_cluster_scores(networkType, network1, network2, id_space, node_ids, mwu):
    dir = get_network_comp_dir(networkType, network1, network2, id_space)
    local_distances = get_local_dist_file(dir)
    return get_cluster_scores(networkType, local_distances, node_ids, mwu)

def calculate_local_scores(networkType, network1, network2, id_space, node_ids):
    dir = get_network_comp_dir(networkType, network1, network2, id_space)
    local_p_values = get_local_p_value_file(dir)
    print(local_p_values)
    return get_local_scores(local_p_values, node_ids)

def calculate(networkType, network1, network2, id_space, disorders, mwu):
    data_dir = get_network_comp_dir(networkType, network1, network2, id_space)
    local_dist_file = get_local_dist_file(data_dir)
    local_p_value_file = get_local_p_value_file(data_dir)

    # compute_empirical_p_values(
    scores = {'local': get_local_scores(local_p_value_file, disorders),
              'global': get_global_scores(get_global_score_files(networkType, data_dir)),
              'cluster': get_cluster_scores(networkType, local_dist_file, disorders, mwu)}

    return scores

    # data = hook.parameters
    # hook.set_progress(0.1, "Executing")
    # result = validate(tar=data["target"], tar_id=data["target_id"], mode="set",
    #                   runs=data["runs"],
    #                   replace=data["replace"], ref=None, ref_id=None, enriched=None,
    #                   background_model=data["background_model"], background_network=None, distance=data["distance"],
    #                   out_dir=data["out"],
    #                   uid=data["uid"], set_progress=hook.set_progress)
    # hook.set_files(files=result["files"], uid=data["uid"])
    # hook.set_results(results=result["result"])


"""Add qcengine to required programs

Revision ID: b1d1a21895ba
Revises: 97e1d19796c7
Create Date: 2022-07-07 09:17:33.447320

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = "b1d1a21895ba"
down_revision = "97e1d19796c7"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.execute(sa.text("UPDATE task_queue SET required_programs = ARRAY['qcengine'] || required_programs"))
    op.execute(
        sa.text(
            """UPDATE compute_manager SET programs = programs::jsonb || jsonb_build_object('qcengine', qcengine_version)"""
        )
    )
    op.drop_column("compute_manager", "qcengine_version")
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    raise RuntimeError("Cannot downgrade")
    # ### end Alembic commands ###


from sys import stdin
from collections import defaultdict

n, q = map(int, stdin.readline().strip().split())
a = list(map(int, stdin.readline().strip().split()))
differences = defaultdict(list)
j = 0
prev_diff = diff = a[1] - a[0]
for i in range(1, n):
    diff = a[i] - a[i - 1]
    if diff != prev_diff:
        differences[prev_diff].append((j, i, i - j))
        j = i - 1
        prev_diff = diff
differences[diff].append((j, n, n - j))
for _ in range(q):
    l, r, d = map(int, stdin.readline().strip().split())
    l -= 1
    if l == r or d not in differences:
        print(1)
    else:
        ans = 1
        for start, end, length in differences[d]:
            if end <= l:
                continue
            elif l <= start and r >= end:
                ans = max(ans, length)
            elif start <= l <= end:
                ans = max(ans, min(r, end) - l)
            elif start <= r <= end:
                ans = max(ans, r - start)
            else:
                break
        print(ans)

hackerearth/Data Structures/Advanced Data Structures/Segment Trees/Length of arithmetic progressions/solution.py


import common
import wcodegen

class PerlDialogGenerator(wcodegen.PerlWidgetCodeWriter):
    new_signature = [
        '$parent', '$id', '$title', '$pos', '$size', '$style', '$name'
    ]

    def get_code(self, obj):
        return [], [], []

    def get_properties_code(self, obj):
        out = []
        title = obj.properties.get('title')
        if title:
            out.append('$self->SetTitle(%s);\n' %
                       self.codegen.quote_str(title))
        icon = obj.properties.get('icon')
        if icon:
            stmt_icon = self.generate_code_bitmap(icon, obj.preview)
            out.append('my $icon = &Wx::wxNullIcon;\n')
            out.append('$icon->CopyFromBitmap(%s);\n' % stmt_icon)
            out.append('$self->SetIcon($icon);\n')
        out.extend(self.codegen.generate_common_properties(obj))
        return out

    def get_layout_code(self, obj):
        ret = ['$self->Layout();\n']
        try:
            if int(obj.properties['centered']):
                ret.append('$self->Centre();\n')
        except (KeyError, ValueError):
            pass
        return ret

# end of class PerlDialogGenerator

def initialize():
    klass = 'wxDialog'
    common.class_names['EditDialog'] = klass
    common.toplevels['EditDialog'] = 1
    common.register('perl', klass, PerlDialogGenerator(klass))


import itertools
import unittest
from itertools import product
from typing import Optional, Tuple, Union, List, overload

import numpy as np
from scipy import stats
from scipy.spatial import distance

from couleurs.utils.stats import normals_kl_div, normals_js_dist

_ListOfNp = List[np.ndarray]

def _make_positive_semidefinite(matrix: np.ndarray) -> np.ndarray:
    n_dim = matrix.shape[0]
    return np.dot(matrix, matrix.T) / (2 * n_dim ** 2)

@overload
def _generate_normal_params(nb_dim: int, nb_list1: int) -> Tuple[_ListOfNp, _ListOfNp]:
    pass

@overload
def _generate_normal_params(nb_dim: int, nb_list1: int, nb_list2: int) -> Tuple[_ListOfNp, _ListOfNp, _ListOfNp, _ListOfNp]:
    pass

def _generate_normal_params(
        nb_dim: int, nb_list1: int, nb_list2: Optional[int] = None
) -> Union[Tuple[_ListOfNp, _ListOfNp], Tuple[_ListOfNp, _ListOfNp, _ListOfNp, _ListOfNp]]:
    rng = np.random.default_rng(1984)

    # We test with lists of numpy arrays
    mu_list1 = [rng.uniform(-2 + i*2, -1 + i*2, nb_dim) for i in range(nb_list1)]
    sigma_list1 = [_make_positive_semidefinite(rng.uniform(-2 + i*3, 2 + i*3, (nb_dim, nb_dim))) for i in range(nb_list1)]

    if nb_list2 is not None:
        mu_list2 = [rng.uniform(5 + i*2, 6 + i*2, nb_dim) for i in range(nb_list2)]
        sigma_list2 = [_make_positive_semidefinite(rng.uniform(-3 + i*3, 3 + i*3, (nb_dim, nb_dim))) for i in range(nb_list2)]
        return mu_list1, sigma_list1, mu_list2, sigma_list2
    else:
        return mu_list1, sigma_list1

def _calculate_kl_divergence(m1: np.ndarray, m2: np.ndarray, s1: np.ndarray, s2: np.ndarray) -> float:
    """
    Not efficient, but easy to follow procedure to calculate the KL divergence between 2 multivariate normals
    See: https://stats.stackexchange.com/questions/60680/kl-divergence-between-two-multivariate-gaussians
    """
    d = m1.shape[0]
    det1 = np.linalg.det(s1)
    det2 = np.linalg.det(s2)
    s2_inv = np.linalg.inv(s2)
    mu_diff = m2 - m1

    log_term = np.log2(det2 / det1)
    trace_term = np.trace(s2_inv @ s1)
    quad_term = mu_diff @ s2_inv @ mu_diff

    return 0.5 * (log_term - d + trace_term + quad_term)

def _calculate_js_dist(m1: np.ndarray, m2: np.ndarray, s1: np.ndarray, s2: np.ndarray) -> float:
    normal1 = stats.multivariate_normal(m1, s1)
    normal2 = stats.multivariate_normal(m2, s2)
    samples1 = normal1.rvs(50000)
    samples2 = normal2.rvs(50000)
    samples = np.vstack((samples1, samples2))
    probs1 = normal1.pdf(samples)
    probs2 = normal2.pdf(samples)
    return distance.jensenshannon(probs1, probs2, base=2)

class NormalsKLDivTestCase(unittest.TestCase):
    def test_one_set(self):
        rng = np.random.default_rng(2012)

        # Test with two identical distributions
        nb_dim = 3
        mu = rng.uniform(-1, 5, nb_dim)
        sigma = _make_positive_semidefinite(rng.uniform(-2, 2, (nb_dim, nb_dim)))
        mu_list = np.array([mu, mu])
        sigma_list = np.array([sigma, sigma])
        res = normals_kl_div(mu_list, sigma_list)
        self.assertTrue(np.allclose(res, 0))

        # Test with 2 different distributions
        mu1 = mu
        mu2 = rng.uniform(-1, 5, nb_dim)
        sigma1 = sigma
        sigma2 = _make_positive_semidefinite(rng.uniform(-2, 2, (nb_dim, nb_dim)))
        mu_list = np.array([mu1, mu2])
        sigma_list = np.array([sigma1, sigma2])
        res = normals_kl_div(mu_list, sigma_list)
        self.assertAlmostEqual(0, res[0, 0])
        self.assertAlmostEqual(0, res[1, 1])
        expected1 = _calculate_kl_divergence(mu1, mu2, sigma1, sigma2)
        expected2 = _calculate_kl_divergence(mu2, mu1, sigma2, sigma1)
        self.assertAlmostEqual(expected1, res[0, 1])
        self.assertAlmostEqual(expected2, res[1, 0])

    def test_two_sets(self):
        nb_dim = 4
        nb_list1 = 2
        nb_list2 = 3

        mu_list1, sigma_list1, mu_list2, sigma_list2 = _generate_normal_params(nb_dim, nb_list1, nb_list2)
        res = normals_kl_div(mu_list1, sigma_list1, mu_list2, sigma_list2)

        # Check correct size
        self.assertEqual((nb_list1, nb_list2), res.shape)

        # Check all values equal divergences calculated explicitly
        for i, j in product(range(nb_list1), range(nb_list2)):
            with self.subTest(i=i, j=j):
                mu1 = mu_list1[i]
                mu2 = mu_list2[j]
                sigma1 = sigma_list1[i]
                sigma2 = sigma_list2[j]

                expected = _calculate_kl_divergence(mu1, mu2, sigma1, sigma2)
                self.assertAlmostEqual(expected, res[i, j])

class NormalJSDistTestCase(unittest.TestCase):
    def test_one_set(self):
        nb_dim = 4
        nb_list1 = 2

        mu_list1, sigma_list1 = _generate_normal_params(nb_dim, nb_list1)
        res = normals_js_dist(mu_list1, sigma_list1)
        self.assertEqual((nb_list1, nb_list1), res.shape)
        self.assertEqual(0, res[0, 0])
        self.assertEqual(0, res[1, 1])

    def test_two_sets_vert(self):
        nb_dim = 4
        nb_list1 = 3
        nb_list2 = 2

        mu_list1, sigma_list1, mu_list2, sigma_list2 = _generate_normal_params(nb_dim, nb_list1, nb_list2)
        res = normals_js_dist(mu_list1, sigma_list1, mu_list2, sigma_list2, nb_samples=100000)
        self.assertEqual((nb_list1, nb_list2), res.shape)
        self.assertAlmostEqual(res[0, 1], res[1, 0], 4)

        # Check each element if they are equal
        for i, j in itertools.product(range(nb_list1), range(nb_list2)):
            mu1 = mu_list1[i]
            mu2 = mu_list2[j]
            sigma1 = sigma_list1[i]
            sigma2 = sigma_list2[j]
            with self.subTest(i=i, j=j):
                expected = _calculate_js_dist(mu1, mu2, sigma1, sigma2)
                actual = res[i, j]
                self.assertAlmostEqual(actual, expected, 4)

if __name__ == '__main__':
    unittest.main()


class Solution:
    def strStr(self, haystack: str, needle: str) -> int:
        #
        # Solution v1: Brute Force
        #
        # Runtime: 130 ms @ (beats) 16.04%
        # Memory Usage: 14.6 MB @ (beats) 33.63%
        #
        # h_len = len(haystack)
        # n_len = len(needle)
        #
        # if n_len == 0:
        #     return 0
        # elif h_len == 0:
        #     return -1
        # elif h_len < n_len:
        #     return -1
        # elif h_len == n_len:
        #     return 0 if haystack == needle else -1
        # else:
        #     for i in range(h_len - n_len + 1):
        #         sub = haystack[i:(i + n_len)]
        #         if sub == needle:
        #             return i
        #
        # return -1

        try:
            h_len = len(haystack)
            n_len = len(needle)
            haystack.index(needle)
        except:
            return -1


import codecs

infile = "hl_software_2019-09-27_14-25-52_live.log"

phrases = ["lock pos", "today", "yday", "new order", "trade"]
with codecs.open(infile, "rb", encoding='utf-16') as f:
    for line in f:
        for p in phrases:
            if p in line:
                print(line)


import numpy as np
import pandas as pd
import string
import pickle

import sklearn
from sklearn.metrics import classification_report
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk import sent_tokenize

class MeanEmbeddingVectorizer(object):
    def __init__(self, word2vec):
        self.word2vec = word2vec
        # if a text is empty we should return a vector of zeros
        # with the same dimensionality as all the other vectors
        self.dim = 300

    def fit(self, X):
        return self

    def transform(self, X):
        return np.array([
            np.mean([self.word2vec[w] for w in words if w in self.word2vec]
                    or [np.zeros(self.dim)], axis=0)
            for words in X
        ])

def get_combined_review(top_reviews):
    all_reviews = ''
    for review in top_reviews:
        all_reviews += review.strip(' ')
        all_reviews += ' '
    return all_reviews.strip()

def tokenize(text):
    punctuations = string.punctuation.replace("\'", '')

    for ch in punctuations:
        text = text.replace(ch, " " + ch + " ")

    tokens = text.split()
    for i, token in enumerate(tokens):
        if not token.isupper():
            tokens[i] = token.lower()
    return tokens

def n_intersection(q, r):
    return len(set(q).intersection(set(r)))

def w2v_sim(q, r, w2v_vectorizer):
    # dot product of q and r as w2v vectors
    q_vec = w2v_vectorizer.transform([q])
    r_vec = w2v_vectorizer.transform([r])
    return q_vec.dot(r_vec.transpose())[0][0]

def tf_idf_sim(q, r, tfidf_vectorizer):
    # dot product of q and r as tfidf vectors
    q_vec = tfidf_vectorizer.transform([q])
    r_vec = tfidf_vectorizer.transform([r])
    return q_vec.dot(r_vec.transpose()).toarray()[0][0]

def tf_idf_sim_sentence(q, rs, tfidf_vectorizer):
    # max of dot products of q and each sentence in r as tfidf vectors
    q_vec = tfidf_vectorizer.transform([q])
    if len(rs) == 0:
        return 0
    return max([q_vec.dot(tfidf_vectorizer.transform([r]).transpose()).toarray()[0][0] for r in rs])

def w2v_sim_sentence(q, rs, w2v_vectorizer):
    # max of dot products of q and each sentence in r as tfidf vectors
    q_vec = w2v_vectorizer.transform([q])
    if len(rs) == 0:
        return 0
    return max([q_vec.dot(w2v_vectorizer.transform([r]).transpose())[0][0] for r in rs])

def tf_idf_sim_sentence_mean(q, rs, tfidf_vectorizer):
    # max of dot products of q and each sentence in r as tfidf vectors
    q_vec = tfidf_vectorizer.transform([q])
    if len(rs) == 0:
        return 0
    return np.mean([q_vec.dot(tfidf_vectorizer.transform([r]).transpose()).toarray()[0][0] for r in rs])

def w2v_sim_sentence_mean(q, rs, w2v_vectorizer):
    # max of dot products of q and each sentence in r as tfidf vectors
    q_vec = w2v_vectorizer.transform([q])
    if len(rs) == 0:
        return 0
    return np.mean([q_vec.dot(w2v_vectorizer.transform([r]).transpose())[0][0] for r in rs])

def load_classification_model(filename):

    with open(filename, 'rb') as fp:
        model = pickle.load(fp)

    return model

def load_vectorizers(tfidf_vectorizer_filename, w2v_vectorizer_filename):

    with open(tfidf_vectorizer_filename, 'rb') as fp:
        tfidf_vectorizer = pickle.load(fp)

    with open(w2v_vectorizer_filename, 'rb') as fp:
        w2v_vectorizer = pickle.load(fp)

    return {
        'w2v': w2v_vectorizer,
        'tfidf': tfidf_vectorizer
    }

def compute_features(vectorizers, question, reviews):
    q_tokens = tokenize(question)
    r_tokens = tokenize(reviews)
    r_sents = sent_tokenize(reviews)
    n_q = len(q_tokens)
    n_r = len(r_tokens)

    intersection_n = len(set(q_tokens).intersection(set(r_tokens)))
    intr_frac = intersection_n / n_q

    #tfidf = tf_idf_sim(question, reviews, vectorizers['tfidf'])
    #w2v = w2v_sim(question, reviews, vectorizers['w2v'])
    w2v_sent = w2v_sim_sentence(question, r_sents, vectorizers['w2v'])
    tfidf_sent = tf_idf_sim_sentence(question, r_sents, vectorizers['tfidf'])
    w2v_sent_mean = w2v_sim_sentence_mean(question, r_sents, vectorizers['w2v'])
    tfidf_sent_mean = tf_idf_sim_sentence_mean(question, r_sents, vectorizers['tfidf'])

    return np.array([
        [n_q, n_r, intersection_n, intr_frac, w2v_sent, tfidf_sent, w2v_sent_mean, tfidf_sent_mean]
    ])

def predictions_k(model, X, k):
    probs = model.predict_proba(X)
    return probs[:, 1] >= k

def is_answerable(model, vectorizers, question, top_reviews):
    features = compute_features(vectorizers, question, get_combined_review(top_reviews))
    return int(predictions_k(model, features, 0.7)[0])

def is_suggestive(model, vectorizers, question, top_reviews):
    features = compute_features(vectorizers, question, get_combined_review(top_reviews))
    return 1 - int(predictions_k(model, features, 0.5)[0])


# This is a sample Python script.

# Press Shift+F10 to execute it or replace it with your code.
# Press Double Shift to search everywhere for classes, files, tool windows, actions, and settings.
import sys

def recursive(n):
    if n == 1:
        return 1
    else:
        return n*recursive(n-1)

def fibonacci(n):
    if n == 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fibonacci(n-1) + fibonacci(n-2)

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    # 預設 recursion 次數
    print('Python 預設 recursion 次數 : \n' + str(sys.getrecursionlimit()))
    print('')
    # 階乘
    print('階乘:')
    ans = recursive(5)
    print(ans)

    # Fibonacci
    print('')
    print('Fibonacci 數列:')
    print(fibonacci(10))

# See PyCharm help at https://www.jetbrains.com/help/pycharm/


"""
    setup dependencies:
        Sets up NTP's dependencies as defined in setup.json
        Workspace & CMSSW must be set up

    Usage:
        setup dependencies
"""
import logging
from .. import Command as C
from hepshell.interpreter import time_function

LOG = logging.getLogger(__name__)

class Command(C):

    def __init__(self, path=__file__, doc=__doc__):
        super(Command, self).__init__(path, doc)

    @time_function('setup dependencies', LOG)
    def run(self, args, variables):
        if not self.__can_run():
            return False
        self.__prepare(args, variables)
        commands = [
            'cd {CMSSW_SRC}',
            'source /cvmfs/cms.cern.ch/cmsset_default.sh',
            'eval `/cvmfs/cms.cern.ch/common/scram runtime -sh`',
        ]
        from .. import DEPENDENCIES, CMSSW_SRC, CMSSW_VERSION
        from hepshell.interpreter import call

        rcs = []
        for dep in DEPENDENCIES:
            LOG.info('Setting up dependency "{0}"'.format(dep['name']))
            provider = dep['provider']
            source = dep['source']
            destination = dep['destination']
            command = ''
            if provider == 'git':
                command = 'git clone {source} {destination}'.format(
                    source=source, destination=destination)
            elif provider == 'git-cms-merge-topic':
                command = 'git-cms-merge-topic {source}'.format(source=source)
            elif provider == 'pip':
                command = 'pip install -U --user {source}'.format(source=source)
            else:
                LOG.error('Unknown provider "{0}"'.format(provider))
                return False
            if 'setup-cmds' in dep:
                additional_commands = ' && '.join(dep['setup-cmds'])
                command = command + ' && ' + additional_commands

            all_in_one = ' && '.join(commands)
            all_in_one = all_in_one + ' && ' + command + ' \n'
            all_in_one = all_in_one.format(
                CMSSW_VERSION=CMSSW_VERSION,
                CMSSW_SRC=CMSSW_SRC,
                )
            rc = call(all_in_one, logger=LOG, shell=True)
            rcs.append(rc)

        return all(rcs)

    def __can_run(self):
        import os
        from .. import CMSSW_VERSION, WORKSPACE

        if not os.path.exists(WORKSPACE):
            LOG.error('Workspace does not exist: {0}'.format(WORKSPACE))
            return False
        if not os.path.exists(WORKSPACE + '/' + CMSSW_VERSION):
            LOG.error('CMSSW is not set up: {0}'.format(
                WORKSPACE + '/' + CMSSW_VERSION))
            return False
        return True


'''
Created on Aug 4, 2012

@author: lkreczko
'''
CODE = {
    'ENDC':0, # RESET COLOR
    'BOLD':1,
    'UNDERLINE':4,
    'BLINK':5,
    'INVERT':7,
    'CONCEALD':8,
    'STRIKE':9,
    'GREY30':90,
    'GREY40':2,
    'GREY65':37,
    'GREY70':97,
    'GREY20_BG':40,
    'GREY33_BG':100,
    'GREY80_BG':47,
    'GREY93_BG':107,
    'DARK_RED':31,
    'RED':91,
    'RED_BG':41,
    'LIGHT_RED_BG':101,
    'DARK_YELLOW':33,
    'YELLOW':93,
    'YELLOW_BG':43,
    'LIGHT_YELLOW_BG':103,
    'DARK_BLUE':34,
    'BLUE':94,
    'BLUE_BG':44,
    'LIGHT_BLUE_BG':104,
    'DARK_MAGENTA':35,
    'PURPLE':95,
    'MAGENTA_BG':45,
    'LIGHT_PURPLE_BG':105,
    'DARK_CYAN':36,
    'AUQA':96,
    'CYAN_BG':46,
    'LIGHT_AUQA_BG':106,
    'DARK_GREEN':32,
    'GREEN':92,
    'GREEN_BG':42,
    'LIGHT_GREEN_BG':102,
    'BLACK':30,
}

def termcode(num):
    return '\033[%sm' % num

def colorstr(text, color):
    if color in CODE.keys():
        return termcode(CODE[color]) + text + termcode(CODE['ENDC'])
    else:
        return text


import pandas as pd

from ambient import storet
from ambient import sites
from ambient import crosswalk
from ambient.nwis import cols
from ambient.storet.cols import group as group_cols

def to_storet(df, crosswalk_df, storet_site_id=None):
    """ Format an NWIS record for STORET

    Parameter
    ---------
    df : DataFrame
        NWIS record

    storet_site_id : list-like
        List of STORET site id's corresponding with each record in df

    Returns
    -------
    STORET formated record.
    """
    #TODO consider throwing exception for empty DataFrame
    if df.empty:
        return pd.DataFrame()
    out = pd.DataFrame(columns=storet.cols.all_cols)
    out[storet.cols.result] = censored_results(df, inverse=True)
    out[storet.cols.censor] = censored_results(df)

    out[storet.cols.org] = df[cols.org].values
    out[group_cols] = _lookup_storet_cols(df, crosswalk_df)

    out[storet.cols.site_id] = storet_site_id #TODO make this a series
    # otherwise it won't work if it is called before the df is populated

    #crosswalk date and time
    out[storet.cols.time] = df[cols.time]
    out[storet.cols.date] = df[cols.date]
    out[storet.cols.tz] = df[cols.tz]

    # Drop any NaT values
    # TODO: investigate origin of NaT in NWIS records
    out = out[out[storet.cols.time] != 'NaT']

    return out

def conversion_factor(df, units_df, parameter_col):
    """ Returns conversion factor to match units in NWIS record with STORET.

    Examples
    --------
    >>> factor, parm_cd = conversion_factor(df, units_df)
    >>> df['result_va'] = df['result_va'] * factor
    """
    out = df.merge(units_df, how='left',
                   left_on = parameter_col,
                   right_on = cols.parm_cd_NWIS)

    factor = out[cols.factor].copy()
    factor[factor.isna()] = 1

    parm_cd = out[cols.parm_cd_STORET].copy()
    parm_cd[parm_cd.isna()] = out.loc[parm_cd.isna(), cols.parameter_cd]

    return factor.values, parm_cd.values

def crosswalkable(df, crosswalk_df, inverse=False):
    """List all NWIS parameter codes in ambient.misc.param_dict.

    Parameters
    ----------
    df : DataFrame
        NWIS record

    param_lookup_table : DataFrame

    inverse : Boolean
        If True, list parameter codes not in param_dict.

    Returns
    -------
    List of parameter codes
    """
    miss = df.merge(crosswalk_df,
                    how='left',
                    left_on=cols.parameter_cd,
                    right_on=crosswalk.cols.parameter_cd,
                    ).isna().values

    if not inverse:
        return df[~miss].unique().tolist()

    else:
        return df[miss].unique().tolist()

def censored_results(df, inverse=False):
    """
    Parameters
    ----------
    df : DataFrame
        An NWIS record

    inverse : Boolean
        If set True, return uncensored record.

    Returns
    -------
    A Series consisting of censored values and nans in place of uncensored values.
    """
    if not inverse:
        return df[cols.result].where(df[cols.remark] == '<').values

    elif inverse:
        return df[cols.result].where(df[cols.remark] != '<').values

def _lookup_storet_cols(df, crosswalk_df):
    """ Return STORET characteristic names and sample fraction from an NWIS record.

    Parameters
    ----------
    df : DataFrame
        An NWIS record.

    crosswalk_df : DataFrame

    Returns
    -------
    A DataFrame with the following columns: characteristic name, sample fraction, and units.
    """
    # ensure there are no duplicates in crosswalk_df
    assert crosswalk_df.duplicated(subset=crosswalk.cols.group).any() == False

    _check_lookup_table(crosswalk_df)
    out = df.merge(crosswalk_df,
                   left_on=cols.parameter_cd,
                   right_on=crosswalk.cols.parameter_cd,
                   how='left')

    return out[crosswalk.cols.group]

def _lookup_storet_id(df, site_df):
    """
    XXX: this function is not called
    """
    out = df.merge(site_df,
                   left_on=cols.site_id,
                   right_on=sites.cols.usgs_id,
                   how='left')

    return ('IL_EPA_WQX-' + out[sites.cols.storet_id].values)

def _check_lookup_table(crosswalk_df):
    """XXX not sure this is still necessary
    """
    # check if any columns contain nan
    #if crosswalk_df[storet.cols.group].isna().any().any():
    #    raise ValueError("Must import crosswalk table with pd.read_csv(filename, keep_default_na=False)")
    # XXX commented on 20181214, but may introduce bugs
    if cols.parameter_cd not in crosswalk_df.columns:
        raise KeyError("Crosswalk table must have 'parm_cd' column")


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import sys
import ssl
import socket
import time
import datetime
import threading
import asn1crypto.x509
import requests

from .manual import *

DETECT_CIPHERS = True
DEFAULT_TIMEOUT = 5.0

def make_ssocket(protocol, ciphers=ssl._DEFAULT_CIPHERS):
    """Creates a TCP socket wrapped in SSL."""
    return ssl.wrap_socket( socket.socket(),
                            ssl_version=protocol,
                            ciphers=ciphers)

def get_supported_ciphers(dest, protocol):
    """Returns a list of ciphers supported by a host.

    Arguments:
        dest -- the (IP, port) to connect to
        protocol -- the SSL/TLS version to test; it is also
                     the index into the ssl._PROTOCOL_NAMES array

    The returned list contains cipher names.
    """
    chosen_ciphers = []
    # start by enabling all ciphers
    cipher_string = "ALL:aNULL:eNULL"
    while True:
        try:
            ssock = make_ssocket(protocol, cipher_string)
            ssock.connect(dest)
            # the connection will be made with the highest-priority cipher
            chosen_ciphers.append( ssock.cipher()[0] )
            ssock.close()
            # having detected the cipher, disable it
            cipher_string += ":!"+chosen_ciphers[-1]
        except OSError:
            # if no connection can be made with the remaining ciphers,
            #  it means we've enumerated all the supported ones
            break
    return chosen_ciphers

def get_certificate(dest):
    """Returns a DER-encoded certificate (as a bytes object).

    Arguments:
            dest -- the (IP, port) to connect to
    """
    try:
        ssock = make_ssocket(ssl.PROTOCOL_SSLv23)  # connects to SSLv3+
        ssock.connect(dest)
        der_cert = ssock.getpeercert(binary_form=True)
        ssock.close()
        return der_cert
    except OSError:
        return None
    # c = asn1crypto.x509.Certificate.load(der_cert)

########################################################################
########################################################################
########################################################################

def get_certificate_chain(url):
    """Returns a list of DER-encoded certificate (as bytes).

    Arguments: url -- the site to connect to"""
    try:
        sock = socket.socket()
        sock.connect((url, 443))
    except OSError:
        try:
            time.sleep(5)
            sock.settimeout(10)
            sock.connect((url, 443))
        except:
            return []

    hello = create_client_hello(SSLVersion.SSLv3,
                                [cs for cs in CipherSuite],
                                sni_url = url,
                                max_ssl_version = SSLVersion.TLSv1_2)
    try:
        sock.send(hello)
        time.sleep(2)             # to make sure all data has arrived
        resp = sock.recv(100000)
    except OSError:
        return []

    # find a Certificate record
    while resp:
        try:    record = TLSRecord(resp)
        except: return []

        if record.type == RecordType.handshake and record.data[0:1] == HandshakeType.certificate.value:
            return extract_certificate_chain(record)
        resp = resp[5+len(record):]

    return []

def get_HTTP_header_fields(url, fieldnames):
    """Returns the contents of the specified HTTP response header fields.

    Arguments:
        url -- the website to check (in the form '[xyz://]www.example.com/')
        fieldnames -- a list of header field names; can be a single string,
                      in which case it is treated like a list of length one

    Returns:
        field_contents -- a mapping (dict) from fieldnames to their values;
                          empty in case of error or if there are no headers
    """
    url = "https://" + url.split("//", maxsplit=1)[-1]
    if type(fieldnames) is str:
        fieldnames = [fieldnames]

    field_contents = {}

    try:
        resp = requests.head(url, allow_redirects=True)
    except requests.RequestException as e:
        return {}

    return {k:v for (k,v) in resp.headers.items() if k in fieldnames}

class Record:
    """A class for storing the results of a single scan."""
    def __init__(self, host):
        self.hostname, self.IP = host
        self.protocols = []
        self.cipher_suites = []
        self.timestamp = int(time.time())
        self.certificate = None
        self.certificate_chain = []
        self.extensions = []
        self.HTTP_header_fields = {}

    def add_protocol(self, protocol):
        if protocol not in self.protocols:
            self.protocols.append( protocol )

    def add_cipher_suite(self, csuite):
        if csuite not in self.cipher_suites:
            self.cipher_suites.append( csuite )

    def add_to_DB(self, db_cursor):
        SQL_GET_INDEX = "SELECT COALESCE(MAX(indeks),0) FROM certificates;"
        SQL_GET_CERT  = "SELECT indeks FROM certificates WHERE certificate = ?;"
        SQL_INSERT_CERT = "INSERT INTO certificates VALUES (?,?);"
        SQL_INSERT_MAIN = "INSERT INTO scan_results VALUES (?,?,?,?,?,?,?,?);"

        certs = []
        for cert in self.certificate_chain:   # insert any new certificates first

            # get the index of next cert
            db_cursor.execute(SQL_GET_INDEX)
            next_index = db_cursor.fetchone()[0] + 1

            # check whether cert already exists
            db_cursor.execute(SQL_GET_CERT, (cert,));
            index = db_cursor.fetchone()

            if not index:
                certs += [next_index]
                db_cursor.execute(SQL_INSERT_CERT, (next_index, cert))
                next_index += 1
            else:
                certs += [index[0]]

        # Here, `certs` has the certificate chain as indices;
        #  now we add the record into the main table.

        protos_bytes  = SSLVersion.as_bytes(self.protocols)
        csuite_bytes  = CipherSuite.as_bytes(self.cipher_suites)
        ext_bytes     = ExtensionType.as_bytes(self.extensions)
        cchain_string = "$".join(str(i) for i in certs)
        HTTP_fields = "_$_".join(k+":"+v for k,v in self.HTTP_header_fields.items())
        if not protos_bytes:  protos_bytes  = None
        if not csuite_bytes:  csuite_bytes  = None
        if not ext_bytes:     ext_bytes     = None
        if not cchain_string: cchain_string = None
        if not HTTP_fields:   HTTP_fields   = None

        db_cursor.execute(SQL_INSERT_MAIN,
                           (self.hostname, self.IP, self.timestamp,
                            protos_bytes, csuite_bytes, cchain_string,
                            ext_bytes, HTTP_fields))

    def __str__(self):
        s = "host: " + self.hostname + "\n"
        s += "IP: " + self.IP + "\n"
        s += "timestamp: " + datetime.datetime.fromtimestamp(self.timestamp)\
                                              .isoformat(" ") + "\n"
        s += "supports:\n"
        for proto in self.protocols:
            s += "  "+proto.name+"\n"

        for cipher in proto.cipher_suites:
            s += "  "+cipher+"\n"

        return s

def init_db_tables(db_cursor):
    """Create the necessary database tables, if they do not already exist."""
    db_cursor.execute(
        "CREATE TABLE IF NOT EXISTS scan_results(\n"
        "    hostname           TEXT,\n"
        "    ip                 TEXT,\n"
        "    timestamp          INTEGER,\n"
        "    protocols          BLOB,\n"
        "    ciphersuites       BLOB,\n"
        "    certificate_chain  TEXT,\n"
        "    extensions         BLOB,\n"
        "    HTTP_header_fields TEXT,\n"
        "    PRIMARY KEY (hostname, timestamp));\n"
    )

    db_cursor.execute("CREATE TABLE IF NOT EXISTS certificates(\n"
                      "    indeks      INTEGER,\n"
                      "    certificate BLOB);\n"
    )

def process(host):
    hostname, IP = host
    if (hostname=="?"):
        dest = (IP, 443)
    else:
        dest = (hostname, 443)
    url = hostname

    # set default socket timeout
    socket.setdefaulttimeout(DEFAULT_TIMEOUT)
    # a dictionary to record the supported protocols
    supported = {}

    record = Record(host)

    # detect supported protocol versions
    record.protocols = [v for v in SSLVersion if try_protocol(v, url)]

    if record.protocols:
        # detect supported cipher suites
        #  (don't duplicate, try just on the highest-supported protocol)
        if SSLVersion.TLSv1_2 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1_2, url)
        elif SSLVersion.TLSv1_1 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1_1, url)
        elif SSLVersion.TLSv1 in record.protocols:
            cs = scan_all_ciphers(SSLVersion.TLSv1,   url)
        else:
            cs = scan_all_ciphers(SSLVersion.SSLv3,   url)

        # record.cipher_suites = list(set(cs_ssl3 + cs_tls1 + cs_tls1_1 + cs_tls1_2))
        record.cipher_suites = sorted(cs)

        # record the certificate chain
        record.certificate_chain = get_certificate_chain(url)

        # detect supported extensions
        extensions = scan_all_extensions(SSLVersion.TLSv1_2, url)
        record.extensions = sorted(extensions)

        # record certain response header fields
        fields = get_HTTP_header_fields(hostname,
                                        ["Server",
                                        "Strict-Transport-Security",
                                        "Public-Key-Pins",
                                        "Public-Key-Pins-Report-Only"])
        record.HTTP_header_fields = fields;

    return record

class SSLTester(threading.Thread):

    def __init__(self, target_feed, results_feed):
        threading.Thread.__init__(self)
        self.target_feed = target_feed
        self.results_feed = results_feed

    def run(self):
        for target in self.target_feed:
            res = self._scan(target)
            self.results_feed.put(res)

    def _scan(self, target):
        return process(target)


from django.contrib import admin
from django.urls import include, path
from django.views.generic.base import TemplateView
from . import views
#Diferentes direcciones URL que utiliza el proyecto y los views que utiliza
#Los url que se encuentran son aquellos que cargan la vista de lo siguiente:
#Ingreso de Carnet, Instrucciones para descarga de libros, Vista de libros disponibles
#Visualizador de libro de matematica, ingles y fisica
urlpatterns = [
    path('', views.carnet, name='carnet'),
    path('visitante', views.visitante, name='visitante'),
    path('instructions', views.instructions, name='instructions'),
    path('instructions2', views.instructions2, name='instructions2'),
    path('nombre', views.nombre, name='nombre'),
    path('student', views.detail, name='detail'),
    path('students', views.portal, name='portal'),
	path('libro_Fisica1', views.libro_Fisica1, name='libro_Fisica1'),
    path('', include('pwa.urls')),
    path('pdf_view', views.pdf_view, name='pdf_view'),
    path(r'adminactions/', include('adminactions.urls')),
    ]


import torch
from torch.utils.data import Dataset, ConcatDataset, DataLoader

from transformers  import BertTokenizer
from torchvision import transforms
import os
import pandas as pd
import json
import matplotlib.pyplot as plt
from PIL import Image
from tqdm import tqdm, trange
import numpy as np
from datetime import datetime

from predict import get_predictions, compute_score
from model import TriTransModel

import config as argumentparser

if __name__ == '__main__':

    config = argumentparser.ArgumentParser()

    print("PyTorch version: ", torch.__version__)

    tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")

    # Setting
    max_length = [config.qst_len, config.ans_len]

    if config.dataset == 'v7w':
        from dataset.v7w_dataset import v7wDataset, trim_collate
        print('Used the v7w dataset')
        max_boxes = 50
        sampler = None
        train_dset = v7wDataset('train', adaptive=True, max_boxes=max_boxes, maxlen=max_length)
        val_dset = v7wDataset('val', adaptive=True, max_boxes=max_boxes, maxlen=max_length)
        test_dset = v7wDataset('test', adaptive=True, max_boxes=max_boxes, maxlen=max_length)
        trainval_dset = ConcatDataset([train_dset, val_dset])

        trainloader = DataLoader(trainval_dset, config.batch_size, shuffle=sampler, num_workers=8,
                                    collate_fn=trim_collate, pin_memory=True)
        testloader = DataLoader(test_dset, config.batch_size, shuffle=False, sampler=sampler, num_workers=8,
                                    collate_fn=trim_collate, pin_memory=False)
    elif config.dataset == 'vqa1':
        from dataset.vqa1_dataset import trainDataset
        print('Used the vqa1.0 dataset')
        train_dset = trainDataset(max_length)
        trainloader = DataLoader(train_dset, config.batch_size, num_workers=8, drop_last=False, pin_memory=False)
    elif config.dataset == 'vqa2':
        from dataset.vqa2_dataset import vqaDataset
        print('Used the vqa2.0 dataset')
        trainset = vqaDataset('train,nominival', max_length)
        trainloader = DataLoader(
            trainset, batch_size=config.batch_size,
            shuffle=True, num_workers=8,
            drop_last=False, pin_memory=True
        )
        valset = vqaDataset('minival', max_length)
        valloader = DataLoader(
            valset, batch_size=config.batch_size,
            shuffle=True, num_workers=8,
            drop_last=False, pin_memory=True
        )
    elif config.dataset == 'tdiuc':
        from dataset.tdiuc_dataset import tdiuc_Dataset
        print('Used the tdiuc dataset')
        train_dset = tdiuc_Dataset('train', maxlen=max_length)
        val_dset = tdiuc_Dataset('val', maxlen=max_length)
        trainloader = DataLoader(train_dset, config.batch_size, num_workers=8, drop_last=False,
                                    collate_fn=trim_collate, pin_memory=False)
        valloader = DataLoader(val_dset, config.batch_size, shuffle=False, num_workers=8, drop_last=False,
                                    collate_fn=trim_collate, pin_memory=False)
    elif config.dataset == 'gqa':
        from dataset.gqa_dataset import gqaDataset
        print('Used the gqa dataset')
        train_dset = gqaDataset('train', max_length)
        val_dset = gqaDataset('testdev', max_length)
        trainloader = DataLoader(train_dset, config.batch_size, num_workers=8, drop_last=False, pin_memory=False)
        valloader = DataLoader(val_dset, config.batch_size, shuffle=False, num_workers=8, drop_last=False, pin_memory=False)
    else:
        print('PLZ, select the right dataset!')
        os._exit(0)

    print("Successfully loaded data.")

    model = TriTransModel(num_layers=config.layers, ff_dim=config.ff_dim, dropout=config.dropout)
    if config.load:
        print('Loading pretrain ckpt {}'.format(config.load))
        ckpt = torch.load(config.load)
        mod_dict = model.state_dict()
        for k, v in ckpt.items():
            if k in mod_dict:
                mod_dict[k] = v
        model.load_state_dict(mod_dict)
        print('Finish!')
    if config.cuda and torch.cuda.is_available():
        model.cuda()

    # optimizer
    # gamma = 0.1 for lr schduler
    optimizer = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=config.lr)

    lr_decay_step = 2
    lr_decay_rate = 0.25
    gradual_warmup_steps = [0.5 * config.lr, 1.0 * config.lr, 1.5 * config.lr, 2.0 * config.lr]
    lr_decay_epochs = range(10, 20, lr_decay_step)

    # Train the model
    best_score = 0
    for epoch in trange(config.epoch, desc='Epoch', ncols=100):
        model.train()
        run_bce_loss = 0
        total_score = 0

        log = '[epoch %d] ===========================================\n' % (epoch)

        if epoch < len(gradual_warmup_steps):
            optimizer.param_groups[0]['lr'] = gradual_warmup_steps[epoch]
            log += 'gradual warmup lr: %.8f\n' % optimizer.param_groups[0]['lr']
        elif epoch in lr_decay_epochs:
            optimizer.param_groups[0]['lr'] *= lr_decay_rate
            log += 'decreased lr: %.8f\n' % optimizer.param_groups[0]['lr']
        else:
            log += 'lr: %.8f\n' % optimizer.param_groups[0]['lr']

        process_bar = tqdm(trainloader, desc='Train', ncols=100)

        for (_, qst_ids, ans_ids, img_tensor, labels) in process_bar:

            qst_ids, ans_ids, img_tensor, labels = qst_ids.cuda(), ans_ids.cuda(), img_tensor.cuda(), labels.cuda()

            if config.dataset == 'v7w':
                qst_ids = qst_ids.unsqueeze(1).expand(qst_ids.size(0), 4, qst_ids.size(1)).contiguous().view(qst_ids.size(0) * 4, qst_ids.size(1))
                img_tensor = img_tensor.unsqueeze(1).expand(img_tensor.size(0), 4, img_tensor.size(1), img_tensor.size(2)).contiguous().view(img_tensor.size(0) * 4, img_tensor.size(1), img_tensor.size(2))
                ans_ids = ans_ids.view(ans_ids.size(0) * ans_ids.size(1), ans_ids.size(2))
                labels = labels.view(ans_ids.size(0), 1)
                labels = torch.cat([labels, 1-labels], 1)
                labels = labels.float()
            elif config.dataset == 'vqa1':
                qst_ids = qst_ids.unsqueeze(1).expand(qst_ids.size(0), 18, qst_ids.size(1)).contiguous().view(qst_ids.size(0) * 18, qst_ids.size(1))
                img_tensor = img_tensor.unsqueeze(1).expand(img_tensor.size(0), 18, img_tensor.size(1), img_tensor.size(2)).contiguous().view(img_tensor.size(0) * 18, img_tensor.size(1), img_tensor.size(2))
                ans_ids = ans_ids.view(ans_ids.size(0) * 18, ans_ids.size(2))
                labels = labels.view(labels.size(0) * 18, labels.size(2))
            elif config.dataset == 'vqa2' or config.dataset == 'tdiuc' or config.dataset == 'gqa':
                qst_ids = qst_ids.unsqueeze(1).expand(qst_ids.size(0), 4, qst_ids.size(1)).contiguous().view(qst_ids.size(0) * 4, qst_ids.size(1))
                img_tensor = img_tensor.unsqueeze(1).expand(img_tensor.size(0), 4, img_tensor.size(1), img_tensor.size(2)).contiguous().view(img_tensor.size(0) * 4, img_tensor.size(1), img_tensor.size(2))
                ans_ids = ans_ids.view(ans_ids.size(0) * 4, ans_ids.size(2))
                labels = labels.view(labels.size(0) * 4, labels.size(2))

            optimizer.zero_grad()

            # forward pass
            outputs = model(input_ids=qst_ids,
                            ans_ids=ans_ids,
                            labels=labels,
                            img_data=img_tensor)

            bce_loss = outputs[0]
            out_pred = outputs[1]

            bce_loss.backward()
            optimizer.step()

            # Train acc
            if config.dataset == 'vqa1':
                scores, batch_score, prob_preds = compute_score(out_pred, labels, 18)
            else:
                scores, batch_score, prob_preds = compute_score(out_pred, labels, 4)
            total_score += scores
            run_bce_loss += bce_loss.item()

            process_bar.set_postfix(BCE_loss=bce_loss.item(), batch_acc=batch_score)
            process_bar.update()

        train_bce_loss = run_bce_loss / len(trainloader)
        train_acc =  total_score / len(trainloader.dataset)
        log += 'Train acc: %.4f, Loss: %.4f\n' % (train_acc, train_bce_loss)

        if config.save_model:
            torch.save(model.state_dict(), os.path.join(config.output, 'epoch%d.pth'%epoch))

        # valid
        if config.dataset == 'v7w':
            print('Evaluating...')
            # Evaluate
            test_acc, test_bce_loss = get_predictions(config, model, testloader)

            log += 'Test acc: %.4f, Loss: %.4f\n' % (test_acc, test_bce_loss)

            # record the best
            if best_score < test_acc:
                best_score = test_acc
            log += 'Best score: %.4f\n' % best_score

        elif config.dataset == 'tdiuc':
            val_anslist = json.load(open('/home/joel-zhang/file/tdiuc/ban_ans/val_anslist.json'))
            ans_dict = {datum['question_id']: list(datum['answer'].keys()) for datum in val_anslist}
            targets = cPickle.load(open('/home/joel-zhang/file/tdiuc/cache/val_target.pkl', 'rb'))
            labels = {datum['question_id']: datum['labels'] for datum in targets}
            print('Evaluating...')
            # Evaluate
            val_acc = 0
            num = 0
            model.eval()
            for (q_id, qst_ids, ans_ids, img_tensor, _) in tqdm(valloader):
                qst_ids, img_tensor, ans_ids = qst_ids.cuda(), img_tensor.cuda(), ans_ids.cuda()

                qst_ids = qst_ids.unsqueeze(1).expand(qst_ids.size(0), 4, qst_ids.size(1)).contiguous().view(qst_ids.size(0) * 4, qst_ids.size(1))
                img_tensor = img_tensor.unsqueeze(1).expand(img_tensor.size(0), 4, img_tensor.size(1), img_tensor.size(2)).contiguous().view(img_tensor.size(0) * 4, img_tensor.size(1), img_tensor.size(2))
                ans_ids = ans_ids.view(ans_ids.size(0) * 4, ans_ids.size(2))

                with torch.no_grad():
                    outputs = model(input_ids=qst_ids,
                                    ans_ids=ans_ids,
                                    img_data=img_tensor)
                pred = outputs[0]
                pred = pred.reshape(-1, 4, 2)
                for i in range(pred.size(0)):
                    try:
                        s = pred[i][:, 0]
                        _, ans_id = s.max(0)
                        ans = ans_dict[q_id[i].item()][ans_id]
                        target_ans = label2ans[labels[q_id[i].item()]]
                        if ans == target_ans:
                            val_acc += 1
                        num += 1
                    except:
                        print('val error')

            val_acc = val_acc / num
            log += "tritrans Valid acc: %f\n" % val_acc

        elif config.dataset == 'vqa1':
            pass

        elif config.dataset == 'vqa2':
            print('Evaluating...')
            # Evaluate
            model.eval()

            quesid2ans = {}
            ans_list = json.load(open('/home/jyt21/VQA/MIRTT/bilinear_method/result/bertban_vqa2/train_anslist.json'))
            ans_dict = {datum['question_id']: list(datum['answer'].keys()) for datum in ans_list}

            with torch.no_grad():
                for (q_id, ques, ans, img) in tqdm(valloader):
                    ques, img, ans = ques.cuda(), img.cuda(), ans.cuda()
                    ques = ques.unsqueeze(1).expand(ques.size(0), 4, ques.size(1)).contiguous().view(ques.size(0) * 4, ques.size(1))
                    img = img.unsqueeze(1).expand(img.size(0), 4, img.size(1), img.size(2)).contiguous().view(img.size(0) * 4, img.size(1), img.size(2))
                    ans = ans.view(ans.size(0) * 4, ans.size(2))

                    outputs = model(input_ids=ques, ans_ids=ans, img_data=img)
                    pred = outputs[0]
                    pred = pred.reshape(-1, 4, 2)

                    for i in range(pred.size(0)):
                        try:
                            s = pred[i][:, 0]
                            _, ans_id = s.max(0)
                            ans = ans_dict[q_id[i].item()][ans_id]
                            quesid2ans[q_id[i].item()] = ans
                        except:
                            print('val error')

            queslist = json.load(open('/mnt/data/jyt/VQA2.0/minival.json', 'r'))
            id2datum = {
                datum['question_id']: datum
                for datum in queslist
            }
            score = 0.
            for quesid, ans in quesid2ans.items():
                datum = id2datum[quesid]
                label = datum['label']
                if ans in label:
                    score += label[ans]
            log += "tritrans valid acc: %f\n" % (score / len(quesid2ans))

        elif config.dataset == 'gqa':
            print('Evaluating...')
            # Evaluate
            model.eval()

            quesid2ans = {}
            ans_list = json.load(open('/home/joel-zhang/file/jyt/bi_gqa/result/ban/test_anslist.json'))
            ans_dict = {datum['question_id']: list(datum['answer'].keys()) for datum in ans_list}

            with torch.no_grad():
                for (q_id, ques, ans, img) in tqdm(valloader):
                    ques, img, ans = ques.cuda(), img.cuda(), ans.cuda()
                    ques = ques.unsqueeze(1).expand(ques.size(0), 4, ques.size(1)).contiguous().view(ques.size(0) * 4, ques.size(1))
                    img = img.unsqueeze(1).expand(img.size(0), 4, img.size(1), img.size(2)).contiguous().view(img.size(0) * 4, img.size(1), img.size(2))
                    ans = ans.view(ans.size(0) * 4, ans.size(2))

                    outputs = model(input_ids=ques, ans_ids=ans, img_data=img)
                    pred = outputs[0]
                    pred = pred.reshape(-1, 4, 2)

                    for i in range(pred.size(0)):
                        try:
                            s = pred[i][:, 0]
                            _, ans_id = s.max(0)
                            ans = ans_dict[q_id[i]][ans_id]
                            quesid2ans[q_id[i]] = ans
                        except:
                            print('val error')

            queslist = json.load(open('/home/joel-zhang/file/lxmert/data/gqa/testdev.json', 'r'))
            id2datum = {
                datum['question_id']: datum
                for datum in queslist
            }
            score = 0.
            for quesid, ans in quesid2ans.items():
                datum = id2datum[quesid]
                label = datum['label']
                if ans in label:
                    score += label[ans]
            log += "tritrans valid acc: %f\n" % (score / len(quesid2ans))

        with open(os.path.join(config.output, 'log.txt'), 'a') as f:
            f.write(log)


from aiogram.types.reply_keyboard import ReplyKeyboardMarkup

from loader import _

def get_admin_keyboard_markup() -> ReplyKeyboardMarkup:
    markup = ReplyKeyboardMarkup(resize_keyboard=True, selective=True, row_width=2)

    markup.row(_("➕ Добавить Админа"), _("🎁 Выдать VIP"))
    markup.row(_("🔖 Рассылка"), _("📫 Личка"))
    markup.row(_("🔢 Количество пользователей"), _("🗂 Таблицы"))
    markup.row(_("🧾 Меню"))

    return markup


import os, csv

# Read CSV file into a list of lists
def ReadToList(csvPath):
    try:
        with open(csvPath, 'r') as csvFile:
            reader = csv.reader(csvFile)
            return list(reader)
    except:
        print("The CSV could not be read.")
        return []

def AddRow(csvPath, rowToAdd):
    alreadyExists = False
    if os.path.exists(csvPath):
        try:
            with open(csvPath, 'r') as csvFile:
                for row in csv.reader(csvFile):
                    if row == rowToAdd:
                        alreadyExists = True
        except:
            print("The CSV could not be read.")

    if not alreadyExists:
        with open(csvPath, 'a') as csvFile:
            csvFile.write(','.join(rowToAdd) + "\n")

    return not alreadyExists

def DeleteRow(csvPath, rowToRemove):
    csvList = ReadToList(csvPath)
    with open(csvPath, 'w', newline='') as csvOut:
        writer = csv.writer(csvOut)
        for row in csvList:
            if row != rowToRemove:
                writer.writerow(row)

def _TestAddRow():
    csvList = ReadToList('local.csv')
    rowToAdd = csvList[1]
    rowToAdd[0] = 'ElBarfo'
    if AddRow('local.csv', rowToAdd):
        print('line added')
    else:
        print('line was already present')
    csvList = ReadToList('local.csv')
    for line in csvList:
        print(line)

def _TestDeleteRow():
    csvList = ReadToList('local.csv')
    rowToRemove = csvList[1]
    DeleteRow('local.csv', rowToRemove)
    csvList = ReadToList('local.csv')
    for line in csvList:
        print(line)

if __name__ == "__main__":
    _TestDeleteRow()


import os, unittest
from deterministic_encryption_utils.encryption.extensions.ModificationTimeFileSaltProvider import ModificationTimeFileSaltProvider
from tests.SaltProviderTestBase import SaltProviderTestBase

class TestModificationTimeFileSaltProvider(SaltProviderTestBase, unittest.TestCase):

    def constructSubject(self):
        return ModificationTimeFileSaltProvider()

    def getId(self):
        return 'mtime'

    def testGivenSaltForFile(self):
        tmpFile = os.path.join(self.tmpDir, 'afile')
        open(tmpFile, 'a').close()
        salt = self.subject.getSaltFor(tmpFile)
        self.assertEqual(str(os.path.getmtime(tmpFile)), salt)

    def testDirectoryPath(self):
        tmpDir = os.path.join(self.tmpDir, 'adir')
        os.mkdir(tmpDir)
        with self.assertRaises(ValueError) as _:
            self.subject.getSaltFor(tmpDir)

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()