#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys

try:
    reload(sys).setdefaultencoding("UTF-8")
except:
    pass

try:
    from setuptools import setup
except ImportError:
    print('Please install or upgrade setuptools or pip to continue')
    sys.exit(1)

if sys.version_info < (3, 3):
   print('PixeLINKds requires Python 3.3+')
   sys.exit(1)

import codecs

def read(filename):
    return codecs.open(filename, encoding='utf-8').read()

long_description = '\n\n'.join([read('README'),
                                read('AUTHORS'),
                                read('CHANGES')])

__doc__ = long_description

setup(
    name='pixelinkds',
    version='0.2.dev0',
    description='PixeLINK Data Steam (PDS) Files in Python.',
    long_description=long_description,
    keywords='PixeLINK Data Stream PDS image movie',
    author='Hernan E. Grecco',
    author_email='hernan.grecco@gmail.com',
    url='https://github.com/hgrecco/pixelinkds',
    #test_suite='pint.testsuite.testsuite',
    zip_safe=True,
    packages=['pixelinkds'],
    license='BSD',
    classifiers=[
        'Development Status :: 4 - Beta',
        'Intended Audience :: Developers',
        'Intended Audience :: Science/Research',
        'License :: OSI Approved :: BSD License',
        'Operating System :: MacOS :: MacOS X',
        'Operating System :: Microsoft :: Windows',
        'Operating System :: POSIX',
        'Programming Language :: Python',
        'Topic :: Scientific/Engineering',
        'Topic :: Software Development :: Libraries',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
    ])


#!/usr/bin/env python
import hashlib

import time

import click
import sys

@click.command()
@click.option("-s", "--settle-duration", default=5.0, show_default=True,
              help="Duration without change after which file is considered settled.")
@click.option("-t", "--timeout", default=60, show_default=True,
              help="Total timeout after which to abort waiting for file to settle")
@click.argument("filename", type=click.Path(exists=True, dir_okay=False))
def main(settle_duration, timeout, filename):
    window = start = time.time()
    last_sum = ""
    while time.time() - window < settle_duration:
        with open(filename) as file_:
            file_sum = hashlib.sha1(file_.read()).hexdigest()
        if file_sum != last_sum:
            window = time.time()
            last_sum = file_sum
        time.sleep(.2)

        if time.time() - start > timeout:
            print("File '{}' hasn't settled after {} s. Aborting.".format(filename, timeout))
            sys.exit(1)

if __name__ == "__main__":
    main()


import json, re
import requests
from django.core import serializers
from django.http import JsonResponse
from django.template.response import TemplateResponse
from app.distance_of_points import distance_checker
from app.dublin_bikes_api import json_data_import
from app.models import DublinBikes
from .sql_selects import SELECT_ALL_DATA_VALUES

def json_all_stations(request):
    lat = request.GET['lat']
    long = request.GET['long']

    json_data = json_data_import()
    for i in json_data:
        DublinBikes.objects.all()
    all_data = SELECT_ALL_DATA_VALUES
    results = {}
    # for every queryset in the list serialize it.
    data = serializers.serialize("json", all_data)
    results[0] = json.loads(data)
    regex = r'\(([^)]+)\)'

    for i in results[0]:
        coords = i['fields']['position']
        res = re.findall(regex, coords)
        blong, blat = res[0].split(' ')
        dist = distance_checker(lat, long, blat, blong)
        dist = float(dist) / 1000
        dist = round(dist, 2)
        i['distance'] = dist
    return JsonResponse(results)

def index(request):
    return TemplateResponse(request, 'app/index.html')

#Uncomment code and run before demo
def all_stations(request):
    # json_data = json_data_import()
    # for i in json_data:
    #     DublinBikes.objects.filter(stand_number=i['number']).update(
    #         available_bike_stands=i['available_bike_stands'], available_bikes=i['available_bikes'],
    #         last_update=i['last_update'])
    # all_data = SELECT_ALL_DATA_VALUES
    return TemplateResponse(request, 'app/all_bike_stations.html')

def find_nearest_station(request):
    # json_data = json_data_import()
    # for i in json_data:
    #     DublinBikes.objects.filter(stand_number=i['number']).update(
    #         available_bike_stands=i['available_bike_stands'], available_bikes=i['available_bikes'],
    #         last_update=i['last_update'])
    # all_data = SELECT_ALL_DATA_VALUES
    return TemplateResponse(request, 'app/find_nearset_station.html')

def json_nearest_station(request):
    # Request address from form input
    address = request.GET.get('address')
    address = address + " dublin ireland"
    google_url = "https://maps.googleapis.com/maps/api/geocode/json?mode=walking&address={0}".format(address)
    # Google api requires underscores so replaces space default + symbol
    google_url = google_url.replace("+", "_")
    google_reply = requests.get(google_url)
    # Get the google reply request (string format) Make it into JSON
    location_data = json.loads(google_reply.text)
    # Parse lat and lng data
    lat = location_data['results'][0]['geometry']['location']['lat']
    lng = location_data['results'][0]['geometry']['location']['lng']

    json_data = json_data_import()
    for i in json_data:
        DublinBikes.objects.all()

    all_data = SELECT_ALL_DATA_VALUES
    results = {}
    # for every queryset in the list serialize it.
    data = serializers.serialize("json", all_data)
    results[0] = json.loads(data)
    regex = r'\(([^)]+)\)'

    # Distance checker for bike locations lat and long to users location
    for i in results[0]:
        coords = i['fields']['position']
        res = re.findall(regex, coords)
        blong, blat = res[0].split(' ')
        dist = distance_checker(lat, lng, blat, blong)
        dist = float(dist) / 1000
        dist = round(dist, 2)
        i['distance'] = dist
    results['lat'] = lat
    results['lng'] = lng
    return JsonResponse(results)


import unittest

from . import day5

class MyTestCase(unittest.TestCase):
    def test_day2(self):
        programStr = "1,9,10,3,2,3,11,0,99,30,40,50"
        program = [int(x) for x in programStr.split(',')]
        self.assertEqual(day5.run(program), 3500)

    def test_day5(self):
        programStr = "3,0,4,0,99"
        program = [int(x) for x in programStr.split(',')]
        self.assertEqual(day5.run(program, [1]), 1)

if __name__ == '__main__':
    unittest.main()


import unittest

from france import Country

class CountryTest(unittest.TestCase):
    def testCountry(self):
        GDP : float = 170000000
        continent : str = "Europe"
        country : Country = Country(continent, GDP)

        self.assertEqual(country.continent,continent)
        self.assertEqual(country.GDP,GDP)

if __name__ == '__main__':
    unittest.main()


from __future__ import print_function

import time

import numpy as np
import sympy as sy

from bokeh.objects import Plot, DataRange1d, LinearAxis, ColumnDataSource, Glyph, Grid, Legend
from bokeh.widgetobjects import Slider, TextInput, HBox, VBox, Dialog
from bokeh.glyphs import Patch, Line, Text
from bokeh.document import Document
from bokeh.session import Session

from requests.exceptions import ConnectionError

document = Document()
session = Session()
session.use_doc('taylor_server')
session.load_document(document)

xs = sy.Symbol('x')
expr = sy.exp(-xs)*sy.sin(xs)
order = 1

def taylor(fx, xs, order, x_range=(0, 1), n=200):
    x0, x1 = x_range
    x = np.linspace(float(x0), float(x1), n)

    fy = sy.lambdify(xs, fx, modules=['numpy'])(x)
    tx = fx.series(xs, n=order).removeO()

    if tx.is_Number:
        ty = np.zeros_like(x)
        ty.fill(float(tx))
    else:
        ty = sy.lambdify(xs, tx, modules=['numpy'])(x)

    return x, fy, ty

def update_data():
    x, fy, ty = taylor(expr, xs, order, (-2*sy.pi, 2*sy.pi), 200)

    plot.title = "%s vs. taylor(%s, n=%d)" % (expr, expr, order)
    legend.legends = {
        "%s"         % expr: [line_f_glyph],
        "taylor(%s)" % expr: [line_t_glyph],
    }
    source.data = dict(x=x, fy=fy, ty=ty)
    slider.value = order

    session.store_document(document)

source = ColumnDataSource(data=dict(
    x  = [],
    fy = [],
    ty = [],
))

xdr = DataRange1d(sources=[source.columns("x")])
ydr = DataRange1d(sources=[source.columns("fy")])

plot = Plot(data_sources=[source], x_range=xdr, y_range=ydr, plot_width=800, plot_height=400)

line_f = Line(x="x", y="fy", line_color="blue", line_width=2)
line_f_glyph = Glyph(data_source=source, xdata_range=xdr, ydata_range=ydr, glyph=line_f)
plot.renderers.append(line_f_glyph)

line_t = Line(x="x", y="ty", line_color="red", line_width=2)
line_t_glyph = Glyph(data_source=source, xdata_range=xdr, ydata_range=ydr, glyph=line_t)
plot.renderers.append(line_t_glyph)

xaxis = LinearAxis(plot=plot, location="bottom")
plot.below.append(xaxis)
yaxis = LinearAxis(plot=plot, location="left")
plot.left.append(yaxis)

xgrid = Grid(plot=plot, dimension=0, axis=xaxis)
ygrid = Grid(plot=plot, dimension=1, axis=yaxis)

legend = Legend(plot=plot, orientation="bottom_left")
plot.renderers.append(legend)

def on_slider_value_change(obj, attr, old, new):
    global order
    order = int(new)
    update_data()

def on_text_value_change(obj, attr, old, new):
    try:
        global expr
        expr = sy.sympify(new, dict(x=xs))
    except (sy.SympifyError, TypeError, ValueError) as exception:
        dialog.content = str(exception)
        dialog.visible = True
        session.store_objects(dialog)
    else:
        update_data()

dialog = Dialog(title="Invalid expression", buttons=["Close"])

slider = Slider(start=1, end=20, value=order, step=1, title="Order:")
slider.on_change('value', on_slider_value_change)

text = TextInput(value=str(expr), title="Expression:")
text.on_change('value', on_text_value_change)

inputs = HBox(children=[slider, text])
layout = VBox(children=[inputs, plot, dialog])

document.add(layout)
update_data()

if __name__ == "__main__":
    link = session.object_link(document._plotcontext)
    print("Please visit %s to see the plots" % link)

    try:
        while True:
            session.load_document(document)
            time.sleep(0.5)
    except KeyboardInterrupt:
        print()
    except ConnectionError:
        print("Connection to bokeh-server was terminated")


import sys

import pandas as pd
import sqlite3
from sqlalchemy import create_engine

def load_data(messages_filepath, categories_filepath):
    """ Loding data from .csv files and cleaming """
    messages = pd.read_csv(messages_filepath)
    categories = pd.read_csv(categories_filepath)

    cat_df = pd.DataFrame.from_records(list(categories.categories.str.split(';')))
    for i in range(36):
        cat_df[i] = cat_df[i].str.replace('-\d', '')
    cat_df = cat_df.drop_duplicates()
    cat_clos = list(cat_df.loc[0,:])

    categories.categories = categories.categories.str.split(';')
    categories[cat_clos] = pd.DataFrame(categories.categories.values.tolist(), index=categories.index)
    categories = categories.drop(columns=['categories'])
    for col in cat_clos:
        categories[col] = categories[col].str.replace('\D', '').astype(int)

    df = messages.merge(categories, on='id')

    return df

def clean_data(df):
    """ Dropping duplicates """
    df = df.drop_duplicates()

    return df

def save_data(df, database_filename):
    """ Saving data into SQLite database """
    engine = create_engine('sqlite:///{}'.format(database_filename))
    df.to_sql('mess_cat', engine, index=False)

def main():
    if len(sys.argv) == 4:

        messages_filepath, categories_filepath, database_filepath = sys.argv[1:]

        print('Loading data...\n    MESSAGES: {}\n    CATEGORIES: {}'
              .format(messages_filepath, categories_filepath))
        df = load_data(messages_filepath, categories_filepath)

        print('Cleaning data...')
        df = clean_data(df)

        print('Saving data...\n    DATABASE: {}'.format(database_filepath))
        save_data(df, database_filepath)

        print('Cleaned data saved to database!')

    else:
        print('Please provide the filepaths of the messages and categories '\
              'datasets as the first and second argument respectively, as '\
              'well as the filepath of the database to save the cleaned data '\
              'to as the third argument. \n\nExample: python process_data.py '\
              'disaster_messages.csv disaster_categories.csv '\
              'DisasterResponse.db')

if __name__ == '__main__':
    main()


import json
import subprocess
from contextlib import suppress

from bbot.modules.base import BaseModule

class masscan(BaseModule):
    flags = ["active", "portscan", "aggressive"]
    watched_events = ["SCAN"]
    produced_events = ["OPEN_TCP_PORT"]
    meta = {"description": "Port scan IP subnets with masscan"}
    # 600 packets/s ~= entire private IP space in 8 hours
    options = {"ports": "80,443", "rate": 600, "wait": 10, "ping_first": False, "use_cache": False}
    options_desc = {
        "ports": "Ports to scan",
        "rate": "Rate in packets per second",
        "wait": "Seconds to wait for replies after scan is complete",
        "ping_first": "Only portscan hosts that reply to pings",
        "use_cache": "Instead of scanning, use the results from the previous scan",
    }
    deps_ansible = [
        {
            "name": "install dev tools",
            "package": {"name": ["gcc", "git", "make"], "state": "present"},
            "become": True,
            "ignore_errors": True,
        },
        {
            "name": "Download masscan source code",
            "git": {
                "repo": "https://github.com/robertdavidgraham/masscan.git",
                "dest": "#{BBOT_TEMP}/masscan",
                "single_branch": True,
                "version": "master",
            },
        },
        {
            "name": "Build masscan",
            "command": {
                "chdir": "#{BBOT_TEMP}/masscan",
                "cmd": "make -j",
                "creates": "#{BBOT_TEMP}/masscan/bin/masscan",
            },
        },
        {
            "name": "Install masscan",
            "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
        },
    ]
    _qsize = 100

    async def setup(self):
        self.ports = self.config.get("ports", "80,443")
        self.rate = self.config.get("rate", 600)
        self.wait = self.config.get("wait", 10)
        self.ping_first = self.config.get("ping_first", False)
        self.alive_hosts = dict()
        # make a quick dry run to validate ports etc.
        self._target_findkey = "9.8.7.6"
        if not self.helpers.in_tests:
            try:
                dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True)
                dry_run_result = await self.helpers.run(dry_run_command)
                self.masscan_config = dry_run_result.stdout
                self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l)
            except subprocess.CalledProcessError as e:
                self.warning(f"Error in masscan: {e.stderr}")
                return False

        self.run_time = self.helpers.make_date()
        self.use_cache = self.config.get("use_cache", False)
        self.ping_cache = self.scan.home / f"masscan_ping.txt"
        self.syn_cache = self.scan.home / f"masscan_syn.txt"
        if self.use_cache:
            files_exist = self.ping_cache.is_file() or self.syn_cache.is_file()
            files_empty = self.helpers.filesize(self.ping_cache) == 0 and self.helpers.filesize(self.syn_cache) == 0
            if not files_exist:
                return (
                    False,
                    f"use_cache is True but could not find cache file at {self.ping_cache} or {self.syn_cache}",
                )
            if files_empty:
                return (
                    False,
                    f"use_cache is True but could cached files {self.ping_cache} and {self.syn_cache} are empty",
                )
        else:
            self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
        self.ping_cache_fd = None
        self.syn_cache_fd = None
        return True

    async def handle_event(self, event):
        if self.use_cache:
            self.emit_from_cache()
        else:
            exclude, invalid_exclude = self._build_targets(self.scan.blacklist)
            targets, invalid_targets = self._build_targets(self.scan.whitelist)
            if invalid_exclude > 0:
                self.warning(
                    f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)"
                )
            if invalid_targets > 0:
                self.warning(
                    f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
                )

            if not targets:
                self.warning("No targets specified")
                return

            # ping scan
            if self.ping_first:
                self.verbose("Starting masscan (ping scan)")

                await self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True)
                targets = ",".join(str(h) for h in self.alive_hosts)
                if not targets:
                    self.warning("No hosts responded to pings")
                    return

            # TCP SYN scan
            if self.ports:
                self.verbose("Starting masscan (TCP SYN scan)")
                await self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude)
            else:
                self.verbose("No ports specified, skipping TCP SYN scan")
            # save memory
            self.alive_hosts.clear()

    async def masscan(self, targets, result_callback, exclude=None, ping=False):
        # config file
        masscan_config = self.masscan_config.replace(self._target_findkey, targets)
        self.debug("Masscan config:")
        for line in masscan_config.splitlines():
            self.debug(line)
        config_file = self.helpers.tempfile(masscan_config)
        # output file
        # process_output = functools.partial(self.process_output, result_callback=result_callback)
        # json_output_file = self.helpers.tempfile_tail(process_output)
        # command
        command = self._build_masscan_command(config=config_file, exclude=exclude, ping=ping)
        # execute
        stats_file = self.helpers.tempfile_tail(callback=self.verbose)
        try:
            with open(stats_file, "w") as stats_fh:
                async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh):
                    self.process_output(line, result_callback=result_callback)
        finally:
            stats_file.unlink()

    def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_run=False, ping=False):
        command = ("masscan", "--rate", self.rate, "--wait", self.wait, "--open-only", "-oJ", "-")
        if targets is not None:
            command += (targets,)
        if config is not None:
            command += ("-c", config)
        if ping:
            command += ("--ping",)
        elif not dry_run:
            command += ("-p", self.ports)
        if exclude is not None:
            command += ("--exclude", exclude)
        if dry_run:
            command += ("--echo",)
        return command

    def process_output(self, line, result_callback):
        try:
            j = json.loads(line)
        except Exception:
            return
        ip = j.get("ip", "")
        if not ip:
            return
        ports = j.get("ports", [])
        if not ports:
            return
        for p in ports:
            proto = p.get("proto", "")
            port_number = p.get("port", "")
            if proto == "" or port_number == "":
                continue
            result = str(ip)
            source = None
            with suppress(KeyError):
                source = self.alive_hosts[ip]
            if proto != "icmp":
                result += f":{port_number}"
                if source is None:
                    source = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
                    self.emit_event(source)
            result_callback(result, source=source)

    def append_alive_host(self, host, source):
        host_event = self.make_event(host, "IP_ADDRESS", source=self.get_source_event(host))
        self.alive_hosts[host] = host_event
        self._write_ping_result(host)
        self.emit_event(host_event)

    def emit_open_tcp_port(self, data, source):
        self._write_syn_result(data)
        self.emit_event(data, "OPEN_TCP_PORT", source=source)

    def emit_from_cache(self):
        ip_events = {}
        # ping scan
        if self.ping_cache.is_file():
            cached_pings = list(self.helpers.read_file(self.ping_cache))
            if cached_pings:
                self.success(f"{len(cached_pings):,} hosts loaded from previous ping scan")
            else:
                self.verbose(f"No hosts cached from previous ping scan")
            for ip in cached_pings:
                if self.scan.stopping:
                    break
                ip_event = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip))
                ip_events[ip] = ip_event
                self.emit_event(ip_event)
        # syn scan
        if self.syn_cache.is_file():
            cached_syns = list(self.helpers.read_file(self.syn_cache))
            if cached_syns:
                self.success(f"{len(cached_syns):,} hosts loaded from previous SYN scan")
            else:
                self.warning(f"No hosts cached from previous SYN scan")
            for line in cached_syns:
                if self.scan.stopping:
                    break
                host, port = self.helpers.split_host_port(line)
                host = str(host)
                source_event = ip_events.get(host)
                if source_event is None:
                    self.verbose(f"Source event not found for {line}")
                    source_event = self.make_event(line, "IP_ADDRESS", source=self.get_source_event(line))
                    self.emit_event(source_event)
                self.emit_event(line, "OPEN_TCP_PORT", source=source_event)

    def get_source_event(self, host):
        source_event = self.scan.whitelist.get(host)
        if source_event is None:
            source_event = self.scan.root_event
        return source_event

    def _build_targets(self, target):
        invalid_targets = 0
        targets = []
        for t in target:
            t = self.helpers.make_ip_type(t.data)
            if isinstance(t, str):
                invalid_targets += 1
            else:
                targets.append(t)
        return ",".join(str(t) for t in targets), invalid_targets

    async def cleanup(self):
        if self.ping_first:
            with suppress(Exception):
                self.ping_cache_fd.close()
        with suppress(Exception):
            self.syn_cache_fd.close()

    def _write_ping_result(self, host):
        if self.ping_cache_fd is None:
            self.helpers.backup_file(self.ping_cache)
            self.ping_cache_fd = open(self.ping_cache, "w")
        self.ping_cache_fd.write(f"{host}\n")
        self.ping_cache_fd.flush()

    def _write_syn_result(self, data):
        if self.syn_cache_fd is None:
            self.helpers.backup_file(self.syn_cache)
            self.syn_cache_fd = open(self.syn_cache, "w")
        self.syn_cache_fd.write(f"{data}\n")
        self.syn_cache_fd.flush()


# -*- coding: utf-8 -*-
"""
@File: logger
@Time: 2020-07-01 14:37:16
@Auth: tang
"""
import datetime
from gli.logger import Logger
from gli.connect.mongodb import MongoDB
from datetime import datetime, timedelta

class ProxyIP(object):
    """Proxy IP
    """
    def __init__(self):
        self.logger = Logger(folder="zhima")
        client = MongoDB()
        if "local_rw" in client:
            self.reader = client["local_rw"]["proxies"]
        else:
            raise RuntimeError("The specified configuration item could not be found.")
        if "local_rw" in client:
            self.writer = client["local_rw"]["proxies"]
        else:
            raise RuntimeError("The specified configuration item could not be found.")

    def inc_used(self, id: object, field: str):
        """ Increase the number of used.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.writer["proxy"].update_one(
                { "_id": id },
                { "$set": { "update_time": datetime.now() }, "$inc": { "used": int(1), "spider.%s.used" % field: int(1) } })
        except Exception as err:
            self.logger.error(err)

    def inc_fail(self, id:object, field:str):
        """ Increase the number of fail.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.writer["proxy"].update_one(
                { "_id": id },
                { "$set": { "update_time": datetime.now() }, "$inc": { "fail": int(1), "spider.%s.fail" % field: int(1) } })
        except Exception as err:
            self.logger.error(err)

    def modify_update_time(self):
        """ Modify update time.
        Args:
            id: Proxy ID.
            field: Field name in proxy information.
        """
        try:
            self.reader["proxy"].find_one_and_update(
                {},
                { "$set": { "update_time": datetime.now() } },
                sort = [("update_time", -1)])
        except Exception as err:
            self.logger.error(err)

    def run(self, field:str, fail_weight: int = 20, margin: int = 10):
        """Get proxy IP address.
        Args:
            field: Field name in proxy information.
            fail_weight: Reduce selection probability after failure.
            margin: Expiration time margin of proxy IP.
        Returns:
            Success returns { "_id", "ip", "port" }, otherwise None.
        """
        try:
            cursor = self.reader["proxy"].aggregate([
                { "$match": { "expire_time": { "$gt": datetime.now() + timedelta(seconds=margin) } } },
                { "$addFields": { "count": { "$add": [ "$spider.%s.used" % field, { "$multiply": [ "$spider.%s.fail" % field, fail_weight ] } ] } } },
                { "$sort": { "count": 1 } },
                { "$project": { "ip": 1, "port": 1 } },
                { "$limit": 1 } ],
                allowDiskUse=False)
            for doc in cursor:
                # Increase the number of used.
                self.inc_used(doc["_id"], field)
                return doc
            # Modify update time.
            self.modify_update_time()
        except Exception as err:
            self.logger.error(err)


import sqlite3
from sqlite3 import Error

class DatabaseFiller:
    def __init__(self) -> None:
        self.db_filename = "game_stats.db"
        self.create_connection()
        self.create__map_name_table()
        self.create_table()
        pass

    def create_connection(self):
        """ create a database connection to a SQLite database """
        conn = None
        try:
            conn = sqlite3.connect(self.db_filename)
            print(sqlite3.version)
        except Error as e:
            print(e)
        finally:
            if conn:
                conn.close()

    def create__map_name_table(self):
        conn = sqlite3.connect(self.db_filename)
        c = conn.cursor()
        c.execute("""CREATE TABLE IF NOT EXISTS map_name (
                                                        id integer PRIMARY KEY,
                                                        map_name text NOT NULL,
                                                        original_console text NOT NULL
                                                        )""")
        # if the table map_name is empty, insert the data from the Mario_kart8deluxe_maps_with_dlcs_map.csv file
        if c.execute("""SELECT COUNT(*) FROM map_name""").fetchone()[0] == 0:
            # read a csv file and insert the data in the table map_name
            with open('Mario_kart8deluxe_maps_with_dlcs_map.csv', 'r') as file:
                for line in file:
                    line = line.split(',')
                    c.execute(
                        "INSERT INTO map_name (map_name,original_console) VALUES (?,?)", (line[0], line[1]))

        conn.commit()
        conn.close()

    def create_table(self):
        conn = sqlite3.connect(self.db_filename)
        c = conn.cursor()

        c.execute("""CREATE TABLE IF NOT EXISTS game (
                                                        id integer PRIMARY KEY,
                                                        date text NOT NULL
                                                        )""")
        c.execute("""CREATE TABLE IF NOT EXISTS leaderboard (
                                                id integer PRIMARY KEY,
                                                game_id integer NOT NULL,
                                                player_name text NOT NULL,
                                                player_score integer NOT NULL,
                                                FOREIGN KEY (game_id) REFERENCES game(id)
                                                )""")
        c.execute("""CREATE TABLE IF NOT EXISTS maps_played (
                                                id integer PRIMARY KEY,
                                                game_id integer NOT NULL,
                                                map_name text NOT NULL,
                                                FOREIGN KEY (game_id) REFERENCES game(id)
                                                )""")
        conn.commit()
        conn.close()

    def insert_data(self, map_name, players, scores, date):
        conn = sqlite3.connect(self.db_filename)
        c = conn.cursor()
        # insert the data in the table game
        c.execute("INSERT INTO game (date) VALUES (?)", (date,))
        game_id = c.lastrowid
        for player, score in zip(players, scores):
            c.execute("INSERT INTO leaderboard (game_id, player_name, player_score) VALUES (?,?,?)", (game_id, player, score))
        for map in map_name:
            c.execute("INSERT INTO maps_played (game_id, map_name) VALUES (?,?)", (game_id, map))
        conn.commit()
        conn.close()

if __name__ == '__main__':
    db = DatabaseFiller()


def lossy_fuse(list_1, list_2):
    '''  Return a new list containing all the elements of list 1 except the
    last followed by all the elements of list 2 except the first '''
    temp = []
    for i in range(0, len(list_1) - 1):
        temp.append(list_1[i])
    for j in range(1, len(list_2)):
        temp.append(list_2[j])
    return temp

def should_shutdown(battery_level, time_on):
    ''' Return true or false according to the two factors '''
    if time_on < 60 and battery_level < 4.7:
        return 1 > 0
    elif time_on >= 60 and battery_level < 4.8:
        return 1 > 0
    else:
        return 1 < 0

def print_wedge(size):
    ''' Print a wedge of asterisks of a given size '''
    star = '*' * size
    while size > 0:
        print(star)
        size = size - 1
        star = '*' * size

def num_doublings(initial_population, final_population):
    ''' Count how many doublings '''
    i = 0
    while initial_population < final_population:
        i = i + 1
        initial_population = initial_population * 2
    return i

def sum_numbers_in_file(filename):
    ''' Read the numbers and return the sum '''
    infile = open(filename)
    data = infile.readlines()
    data = map(int, data)
    data_sum = sum(data)
    return data_sum

def max_num_in_file(filename):
    ''' Return the largest integer '''
    infile = open(filename)
    data = infile.readlines()
    data = map(int, data)
    data_max = max(data)
    return data_max

def print_numbered_lines(filename):
    ''' Print out all the lines of the file with the line number '''
    infile = open(filename)
    data = infile.readlines()
    for i in range(0, len(data)):
        print(str(i + 1) + ':', data[i].rstrip())

def print_reversed(filename):
    ''' Print out the lines from the file in reversed order '''
    infile = open(filename)
    data = infile.readlines()
    for i in range(len(data) - 1, -1, -1):
        print(data[i].rstrip())

def print_daily_totals(filename):
    ''' Print out the date and the total amount of rainfall '''
    infile = open(filename)
    data = infile.readlines()
    for i in range(0, len(data)):
        item = data[i]
        string = item.split(',')
        date = string[0]
        string[-1] = string[-1].rstrip()
        amount = string[1: ]
        amount = map(float, amount)
        total = sum(amount)
        print(date, '=', '{0:.2f}'.format(total))

def file_size(filename):
    ''' Return the count of letters '''
    infile = open(filename)
    data = infile.readlines()
    total = 0
    for i in range(0, len(data)):
        total = total + len(data[i])
    return total

def generate_daily_totals(input_filename, output_filename):
    ''' Generate a new file with the date and the total amount of rainfall '''
    infile = open(input_filename)
    outfile = open(output_filename, 'w')
    data = infile.readlines()
    for i in range(0, len(data)):
        item = data[i]
        string = item.split(',')
        date = string[0]
        string[-1] = string[-1].rstrip()
        amount = string[1: ]
        amount = map(float, amount)
        total = sum(amount)
        outfile.write(date + ' = ' + '{0:.2f}'.format(total) + '\n')

def write_reversed_file(input_filename, output_filename):
    ''' Generate a new file with the lines from the file in reversed order '''
    infile = open(input_filename)
    outfile = open(output_filename, 'w')
    data = infile.readlines()
    for i in range(len(data) - 1, -1, -1):
        invdata = data[i].rstrip()
        outfile.write(invdata + '\n')


import pickle
import os
import pathlib

class Geoip:

    def __init__(self, file=None):
        try:
            if file and os.path.isfile(file):
                path = os.path.abspath(file)

            elif file == None:
                path = pathlib.Path(__file__).parent.resolve()
                path = os.path.join(path, 'geoip.bin')

            else:
                print("nothing works here")
                exit()

            with open(path, 'rb') as fp:
                self.data = pickle.load(fp)

        except Exception as e:
            print("could not find geoip data")
            print(pathlib.Path(__file__).parent.resolve())
            print(e)
            exit()
            self.data = None

    def test(self):
        print()
        print("test..")
        if self.data:
            print(len(self.data))
            print(self.data[0])
            print(self.data[1000])
            print(self.data[10000])
            print()
        else:
            print("no data found..")

    def ip_int(self, ip:str) -> int:
        a3,a2,a1,a0 = ip.split('.')
        a3,a2,a1,a0 = int(a3),int(a2),int(a1),int(a0)
        return 256**3*a3 + 256**2*a2 + 256*a1 + a0

    def search(self, ip):
        int_ip = self.ip_int(ip)

        for data in self.data:
            if int_ip >= data['start'] and int_ip <= data['stop']:
                return data
                #return {"code": data['code'], "country": data['country']}

        return None

if __name__ == "__main__":
    ## test it..
    geoip = Geoip()
    geoip.test()
    print(geoip.search("61.177.173.39"))


#! /usr/bin/env python
#-*- coding: utf-8 -*-
# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79:

"""
    Views for Microbe app
"""

__author__ = 'TROUVERIE Joachim'

import os.path as op
import shelve
import xml.etree.ElementTree as ET
from datetime import datetime
from werkzeug.contrib.atom import AtomFeed
from urlparse import urljoin

from microbe import app, contents, babel
from microbe.utils import render, render_list
from microbe.mods.search import search_query
from microbe.flatcontent.forms import CommentForm
from microbe.mods.links.models import Links
from microbe.mods.search.forms import SearchForm

from flask.ext.babel import format_datetime, lazy_gettext
from flask.ext.themes2 import static_file_url, get_theme
from flask import (g, request, abort, url_for, make_response,
                  render_template, redirect)

@app.errorhandler(404)
def page_not_found(e):
    """
        Page not found error handler
    """
    return render('404.html'), 404

@babel.localeselector
def get_locale() :
    """
        Get locale for page translations
    """
    return app.config.get('LANGUAGE')

@app.template_filter('date')
def date_filter(date, format = None) :
    """
        Date filter to use in Jinja2 templates

        :param date : datetime object to convert
        :param format: format to convert date object
        :type date: datetime
        :type format: str

        .. warnings:: Format is in flask.ext.babel format
    """
    if date :
        if format :
            return format_datetime(date, format)
        else :
            return format_datetime(date, 'dd MM yyyy')
    else :
        return ''

@app.before_request
def before_request() :
    """
        Refresh global vars before each requests
    """
    # update config
    path = app.config['SHELVE_FILENAME']
    db = shelve.open(path)
    app.config.update(db)
    # close db
    db.close()
    # posts list
    g.posts  = sorted(
                [c for c in contents if c.content_type == 'posts'
                and not c.draft],
                key = lambda x : x.published,
                reverse = True
                )
    # posts categories
    g.categories = set([c.category for c in g.posts if c.category])
    # static pages
    g.static_pages = [c for c in contents if c.content_type == 'pages'
                     and not c.draft]
    # links
    g.links = Links.get_all()
    # search form
    if not hasattr(g, 'search_form') :
        g.search_form = SearchForm()

@app.route('/')
def index():
    """
        Main page of the app

        List of blog posts summaries
    """
    return render_list('index.html', g.posts)

@app.route('/sitemaps.xml')
def sitemap() :
    """
        Site sitemap
    """
    # list all contents
    sitemap_contents = [c for c in contents if not c.draft]
    # root
    root = ET.Element('urlset')
    root.set('xmlns', 'http://www.sitemaps.org/schemas/sitemap/0.9')
    for content in sitemap_contents :
        url = ET.SubElement(root, 'url')
        loc = ET.SubElement(url, 'loc')
        loc.text = urljoin(request.url_root, content.path)
        lastmod = ET.SubElement(url, 'lastmod')
        lastmod.text = format_datetime(content.published, 'yyyy-MM-dd')
    text = ET.tostring(root)
    response = make_response(text, 200)
    response.headers['Content-Type'] = 'application/xml'
    return response

@app.route('/favicon.ico')
def favicon() :
    """
        App favicon
    """
    default = app.config['DEFAULT_THEME']
    theme_id = app.config.get(u'THEME', default)
    theme = get_theme(theme_id)
    path = op.join(theme.static_path, 'img', 'favicon.png')
    if op.exists(path) :
        url = static_file_url(theme, 'img/favicon.png')
    else :
        url = url_for('static', filename='img/favicon.png')
    return redirect(url)

@app.route('/robots.txt')
def robots() :
    """
        App robots.txt
    """
    try :
        url = url_for('static', filename='media/robots.txt')
        return redirect(url)
    except :
        abort(404)

@app.route('/<path:path>/', methods = ['GET', 'POST'])
def page(path):
    """
        Get access for page from its path.
        If path is not valid it will return a 404 error page

        :param path: Valid content path
    """
    content = contents.get_or_404(path)
    form = None
    # enable comments for posts only
    if content.content_type == 'posts' :
        form = CommentForm()
    # form management
    if form and form.validate_on_submit() :
        author = form.name.data
        body = form.content.data
        content.add_comment(author, body)
    return render('page.html', page = content, form = form)

@app.route('/category/<category>')
def category(category) :
    """
        Filter posts by category

        :param category: Category to filter contents
        :type category: str
    """
    posts = [p for p in g.posts if p.category == category]
    return render_list('index.html', posts, title = category)

@app.route('/tag/<tag>')
def tag(tag) :
    """
        Filter posts by tag

        :param tag: Category to filter contents
        :type tag: str
    """
    posts = [p for p in g.posts if tag in p.tags.split(',')]
    return render_list('index.html', posts, title = tag)

@app.route('/archives')
def archives() :
    """
       List all contents order by reverse date
    """
    # sort pages by reverse date
    sorted_pages = sorted(
                        contents,
                        key = lambda x : x.published,
                        reverse = True
                    )
    return render_list('archive.html', sorted_pages, per_page=10)

@app.route('/search/', methods = ['POST'])
def search() :
    """
        Search in contents
    """
    if not g.search_form.validate_on_submit() :
        return redirect(url_for('index'))
    query = g.search_form.search.data
    contents = search_query(query)
    # sort not draft contents by reverse date
    sorted_contents = sorted(
                            [c for c in contents if not c.draft],
                            key = lambda x : x.published,
                            reverse = True
                       )
    return render_list('index.html', sorted_contents, per_page=10)

@app.route('/feed.atom')
def feed() :
    """
        Generate 20 last content atom feed
    """
    if app.config.get('RSS', 'NO') != 'YES' :
        abort(404)
    name = app.config['SITENAME']
    feed = AtomFeed(name,feed_url=request.url, url=request.url_root)
    # sort posts by reverse date
    for post in g.posts[:20] :
        feed.add( post.title,
                unicode(post.summary),
                content_type = 'html',
                author = post.meta.get('author', ''),
                url = urljoin(request.url_root, post.path),
                updated = post.published)
    return feed.get_response()


from components.isatsounds import *
from game import GameState
import unittest

class TestSounds(unittest.TestCase):
    def setUp(self):
        self.mock_1 = GameState(['AB', 'CD'], 301, 'fake1', True)
        self.mock_1.advance_player()
        self.mock_1.add_dart('T20', 60)
        self.mock_1.add_dart('T20', 60)
        self.snd = ISATSounds(test=True)

    def test_events_0(self):
        methods = ['DartStuck', 'ManualNextPlayer']
        for m in methods:
            getattr(self.snd, m)()

    def test_events_1(self):
        methods = ['FrameStarted']
        for m in methods:
            getattr(self.snd, m)(self.mock_1)

    def test_events_2(self):
        methods = ['Hit', 'HitBust', 'HitWinner', 'EnterHold']
        for m in methods:
            getattr(self.snd, m)(self.mock_1, 'T20')


import contextlib
from argparse import Namespace
from logging import setLogRecordFactory
import math
from typing import Any, List
import random
from fairseq.data.dictionary import Dictionary

import torch
import torch.nn as nn
from dataclasses import dataclass, field
from fairseq import checkpoint_utils, tasks, utils
from fairseq.dataclass import FairseqDataclass
from fairseq.dataclass.utils import convert_namespace_to_omegaconf
from fairseq.models import BaseFairseqModel, FairseqEncoder, register_model
from fairseq.models.transformer import TransformerEncoder, TransformerDecoder
from fairseq.models.hubert.hubert import MASKING_DISTRIBUTION_CHOICES
from fairseq.tasks import FairseqTask
from omegaconf import II, MISSING
from fairseq.modules import transformer_layer
from fairseq.modules.checkpoint_activations import checkpoint_wrapper
from fairseq.distributed import fsdp_wrap
from fairseq.models.masked_lm import MaskedLMEncoder
from fairseq.tasks.optimize_ali_speech_language import OptimizingAlignmentTask
from fairseq.data.data_utils import compute_mask_indices
from fairseq.models.wav2vec.wav2vec2 import TransformerSentenceEncoderLayer
from fairseq.models.hubert.hubert_asr import HubertAsrConfig, HubertCtcConfig, HubertSeq2SeqConfig

class HubertEncoder(FairseqEncoder):
    def __init__(self, cfg: HubertAsrConfig, task, tgt_dict=None):
        self.apply_mask = cfg.apply_mask

        arg_overrides = {
            "dropout": cfg.dropout,
            "activation_dropout": cfg.activation_dropout,
            "dropout_input": cfg.dropout_input,
            "attention_dropout": cfg.attention_dropout,
            "mask_length": cfg.mask_length,
            "mask_prob": cfg.mask_prob,
            "mask_selection": cfg.mask_selection,
            "mask_other": cfg.mask_other,
            "no_mask_overlap": cfg.no_mask_overlap,
            "mask_channel_length": cfg.mask_channel_length,
            "mask_channel_prob": cfg.mask_channel_prob,
            "mask_channel_selection": cfg.mask_channel_selection,
            "mask_channel_other": cfg.mask_channel_other,
            "no_mask_channel_overlap": cfg.no_mask_channel_overlap,
            "encoder_layerdrop": cfg.layerdrop,
            "feature_grad_mult": cfg.feature_grad_mult,
        }

        if cfg.w2v_args is None:
            state = checkpoint_utils.load_checkpoint_to_cpu(
                cfg.w2v_path, arg_overrides
            )
            w2v_args = state.get("cfg", None)
            if w2v_args is None:
                w2v_args = convert_namespace_to_omegaconf(state["args"])
            cfg.w2v_args = w2v_args
        else:
            state = None
            w2v_args = cfg.w2v_args
            if isinstance(w2v_args, Namespace):
                cfg.w2v_args = w2v_args = convert_namespace_to_omegaconf(
                    w2v_args
                )
        assert cfg.normalize == w2v_args.task.normalize, (
            "Fine-tuning works best when data normalization is the same. "
            "Please check that --normalize is set or unset for "
            "both pre-training and here"
        )

        w2v_args.task.data = cfg.data

        task = tasks.setup_task(w2v_args.task)
        if state is not None and "task_state" in state:
            # This will load the stored "dictionaries" object
            task.load_state_dict(state["task_state"])
        model = task.build_model(w2v_args.model)
        model.remove_pretraining_modules()

        if state is not None and not cfg.no_pretrained_weights:
            # set strict=False because we omit some modules
            model.load_state_dict(state["model"], strict=False)

        super().__init__(task.source_dictionary)

        d = w2v_args.model.encoder_embed_dim

        if hasattr(model, 'speech_encoder'):
            self.w2v_model = model.speech_encoder
        else:
            self.w2v_model = model

        self.w2v_model.remove_pretraining_modules()
        self.final_dropout = nn.Dropout(cfg.final_dropout)
        self.freeze_finetune_updates = cfg.freeze_finetune_updates
        self.num_updates = 0

        if tgt_dict is not None:
            self.proj = Linear(d, len(tgt_dict))
        elif getattr(cfg, "decoder_embed_dim", d) != d:
            self.proj = Linear(d, cfg.decoder_embed_dim)
        else:
            self.proj = None

    def set_num_updates(self, num_updates):
        """Set the number of parameters updates."""
        super().set_num_updates(num_updates)
        self.num_updates = num_updates

    def forward(self, source, padding_mask, tbc=True, **kwargs):

        w2v_args = {
            "source": source,
            "padding_mask": padding_mask,
            "mask": self.apply_mask and self.training,
        }

        ft = self.freeze_finetune_updates <= self.num_updates

        with torch.no_grad() if not ft else contextlib.ExitStack():

            res  = self.w2v_model.forward(**w2v_args)
            x = res["x"]
            padding_mask = res["padding_mask"]
            pos_bias = res["layer_result"][-1][2]
            if tbc:
                # B x T x C -> T x B x C
                x = x.transpose(0, 1)

        x = self.final_dropout(x)

        # if self.proj:
        #     x_ctc = self.proj(x)

        return {
            "encoder_out": x,  # T x B x C
            "encoder_padding_mask": padding_mask,  # B x T
            "padding_mask": padding_mask,
            "pos_bias": pos_bias
        }

    def reorder_encoder_out(self, encoder_out, new_order):
        if encoder_out["encoder_out"] is not None:
            encoder_out["encoder_out"] = encoder_out[
                "encoder_out"
            ].index_select(1, new_order)
        if encoder_out["encoder_padding_mask"] is not None:
            encoder_out["encoder_padding_mask"] = encoder_out[
                "encoder_padding_mask"
            ].index_select(0, new_order)
        return encoder_out

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        return None

    def upgrade_state_dict_named(self, state_dict, name):
        return state_dict

    def get_normalized_probs(self, net_output, log_probs):
        """Get normalized probabilities (or log probs) from a net's output."""

        logits = net_output["encoder_out"]
        if log_probs:
            return utils.log_softmax(logits.float(), dim=-1)
        else:
            return utils.softmax(logits.float(), dim=-1)

def Embedding(num_embeddings, embedding_dim, padding_idx):
    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
    nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m

def Linear(in_features, out_features, bias=True):
    m = nn.Linear(in_features, out_features, bias)
    nn.init.xavier_uniform_(m.weight)
    if bias:
        nn.init.constant_(m.bias, 0.0)
    return m

@dataclass
class HubertTextMTLConfig(HubertCtcConfig):
    # text
    text_encoder_layers: int = field(
        default=4, metadata={"help": "num encoder layers in the text encoder"}
    )

    # masking
    text_encoder_apply_mask: bool = field(
        default=False, metadata={"help": "apply masking during fine-tuning"}
    )
    text_encoder_mask_length: int = field(
        default=10, metadata={"help": "repeat the mask indices multiple times"}
    )
    text_encoder_mask_prob: float = field(
        default=0.5,
        metadata={
            "help": "probability of replacing a token with mask "
            "(normalized by length)"
        },
    )
    text_encoder_mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
        default="static", metadata={"help": "how to choose masks"}
    )
    text_encoder_mask_other: float = field(
        default=0,
        metadata={
            "help": "secondary mask argument "
            "(used for more complex distributions), "
            "see help in compute_mask_indices"
        },
    )
    text_encoder_no_mask_overlap: bool = field(
        default=False, metadata={"help": "whether to allow masks to overlap"}
    )
    text_encoder_mask_min_space: int = field(
        default=1,
        metadata={
            "help": "min space between spans (if no overlap is enabled)"
        },
    )
    # model
    shared_encoder_layer: int = field(
        default=1,
        metadata={"help": "the number of shared encoder layers"},
    )
    embedding_aligner_dim: int = field(
        default=1024,
        metadata={"help": "the dimension of embedding aligner"}
    )
    swap_embedding_ratio: float = field(
        default=0.2,
        metadata={"help": "the probability of embedding swapping"}
    )
    swap_embedding_phoneme_aware: bool = field(
        default=True,
        metadata={"help": "swap embedding with phoneme aware"}
    )

class MaskedTextEncoder(BaseFairseqModel):
    def __init__(
        self,
        cfg:HubertTextMTLConfig,
        dictionary,

    ):
        super().__init__()
        # 1. token embedding
        self.encoder_embed_dim= cfg.encoder_embed_dim
        self.token_embedding = self.build_embedding(cfg,dictionary,self.encoder_embed_dim)
        # 2. text encoder
        self.MASK = 0
        self.embedding_dim = self.encoder_embed_dim
        self.dropout = cfg.dropout
        self.encoder_layers = nn.ModuleList(
            [
                TransformerSentenceEncoderLayer(
                    embedding_dim=self.embedding_dim,
                    ffn_embedding_dim=cfg.encoder_ffn_embed_dim,
                    num_attention_heads=cfg.encoder_attention_heads,
                    dropout=self.dropout,
                    attention_dropout=cfg.attention_dropout,
                    activation_dropout=cfg.activation_dropout,
                    activation_fn=cfg.activation_fn,
                    layer_norm_first=cfg.layer_norm_first,
                )
                for _ in range(cfg.text_encoder_layers)
            ]
        )
        self._dictionary = dictionary
        self._mask_prob = cfg.text_encoder_mask_prob
        self._apply_mask = cfg.text_encoder_apply_mask
        self._mask_length = cfg.text_encoder_mask_length
        self._mask_selection = cfg.text_encoder_mask_selection
        self._mask_other = cfg.text_encoder_mask_other
        self._no_mask_overlap = cfg.text_encoder_no_mask_overlap
        self._mask_min_space = cfg.text_encoder_mask_min_space

    def forward(
        self,
        prev_phoneme,
        prev_phoneme_mask,
        apply_mask,
    ):
        # 1. apply mask

        if apply_mask:
            prev_phoneme, _ = self.apply_mask(prev_phoneme, prev_phoneme_mask, self._dictionary)
        # 2. embedding
        prev_phoneme = self.token_embedding(prev_phoneme)
        prev_phoneme = prev_phoneme.transpose(0,1)
        # 3. encoder
        pos_bias=None
        for transformer in self.encoder_layers:
            prev_phoneme,_,pos_bias = transformer(prev_phoneme, self_attn_padding_mask=prev_phoneme_mask, pos_bias=pos_bias)
        prev_phoneme = prev_phoneme.transpose(0,1)
        return {
            "encoder_out": prev_phoneme,
            "padding_mask": prev_phoneme_mask,
            "pos_bias": pos_bias
        }

    def apply_mask(self, x, padding_mask, target_list):
        B, T = x.shape
        if self._mask_prob > 0:
            mask_indices = compute_mask_indices(
                (B, T),
                padding_mask,
                self._mask_prob,
                self._mask_length,
                self._mask_selection,
                self._mask_other,
                min_masks=2,
                no_overlap=self._no_mask_overlap,
                min_space=self._mask_min_space,
            )
            mask_indices = torch.from_numpy(mask_indices).to(x.device)
            x[mask_indices] = self.MASK
        else:
            mask_indices = None
        return x, mask_indices
    @classmethod
    def build_embedding(cls, args, dictionary, embed_dim, path=None):
        num_embeddings = len(dictionary)
        padding_idx = dictionary.pad()

        emb = Embedding(num_embeddings, embed_dim, padding_idx)
        # if provided, load from preloaded dictionaries
        if path:
            embed_dict = utils.parse_embedding(path)
            utils.load_embedding(embed_dict, dictionary, emb)
        return emb

    def get_normalized_probs(self, net_output, log_probs):
        """Get normalized probabilities (or log probs) from a net's output."""

        logits = net_output["encoder_out"]
        if log_probs:
            return utils.log_softmax(logits.float(), dim=-1)
        else:
            return utils.softmax(logits.float(), dim=-1)

@register_model("hubert_text_mtl", dataclass=HubertTextMTLConfig)
class HubertTextMTL(BaseFairseqModel):
    def __init__(
        self,
        cfg: HubertTextMTLConfig,
        w2v_encoder: BaseFairseqModel,
        text_encoder:BaseFairseqModel,
        embedding_aligner,
        has_shared_encoder
    ):
        super().__init__()
        self.cfg = cfg
        arg_overrides = cfg.w2v_args.model.copy()
        arg_overrides["encoder_layers"] = cfg.shared_encoder_layer
        # 1. audio encoder
        self.w2v_encoder = w2v_encoder
        # 2. text encoder
        self.text_encoder = text_encoder
        # 4. shared encoder
        self.embedding_dim = cfg.w2v_args.model.encoder_embed_dim
        self.dropout = cfg.w2v_args.model.dropout
        if has_shared_encoder:
            self.shared_encoder = self.w2v_encoder.w2v_model.encoder.layers[-(self.cfg.shared_encoder_layer):]
        else:
            self.shared_encoder = None
        # 5. embedding aligner
        self.embedding_aligner = embedding_aligner
        # 6. ctc proj
        self.swap_embedding_ratio = cfg.swap_embedding_ratio
        self.swap_embedding_phoneme_aware = cfg.swap_embedding_phoneme_aware

    def upgrade_state_dict_named(self, state_dict, name):
        super().upgrade_state_dict_named(state_dict, name)
        return state_dict

    @classmethod
    def build_embedding(cls, args, dictionary, embed_dim, path=None):

        num_embeddings = len(dictionary)
        padding_idx = dictionary.pad()

        emb = Embedding(num_embeddings, embed_dim, padding_idx)
        # if provided, load from preloaded dictionaries
        if path:
            embed_dict = utils.parse_embedding(path)
            utils.load_embedding(embed_dict, dictionary, emb)
        return emb

    @classmethod
    def build_model(cls, cfg: HubertTextMTLConfig, task: FairseqTask):
        """Build a new model instance."""
        # 1. audio encoder
        w2v_encoder = HubertEncoder(cfg, task, task.target_dictionary)
        if task.has_text_encoder:
            # 2. text encoder
            text_encoder = MaskedTextEncoder(
                cfg,
                task.phoneme_dictionary
            )
        else:
            text_encoder = None
        # embedding_aligner
        embedding_aligner = nn.parameter.Parameter(
            nn.init.uniform_(torch.empty(
                (len(task.phoneme_dictionary), cfg.w2v_args.model.encoder_embed_dim)
            ))
        )

        return cls(cfg, w2v_encoder, text_encoder, embedding_aligner, task.has_shared_encoder)

    def forward(
        self,
        source,
        padding_mask,
        phf_input,
        phf_padding_mask,
        lengths,
        mode: str = "speech",
    ):
        if mode == "speech":
            return self.forward_speech(
                source,
                padding_mask,
                phf_input,
                phf_padding_mask
            )
        elif mode == "text":
            return self.forward_text(
                phf_input,
                phf_padding_mask
            )

    def swap_embedding(self,audio_embedding, text_embedding, accum_alignment):
        # audio_embedding is B*T*D
        # text_embedding is B*T*D
        # assert the length of audio embedding is the same as text embedding
        assert(audio_embedding.shape[1] == text_embedding.shape[1]), str((audio_embedding.shape,text_embedding.shape))
        # building mask
        bsz = audio_embedding.shape[0]
        channel_size = audio_embedding.shape[2]
        for i in range(bsz):
            if self.swap_embedding_phoneme_aware:
                mask = torch.ones((audio_embedding.shape[1]), device=audio_embedding.device)

                indices = random.sample(list(range(1,len(accum_alignment[i]))),
                    math.ceil(len(accum_alignment[i])* self.swap_embedding_ratio))
                for index in indices:
                    start,end = accum_alignment[i][index-1], accum_alignment[i][index]
                    mask[start:end] = 0
            else:
                mask = (
                    torch.randn(
                        text_embedding[i].shape[0],
                        device=text_embedding.device
                    ).uniform() > self.swap_embedding_ratio
                ).float()
            mask = mask.unsqueeze(1).expand(mask.shape[0],channel_size)
            text_embedding_tmp = text_embedding[i] * mask + audio_embedding[i] * (1 - mask)
            audio_embedding[i] = audio_embedding[i] * mask + text_embedding[i] * (1 - mask)
            text_embedding[i] = text_embedding_tmp

    def get_accum_from_phoneme_seq(self, phoneme_seq, phoneme_padding_mask):
        bsz = phoneme_seq.shape[0]
        accum_lists = []
        for i in range(bsz):
            accum = [indice+1 for indice,j in enumerate(range(phoneme_seq[i].shape[0]-1))
                if phoneme_padding_mask[i][j] == False and phoneme_seq[i][j]!=phoneme_seq[i][j+1] ]
            accum_lists.append(accum)
        return accum_lists

    def forward_speech(
        self,
        x,
        padding_mask,
        xt,
        phoneme_padding_mask
    ):
        # 1. audio encoder
        # assert audio input is feature
        assert(len(x.shape)==2)

        x_dict = self.w2v_encoder(x, padding_mask, False, output_layer = (len(self.w2v_encoder.w2v_model.encoder.layers) - self.cfg.shared_encoder_layer) )
        # x = x_dict["encoder_out"]
        if self.text_encoder is not None and xt is not None and self.training:
            # because of w2v downsample we do downsample here
            '''
                notice!!!!!
                if we change the structure of w2v conv, we should change the downsample here
                this downsample is compute as follow:
                    w2v conv structure [(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2
                the field we can seen is
                    { (x[i]-1)*2+2 = x[i-1] }*2
                    { (x[i]-1)*2+3 = x[i-1] }*4
                    { (x[i]-1)*5+10 = x[i-1] }*1
                then we get 400 which is the same as feature extraction of fbank
                then we can compute overlap from the top (512,3,2)
                    { (x[i]-1)*2+3 = x[i-1] }*4
                    { (x[i]-1)*5+10 = x[i-1] }*1
                finally we get 80 and ( 400 - 80 ) = 320 = 160 * 2,
                so the conv structure can map to the kaldi fbank feature by the kaldi fbank
                feature downsampling of twice
            '''
            xt = xt[:,::2]
            phoneme_padding_mask = phoneme_padding_mask[:,::2]
            # 2. text_encoder
            accum_list = self.get_accum_from_phoneme_seq(xt, phoneme_padding_mask)
            # for swapping embedding we do not mask the input
            xt_dict = self.text_encoder(xt,phoneme_padding_mask, apply_mask=False )
            # 3. text_encoder -> swap embedding
            assert(x_dict["encoder_out"].shape[1] == xt_dict["encoder_out"].shape[1]), str((x_dict["encoder_out"].shape,xt_dict["encoder_out"].shape))+ ", "+ str(x.shape)+ ", "+str(xt.shape)
            self.swap_embedding(
                x_dict["encoder_out"],
                xt_dict["encoder_out"],
                accum_list
            )
        # 4. audio encoder -> embedding aligner -> ctc prob
        #    text encoder -> embedding aligner -> mlm prob
        x = x_dict["encoder_out"]
        pos_bias = x_dict["pos_bias"]
        x_out = None
        if self.shared_encoder:
            x_out = utils.log_softmax(torch.cdist(x,self.embedding_aligner), -1)
            x = x.transpose(0,1)
            # 5. audio encoder -> shared encoder

            for transformer in self.shared_encoder:
                x,_,pos_bias = transformer(x, self_attn_padding_mask=x_dict["encoder_padding_mask"],pos_bias=pos_bias)
            x = x.transpose(0,1)
        x = self.w2v_encoder.proj(x)
        return {
            "ctc_prob": x_out,
            "final_ctc_prob": x,
            "padding_mask": x_dict["encoder_padding_mask"],
        }

    def forward_text(
        self,
        prev_phoneme,
        phoneme_padding_mask
    ):
        # 1. text encoder
        out_dict = self.text_encoder(prev_phoneme, phoneme_padding_mask, apply_mask=True)
        phoneme_padding_mask = out_dict["padding_mask"]
        # 2. audio encoder -> embedding aligner -> MLM prob
        # x = out_dict["encoder_out"]

        # 4. audio encoder -> shared encoder
        out = out_dict["encoder_out"]
        out = out.transpose(0,1)
        pos_bias=out_dict["pos_bias"]
        for transformer in self.shared_encoder:
            out,_,pos_bias = transformer(out, self_attn_padding_mask=phoneme_padding_mask,pos_bias=pos_bias)
        out = out.transpose(0,1)
        x = torch.cdist(out,self.embedding_aligner)
        out = self.w2v_encoder.proj(out)
        return {
            "mlm_prob": x,
            "final_ctc_prob": out,
            "phoneme_padding_mask": phoneme_padding_mask
        }

    def get_normalized_probs(self, net_output,name, log_probs):
        """Get normalized probabilities (or log probs) from a net's output."""

        logits = net_output[name].transpose(0,1)
        if log_probs:
            return utils.log_softmax(logits.float(), dim=-1)
        else:
            return utils.softmax(logits.float(), dim=-1)


#Your Function should return a list containing all possible IP address
#No need to worry about order
class Solution:
    def genIp(self, s):
        #Code here
        res=[]
        if len(s)>12:
            return res
        if len(s)<4:
            return res
        def backtrack(i,dots,curIP):
            if dots ==4 and i==len(s):
                res.append(curIP[:-1])
            if dots>4:
                return
            for j in range(i,min(i+3,len(s))):
                if (int(s[i:j+1])<256 and ((i==j) or s[i]!='0')):
                    backtrack(j+1,dots+1,curIP+s[i:j+1]+'.')
        backtrack(0,0,'')
        return res

#{
 # Driver Code Starts
#Main
if __name__=='__main__':
    t = int(input())
    for i in range(t):
        s = input().strip()
        res = Solution().genIp(s)
        res.sort()
        if(len(res)):
            for u in res:
                print(u)
        else:
            print(-1)
# } Driver Code Ends


# -*- Mode: Python3; coding: utf-8; indent-tabs-mpythoode: nil; tab-width: 4 -*-

import os
import numpy as np
from random import randrange

# módulo de manipulação da imagem
from RGB import RGB

PATH = "../Images/"

def view(data, num_out_values=10):

    # Formatar
    h, w, c = data.shape
    d = data.flatten()
    rgb_hex = [RGB(int(d[i]), int(d[i+1]), int(d[i+2])).toHex()
               for i in range(0, h * w * c, c)]

    # Contar
    u_hex, counts = np.unique(rgb_hex, return_counts=True)

    # Listar
    ranking = {u_hex[i]: counts[i] for i in range(len(u_hex))}
    ranking = sorted(ranking.items(), key=lambda v: v[1], reverse=True)
    max_pos = num_out_values if num_out_values < len(ranking) else 10

    print("Frequency:")
    print("RGB             Hexa     Integer    Num/Total %")
    for v in ranking:
        percent = v[1] / (h * w)
        print("{} {}{:>10} {:>3}/{} {:.2f}%".format(RGB(v[0]),
              RGB(v[0]).toHex(), RGB(v[0]).toInt(), v[1], str(h * w),
              percent * 100))
        max_pos = max_pos - 1
        if max_pos < 0:
            break
    print("-" * 50)

def test(filename, num_out_values=10):

    img_np = PATH + filename + ".npy"
    print("Data: ", img_np)

    if not os.path.exists(img_np):
        print("File not found!")
        return

    data = np.load(img_np)

    h, w, c = data.shape
    if c > 3:
        data = data[:, :, :3]

    view(data, num_out_values)

if __name__ == '__main__':

    # ndArray
    px1 = [10, 10, 10]
    px2 = [255, 255, 255]
    pixels = np.array(px1 * 15 + px2 * 5)
    pixels = np.array(pixels).reshape(5, 4, 3)
    view(pixels)

    h, w, c = 5, 4, 3
    pixels = []
    for i in range(h * w):
        r = randrange(0, 255)
        g = randrange(0, 255)
        b = randrange(0, 255)
        pixels += [[r, g, b]]
    pixels = np.array(pixels).reshape(h, w, c)
    view(pixels, 3)

    # ndArray (Imagem)
    test("folha_cafeeiro_doente", 5)
    test("cafeeiro_campo", 10)


import socket, random

running = True

print("launching server")
IP = "127.0.0.1"
PORT = random.randint(8000, 9000)

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((IP, PORT))
print(f"server connected\nPort: {PORT}")

windows = []
try:
    while True:
        data, addr = sock.recvfrom(1024)
        data = data.decode().split("|")
        if data[0] == "window":
            windows.append(addr)
            #sock.sendto(str.encode("1"), addr)  # enable if data loss present
        elif data[0] == "client":
            for window in windows:
                sock.sendto(str.encode(data[1]), window)
except KeyboardInterrupt:
    sock.shutdown()
    sock.close()


import FreeCAD as App
import math
import Draft

Gui.activateWorkbench("DraftWorkbench")

#distant to move point
n = 2

#does need to mirror normal:
# 1 - mirror
# 0 - do not mirror
mirror = 1

groupDefinded = False

objSelected = Gui.Selection.getSelection()
objInSelection = len(objSelected)
for c_obj in range(0, objInSelection):
  print("c_obj ",c_obj);
  object = objSelected[c_obj]
  print("[BSpline extender] Modifying ", object.Label)
  points = object.Points
  numPoints = len(points)
  pointA = points[0]*1
  pointA[2] = pointA[2] - 1
  for c_pts in range(0, numPoints):
    pointB = points[c_pts]*1
    if c_pts == 19:
      pointC = points[19]*1
      pointC[2] = pointC[2] - 1
    else:
      pointC = points[c_pts+1]*1
    #deep coping points
    AX = pointA[0]*1
    AY = pointA[1]*1
    AZ = pointA[2]*1
    pointA = pointB*1
    BX = pointB[0]*1
    BY = pointB[1]*1
    BZ = pointB[2]*1
    CX = pointC[0]*1
    CY = pointC[1]*1
    CZ = pointC[2]*1
    # ToDo: next calculation made fo XZ plane only. need to use cselected plane.
    #calculate angel for normal in midle point
    AB = math.sqrt(math.pow((AX-BX),2)+math.pow((AZ-BZ),2))
    BC = math.sqrt(math.pow((BX-CX),2)+math.pow((BZ-CZ),2))
    #print("AB=",AB, " BC=",BC)
    a = math.asin((BZ-AZ)/AB)*180/3.141593
    g = math.asin((CZ-BZ)/BC)*180/3.141593
    b = 180 - g + a
    p = g + b / 2 - 180 * mirror
    #calculate new point coordinates
    NX = BX - n * math.cos(p*3.141593/180)
    NZ = BZ - n * math.sin(p*3.141593/180)
    NY = BY
    #creeating new point
    point = points[c_pts]
    point = FreeCAD.Vector(NX, NY, NZ)
    points[c_pts] = point*1
    #ToDo: make parent linkage of points of two splines
  bspline = Draft.makeBSpline(points,False,object.Placement,True)
  bspline.Label=object.Label+"_Ext"
  listFolders = object.InList
  selFolder = listFolders[len(listFolders)-1]
  if groupDefinded == False:
    # Checking if Label "_Ext" exists,
    label = "_Ext"
    numeric=0
    groups = selFolder.Group
    numgroups = len(groups)
    for numobjs in range(0, numgroups):
      tmpobj = selFolder.Group[numobjs]
      if (tmpobj.TypeId == 'App::DocumentObjectGroup') | (tmpobj.TypeId == 'App::DocumentObjectGroupPython'):
        leftside=tmpobj.Label[0:4]
        if leftside == label:
          length = len(tmpobj.Label)
          rightside = tmpobj.Label[4:length]
          if rightside == '':
            rightside = '0'
            int(rightside)
          numeric = max(numeric, int(rightside))
    numeric = numeric + 1
    label=label+str(numeric)
    print("-")
    # Now 'label' = first free name started with _Ext and numeric suffix
    # Creating Group with new name 'label'
    #selFolder.Label
    # selFolder.Group - contains all objects in group
    # Creating new Group
    doc = App.ActiveDocument
    newGroup = doc.addObject("App::DocumentObjectGroupPython", label)
    selFolder.addObject(newGroup)
    groupDefinded = True
  #Adding created spline to new group
  newGroup.addObject(bspline)
  #Creating linkage of splines
  bspline.addProperty("App::PropertyLink","LinkToParent")
  bspline.LinkToParent = object


import nltk
import random
from nltk.corpus import movie_reviews
from nltk.tokenize import word_tokenize
from nltk.classify.scikitlearn import SklearnClassifier
from sklearn.naive_bayes import MultinomialNB

# nltk.download('movie_reviews')
# nltk.download('punkt')
def extract_features(words):
    return dict([(word, True) for word in words])

documents = [(list(movie_reviews.words(fileid)), category)
for category in movie_reviews.categories()
for fileid in movie_reviews.fileids(category)]

random.shuffle(documents)

featuresets = [(extract_features(words), category) for (words, category) in documents]

train_set, test_set = featuresets[100:], featuresets[:100]
classifier = SklearnClassifier(MultinomialNB())
classifier.train(train_set)

print("Accuracy:", nltk.classify.accuracy(classifier, test_set))
input_sentence = input("Enter a sentence: ")
words = word_tokenize(input_sentence)
features =extract_features(words)
print("Predicted sentiment:", classifier.classify(features))


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from db_methods import DatabaseS
from json import dumps
from models import FilterData, FilterDataTT
# from starlette.status import HTTP_403_FORBIDDEN
# import json

app = FastAPI()

origins = ["*",]

app.add_middleware(
    CORSMiddleware,
    allow_origins=origins,
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

@app.get("/")
async def root():
    return ""

@app.get('/filters_main/{typee}')
async def get_filters_main(typee: str):
    return dumps({'result': DatabaseS().get_filters_main(typee=typee)})

@app.put('/filter_data/')
async def get_filter_data(data: FilterData):
    return DatabaseS().filter_data(data)

@app.put('/filter_data_tt/')
async def get_filter_data_tt(data: FilterDataTT):
    return DatabaseS().filter_data_tt(data)

@app.delete('/remove_filter')
async def remove_filter(data: FilterData):
    DatabaseS().remove_filter(data)
    return {'removed': True}

@app.get('/get_additions')
async def get_additions():
    return DatabaseS().get_additions()

@app.get('/get_uniqe_typees')
async def get_uniqe_typees():
    return DatabaseS().get_uniqe_typees()

@app.get('/get_uniqe_targets')
async def get_uniqe_targets():
    return DatabaseS().get_uniqe_targets()


#TIENES QUE ESTAR PENDIENTE DE LOS ESPACIOS PORQUE ESA VAINA SI IMPORTA COÑO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
acum=0
mayor=None
menor=None
while True:
    print("ingrese numero o done para salir:")
    inp=input()
    if inp=='done':
        break
    try:
        inp=int(inp)
    except :
        print('Invalid input')
        inp=input()
        inp=int(inp)
    if mayor==None:
        mayor=inp
    elif inp>mayor:
        mayor=inp
    if menor==None:
        menor=inp
    elif menor>inp:
        menor=inp
    acum+=inp

print('acumulado', acum)
print('mayor',mayor)
print('menor', menor)


from sklearn.preprocessing import OneHotEncoder,LabelEncoder

lgb_train = lgb.Dataset(df_train[NUMERIC_COLS], df_train['bad_ind'], free_raw_data=False)
params = {
    'num_boost_round': 50,
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'num_leaves': 2,
    'metric': 'auc',
    'max_depth':1,
    'feature_fraction':1,
    'bagging_fraction':1,
}
model = lgb.train(params,lgb_train)
leaf = model.predict(df_train[NUMERIC_COLS],pred_leaf=True)
lgb_enc = OneHotEncoder()
lgb_enc.fit(leaf)
data_leaf = np.hstack((lgb_enc.transform(leaf).toarray(),df_train[NUMERIC_COLS]))

train, val, train_y, val_y = train_test_split(data_leaf,df_train['bad_ind'],test_size=0.2, random_state=random.choice(range(10000)))

lgb_lm = LogisticRegression(penalty='l1',C = 0.3,solver='liblinear')
lgb_lm.fit(train, train_y)

y_pred_lgb_lm_train = lgb_lm.predict_proba(train)[:, 1]
fpr_lgb_lm_train, tpr_lgb_lm_train, _ = roc_curve(train_y, y_pred_lgb_lm_train)

y_pred_lgb_lm = lgb_lm.predict_proba(val)[:, 1]
fpr_lgb_lm, tpr_lgb_lm, _ = roc_curve(val_y, y_pred_lgb_lm)


# -*- coding: utf-8 -*-
from keras.layers import Activation, UpSampling2D, Convolution2D, MaxPooling2D, BatchNormalization, Dropout, concatenate, add

def conv_bn_relu_gen(nb_filter, rk, ck, st, names, trainable_label = True):
    def f(input):
        conv = Convolution2D(nb_filter, kernel_size=(rk, ck), strides=(st,st),padding="same",
                             use_bias=False,kernel_initializer="truncated_normal",name='conv-'+names)(input)
        conv_norm = BatchNormalization(name='BN-'+names, trainable=trainable_label)(conv)
        conv_norm_relu = Activation(activation = 'relu',name='Relu-'+names)(conv_norm)
        return conv_norm_relu
    return f

def conv_bn(nb_filter, rk, ck, st, names):
    def f(input):
        conv = Convolution2D(nb_filter, kernel_size=(rk, ck), strides=(st,st),padding="same",
                             use_bias=False,kernel_initializer="truncated_normal",name='conv-'+names)(input)
        return conv
    return f

def ResBlock(nb_filter, rk, ck, st, res_names, trainable_label):
    def f(input):
        Node1 = conv_bn_relu_gen(nb_filter, rk, ck, 1, 'Nodes1'+res_names,trainable_label)(input)
        Node2 = conv_bn(nb_filter, rk, ck, 1, 'Nodes2'+res_names)(Node1)
        Node3 = conv_bn(nb_filter, 1, 1, 1, 'Nodes3'+res_names)(input)
        Node4 = add([Node2,Node3])
        conv_norm = BatchNormalization(name='BN_act-'+res_names, trainable=trainable_label)(Node4)
        Res = Activation(activation = 'relu',name='Nodes_act-'+res_names)(conv_norm)
        return Res
    return f

def ResBlock2(nb_filter, rk, ck, st, res_names,trainable_label):
    def f(input):
        Node1 = conv_bn_relu_gen(nb_filter, rk, ck, 1, 'Nodes1'+res_names,trainable_label)(input)
        Node2 = conv_bn(nb_filter, rk, ck, 1, 'Nodes2'+res_names)(Node1)
        Node4 = add([Node2,input])
        conv_norm = BatchNormalization(name='BN_act-'+res_names, trainable=trainable_label)(Node4)
        Res = Activation(activation = 'relu',name='Nodes_act-'+res_names)(conv_norm)
        return Res
    return f

def DeepSNiF_net(input, names, loss_func,trainable_label = True):
    filter_num = 64
    Features1 = ResBlock(filter_num, 3, 3, 1, res_names=names+'ResBlock1',trainable_label=trainable_label)(input)

    pool1 = MaxPooling2D(pool_size=(2,2),name=names+'Pool1')(Features1)
    Features2 = ResBlock(filter_num*2, 3, 3, 1, res_names=names+'ResBlock2',trainable_label=trainable_label)(pool1)

    pool2 = MaxPooling2D(pool_size=(2, 2),name=names+'Pool2')(Features2)
    Features3 = ResBlock(filter_num*4, 3, 3, 1, res_names=names+'ResBlock3',trainable_label=trainable_label)(pool2)

    pool3 = MaxPooling2D(pool_size=(2, 2),name=names+'Pool3')(Features3)
    Features4 = ResBlock(filter_num*8, 3, 3, 1, res_names=names+'ResBlock4',trainable_label=trainable_label)(pool3)
    drop2 = Dropout(0.5)(Features4)

    pool4 = MaxPooling2D(pool_size=(2, 2),name=names+'Pool4')(drop2)
    Features5 = ResBlock(filter_num*16, 3, 3, 1, res_names=names+'ResBlock5',trainable_label=trainable_label)(pool4)
    drop1 = Dropout(0.5)(Features5)

    up1 = UpSampling2D(size=(2, 2),name=names+'Upsample1')(drop1)
    merge1 = concatenate([drop2,up1], axis = 3)
    Features6 = ResBlock(filter_num*8, 3, 3, 1, res_names=names+'ResBlock6',trainable_label=trainable_label)(merge1)

    up2 = UpSampling2D(size=(2, 2),name=names+'Upsample2')(Features6)
    merge2 = concatenate([Features3,up2], axis = 3)
    Features7 = ResBlock(filter_num*4, 3, 3, 1, res_names=names+'ResBlock7',trainable_label=trainable_label)(merge2)

    up3 = UpSampling2D(size=(2, 2),name=names+'Upsample3')(Features7)
    merge3 = concatenate([Features2,up3], axis = 3)
    Features8 = ResBlock(filter_num*2, 3, 3, 1, res_names=names+'ResBlock8',trainable_label=trainable_label)(merge3)

    up4 = UpSampling2D(size=(2, 2),name=names+'Upsample4')(Features8)
    merge4 = concatenate([Features1,up4], axis = 3)
    Features9 = ResBlock(filter_num, 3, 3, 1, res_names=names+'ResBlock9',trainable_label=trainable_label)(merge4)

    if loss_func == "I_divergence":
        Features10 = Convolution2D(1, kernel_size=(1, 1), strides=(1, 1), padding="same",
                               activation="softplus", use_bias = False,
                               kernel_initializer="truncated_normal",
                               name='Prediction_softplus')(Features9)
    elif loss_func == "mse":
        Features10 = Convolution2D(1, kernel_size=(1, 1), strides=(1, 1), padding="same",
                               activation="linear", use_bias = False,
                               kernel_initializer="truncated_normal",
                               name='Prediction_linear')(Features9)

    elif loss_func == "mse_relu":
        Features10 = Convolution2D(1, kernel_size=(1, 1), strides=(1, 1), padding="same",
                               activation="relu", use_bias = False,
                               kernel_initializer="truncated_normal",
                               name='Prediction_relu')(Features9)

    return Features10