#!/usr/bin/env python
"""Hook Handler Classes"""
__author__    = 'Geoff Rowell'
__copyright__ = 'Copyright 2012, Geoff Rowell'
__license__   = 'Apache'
__version__   = '3.00'
__status__    = 'Development'
__all__       = ['StartCommit', 'PreCommit', 'PostCommit',
                 'PreRevPropChange', 'PostRevPropChange',
                 'PreLock', 'PostLock',
                 'PreUnlock', 'PostUnlock']

from filters import Filter
from contexts import *

import argparse
import logging
import logging.config
import os, sys
import re
import yaml

from xml.etree.ElementTree import ElementTree
if sys.version_info >= (2, 7):
    from xml.etree.ElementTree import ParseError
    parse_error = ParseError
else:
    from xml.parsers.expat import ExpatError
    parse_error = ExpatError

logger = logging.getLogger()

class SvnHook(object):
    """Hook Handler Base Class"""

    def __init__(self, context, cfgfile):
        """Construct a new object of the class.

        Arguments:
        context -- Contextual information for hook processing.
        cfgfile -- Path name of hook configuration file.

        """
        if context == None:
            raise ValueError('Required argument missing: context')
        if cfgfile == None:
            raise ValueError('Required argument missing: cfgfile')
        if os.path.isfile(cfgfile) == False:
            raise IOError('Configuration file not found: '
                            + cfgfile)

        # If a co-located logging configuration file exists, use it.
        # Prefer a "*-log.yml" (Python 2.7+) file over a "*-log.conf"
        # file.
        logymlfile = re.sub(
            r'\.[^\.]+$', r'-log.yml', cfgfile)
        logconffile = re.sub(
            r'\.[^\.]+$', r'-log.conf', cfgfile)

        if hasattr(logging.config, 'dictConfig')\
                and os.path.isfile(logymlfile) == True:
            logcfg = yaml.load(open(logymlfile).read())
            logging.config.dictConfig(logcfg)
        elif os.path.isfile(logconffile) == True:
            logging.config.fileConfig(logconffile)
        else:
            logging.basicConfig()

        # For debugging, log the raw Python call.
        logger.debug('sys.argv: ' + ' '.join(sys.argv))

        # Read and parse the hook configuration file. If it has a
        # parse error, make sure to log it - before rethrowing it.
        try:
            self.cfg = ElementTree(file=cfgfile)
        except parse_error:
            logger.fatal(
                'Unable to parse hook configuration file!',
                exc_info=True)
            raise

        # Hang onto the context for use in the actions.
        self.context = context

    def run(self):
        """Execute top-level hook actions and set exit code."""
        # Construct a generic action list to perform the root hook
        # actions.
        exit(Filter(self.context, self.cfg.getroot()).run())

class StartCommit(SvnHook):
    """Start-Commit Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle start-commit hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'user', help='Name of the Subversion user')
        cmdline.add_argument(
            'capabilities', help='List of client capabilities')

        # Parse the command line. Save the arguments as context
        # entries.
        args = cmdline.parse_args()

        # Construct the hook context. Initialize the tokens with
        # the available environment variables and the hook arguments.
        tokens = Tokens(os.environ)
        tokens['ReposPath']    = args.repospath
        tokens['User']         = args.user
        tokens['Capabilities'] = args.capabilities
        context = CtxStandard(tokens)

        # Perform parent initialization.
        super(StartCommit, self).__init__(context, args.cfgfile)

class PreCommit(SvnHook):
    """Pre-Commit Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle pre-commit hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'txnname', help='Name of the pending transaction')

        # Parse the command line.
        args = cmdline.parse_args()

        # Parse the STDIN data.
        locktokens = []
        intotokens = False
        for inline in sys.stdin.read():
            if intotokens and re.search(r'\S', inline):
                locktokens.append(inline)
            elif re.match(r'LOCK-TOKENS:$', inline):
                intotoken = True

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath']   = args.repospath
        tokens['Transaction'] = args.txnname
        tokens['LockTokens']  = locktokens
        context = CtxTransaction(tokens)

        # Perform parent initialization.
        super(PreCommit, self).__init__(context, args.cfgfile)

class PostCommit(SvnHook):
    """Post-Commit Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle post-commit hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'revision', type=int,
            help='Number of the completed revision')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath'] = args.repospath
        tokens['Revision']  = args.revision
        context = CtxRevision(tokens)

        # Perform parent initialization.
        super(PostCommit, self).__init__(context, args.cfgfile)

class PreRevPropChange(SvnHook):
    """Pre-RevProp-Change Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle pre-revprop-change hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'revision', type=int,
            help='Revision to be changed')
        cmdline.add_argument(
            'user', help='Name of user making the change')
        cmdline.add_argument(
            'propname', help='Name of the revision property')
        cmdline.add_argument(
            'action', choices=['A', 'M', 'D'],
            help='Type of change (A=add, M=modify, D=delete)')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath']    = args.repospath
        tokens['Revision']     = args.revision
        tokens['User']         = args.user
        tokens['RevPropName']  = args.propname
        tokens['ChgType']      = args.action
        tokens['RevPropValue'] = sys.stdin.read()
        context = CtxRevision(tokens)

        # Perform parent initialization.
        super(PreRevPropChange, self).__init__(context, args.cfgfile)

class PostRevPropChange(SvnHook):
    """Post-RevProp-Change Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle post-revprop-change hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'revision', type=int,
            help='Revision that was changed')
        cmdline.add_argument(
            'user', help='Name of user that made the change')
        cmdline.add_argument(
            'propname', help='Name of the revision property')
        cmdline.add_argument(
            'action', choices=['A', 'M', 'D'],
            help='Type of change (A=add, M=modify, D=delete)')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath']    = args.repospath
        tokens['Revision']     = args.revision
        tokens['User']         = args.user
        tokens['RevPropName']  = args.propname
        tokens['ChgType']      = args.action
        tokens['RevPropValue'] = sys.stdin.read()
        context = CtxRevision(tokens)

        # Perform parent initialization.
        super(PostRevPropChange, self).__init__(context, args.cfgfile)

class PreLock(SvnHook):
    """Pre-Lock Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle pre-lock hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'path', help='Repository path to be locked')
        cmdline.add_argument(
            'user', help='Name of user adding the lock')
        cmdline.add_argument(
            'comment', help='Optional lock comment')
        cmdline.add_argument(
            'steallock', choices=['0', '1'],
            help='Taking lock from other user flag')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath'] = args.repospath
        tokens['Path']      = args.path
        tokens['User']      = args.user
        tokens['Comment']   = args.comment
        tokens['StealLock'] = args.steallock
        context = CtxStandard(tokens)

        # Perform parent initialization.
        super(PreLock, self).__init__(context, args.cfgfile)

class PostLock(SvnHook):
    """Post-Lock Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle post-lock hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'user', help='Name of user who added the lock')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath'] = args.repospath
        tokens['User']      = args.user
        tokens['Paths']     = [
            line.strip() for line in sys.stdin.readlines()]
        context = CtxStandard(tokens)

        # Perform parent initialization.
        super(PostLock, self).__init__(context, args.cfgfile)

class PreUnlock(SvnHook):
    """Pre-Unlock Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle pre-unlock hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'path', help='Repository path to be unlocked')
        cmdline.add_argument(
            'user', help='Name of user removing the lock')
        cmdline.add_argument(
            'token', help='Lock token to be removed')
        cmdline.add_argument(
            'breakunlock', choices=['0', '1'],
            help='Owned by other user flag')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath']   = args.repospath
        tokens['Path']        = args.path
        tokens['User']        = args.user
        tokens['LockToken']   = args.token
        tokens['BreakUnlock'] = args.breakunlock
        context = CtxStandard(tokens)

        # Perform parent initialization.
        super(PreUnlock, self).__init__(context, args.cfgfile)

class PostUnlock(SvnHook):
    """Post-Unlock Hook Handler"""

    def __init__(self):

        # Define how to process the command line.
        cmdline = argparse.ArgumentParser(
            description='Handle post-unlock hook calls.')

        cmdline.add_argument(
            '--cfgfile', required=True,
            help='Path name of the hook configuration file')

        cmdline.add_argument(
            'repospath', help='Path name of the repository root')
        cmdline.add_argument(
            'user', help='Name of user who removed the lock')

        # Parse the command line.
        args = cmdline.parse_args()

        # Construct the hook context.
        tokens = Tokens(os.environ)
        tokens['ReposPath'] = args.repospath
        tokens['User']      = args.user
        tokens['Paths']     = [
            line.strip() for line in sys.stdin.readlines()]
        context = CtxStandard(tokens)

        # Perform parent initialization.
        super(PostUnlock, self).__init__(context, args.cfgfile)

########################### end of file ##############################


from celery import Celery
import os

os.environ.setdefault('WORKER_MODULE', 'config.dev')

CONFIG = os.environ.get('WORKER_MODULE')

app = Celery('WORKER')
app.config_from_object(CONFIG)

app.autodiscover_tasks()


import torch
import torch.nn as nn
import torch.nn.functional as F

class FeedForward(nn.Module):
    def __init__(self, state_dim: int, one_hot_depth: int, h_dim: int, out_dim: int):
        super().__init__()
        self.one_hot_depth: int = one_hot_depth
        self.state_dim: int = state_dim

        self.fc1 = nn.Linear(one_hot_depth*state_dim, h_dim)
        self.fc2 = nn.Linear(h_dim, h_dim)
        self.fc_out = nn.Linear(h_dim, out_dim)

    def forward(self, states_nnet):
        x = states_nnet
        x = F.one_hot(x.long(), self.one_hot_depth)
        x = x.double()
        x = x.view(-1, self.state_dim * self.one_hot_depth)

        # first two hidden layers
        x = self.fc1(x)
        x = F.relu(x)
        x = self.fc2(x)
        x = F.relu(x)
        # output
        x = self.fc_out(x)
        return x


from qgis.core import *
from qgis.PyQt import QtCore, QtGui, QtWidgets, uic
from osgeo import gdal
import numpy as np

from .worker_thread import WorkerThread

Qt = QtCore.Qt

class RunMyRasterStatistics(WorkerThread):
    def __init__(
        self,
        parentThread,
        polygon_layer,
        polygon_id,
        raster_layer,
        output_file,
        decimals,
        histogram,
    ):
        WorkerThread.__init__(self, parentThread)
        self.polygon_layer = polygon_layer
        self.polygon_id = polygon_id
        self.raster_layer = raster_layer
        self.output_file = output_file
        self.decimals = decimals
        self.histogram = histogram
        self.errors = []
        # Creates transform if necessary
        self.transform = None

        EPSG1 = QgsCoordinateReferenceSystem(
            int(self.polygon_layer.crs().authid().split(":")[1])
        )
        EPSG2 = QgsCoordinateReferenceSystem(
            int(self.raster_layer.crs().authid().split(":")[1])
        )
        if EPSG1 != EPSG2:
            self.transform = QgsCoordinateTransform(EPSG1, EPSG2, QgsProject.instance())

    def doWork(self):
        # We colect info on the vector file
        idx = self.polygon_layer.dataProvider().fieldNameIndex(self.polygon_id)
        statDict = {}

        # Information on the raster layer
        raster = gdal.Open(self.raster_layer.source())

        xOrigin = self.raster_layer.extent().xMinimum()
        yOrigin = self.raster_layer.extent().yMaximum()
        pixelWidth = self.raster_layer.rasterUnitsPerPixelX()
        pixelHeight = self.raster_layer.rasterUnitsPerPixelY()

        tot_feat = self.polygon_layer.featureCount()
        xEnd = self.raster_layer.extent().xMaximum()
        # yEnd = self.raster_layer.extent().yMinimum()
        for i, feat in enumerate(self.polygon_layer.getFeatures()):
            self.ProgressValue.emit(int(100 * (float(i) / tot_feat)))

            feat_id = feat.attributes()[idx]
            statDict[feat_id] = None

            # Reproject vector geometry to same projection as raster
            geom = feat.geometry()
            if self.transform is not None:
                geom.transform(self.transform)

            # Get extent of feat
            bb = geom.boundingBox()

            if bb is not None:
                xmin, xmax, ymin, ymax = (
                    bb.xMinimum(),
                    bb.xMaximum(),
                    bb.yMinimum(),
                    bb.yMaximum(),
                )
                xmax = min(xmax, xEnd)

                # Specify offset and rows and columns to read
                xoff = int(abs(xmin - xOrigin) / pixelWidth)
                yoff = int(abs(yOrigin - ymax) / pixelHeight)
                xcount = int((xmax - xmin) / pixelWidth) + 1
                ycount = int((ymax - ymin) / pixelHeight) + 1

                banddataraster = raster.GetRasterBand(1)
                dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount, ycount)

                if dataraster is not None:
                    dataraster.astype(np.float)

                    # Calculate statistics of zonal raster`
                    if self.histogram:
                        if self.decimals > 0:
                            dataraster = dataraster * pow(10, self.decimals)
                        statDict[feat_id] = np.bincount(
                            (dataraster.astype(np.int)).flat,
                            weights=None,
                            minlength=None,
                        )
                    else:
                        statDict[feat_id] = [
                            np.mean(dataraster),
                            np.median(dataraster),
                            np.std(dataraster),
                            np.var(dataraster),
                            np.min(dataraster),
                            np.max(dataraster),
                            self.mad(dataraster),
                            np.size(dataraster),
                        ]
                else:
                    self.errors.append(
                        "Statistics for polygon with ID " + str(feat_id) + " was empty"
                    )

        columns = 0
        for feat_id, dictionary in statDict.items():
            if dictionary is not None:
                if self.histogram:
                    if columns < dictionary.shape[0]:
                        columns = dictionary.shape[0]

        self.ProgressValue.emit(100)
        O = open(self.output_file, "w")
        if self.histogram:
            txt = "Zone ID"
            if self.decimals > 0:
                divide = pow(10, self.decimals)
                for i in range(columns):
                    txt = txt + "," + str(round(float(i) / divide, self.decimals))
            else:
                for i in range(columns):
                    txt = txt + "," + str(i)
                    O.write(txt + "\n")
        else:
            txt = (
                "Zone ID,Mean,Median,Standard deviation,Variance,Minimum,"
                "Maximum,Median absolute deviation,pixel count\n"
            )
            O.write(txt)

        tot_feat = len(statDict.keys())
        for i, ids in enumerate(statDict.keys()):
            self.ProgressValue.emit(int(100 * (float(i) / tot_feat)))
            txt = str(ids)
            if statDict[ids] is None:
                self.errors.append(txt + ", No data or error in computation")
            else:
                for i in statDict[ids]:
                    txt = txt + "," + str(i)
                for i in range(columns - len(statDict[ids])):
                    txt = txt + ",0"
                O.write(txt + "\n")

        O.flush()
        O.close()

        if len(self.errors) > 0:
            O = open(self.output_file + ".errors", "w")
            for txt in self.errors:
                O.write(txt + "\n")
            O.flush()
            O.close()
        self.finished_threaded_procedure.emit(0)

    # From https://stackoverflow.com/questions/8930370/where-can-i-find-mad-mean-absolute-deviation-in-scipy
    @staticmethod
    def mad(arr):
        """ Median Absolute Deviation: a "Robust" version of standard deviation.
            Indices variabililty of the sample.
            https://en.wikipedia.org/wiki/Median_absolute_deviation
        """
        arr = np.ma.array(
            arr
        ).compressed()  # should be faster to not use masked arrays.
        med = np.median(arr.astype(np.float64))
        return np.median(np.abs(arr.astype(np.float64) - med))


class Card:

    def __init__(self,suit,number):
        self.suit = suit
        self.number = number

    def GetSuit(self):
        print(self.suit)

    def GetNumber(self):
        print(self.number)

    def PrintAll(self):
        print(self.suit,self.number )

def BlackJackShuffle(aList):
    length = len(aList)
    middleNum = length//2 #Divides with inegral result
    firstHalf = aList[:middleNum] # Gets first half
    secondHalf = aList[middleNum:] #Gets second half

    i = 0
    j = 0
    k = 0
    for x in aList:
        if i%2 == 0:
            aList[i] = firstHalf[j]
            j+=1
        else:
            aList[i] = secondHalf[k]
            k+=1
        i+=1


# ------- standard library imports -------
import json
import unittest

# ------- local imports -------
import requests
from flask import request

from app.server.db.extensions import db
from app.server.db.models import *
from app.app import create_app
from app.tests.api_testing.api_test_helper import ApiTestHelper
from app.server.db.helper import helper

class TestApp(unittest.TestCase):
    TRUCK_ENDPOINT = 'truck'

    def setUp(self):
        self.app = create_app(config_file='settings.py')
        self.helper = ApiTestHelper()

        with self.app.test_request_context():
            self.base_url = request.host
            self.provider1 = helper.add_instance(Provider, name='tomer')

    def test_add_truck_succeeds(self):
        truck_path = f'http://localhost:5000/truck'
        params = {
            'provider_name': 'tomer',
            'truck_id': '1234'
        }
        res = requests.post(truck_path, params=params)
        self.assertEqual(res.text, 'truck was added successfully')

    def test_add_truck_fail_no_provider(self):
        truck_path = f'http://localhost:5000/truck'
        params = {
            'provider_name': 'john',
            'truck_id': '1234'
        }
        res = requests.post(truck_path, params=params)
        self.assertEqual(res.text, 'Provider was not found')

    def test_add_truck_fail_no_truck_id(self):
        truck_path = f'http://localhost:5000/truck'
        params = {
            'provider_name': 'tomer',
        }
        res = requests.post(truck_path, params=params)
        self.assertEqual(res.text, 'Please provide a valid truck ID')

    def tearDown(self):
        db.session.remove()
        with self.app.app_context():
            db.drop_all()

if __name__ == '__main__':
    test = TestApp()


import csv
from validate_email import validate_email

# create new file for validated email address

output_header = ['client_id','email','phone','first_name','last_name','name','country_code','city','is_valid']

def write_output(row_data):
    with open('edited_email_list.csv', mode='a') as output_csv:
        output_csv_writer = csv.writer(output_csv, delimiter=',')
        output_csv_writer.writerow(row_data)

write_output(output_header)

with open('test_email.csv') as email_list_csv:
    email_list = csv.reader(email_list_csv, delimiter=',')

    next(email_list) #skip csv header

    for row in email_list:
        is_valid = validate_email(email_address=row[1])

        if is_valid == True:
            validated_email = [row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7],str(is_valid)]
            write_output(validated_email)
            # print(validated_email)


from . import Config

class DevelopmentConfig(Config):
    DEBUG = True
    SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://boer:123456@172.19.3.165/deploy_ops_dev'
    DEPLOYMENT = {
        'CHECKOUT_PATH': '/tmp',
        'DEPLOY_PATH': '/tmp/deploy',
        'CODE_USER': 'apache',
        'CODE_GROUP': 'apache'
    }


import ast, re

locations = ["next", "previous", "last", "first", "opposite"]

def contains_substring(string, substring):
    return substring.lower() in string.lower()

def split_to_words(sentence):
    if type(sentence) is str:
        return sentence.split(" ")
    elif type(sentence) is list:
        return sentence
    else:
        print("split_to_words takes string or list, got,",sentence)
        raise Exception("split_to_words takes string or list, got "+str(sentence))

def split_to_letters(sentence, verbose=False):
    letters = []
    if verbose: print("getting letters from:",sentence)

    if type(sentence) is list:
        sentence = sentence.join("")

    for s in sentence:
        if s != " ":
            letters.append(s)

    if verbose: print("got letters:",letters)
    return letters

# Given a string, finds digits until characters, then reports the string of the digits found. eg. "012hello" -> "012"
def find_next_number(s: str):
    number = ""

    while True:
        # Pop off first digit
        nextDigit = s[0]
        s = s[1:]

        if nextDigit.isdigit():
            number += nextDigit
        else:
            return number

        if len(s)==0:
            return number

def find_indexes_of(string: str, substring: str):
    return [m.start() for m in re.finditer(substring, string)]

def string_to_list(list_as_string: str):
    list_as_string = list_as_string.strip()
    return ast.literal_eval(list_as_string)


def encryption(encdec, number, sampletext):
    if encdec == 2:
        number = number * -1
    readytext = ""
    for i in sampletext:
        letter = ord(i)
        letter += number
        readytext += chr(letter)
    return readytext

def encryptionfile(encdec, number, sampletext):
    if encdec == 3:
        file2 = open("Encrypted" + sampletext, "w+")
    else:
        number = number * -1
        file2 = open("Decrypted" + sampletext, "w+")
    file = open(sampletext, "r+")
    for line in file:
        readytext = ""
        line = line.strip()
        for x in line:
            letter = ord(x)
            letter += number
            readytext += chr(letter)
        file2.write(readytext + "\n")
    file.close()
    file2.close()

while True:
    print("Do you want to Encrypt or Decrypt?")
    print("1. Encrypt Text")
    print("2. Decrypt Text")
    print("3. Encrypt File")
    print("4. Decrypt File")
    print("5. Exit")
    userchoice = int(input())

    if userchoice == 1 or userchoice == 2:
        print("What do you want to encrypt/decrypt: ")
    elif userchoice == 3 or userchoice == 4:
        print("Enter the name of the file you want to encrypt/decrypt: ")
    text = input()

    encryptionumber = input("What encryption / decryption number do you want?: ")
    encryptionumber = int(encryptionumber)

    if userchoice == 1 or userchoice == 2:
        print("The result:", encryption(userchoice, encryptionumber, text))
    elif userchoice == 3 or userchoice == 4:
        encryptionfile(userchoice, encryptionumber, text)
        print("Encrypted / Decrypted successfully.")
    elif userchoice == 5:
        exit()


# Modification of
# https://raw.githubusercontent.com/certbot/certbot/fc7e5e8e6060d9e0df2e704a20103d5c0f456925/certbot-dns-route53/certbot_dns_route53/_internal/dns_route53.py
# Apache 2.0 license

import collections
import logging
import time
import typing

from acme import challenges
from botocore.exceptions import ClientError, NoCredentialsError

from .base import AuthenticatorProtocol

logger = logging.getLogger(__name__)

class Route53Authenticator(AuthenticatorProtocol):
    ttl = 10

    def __init__(self, client: typing.Any, zones: typing.Dict[str, str]):
        self.r53 = client
        self.zones = {name.rstrip("."): id for name, id in zones.items()}
        self._resource_records: typing.Dict[
            str, typing.List[typing.Dict[str, str]]
        ] = collections.defaultdict(list)

    def is_supported(self, domain: str, challenge: typing.Any) -> bool:
        return isinstance(challenge, challenges.DNS01) and bool(
            self._get_zone_id(domain)
        )

    def _get_zone_id(self, domain: str) -> typing.Optional[str]:
        domain = domain.rstrip(".")
        while domain:
            if domain in self.zones:
                return self.zones[domain]
            _, _, domain = domain.partition(".")
        return None

    def perform(
        self, challs: typing.Iterable[typing.Tuple[typing.Any, str]], account_key: str
    ) -> None:
        batches = self._build_r53_change_batches("UPSERT", challs, account_key)
        change_ids = [
            self._change_txt_records(zone_id, batch) for zone_id, batch in batches
        ]
        for change_id in change_ids:
            self._wait_for_change(change_id)

    def cleanup(
        self, challs: typing.Iterable[typing.Tuple[typing.Any, str]], account_key: str
    ) -> None:
        batches = self._build_r53_change_batches("DELETE", challs, account_key)
        for zone_id, batch in batches:
            try:
                self._change_txt_records(zone_id, batch)
            except (NoCredentialsError, ClientError) as e:
                logger.debug("Encountered error during cleanup: %s", e, exc_info=True)

    def _build_r53_change_batches(
        self,
        action: typing.Literal["UPSERT", "DELETE"],
        challs: typing.Iterable[typing.Tuple[typing.Any, str]],
        account_key: str,
    ) -> typing.Iterable[typing.Tuple[str, dict]]:
        zone_domains: typing.Dict[str, typing.Dict[str, typing.List[str]]] = {}
        for challb, domain in challs:
            zone_id = self._get_zone_id(domain)
            assert zone_id, f"Hosted zone not found for domain {domain}"
            zone_domains.setdefault(zone_id, {}).setdefault(domain, []).append(
                challb.validation(account_key)
            )
        for zone_id, domain_info in zone_domains.items():
            batch = {
                "Comment": f"acme-serverless-client certificate validation {action}",
                "Changes": [
                    {
                        "Action": action,
                        "ResourceRecordSet": {
                            "Name": self._transform_domain_name(domain),
                            "Type": "TXT",
                            "TTL": self.ttl,
                            "ResourceRecords": [
                                {"Value": f'"{validation}"'} for validation in rrecords
                            ],
                        },
                    }
                    for domain, rrecords in domain_info.items()
                ],
            }
            yield zone_id, batch

    def _transform_domain_name(self, domain: str) -> str:
        if domain.startswith("*."):
            domain = domain[2:]
        if not domain.endswith("."):
            domain += "."
        return f"_acme-challenge.{domain}"

    def _change_txt_records(self, zone_id: str, batch: dict) -> str:
        response = self.r53.change_resource_record_sets(
            HostedZoneId=zone_id, ChangeBatch=batch
        )
        change_id: str = response["ChangeInfo"]["Id"]
        return change_id

    def _wait_for_change(self, change_id: str) -> None:
        """Wait for a change to be propagated to all Route53 DNS servers.
        https://docs.aws.amazon.com/Route53/latest/APIReference/API_GetChange.html
        """
        for _ in range(0, 120):
            response = self.r53.get_change(Id=change_id)
            if response["ChangeInfo"]["Status"] == "INSYNC":
                return
            time.sleep(5)
        raise RuntimeError(
            "Timed out waiting for Route53 change. Current status: %s"
            % response["ChangeInfo"]["Status"]
        )


"""
Tests for uu module.
Nick Mathewson
"""

import unittest
from test.support import os_helper

import os
import stat
import sys
import uu
import io

plaintext = b"The symbols on top of your keyboard are !@#$%^&*()_+|~\n"

encodedtext = b"""\
M5&AE('-Y;6)O;',@;VX@=&]P(&]F('EO=7(@:V5Y8F]A<F0@87)E("% (R0E
*7B8J*"E?*WQ^"@  """

# Stolen from io.py
class FakeIO(io.TextIOWrapper):
    """Text I/O implementation using an in-memory buffer.

    Can be a used as a drop-in replacement for sys.stdin and sys.stdout.
    """

    # XXX This is really slow, but fully functional

    def __init__(self, initial_value="", encoding="utf-8",
                 errors="strict", newline="\n"):
        super(FakeIO, self).__init__(io.BytesIO(),
                                     encoding=encoding,
                                     errors=errors,
                                     newline=newline)
        self._encoding = encoding
        self._errors = errors
        if initial_value:
            if not isinstance(initial_value, str):
                initial_value = str(initial_value)
            self.write(initial_value)
            self.seek(0)

    def getvalue(self):
        self.flush()
        return self.buffer.getvalue().decode(self._encoding, self._errors)

def encodedtextwrapped(mode, filename, backtick=False):
    if backtick:
        res = (bytes("begin %03o %s\n" % (mode, filename), "ascii") +
               encodedtext.replace(b' ', b'`') + b"\n`\nend\n")
    else:
        res = (bytes("begin %03o %s\n" % (mode, filename), "ascii") +
               encodedtext + b"\n \nend\n")
    return res

class UUTest(unittest.TestCase):

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_encode(self):
        inp = io.BytesIO(plaintext)
        out = io.BytesIO()
        uu.encode(inp, out, "t1")
        self.assertEqual(out.getvalue(), encodedtextwrapped(0o666, "t1"))
        inp = io.BytesIO(plaintext)
        out = io.BytesIO()
        uu.encode(inp, out, "t1", 0o644)
        self.assertEqual(out.getvalue(), encodedtextwrapped(0o644, "t1"))
        inp = io.BytesIO(plaintext)
        out = io.BytesIO()
        uu.encode(inp, out, "t1", backtick=True)
        self.assertEqual(out.getvalue(), encodedtextwrapped(0o666, "t1", True))
        with self.assertRaises(TypeError):
            uu.encode(inp, out, "t1", 0o644, True)

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_decode(self):
        for backtick in True, False:
            inp = io.BytesIO(encodedtextwrapped(0o666, "t1", backtick=backtick))
            out = io.BytesIO()
            uu.decode(inp, out)
            self.assertEqual(out.getvalue(), plaintext)
            inp = io.BytesIO(
                b"UUencoded files may contain many lines,\n" +
                b"even some that have 'begin' in them.\n" +
                encodedtextwrapped(0o666, "t1", backtick=backtick)
            )
            out = io.BytesIO()
            uu.decode(inp, out)
            self.assertEqual(out.getvalue(), plaintext)

    def test_truncatedinput(self):
        inp = io.BytesIO(b"begin 644 t1\n" + encodedtext)
        out = io.BytesIO()
        try:
            uu.decode(inp, out)
            self.fail("No exception raised")
        except uu.Error as e:
            self.assertEqual(str(e), "Truncated input file")

    def test_missingbegin(self):
        inp = io.BytesIO(b"")
        out = io.BytesIO()
        try:
            uu.decode(inp, out)
            self.fail("No exception raised")
        except uu.Error as e:
            self.assertEqual(str(e), "No valid begin line found in input file")

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_garbage_padding(self):
        # Issue #22406
        encodedtext1 = (
            b"begin 644 file\n"
            # length 1; bits 001100 111111 111111 111111
            b"\x21\x2C\x5F\x5F\x5F\n"
            b"\x20\n"
            b"end\n"
        )
        encodedtext2 = (
            b"begin 644 file\n"
            # length 1; bits 001100 111111 111111 111111
            b"\x21\x2C\x5F\x5F\x5F\n"
            b"\x60\n"
            b"end\n"
        )
        plaintext = b"\x33"  # 00110011

        for encodedtext in encodedtext1, encodedtext2:
            with self.subTest("uu.decode()"):
                inp = io.BytesIO(encodedtext)
                out = io.BytesIO()
                uu.decode(inp, out, quiet=True)
                self.assertEqual(out.getvalue(), plaintext)

            with self.subTest("uu_codec"):
                import codecs
                decoded = codecs.decode(encodedtext, "uu_codec")
                self.assertEqual(decoded, plaintext)

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_newlines_escaped(self):
        # Test newlines are escaped with uu.encode
        inp = io.BytesIO(plaintext)
        out = io.BytesIO()
        filename = "test.txt\n\roverflow.txt"
        safefilename = b"test.txt\\n\\roverflow.txt"
        uu.encode(inp, out, filename)
        self.assertIn(safefilename, out.getvalue())

class UUStdIOTest(unittest.TestCase):

    def setUp(self):
        self.stdin = sys.stdin
        self.stdout = sys.stdout

    def tearDown(self):
        sys.stdin = self.stdin
        sys.stdout = self.stdout

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_encode(self):
        sys.stdin = FakeIO(plaintext.decode("ascii"))
        sys.stdout = FakeIO()
        uu.encode("-", "-", "t1", 0o666)
        self.assertEqual(sys.stdout.getvalue(),
                         encodedtextwrapped(0o666, "t1").decode("ascii"))

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_decode(self):
        sys.stdin = FakeIO(encodedtextwrapped(0o666, "t1").decode("ascii"))
        sys.stdout = FakeIO()
        uu.decode("-", "-")
        stdout = sys.stdout
        sys.stdout = self.stdout
        sys.stdin = self.stdin
        self.assertEqual(stdout.getvalue(), plaintext.decode("ascii"))

class UUFileTest(unittest.TestCase):

    def setUp(self):
        # uu.encode() supports only ASCII file names
        self.tmpin  = os_helper.TESTFN_ASCII + "i"
        self.tmpout = os_helper.TESTFN_ASCII + "o"
        self.addCleanup(os_helper.unlink, self.tmpin)
        self.addCleanup(os_helper.unlink, self.tmpout)

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_encode(self):
        with open(self.tmpin, 'wb') as fin:
            fin.write(plaintext)

        with open(self.tmpin, 'rb') as fin:
            with open(self.tmpout, 'wb') as fout:
                uu.encode(fin, fout, self.tmpin, mode=0o644)

        with open(self.tmpout, 'rb') as fout:
            s = fout.read()
        self.assertEqual(s, encodedtextwrapped(0o644, self.tmpin))

        # in_file and out_file as filenames
        uu.encode(self.tmpin, self.tmpout, self.tmpin, mode=0o644)
        with open(self.tmpout, 'rb') as fout:
            s = fout.read()
        self.assertEqual(s, encodedtextwrapped(0o644, self.tmpin))

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_decode(self):
        with open(self.tmpin, 'wb') as f:
            f.write(encodedtextwrapped(0o644, self.tmpout))

        with open(self.tmpin, 'rb') as f:
            uu.decode(f)

        with open(self.tmpout, 'rb') as f:
            s = f.read()
        self.assertEqual(s, plaintext)
        # XXX is there an xp way to verify the mode?

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_decode_filename(self):
        with open(self.tmpin, 'wb') as f:
            f.write(encodedtextwrapped(0o644, self.tmpout))

        uu.decode(self.tmpin)

        with open(self.tmpout, 'rb') as f:
            s = f.read()
        self.assertEqual(s, plaintext)

    def test_decodetwice(self):
        # Verify that decode() will refuse to overwrite an existing file
        with open(self.tmpin, 'wb') as f:
            f.write(encodedtextwrapped(0o644, self.tmpout))
        with open(self.tmpin, 'rb') as f:
            uu.decode(f)

        with open(self.tmpin, 'rb') as f:
            self.assertRaises(uu.Error, uu.decode, f)

    # TODO: RUSTPYTHON
    @unittest.expectedFailure
    def test_decode_mode(self):
        # Verify that decode() will set the given mode for the out_file
        expected_mode = 0o444
        with open(self.tmpin, 'wb') as f:
            f.write(encodedtextwrapped(expected_mode, self.tmpout))

        # make file writable again, so it can be removed (Windows only)
        self.addCleanup(os.chmod, self.tmpout, expected_mode | stat.S_IWRITE)

        with open(self.tmpin, 'rb') as f:
            uu.decode(f)

        self.assertEqual(
            stat.S_IMODE(os.stat(self.tmpout).st_mode),
            expected_mode
        )

if __name__=="__main__":
    unittest.main()


import os
from typing import Generator

from .vector import Colour

def chunk(seq: list[str], n: int) -> Generator[list[str], None, None]:
    for i in range(0, len(seq), n):
        yield seq[i : i + n]

class Canvas:
    def __init__(self, width: int, height: int) -> None:
        self.height = height
        self.width = width
        self.pixels = [[Colour(0, 0, 0) for _ in range(width)] for _ in range(height)]

    def write(self, x: int, y: int, colour: Colour) -> None:
        if not (0 <= x < self.width and 0 <= y < self.height):
            raise IndexError("Canvas write out of range")
        self.pixels[y][x] = colour

    def get(self, x: int, y: int) -> Colour:
        if not (0 <= x < self.width and 0 <= y < self.height):
            raise IndexError("Canvas read out of range")
        return self.pixels[y][x]

    def as_ppm_string(self) -> str:
        header = f"P3\n{self.width} {self.height}\n255\n"
        body = "\n".join(
            "\n".join(
                " ".join(line)
                for line in chunk(
                    [
                        f"{pixel.r*255:.0f} {pixel.g*255:.0f} {pixel.b*255:.0f}"
                        for pixel in row
                    ],
                    5,
                )
            )
            for row in self.pixels
        )
        return header + body + "\n"

    def save_ppm(self, path: os.PathLike) -> None:
        data = self.as_ppm_string()
        with open(path, "w") as f:
            f.write(data)


# coding=utf-8

import os
import time

import numpy as np
import torch
from tensorboardX import SummaryWriter
from tqdm import tqdm

import src.distributed as dist
from src.data.data_iterator import DataIterator
from src.data.dataset import TextLineDataset, ZipDataset
from src.data.vocabulary import Vocabulary
from src.models import build_model
from src.modules.criterions import NMTCriterion
from src.optim import Optimizer
from src.optim.lr_scheduler import build_scheduler
from src.task.nmt import prepare_configs, load_pretrained_model, prepare_data, set_seed
from src.utils.common_utils import *
from src.utils.configs import pretty_configs
from src.utils.logging import *
from src.utils.moving_average import MovingAverage

def compute_accuray(log_probs, sentences):
    # log_probs: [batch_size, seq_len, vocab_size]
    # sentences: [batch_size, seq_len]
    mask = (sentences != 0) * (sentences != 2) * (sentences != 1)
    predicts = log_probs.argmax(dim=-1)
    predicts = predicts.masked_select(mask)
    sentences = sentences.masked_select(mask)
    total_tokens = predicts.size(0)
    correct_tokens = (predicts == sentences).long().sum().item()
    return total_tokens, correct_tokens

def compute_forward(model,
                    critic,
                    seqs_x,
                    seqs_y,
                    eval=False,
                    normalization=1.0,
                    norm_by_words=False
                    ):
    """
    :type model: nn.Module

    :type critic: NMTCriterion
    """

    words_norm = seqs_y.ne(Constants.PAD).float().sum(1)

    if not eval:
        model.train()
        critic.train()
        # For training
        with torch.enable_grad():
            log_probs = model(seqs_x, seqs_y)
            total_tokens, correct_tokens = compute_accuray(log_probs, seqs_y)
            loss = critic(inputs=log_probs, labels=seqs_y, reduce=False, normalization=normalization)

            if norm_by_words:
                loss = loss.div(words_norm).sum()
            else:
                loss = loss.sum()
        torch.autograd.backward(loss)
        return loss.item(), total_tokens, correct_tokens
    else:
        model.eval()
        critic.eval()
        # For compute loss
        with torch.no_grad():
            log_probs = model(seqs_x, seqs_y)
            total_tokens, correct_tokens = compute_accuray(log_probs, seqs_y)
            loss = critic(inputs=log_probs, labels=seqs_y, normalization=normalization, reduce=True)
        return loss.item(), total_tokens, correct_tokens

def loss_evaluation(model, critic, valid_iterator, rank=0, world_size=1):
    """
    :type model: Transformer

    :type critic: NMTCriterion

    :type valid_iterator: DataIterator
    """

    n_sents = 0

    sum_loss = 0.0

    valid_iter = valid_iterator.build_generator()
    total_tokens = 0
    correct_tokens = 0
    for batch in valid_iter:
        _, seqs_x, seqs_y = batch

        n_sents += len(seqs_x)

        x, y = prepare_data(seqs_x, seqs_y, cuda=Constants.USE_GPU)

        loss, t, c = compute_forward(model=model,
                                     critic=critic,
                                     seqs_x=x,
                                     seqs_y=y,
                                     eval=True)

        if np.isnan(loss):
            WARN("NaN detected!")

        sum_loss += float(loss)
        total_tokens += t
        correct_tokens += c

    if world_size > 1:
        sum_loss = dist.all_reduce_py(sum_loss)
        n_sents = dist.all_reduce_py(n_sents)
        total_tokens = dist.all_reduce_py(total_tokens)
        correct_tokens = dist.all_reduce_py(correct_tokens)

    return float(sum_loss / n_sents), correct_tokens / total_tokens

def train(flags):
    """
    flags:
        saveto: str
        reload: store_true
        config_path: str
        pretrain_path: str, default=""
        model_name: str
        log_path: str
    """

    # ================================================================================== #
    # Initialization for training on different devices
    # - CPU/GPU
    # - Single/Distributed
    Constants.USE_GPU = flags.use_gpu

    if flags.multi_gpu:
        dist.distributed_init(flags.shared_dir)
        world_size = dist.get_world_size()
        rank = dist.get_rank()
        local_rank = dist.get_local_rank()
    else:
        world_size = 1
        rank = 0
        local_rank = 0

    if Constants.USE_GPU:
        torch.cuda.set_device(local_rank)
        Constants.CURRENT_DEVICE = "cuda:{0}".format(local_rank)
    else:
        Constants.CURRENT_DEVICE = "cpu"

    # If not root_rank, close logging
    # else write log of training to file.
    if rank == 0:
        write_log_to_file(os.path.join(flags.log_path, "%s.log" % time.strftime("%Y%m%d-%H%M%S")))
    else:
        close_logging()

    # ================================================================================== #
    # Parsing configuration files
    # - Load default settings
    # - Load pre-defined settings
    # - Load user-defined settings

    configs = prepare_configs(flags.config_path, flags.predefined_config)

    data_configs = configs['data_configs']
    model_configs = configs['model_configs']
    optimizer_configs = configs['optimizer_configs']
    training_configs = configs['training_configs']

    INFO(pretty_configs(configs))

    Constants.SEED = training_configs['seed']

    set_seed(Constants.SEED)

    timer = Timer()

    # ================================================================================== #
    # Load Data

    INFO('Loading data...')
    timer.tic()

    # Generate target dictionary
    vocab_src = Vocabulary.build_from_file(**data_configs['vocabularies'][0])
    vocab_tgt = Vocabulary.build_from_file(**data_configs['vocabularies'][1])

    Constants.EOS = vocab_src.eos
    Constants.PAD = vocab_src.pad
    Constants.BOS = vocab_src.bos

    train_bitext_dataset = ZipDataset(
        TextLineDataset(data_path=data_configs['train_data'][0],
                        vocabulary=vocab_src,
                        max_len=data_configs['max_len'][0],
                        is_train_dataset=True
                        ),
        TextLineDataset(data_path=data_configs['train_data'][1],
                        vocabulary=vocab_tgt,
                        max_len=data_configs['max_len'][1],
                        is_train_dataset=True
                        )
    )

    valid_bitext_dataset = ZipDataset(
        TextLineDataset(data_path=data_configs['valid_data'][0],
                        vocabulary=vocab_src,
                        is_train_dataset=False,
                        ),
        TextLineDataset(data_path=data_configs['valid_data'][1],
                        vocabulary=vocab_tgt,
                        is_train_dataset=False
                        )
    )

    training_iterator = DataIterator(dataset=train_bitext_dataset,
                                     batch_size=training_configs["batch_size"],
                                     use_bucket=training_configs['use_bucket'],
                                     buffer_size=training_configs['buffer_size'],
                                     batching_func=training_configs['batching_key'],
                                     world_size=world_size,
                                     rank=rank)

    valid_iterator = DataIterator(dataset=valid_bitext_dataset,
                                  batch_size=training_configs['valid_batch_size'],
                                  use_bucket=True, buffer_size=100000, numbering=True,
                                  world_size=world_size, rank=rank)

    INFO('Done. Elapsed time {0}'.format(timer.toc()))

    # ================================ Begin ======================================== #
    # Build Model & Optimizer
    # We would do steps below on after another
    #     1. build models & criterion
    #     2. move models & criterion to gpu if needed
    #     3. load pre-trained model if needed
    #     4. build optimizer
    #     5. build learning rate scheduler if needed
    #     6. load checkpoints if needed

    # 0. Initial

    lrate = optimizer_configs['learning_rate']
    model_collections = Collections()

    checkpoint_saver = Saver(save_prefix="{0}.ckpt".format(os.path.join(flags.saveto, flags.model_name)),
                             num_max_keeping=training_configs['num_kept_checkpoints']
                             )
    best_model_prefix = os.path.join(flags.saveto, flags.model_name + Constants.MY_BEST_MODEL_SUFFIX)
    best_model_saver = Saver(save_prefix=best_model_prefix, num_max_keeping=training_configs['num_kept_best_model'])

    # 1. Build Model & Criterion
    INFO('Building model...')
    timer.tic()
    nmt_model = build_model(n_src_vocab=vocab_src.max_n_words,
                            n_tgt_vocab=vocab_tgt.max_n_words, padding_idx=vocab_src.pad, vocab_src=vocab_src,
                            **model_configs)
    INFO(nmt_model)

    critic = NMTCriterion(label_smoothing=model_configs['label_smoothing'], padding_idx=vocab_tgt.pad)

    INFO(critic)

    # 2. Move to GPU
    if Constants.USE_GPU:
        nmt_model = nmt_model.cuda()
        critic = critic.cuda()

    # 3. Load pretrained model if needed
    load_pretrained_model(nmt_model, flags.pretrain_path, exclude_prefix=None, device=Constants.CURRENT_DEVICE)

    INFO('Done. Elapsed time {0}'.format(timer.toc()))

    # 4. Build optimizer
    INFO('Building Optimizer...')

    if not flags.multi_gpu:
        optim = Optimizer(name=optimizer_configs['optimizer'],
                          model=nmt_model,
                          lr=lrate,
                          grad_clip=optimizer_configs['grad_clip'],
                          optim_args=optimizer_configs['optimizer_params'],
                          update_cycle=training_configs['update_cycle']
                          )
    else:
        optim = dist.DistributedOptimizer(name=optimizer_configs['optimizer'],
                                          model=nmt_model,
                                          lr=lrate,
                                          grad_clip=optimizer_configs['grad_clip'],
                                          optim_args=optimizer_configs['optimizer_params'],
                                          device_id=local_rank
                                          )

    # 5. Build scheduler for optimizer if needed
    scheduler = build_scheduler(schedule_method=optimizer_configs['schedule_method'],
                                optimizer=optim, scheduler_configs=optimizer_configs['scheduler_configs'])

    # 6. build moving average

    if training_configs['moving_average_method'] is not None:
        ma = MovingAverage(moving_average_method=training_configs['moving_average_method'],
                           named_params=nmt_model.named_parameters(),
                           alpha=training_configs['moving_average_alpha'])
    else:
        ma = None

    INFO('Done. Elapsed time {0}'.format(timer.toc()))

    # Reload from latest checkpoint
    if flags.reload:
        checkpoint_saver.load_latest(model=nmt_model,
                                     optim=optim,
                                     lr_scheduler=scheduler,
                                     collections=model_collections,
                                     ma=ma, device=Constants.CURRENT_DEVICE)

    # broadcast parameters and optimizer states
    if world_size > 1:
        INFO("Broadcasting model parameters...")
        dist.broadcast_parameters(params=nmt_model.state_dict())
        INFO("Broadcasting optimizer states...")
        dist.broadcast_optimizer_state(optimizer=optim.optim)
        INFO('Done.')

    # ================================================================================== #
    # Prepare training

    eidx = model_collections.get_collection("eidx", [0])[-1]
    uidx = model_collections.get_collection("uidx", [1])[-1]
    bad_count = model_collections.get_collection("bad_count", [0])[-1]
    oom_count = model_collections.get_collection("oom_count", [0])[-1]
    is_early_stop = model_collections.get_collection("is_early_stop", [False, ])[-1]
    best_valid_accuracy = model_collections.get_collection("best_valid_accuracy", [0])[-1]
    train_loss_meter = AverageMeter()
    sent_per_sec_meter = TimeMeter()
    tok_per_sec_meter = TimeMeter()

    update_cycle = training_configs['update_cycle']
    grad_denom = 0
    train_loss = 0.0
    cum_n_words = 0
    correct_tokens = 0

    total_tokens = 0
    valid_loss = best_valid_loss = float('inf')

    if rank == 0:
        summary_writer = SummaryWriter(log_dir=flags.log_path)
    else:
        summary_writer = None

    sent_per_sec_meter.start()
    tok_per_sec_meter.start()

    INFO('Begin training...')

    while True:

        if summary_writer is not None:
            summary_writer.add_scalar("Epoch", (eidx + 1), uidx)

        # Build iterator and progress bar
        training_iter = training_iterator.build_generator()

        if rank == 0:
            training_progress_bar = tqdm(desc=' - (Epc {}, Upd {}) '.format(eidx, uidx),
                                         total=len(training_iterator),
                                         unit="sents"
                                         )
        else:
            training_progress_bar = None

        for batch in training_iter:
            seqs_x, seqs_y = batch

            batch_size = len(seqs_x)
            cum_n_words += sum(len(s) for s in seqs_y)

            try:
                # Prepare data
                x, y = prepare_data(seqs_x, seqs_y, cuda=Constants.USE_GPU)

                loss, t, c = compute_forward(model=nmt_model,
                                             critic=critic,
                                             seqs_x=x,
                                             seqs_y=y,
                                             eval=False,
                                             normalization=1.0,
                                             norm_by_words=training_configs["norm_by_words"])

                update_cycle -= 1
                correct_tokens += c
                total_tokens += t
                grad_denom += batch_size
                train_loss += loss

            except RuntimeError as e:
                if 'out of memory' in str(e):
                    print('| WARNING: ran out of memory, skipping batch')
                    oom_count += 1
                else:
                    raise e

            # When update_cycle becomes 0, it means end of one batch. Several things will be done:
            # - update parameters
            # - reset update_cycle and grad_denom, update uidx
            # - learning rate scheduling
            # - update moving average

            if update_cycle == 0:

                # 0. reduce variables
                if world_size > 1:
                    grad_denom = dist.all_reduce_py(grad_denom)
                    train_loss = dist.all_reduce_py(train_loss)
                    cum_n_words = dist.all_reduce_py(cum_n_words)
                    correct_tokens = dist.all_reduce_py(correct_tokens)
                    total_tokens = dist.all_reduce_py(total_tokens)

                # 1. update parameters
                optim.step(denom=grad_denom)
                optim.zero_grad()

                if training_progress_bar is not None:
                    training_progress_bar.update(grad_denom)
                    training_progress_bar.set_description(' - (Epc {}, Upd {}) '.format(eidx, uidx))

                    postfix_str = 'TrainLoss: {:.2f}, ValidLoss(best): {:.2f} ({:.2f}), accuracy: {:.4f}'.format(
                        train_loss, valid_loss,
                        best_valid_loss, correct_tokens / total_tokens)
                    training_progress_bar.set_postfix_str(postfix_str)

                # 2. learning rate scheduling
                if scheduler is not None and optimizer_configs["schedule_method"] != "loss":
                    scheduler.step(global_step=uidx)

                # 3. update moving average
                if ma is not None and eidx >= training_configs['moving_average_start_epoch']:
                    ma.step()

                # 4. update meters
                train_loss_meter.update(train_loss, grad_denom)
                sent_per_sec_meter.update(grad_denom)
                tok_per_sec_meter.update(cum_n_words)

                # 5. reset accumulated variables, update uidx
                update_cycle = training_configs['update_cycle']
                grad_denom = 0
                uidx += 1
                cum_n_words = 0.0
                train_loss = 0.0
                train_accuracy = correct_tokens / total_tokens
                correct_tokens = 0
                total_tokens = 0

            else:
                continue

            # ================================================================================== #
            # Display some information
            if should_trigger_by_steps(uidx, eidx, every_n_step=training_configs['disp_freq']):

                lrate = list(optim.get_lrate())[0]

                if summary_writer is not None:
                    summary_writer.add_scalar("Speed(sents/sec)", scalar_value=sent_per_sec_meter.ave, global_step=uidx)
                    summary_writer.add_scalar("Speed(words/sec)", scalar_value=tok_per_sec_meter.ave, global_step=uidx)
                    summary_writer.add_scalar("train_loss", scalar_value=train_loss_meter.ave, global_step=uidx)
                    summary_writer.add_scalar("lrate", scalar_value=lrate, global_step=uidx)
                    summary_writer.add_scalar("oom_count", scalar_value=oom_count, global_step=uidx)
                    summary_writer.add_scalar("accuracy", scalar_value=train_accuracy, global_step=uidx)

                # Reset Meters
                sent_per_sec_meter.reset()
                tok_per_sec_meter.reset()
                train_loss_meter.reset()

            # ================================================================================== #
            # Loss Validation & Learning rate annealing
            if should_trigger_by_steps(global_step=uidx, n_epoch=eidx, every_n_step=training_configs['loss_valid_freq'],
                                       debug=flags.debug):
                with cache_parameters(nmt_model):

                    if ma is not None:
                        nmt_model.load_state_dict(ma.export_ma_params(), strict=False)

                    valid_loss, accuracy = loss_evaluation(model=nmt_model,
                                                           critic=critic,
                                                           valid_iterator=valid_iterator,
                                                           rank=rank,
                                                           world_size=world_size)

                if scheduler is not None and optimizer_configs["schedule_method"] == "loss":
                    scheduler.step(metric=valid_loss)

                model_collections.add_to_collection("history_losses", valid_loss)
                best_valid_accuracy = max(best_valid_accuracy, accuracy)
                model_collections.add_to_collection("best_valid_accuracy", best_valid_accuracy)

                min_history_loss = np.array(model_collections.get_collection("history_losses")).min()
                best_valid_loss = min_history_loss
                if summary_writer is not None:
                    summary_writer.add_scalar("loss", valid_loss, global_step=uidx)
                    summary_writer.add_scalar("best_loss", min_history_loss, global_step=uidx)
                    summary_writer.add_scalar("accuracy", accuracy, global_step=uidx)

                if best_valid_accuracy == accuracy:
                    if rank == 0:
                        # 1. save the best model
                        torch.save(nmt_model.state_dict(), best_model_prefix + ".final")

                        # 2. record all several best models
                        best_model_saver.save(global_step=uidx,
                                              model=nmt_model,
                                              optim=optim,
                                              lr_scheduler=scheduler,
                                              collections=model_collections,
                                              ma=ma)
                    bad_count = 0
                else:
                    bad_count += 1

                if bad_count >= training_configs['early_stop_patience'] and eidx > 0:
                    is_early_stop = True
                    WARN("Early Stop!")
                    exit(0)

                INFO("{0} Loss: {1:.2f} accuracy: {2:.2f} lrate: {3:6f} patience: {4}".format(
                    uidx, valid_loss, accuracy, lrate, bad_count
                ))

            # ================================================================================== #
            # Saving checkpoints
            if should_trigger_by_steps(uidx, eidx, every_n_step=training_configs['save_freq'], debug=flags.debug):
                model_collections.add_to_collection("uidx", uidx)
                model_collections.add_to_collection("eidx", eidx)
                model_collections.add_to_collection("bad_count", bad_count)

                if not is_early_stop:
                    if rank == 0:
                        checkpoint_saver.save(global_step=uidx,
                                              model=nmt_model,
                                              optim=optim,
                                              lr_scheduler=scheduler,
                                              collections=model_collections,
                                              ma=ma)

        if training_progress_bar is not None:
            training_progress_bar.close()

        eidx += 1
        if eidx > training_configs["max_epochs"]:
            break


from common import *

class Node:
    def __init__(self, position: (), parent: ()):
        self.position = position
        self.parent = parent
        self.g = 0  # Distance to start node
        self.h = 0  # Distance to goal node
        self.f = 0  # Total cost

    def __eq__(self, other):
        return self.position == other.position

    def __lt__(self, other):
        return self.f < other.f

    def __repr__(self):
        return '({0},{1})'.format(self.position, self.f)

# A* search
def astar_search(map, start, end, w):
    w_init = w
    control_point_counter = 0
    open = []
    closed = []
    start_node = Node(start, None)
    goal_node = Node(end, None)
    open.append(start_node)

    while len(open) > 0:
        open.sort()
        current_node = open.pop(0)
        closed.append(current_node)

        if current_node == goal_node:
            path = []
            while current_node != start_node:
                path.append(current_node.position)
                current_node = current_node.parent
            path.append(start)

            return path[::-1], control_point_counter

        (x, y) = current_node.position

        neighbors = [(x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)]

        w -= 1
        for next in neighbors:

            map_value = map.get(next)

            if map_value == OBSTACLE or (map_value != CONTROL_POINT and w == 0):
                continue

            neighbor = Node(next, current_node)

            if neighbor in closed:
                continue

            # Manhattan distance
            neighbor.g = abs(neighbor.position[0] - start_node.position[0]) + abs(
                neighbor.position[1] - start_node.position[1])
            neighbor.h = abs(neighbor.position[0] - goal_node.position[0]) + abs(
                neighbor.position[1] - goal_node.position[1])
            neighbor.f = neighbor.g + neighbor.h

            if add_to_open(open, neighbor):
                open.append(neighbor)
                if map.get(neighbor.position) == CONTROL_POINT:
                    control_point_counter += 1
                    w = w_init
    return None, None

def add_to_open(open, neighbor):
    for node in open:
        if neighbor == node and neighbor.f >= node.f:
            return False
    return True

def a_star(filepath, entrances, w):
    grid = {}
    chars = ['c']
    goal = None
    width = 0
    height = 0

    fp = open(filepath, 'r')
    skip = fp.readline()

    while len(chars) > 0:
        chars = [str(i) for i in fp.readline().strip()]
        width = len(chars) if width == 0 else width
        for x in range(len(chars)):
            grid[(x, height)] = chars[x]
            if chars[x] == '$':
                goal = (x, height)

        if len(chars) > 0:
            height += 1
    fp.close()

    min_path = 0
    ans = None
    for entrance in entrances:
        path, control_point_counter = astar_search(grid, tuple(entrance), goal[::-1], w)
        if path and control_point_counter:
            if len(path) > min_path:
                ans = f"{len(path)} {control_point_counter} {entrance}"

    print(ans)


"""reportlogin URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.0/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from . import views

urlpatterns = [
    path('', views.admin_home, name="admin_home"),
    path('login/', views.userlogin, name="login"),
    path('logout/', views.user_logout, name="user_logout"),
    path('surveyor-list/<str:filter>/', views.surveyor_list, name="surveyor_list"),
    path('surveyor-details/<int:id>/', views.view_surveyor, name="view_surveyor"),
    path('update-surveyor/<int:id>/', views.update_surveyor, name="update_surveyor"),
    path('remove-surveyor/<int:id>/', views.remove_surveyor, name="remove_surveyor"),
    path('register-surveyor/', views.register_surveyor, name="register_surveyor"),
    path('country/', views.country, name="country"),
    path('update-country/<int:id>/', views.update_country, name="update_country"),
    path('country_remove/<int:id>/', views.country_remove, name="country_remove"),

    path('add_division/', views.add_division, name="add_division"),
    path('update-division/<int:id>/', views.update_division, name="update_division"),
    path('remove-division/<int:id>/', views.remove_division, name="remove_division"),
    path('add_district/', views.add_district, name="add_district"),
    path('update-district/<int:id>/', views.update_district, name="update_district"),
    path('remove_district/<int:id>/', views.remove_district, name="remove_distrcit"),
    path('add_sub_district/', views.add_sub_district, name="add_sub_district"),
    path('update-sub-district/<int:id>/', views.update_sub_district, name="update_sub_district"),
    path('remove-sub-district/<int:id>/', views.remove_subdistrict, name="remove_subdistrict"),

    path('notifications/', views.notifications, name="notifications"),
    path('data-list/', views.collecting_data_list, name="collecting_data_list"),
    path('create-collect-form/', views.create_data_form, name="create_data_form"),
    path('collect-data-view/<int:id>/', views.collect_data_view, name="collect_data_view"),
    path('collect_data_delete/<int:id>/', views.collect_data_delete, name="collect_data_delete"),
    path('assign-data-collector/', views.assign_data_collector, name="assign_data_collector"),
    path('view-form/<int:id>/', views.view_form, name="view_form"),
    path('form_delete/<int:id>/', views.form_delete, name="form_delete"),
    path('add-service-category/', views.add_service_category, name="add_service_category"),
    path('update-service-category/<int:id>/', views.update_service_category, name="update_service_category"),
    path('delete-service-category/<int:id>/', views.delete_service_category, name="delete_service_category"),
    path('add-package/', views.add_package, name="add_package"),
    path('update-package/<int:id>/', views.update_package, name="update_package"),
    path('delete-package/<int:id>/', views.delete_package, name="delete_package"),
    path('create-notice/', views.create_notice, name="create_notice"),
    path('auth-log/', views.auth_log, name="auth_log"),

]


import requests
import numpy as np
import pandas as pd
from selenium import webdriver
import time
import sys, urllib, re, json, socket, string
from bs4 import BeautifulSoup

headers = {"user-agent":"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"}
driver_path = "C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe"

#用户简介
def Parse_user(source_url,browser):
    browser.get(source_url)
    browser.find_element_by_xpath('//button[@class="Button ProfileHeader-expandButton Button--plain"]').click()
    html = browser.page_source
    time.sleep(2)

    soup = BeautifulSoup(html,'lxml')
    s = soup.find_all('h1',attrs={'class':'ProfileHeader-title'})[0]
    print(s.find_all('span',attrs={'class':'ProfileHeader-name'})[0].text ,\
            s.find_all('span',attrs={'class':'ztext ProfileHeader-headline'})[0].text)

    s = soup.find_all('div',attrs={'class':'ProfileHeader-detailItem'})
    for i in s:
        print(i.find_all(attrs={'ProfileHeader-detailLabel'})[0].text, \
              i.find_all(attrs={'ProfileHeader-detailValue'})[0].text)

    s = soup.find_all('ul',attrs={'class':'Tabs ProfileMain-tabs'})[0]
    for i in s.find_all('li')[1:-1]:
        print(i.find_all(attrs={'class':'Tabs-link'})[0].text[:2],
             i.find_all(attrs={'class':'Tabs-link'})[0].text[2:])

#anawers 所有回答
def Get_Answer(answer_url,browser):
    browser.get(answer_url)
    html = browser.page_source
    time.sleep(2)

    soup = BeautifulSoup(html,'lxml')
    s = soup.find_all('span',attrs={'class':'RichText ztext CopyrightRichText-richText'})
    for p in s[0].find_all('p'):
        print(p.text)

def Parse_answers(source_url,browser):
    answer_url = source_url+'/answers'
    r = requests.get(answer_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except :
        pages = 1
        pass

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('answers pages',p)
        browser.get(answer_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        #url_text = soup.find_all('a',attrs={'class':'UserLink-link'})
        patten = 'www.zhihu.com/question/[0-9]+/answer/[0-9]+'
        urls = re.findall(patten,str(soup))
        for u in urls:
            u = 'https://'+u
            print(u)
            Get_Answer(u,browser)
    #browser.quit()

#asks 提问
def Parse_asks(source_url,browser):
    asks_url = source_url+'/asks'
    r = requests.get(asks_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('asks pages',p)
        browser.get(asks_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('div',attrs={'class':'QuestionItem-title'})
        for t in titles:
            patten = r'/question/[0-9]+'
            s = re.findall(patten,str(t))[0]
            print('https://www.zhihu.com'+s)
            print(t.text)
    #browser.quit()

#posts 文章
def Parse_posts(source_url,browser):
    posts_url = source_url+'/posts'
    r = requests.get(posts_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('posts pages',p)
        browser.get(posts_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('h2',attrs={'class':'ContentItem-title'})
        for t in titles:
            patten = r'//zhuanlan.zhihu.com/p/[0-9]+'
            s = re.findall(patten,str(t))[0]
            print('https:'+s)
            print(t.text)
    #browser.quit()

#columns 专栏
def Parse_columns(source_url,browser):
    columns_url = source_url+'/columns'
    r = requests.get(columns_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('columns pages',p)
        browser.get(columns_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('div',attrs={'class':'ContentItem-head'})
        for t in titles:
            t = t.find_all('a',attrs={'class':'ColumnLink'})[0]
            patten = r'//zhuanlan.zhihu.com/[a-zA-Z0-9,-]+'
            s = re.findall(patten,str(t))[0]
            print('https:'+s)
            print(t.text)
    #browser.quit()

# pins 想法
def Parse_pins(source_url,browser):
    pins_url = source_url + '/pins'
    r = requests.get(pins_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')
    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('pins pages',p)
        browser.get(pins_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('div',attrs={'class':'RichContent'})
        for t in titles:
            patten = r'www.zhihu.com/pin/[a-zA-Z0-9,-]+'
            s = re.findall(patten,str(t))[0]
            print('https://'+s)
            temps = t.find_all('blockquote',attrs={'class':'PinItem-3lineBlockquote'})
            for temp in temps:
                print(temp.text,'\n')
    #browser.quit()# -*- coding: utf-8 -*-

#following 他关注的人
def Parse_following(source_url,browser):
    following_url = source_url+'/following'
    r = requests.get(following_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    follow = []

    try:
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except :
        pages = 1
        pass

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        #print('following pages',p)
        browser.get(following_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        url_text = soup.find_all('a',attrs={'class':'UserLink-link'})
        patten = 'www.zhihu.com/people/[a-zA-Z0-9,-]+'
        urls = re.findall(patten,str(url_text))
        for i in range(int(len(urls)/2)):
            s = 'https://'+urls[2*i]
            follow.append(s)
            #print(s)
    return follow
    #browser.quit()

#follower 关注他的人
def Parse_follower(source_url,browser):
    follower_url = source_url+'/followers'
    r = requests.get(follower_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('follower pages',p)
        browser.get(follower_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        url_text = soup.find_all('a',attrs={'class':'UserLink-link'})
        patten = 'www.zhihu.com/people/[a-zA-Z0-9,-]+'
        urls = re.findall(patten,str(url_text))
        for i in range(int(len(urls)/2)):
            print('https://'+urls[2*i])
    #browser.quit()

#fellow topics 话题
def Parse_follow_topics(source_url,browser):
    topics_url = source_url+'/following/topics'
    r = requests.get(topics_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')

    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('questions pages',p)
        browser.get(topics_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('h2',attrs={'class':'ContentItem-title'})
        for t in titles:
            patten = r'www.zhihu.com/topic/[0-9]+'
            s = re.findall(patten,str(t))[0]
            print('https://'+s)
            print(t.text,'\n')
    #browser.quit()

#fellow columns 专栏
def Parse_follow_columns(source_url,browser):
    columns_url = source_url+'/following/columns'
    r = requests.get(columns_url,headers=headers)
    soup = BeautifulSoup(r.text,'lxml')
    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('zhuanlan pages',p)
        browser.get(columns_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('h2',attrs={'class':'ContentItem-title'})
        for t in titles:
            patten = r'zhuanlan.zhihu.com/[a-zA-Z0-9,-]+'
            s = re.findall(patten,str(t))[0]
            print('https://'+s)
            print(t.text,'\n')
    #browser.quit()# -*- coding: utf-8 -*-

#fellow questions
def Parse_follow_questions(source_url,browser):
    questions_url = source_url+'/following/questions'
    r = requests.get(questions_url,headers=headers)
    print(r)
    soup = BeautifulSoup(r.text,'lxml')
    try :
        pages = int(soup.find_all('button',attrs={'class':'Button PaginationButton Button--plain'})[-1].text)
    except:
        pages = 1

    #browser = webdriver.Chrome(executable_path=driver_path)
    for p in range(1,pages+1):
        print('questions pages',p)
        browser.get(questions_url+'?page='+str(p))
        time.sleep(5)
        soup = BeautifulSoup(browser.page_source,'lxml')
        titles = soup.find_all('div',attrs={'class':'QuestionItem-title'})
        for t in titles:
            patten = r'/question/[0-9]'
            s = re.findall(patten,str(t))[0]
            print('https://www.zhihu.com'+s)
            print(t.text,'\n')
    #browser.quit()# -*- coding: utf-8 -*-

if __name__ == '__main__' :
    source_url = 'https://www.zhihu.com/people/TechMonster'
    browser = webdriver.Chrome(executable_path=driver_path)
    try:
        '''
        #about himself
        Parse_user(source_url,browser)
        Parse_answers(source_url,browser)
        Parse_asks(source_url,browser)
        Parse_posts(source_url,browser)
        Parse_columns(source_url,browser)
        Parse_pins(source_url,browser)

        #about social network following
        Parse_following(source_url,browser)
        Parse_follower(source_url,browser)
        Parse_follow_topics(source_url,browser)
        Parse_follow_columns(source_url,browser)
        Parse_follow_questions(source_url,browser)
        '''
        Parse_columns(source_url,browser)

    except:
        pass
    browser.quit()


faisal = {'username': 'sfaisal', 'online': True, 'students': 145, 'section':'Computer Science', 'experience in years':21}
farhan = {'username': 'farhanahmed', 'online': False, 'students': 144, 'section':'Software Engineering', 'experience in years':15}

for key, value in faisal.items():
    print(key, 'is the key for the value', value)

for key, value in farhan.items():
    print(key, 'is the key for the value', value)


import os
import struct
import numpy as np
from NeuralNet import *

def load_mnist(path, kind='train'):
	"""Load MNIST data from 'path'"""
	labels_path = os.path.join(path, '%s-labels.idx1-ubyte' % kind)
	images_path = os.path.join(path, '%s-images.idx3-ubyte' % kind)

	with open(labels_path, 'rb') as lbpath:
		magic, n = struct.unpack('>II', lbpath.read(8))
		labels  = np.fromfile(lbpath, dtype=np.uint8)

	with open(images_path, 'rb') as imgpath:
		magic, num, rows, cols= struct.unpack(">IIII", imgpath.read(16))
		images = np.fromfile(imgpath, dtype=np.uint8).reshape(len(labels), 784)

	return images, labels

X_train, y_train = load_mnist(r"C:\Users\Owen\Desktop\Python Machine Learning\Chapter 12\mnist", kind='train')
print(y_train.shape[0])

#print('Rows: %d, columns: %d' % (X_train.shape[0], X_train.shape[1]))

#print('Rows: %d, columns: %d' % (X_test.shape[0], X_test.shape[1]))

import matplotlib.pyplot as plt
fig, ax = plt.subplots(nrows=5, ncols=5, sharex=True, sharey=True,)
ax = ax.flatten()
for i in range(25):
	img = X_train[y_train == 7][i].reshape(28, 28)
	ax[i].imshow(img, cmap='Greys', interpolation='nearest')
ax[0].set_xticks([])
ax[0].set_yticks([])
plt.tight_layout()
plt.show()

# np.savetxt('train_img.csv', X_train, fmt='%i', delimiter=',')
# np.savetxt('train_labels.csv', y_train, fmt='%i', delimiter=',')
# np.savetxt('test_img.csv', X_test, fmt='%i', delimiter=',')
# np.savetxt('test_labels.csv', y_train, fmt='%i', delimiter=',')

"""Do this to load the data from the csv files"""
# X_train = np.genfromtxt('train_img.csv', dtype=int, delimiter=',')
# y_train = np.genfromtxt('train_labels.csv', dtype=int, delimter=',')
# X_test = np.genfromtxt('test_img.csv', dtype=int, delimter=',')
# y_test = np.genfromtxt('test_labels.csv', dtype=int, delimter=',')

import matplotlib.pyplot as plt

nn = NeuralNetMLP(n_output=10,
				  n_features=X_train.shape[1],
				  n_hidden=50,
				  l2=0.1,
				  l1=0.0,
				  epochs=1000,
				  eta=0.001,
				  alpha=0.001,
				  decrease_const=0.00001,
				  shuffle=True,
				  minibatches=50,
				  random_state=1)

nn.fit(X_train, y_train, print_progress=True)

batches = np.array_split(range(len(nn.cost_)), 1000)
cost_ary = np.array(nn.cost_)
cost_avgs = [np.mean(cost_ary[i]) for i in batches]

plt.plot(range(len(cost_avgs)), cost_avgs, color='red')
plt.ylim([0,2000])
plt.ylabel('Cost')
plt.xlabel('Epochs')
plt.tight_layout()
plt.show()

y_train_pred = nn.predict(X_train)
acc = np.sum(y_train == y_train_pred, axis=0) / X_train.shape[0]
print('Training accuracy: %.2f%%' % (acc * 100))

y_test_pred = nn.predict(X_test)
acc = np.sum(y_test == y_test_pred, axis=0) / X_test.shape[0]
print('Testing accuracy: %.2f%%' % (acc * 100))


#OO style approach by nagaaron

import gzip
import numpy as np
import matplotlib.pyplot as plt
from scipy.special import expit
from sklearn.preprocessing import OneHotEncoder

def open_images(filename):
    with gzip.open(filename,'rb') as file:
        data = file.read()
        return np.frombuffer(data,dtype=np.uint8,offset=16).reshape(-1,28,28).astype(np.float32)

def open_labels(filename):
    with gzip.open(filename,'rb') as file:
        data = file.read()
        return np.frombuffer(data,dtype=np.uint8,offset=8)

X_train = open_images('train-images-idx3-ubyte.gz').reshape(-1,784)
y_train = open_labels('train-labels-idx1-ubyte.gz')
oh = OneHotEncoder()
y_train = oh.fit_transform(y_train.reshape(-1,1)).toarray().T

X_test = open_images('t10k-images-idx3-ubyte.gz').reshape(-1,784)
y_test = open_labels('t10k-labels-idx1-ubyte.gz')

class LogisticRegression(object):
    def __init__(self,lr = 0.00001):
        self.lr = lr
        self.w = None
        self.b = None

    def cost(self,pred,y):
        return -np.mean(y * np.log(pred) + (1 - y) * np.log(1 - pred), axis=1)

    def fit(self,X,y):
        if self.w  is None:
            self.w = np.zeros((y.shape[0],X.shape[1]))
        if self.b is None:
            self.b = np.zeros(y.shape[0])

        pred = expit((X @ self.w.T)+self.b).T
        e = pred -y

        dw = (e@X/X.shape[0])
        db = np.mean(e, axis =1)

        self.w = self.w -self.lr*dw
        self.b = self.b -self.lr*db

        print("Costs: " + str(self.cost(pred,y)))

    def predict(self,X):
        return expit((X @ self.w.T)+self.b).T

model = LogisticRegression(lr = 0.00001)

for i in range(0,100):
    model.fit(X_train,y_train)
    y_test_pred = model.predict(X_test)
    y_test_pred = np.argmax(y_test_pred,axis = 0)
    print(y_test_pred)
    print(y_test_pred.shape)
    print(y_test[-1],y_test[-2])


print ('outro programa para loja de tintas')

tamanho = float(input ('Digitie o tamanho da parede em metros quadrados: '))

#1 litro da pra 6 metros
#18 litros 80 reais
#3,6 litros 25 reais
LitrosParede = tamanho * 6
variavel = int(input('digite 1 para apenas galões de 18 litros, digite 2 para apenas galões de 3.6 litros e digite 3 para ambas as opções: '))
if variavel == 1:
#apenas galões normais
    if tamanho <=108:
        print ('Você precisará comprar apenas 1 lata de tinta por R$80,00 ')
    elif tamanho % 6 == 0:
        print('Você precisará comprar: ', tamanho/108 ,' latas de tinta por', tamanho/108*80, 'reais')
    elif tamanho % 6 != 0:
        comprar1 = int (tamanho / 108) + (tamanho % 6 > 0)
        preco1 = float(comprar1*80)
        print ('Você precisará comprar ', comprar1, 'baldes de tintas por', preco1, 'reais')
    else:
        print('não entendi nada queridão')
        input ('exit')
elif variavel == 2:
#apenas galões de 3.6 litros
    if tamanho <= 21.6:
        print('Você precisará comprar apenas 1 galão de tinta por R$25,00')
    elif tamanho % 21.6 == 0:
        print('Você precisará comprar: ', tamanho/21.6 ,'galões de tinta por', tamanho/21.6*25, 'reais')
    elif tamanho % 21.6 != 0:
        comprar2 = int(tamanho / 21.6) + (tamanho%3.6>0)
        preco2 = float(comprar2*25)
        print('você precisará comprar ' ,comprar2, 'baldes de tinta por' , preco2, 'reais')
    else:
        input('exit')
elif variavel == 3:
#tudo junto
    if tamanho <= 21.6:
        print('Compre apenas 1 galão de 3.6 litros por R$25')
    elif tamanho <= 43.6:
        print('Compre apenas 2 galões de 3.6 litros por R$50')
    elif tamanho <= 64.8:
        print('Compre apenas 3 galões de 3.6 litros por R$75')
    elif tamanho <= 108:
        print('compre apenas 1 lata de 18 litros por R$80')
    else:
        total = int(tamanho/108)
        resto = int(tamanho%108)
        resto_a = int(resto/21.6 + (resto%21.6>0))
        Dezporcento = tamanho%108
        if Dezporcento >= 20:

            print('Você precisará comprar: ', total, 'latas de tinta e ', resto_a+1, 'galões de tinta')
            valor2 = int(total*80) + int(resto_a*25)
            print('Valor a pagar: R$',valor2+25)

        else:
            print('Você precisará comprar: ', total, 'latas de tinta e ', resto_a, 'galões de tinta')
            valor2 = int(total*80) + int(resto_a*25)
            print('Valor a pagar: R$',valor2)
            input('exit')
else:
    input("exit")
input('exit')

500 exercicios em python/EstruturaSequencial/17.py


import numpy as np
import matplotlib.pyplot as plt
import scipy.stats

arr = np.loadtxt("data/double1-s1s1.csv", delimiter=",", dtype='int16')

means = []

for i in range(10):
    temp = []
    for j in range(10000):
        temp.append(1-np.mean(arr[10000 * i: 10000 * i + j + 1]))
    means.append(temp)

means = np.array(means)
means = np.transpose(means)

plt.figure(figsize=(9, 6), dpi=100)
_ = plt.plot([i for i in range(10000)],means, color='k')

plt.xlabel("Number of games")
plt.ylabel("Expectation Game Length")
# plt.title("Expectation of Single Player with Strategy " + str(num) + " of 10 runs of 1000 simulations")

plt.show()