import json

from .database import Database

QUERIES = {
    'select_row': 'SELECT * FROM user_status WHERE user_id = ?',
    'update_status': 'UPDATE user_status SET status = ?, data = ? WHERE user_id = ?',
    'insert_status': 'INSERT INTO user_status(user_id) SELECT ? WHERE (SELECT Changes() = 0)'
}

class UserStatusTable:
    """
    Class for manipulation of the 'user_status' table in the database.
    """

    def __enter__(self):
        self._connect = Database.open()
        self._cursor = self._connect.cursor()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self._connect.close()

    def _select_row(self, user_id):
        row = self._cursor.execute(QUERIES['select_row'], [user_id]).fetchone()
        return row

    def _update_status(self, user_id, status, data=None):
        self._cursor.execute(QUERIES['update_status'], [status, data, user_id])
        self._cursor.execute(QUERIES['insert_status'], [user_id])

    def status(self, user_id):
        """
        ???
        :param user_id:
        :return:
        """
        row = self._select_row(user_id)
        if row is None:
            return 0, None
        try:
            data = json.loads(row['data'])
        except ValueError:
            data = None
        return int(row['status']), data

    def update(self, user_id, status, data=None):
        """
        ???
        :param user_id:
        :param status:
        :param data:
        :return:
        """
        dump = json.dumps(data)
        with self._connect:
            self._update_status(user_id, status, dump)


# Найти сумму n элементов следующего ряда чисел:
# 1, -0.5, 0.25, -0.125,…
# Количество элементов (n) вводится с клавиатуры.

# в задаче ряд чисел представляет собой последовательность рациональных дробей вида:
# -1/2 в степени (n-1), где n - номер элемента последовательности.

def sequence_sum_recursive(n: int):
    """Рекурсивная функция суммы элементов последовательности.
    n - количество элементов, sum - сумма элементов"""
    # базовый случай - n = 0 (не осталось элементов для суммирования)
    if n == 0:
        return 0
    else:
        current_elem = (-1/2)**(n-1)
        n -= 1
        return sequence_sum_recursive(n) + current_elem

quantity = int(input('введите натуральное число количества элементов последовательности: '))
print('result: ', sequence_sum_recursive(quantity))


import logging as log
import sys, os
from telegram.ext import Application, CommandHandler, MessageHandler, CallbackQueryHandler, filters

def main():
    import handlers

    #Init bot
    TOKEN = os.getenv("TELEGRAM_BOT_TOKEN_PROD")
    bot = Application.builder().token(TOKEN).build()

    # Init input handlers
    bot.add_handler(CommandHandler("start", handlers.start))
    bot.add_handler(CommandHandler("empezar", handlers.start))
    bot.add_handler(CallbackQueryHandler(handlers.InlineKeyboardHandler))
    bot.add_handler(MessageHandler(filters.TEXT, handlers.text_input_handler))
    bot.add_handler(MessageHandler(filters.LOCATION, handlers.location_input_handler))

    bot.run_polling()

if __name__ == '__main__':

    # Init and config logger
    log.basicConfig(
        stream=sys.stdout,
        format='%(asctime)s %(levelname)-8s %(message)s',
        level=log.INFO,
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    main()


# -*- coding: utf-8 -*-
import codecs
import os
import numpy as np
import pandas as pd
import cPickle as pickle
from tqdm import tqdm
from collections import Counter
from gensim import corpora
from six import iteritems
from gensim import corpora,models,similarities
#warning off
pd.options.mode.chained_assignment = None

class tf_idf(object):

    def __init__(self):
        self.dictionary = None
        self.tfidf_model = None

    def file_line_counter(self,file_name):
        """
        Copy of code from model.py file !!! Try to DRY :)
        """
        with codecs.open(file_name, 'r',encoding='utf8') as f:
            for i, l in tqdm(enumerate(f)):
                pass
        print (file_name," contains ",i+1," lines.")
        return i+1

    def train_tfidf_model(self,file_path='../../temp_results/corpus.txt'):
        textfile = codecs.open(file_path, "r", "utf-8")

        print("Reading and Processing Text File")
        first_lines=[]
        for line in textfile:
            first_lines.append(line.strip())

        print ("--------Building Corpora Dictionary---------------" )
        dictionary = corpora.Dictionary(line.split('#|#')[1].split() for line in first_lines)

        #remove words that appear less than 2 times
        #twoids = [tokenid for tokenid,docfreq in iteritems(dictionary.dfs) if docfreq < 2]
        #dictionary.filter_tokens(fiveids)

        #Remove Gaps
        dictionary.compactify()
        dictionary.save_as_text('../../temp_results/tfidf_dictionary.txt',sort_by_word=False)
        dictionary.save('../../temp_results/tfidf_dictionary')
        print("Dictionary Saved")

        print ("--Now Transforming to Bag of Words Vectors on the Fly--")
        class MyCorpus(object):
            def __iter__(self):
                for line in first_lines:
                    yield dictionary.doc2bow(line.split())

        news_corpus  = MyCorpus()
        print("Corpus Built...Now Starting Model Training")
        tfidf_model = models.TfidfModel(news_corpus)
        tfidf_model.save('../../temp_results/tfidf_model')
        print("Model Trained & Saved")

    def load_model_and_dictionary(self):
        self.tfidf_model = models.TfidfModel.load('../../temp_results/tfidf_model')
        self.dictionary = corpora.Dictionary.load('../../temp_results/tfidf_dictionary')
        print ("Dictionary & Model Loaded Successfully")

    def generate_tf_idf_scores(self,test_document_words):
        #Convert test document to bag of words
        test_bow =  self.dictionary.doc2bow(test_document_words)
        tfidf_corpus = self.tfidf_model[test_bow]
        tfidf_scores = [(self.dictionary.get(tokenid),score) for (tokenid,score) in tfidf_corpus]
        return tfidf_scores

    def tf_idf_imp_words(self,sentence,word2_vec_set,top_k):
        document = sentence.strip().split()
        if len(document)<=top_k:
            return u" ".join(document)

        tf_idf_scores = self.generate_tf_idf_scores(document)
        np_tf_idf_scores = np.array(tf_idf_scores)
        data = pd.DataFrame({"word":np_tf_idf_scores[:,0],"tf_idf":np_tf_idf_scores[:,1]})
        word_freq = Counter(document)

        data['freq'] = data['word'].map(word_freq)
#        Clipping to max 5 words
        data.loc[data.freq>5,'freq']=5
        data.sort_values(['tf_idf'],ascending=False,inplace=True)
        data_filtered = data[ data['word'].isin( word2_vec_set) ]
        data_filtered['cum_freq']= data_filtered['freq'].cumsum()
        cum_sum_series = data_filtered[data_filtered['cum_freq'] >= top_k]['cum_freq']

        if len(cum_sum_series) > 0:
            cut_off = cum_sum_series.iloc[0]
            data_filtered = data_filtered[data_filtered['cum_freq'] <= cut_off]
            if  cut_off != top_k and len(data_filtered['cum_freq'])>2:
                data_filtered.iloc[len(data_filtered['cum_freq'])-1,data_filtered.columns.get_loc('freq')]=top_k-data_filtered.iloc[len(data_filtered['cum_freq'])-2]['cum_freq']

        word_freq_array = data_filtered[['word','freq']].values
        dic_final ={}
        for (word,freq) in word_freq_array:
            dic_final[word]=freq
        list_to_return = []
        for word in document:
            if word in dic_final and dic_final[word]>0:
                list_to_return.append(word)
                dic_final[word]-=1
        return u" ".join(list_to_return)

    def process_file(self,file_to_process, processed_file_name,word2_vec_set,top_k,seperator="#|#"):
        file_lines = self.file_line_counter(file_to_process)
        file_to_process_fp = codecs.open(file_to_process,encoding='utf-8')
        with codecs.open(processed_file_name, "w", "utf-8") as fp:
            for i in tqdm(range(file_lines)):
                news_data = file_to_process_fp.readline().strip()
                headline, desc = news_data.split(seperator)
                processed_line = self.tf_idf_imp_words(desc,word2_vec_set,top_k)
                fp.write(headline+seperator+processed_line+"\n")
        file_to_process_fp.close()

if __name__ == '__main__':

    word2vec_file_name = '../../temp_results/word2vec_hindi.txt'
    file_to_process = '../../temp_results/train_corpus.txt'
    processed_file_name = '../../temp_results/tfidf_based_train_corpus.txt'
    #generate top 100000 words from word2vec ...
    top_lines = 100000
    #top words to consider in desc
    top_k = 50
    #read word2vec file
    word2vec_fp = codecs.open(word2vec_file_name,encoding='utf-8')
    dimension_line = word2vec_fp.readline()

    word2vec_top_k_set = set()
    for i in range(top_lines):
        each_word_dimension = word2vec_fp.readline().strip()
        data = each_word_dimension.split()
        word = data[0]
        word2vec_top_k_set.add(word)

    word2vec_fp.close()
    print ("word2vec top {} words loaded in set".format(top_lines))

    ti = tf_idf()
    ti.load_model_and_dictionary()
    ti.process_file(file_to_process,processed_file_name,word2vec_top_k_set,top_k)


import numpy as np
from SimPEG import Survey, Problem, Mesh, Utils
from SimPEG.Regularization import BaseRegularization
import scipy.sparse as sp

def CausalConv(array1, array2, time):
    """
        Evaluate convolution for two causal functions.
        Input

        * array1: array for \\\\(\\\\ f_1(t)\\\\)
        * array2: array for \\\\(\\\\ f_2(t)\\\\)
        * time: array for time

        .. math::

            Out(t) = \int_{0}^{t} f_1(a) f_2(t-a) da

    """

    if array1.shape == array2.shape == time.shape:
        out = np.convolve(array1, array2)
        return out[0:np.size(time)]*(time[1]-time[0])
    else:
        raise Exception("Input same size of 1D arrays!!")

class ExpFitSurvey(Survey.BaseSurvey):
    nD = None
    xyz = None
    uncert = None

    def __init__(self, time, xyz, **kwargs):
        self.time = time
        self.xyz = xyz
        self.ntime = time.size
        self.ntx = self.xyz.shape[0]
        self.nD = self.ntime*self.ntx

    def eval(self, f):
        return f

    def setUncertainty(self, dobs, perc=0.1, floor=0., floorIP=0.):
        self.uncert = np.zeros((self.ntime, self.ntx))
        self.dobs = dobs
        dobs = dobs.reshape((self.ntime, self.ntx), order='F')
        for itx in range(self.ntx):
            ipind = dobs[:, itx] < 0.

            # Set different uncertainty for stations having negative transients
            if (ipind).sum() > 3:
                ip = dobs[ipind, itx]
                self.uncert[:, itx] = perc*abs(dobs[:, itx]) + abs(ip).max()
            else:
                self.uncert[:, itx] = perc*abs(dobs[:, itx])+floor
        self.uncert = Utils.mkvc(self.uncert)

        return self.uncert

class ExpFitProblem(Problem.BaseProblem):

    surveyPair = ExpFitSurvey
    tau = None
    AofT_type = "Impulse"   # Type: "Impulse" or "Impulse_Stack"
    Frequency = 25.         # Frequency
    T = 1./25.              # Period
    currentderiv = None
    current = None

    def __init__(self, mesh, **kwargs):
        Problem.BaseProblem.__init__(self, mesh, **kwargs)
        self.setTau()

    def getCur_from_Curderiv(self):
        self.current = CausalConv(
            self.currentderiv, np.ones_like(self.currentderiv), time_conv
            )

    def getAofT(self, timeconv, tau, dt=None):

        if self.AofT_type == "Step":
            AofT = np.exp(-timeconv/tau)
        elif self.AofT_type == "Impulse":
            AofT = 1./tau*np.exp(-timeconv/tau)
        elif self.AofT_type == "Step_Stack":
            AofT = np.exp(-timeconv/tau)
        elif self.AofT_type == "Impulse_Stack":
            AofT = np.exp(-timeconv/tau)/(1.+np.exp(-self.T/(2*taui)))
        elif self.AofT_type == "Impulse_Conv":
            AofT = CausalConv(
                1./tau*np.exp(-timeconv/tau), -self.currentderiv, timeconv
                )
        return AofT

    # TODO: make general (not it works for db/dt)
    def getG(self, timeconv, t0=0.):
        dt = timeconv[1]-timeconv[0]
        ntime = timeconv.size
        meshtime = Mesh.TensorMesh([dt*np.ones(ntime)], x0=[-dt/2.])
        P = meshtime.getInterpolationMat(self.survey.time+t0, 'CC')
        self.G = np.zeros((self.survey.ntime, self.ntau))
        for j in range(self.ntau):
            self.G[:, j] = P*self.getAofT(timeconv, self.tau[j])

    def setTau(self, minlogtau=-5, maxlogtau=-2, ntau=31):
        self.tau = np.logspace(minlogtau, maxlogtau, ntau)
        self.ntau = ntau

    def fields(self, m, f=None):
        m = m.reshape((self.ntau, self.survey.ntx), order='F')
        pred = self.G.dot(m)
        return Utils.mkvc(pred)

    def Jvec(self, m, v, f=None):
        v = v.reshape((self.ntau, self.survey.ntx), order='F')
        jvec = self.G.dot(v)
        return Utils.mkvc(jvec)

    def Jtvec(self, m, v, f=None):
        v = v.reshape((self.survey.ntime, self.survey.ntx), order='F')
        jtvec = self.G.T.dot(v)
        return Utils.mkvc(jtvec)

class LineRegularization(BaseRegularization):
    xyz_line = None
    alpha_s = 1e0
    alpha_x = 1e-1
    ntau = None

    def __init__(self, mesh, mapping=None, **kwargs):
        BaseRegularization.__init__(self, mesh, mapping=mapping, **kwargs)
        if self.ntau is None:
            raise Exception("Input parameter ntau must be initiated!")
    @property
    def W(self):
        """Regularization matrix W"""
        if getattr(self, '_W', None) is None:
            ntx = self.xyz_line.shape[0]
            meshline = Mesh.TensorMesh([ntx])
            Gx = meshline.cellGradx
            Wx = np.sqrt(self.alpha_x)*sp.kron(Gx, Utils.speye(self.ntau))
            Ws = np.sqrt(self.alpha_s)*Utils.speye(self.ntau*ntx)
            wlist = (Wx, Ws)
            self._W = sp.vstack(wlist)
        return self._W


import unittest
import tempfile
import sys
import os
import stat
import logging

try:
    # python2
    from StringIO import StringIO
except ImportError:
    # python3
    from io import StringIO

import rsl

class TestCommandLineInterface(unittest.TestCase):

    def setUp(self):
        self.temp_files = set()
        logger = logging.getLogger()
        self.log = logging.StreamHandler(StringIO())
        logger.addHandler(self.log)

    def tearDown(self):
        logger = logging.getLogger()
        logger.removeHandler(self.log)

        for temp in self.temp_files:
            temp.close()
            os.remove(temp.name)

    def temp_file(self, mode='w'):
        temp = tempfile.NamedTemporaryFile(mode=mode, delete=False)
        self.temp_files.add(temp)
        return temp

    def test_unsed_arguments(self):
        argv = ['test_unsed_arguments',
                '-nopersist', #don't save database to disk during testing
                '-d', '1',
                '-priority', '32',
                '-lVHs', '-lSCs', '-l2b', '-l2s', '-l3b', '-l3s',
                '-e', 'some_string',
                '-t', 'some_string',
                '-v', 'STMT',
                '-q',
                '-l',
                '-#', '4']
        rsl.main(argv)

    def test_ignoring_arguments(self):
        argv = ['test_ignoring_arguments',
                '-nopersist', #don't save database to disk during testing
                '-ignore_rest', 'some', 'more']
        rsl.main(argv)

    def test_invalid_arguments(self):
        argv = ['test_invalid_arguments',
                'some', 'more']
        self.assertRaises(SystemExit, rsl.main, argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn('ERROR', output)

    def test_help(self):
        argv = ['test_help',
                '-h']

        self.assertRaises(SystemExit, rsl.main, argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn('USAGE', output)

    def test_version(self):
        argv = ['test_version',
                '-version']

        self.assertRaises(SystemExit, rsl.main, argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn(rsl.version.complete_string, output)

    def test_integrity(self):
        schema = self.temp_file(mode='w')
        script = self.temp_file(mode='w')

        schema.file.write('CREATE TABLE Cls (Id STRING);')
        schema.file.write('CREATE UNIQUE INDEX I1 ON Cls (Id);')
        schema.file.flush()

        script.file.write('.create object instance cls of Cls\n')
        script.file.write('.assign cls.Id = "test"\n')

        argv = ['test_integrity',
                '-nopersist',
                '-integrity',
                '-import', schema.name,
                '-arch', script.name]

        self.assertEqual(0, rsl.main(argv))

        # expect two uniqueness constrain violations
        script.file.write('.create object instance cls of Cls\n')
        script.file.write('.assign cls.Id = "test"\n')
        script.file.flush()
        self.assertEqual(1, rsl.main(argv))

    def test_qim(self):
        schema = self.temp_file(mode='w')
        modeldata = self.temp_file(mode='w')
        script = self.temp_file(mode='w')

        schema.file.write('CREATE TABLE Cls (Id STRING, OtherAttr INTEGER);')
        schema.file.write('CREATE UNIQUE INDEX I1 ON Cls (Id);')
        schema.file.flush()

        modeldata.file.write('INSERT INTO Cls VALUES ("myid");')
        modeldata.file.flush()

        script.file.write('')
        script.file.flush()

        argv = ['test_qim',
                '-nopersist',
                '-import', schema.name,
                '-import', modeldata.name,
                '-arch', script.name]
        rsl.main(argv)

        argv = ['test_qim',
                '-nopersist',
                '-qim',
                '-import', schema.name,
                '-import', modeldata.name,
                '-arch', script.name]
        rsl.main(argv)

        mismatches = self.log.stream.getvalue().count('mismatch')
        self.assertEqual(1, mismatches)

    def test_include(self):
        script = self.temp_file(mode='w')
        script.file.write('.print "Hello"\n')
        script.file.write('.include "spam.inc"\n')
        script.file.flush()

        argv = ['test_include',
                '-arch', script.name,
                '-include',
                os.path.dirname(__file__) + os.path.sep + 'test_files',
                '-nopersist']

        rsl.main(argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn('Hello', output)

    def test_nopersist(self):
        db_filename = tempfile.mktemp()
        script = self.temp_file(mode='w')

        script.file.write('.print "Hello"\n')
        script.file.flush()

        argv = ['test_nopersist',
                '-f', db_filename,
                '-arch', script.name,
                '-nopersist']

        rsl.main(argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn('Hello', output)
        self.assertFalse(os.path.exists(db_filename))

    def test_persist(self):
        db = self.temp_file(mode='r')
        schema = self.temp_file(mode='w')
        script = self.temp_file(mode='w')

        schema.file.write('CREATE TABLE Cls (Id UNIQUE_ID);')
        schema.file.flush()

        script.file.write('.create object instance cls of Cls\n')
        script.file.write('.print "Hello"\n')
        script.file.flush()

        argv = ['test_persist',
                '-f', db.name,
                '-import', schema.name,
                '-arch', script.name]

        rsl.main(argv)
        output = sys.stdout.getvalue().strip()
        self.assertIn('Hello', output)

        with open(db.name, 'r') as f:
            s = f.read()
            self.assertIn('CREATE TABLE', s)
            self.assertIn('INSERT INTO', s)

    def test_disable_emit(self):
        emit_filename = tempfile.mktemp().replace('\\', '/')
        script = self.temp_file(mode='w')
        script.file.write('Test\n')
        script.file.write('.emit to file "%s"\n' % emit_filename)
        script.file.flush()

        argv = ['test_disable_emit',
                '-arch', script.name,
                '-emit', 'never',
                '-nopersist']

        rsl.main(argv)
        self.assertFalse(os.path.exists(emit_filename))

    def test_diff(self):
        diff = self.temp_file(mode='r')
        emit = self.temp_file(mode='r')
        script = self.temp_file(mode='w')
        script.file.write('Hello file\n')
        script.file.write('.emit to file "%s"\n' % emit.name.replace('\\', '/'))
        script.file.flush()

        argv = ['test_diff',
                '-arch', script.name,
                '-diff', diff.name,
                '-nopersist']

        rsl.main(argv)
        with open(diff.name, 'r') as f:
            s = f.read()
            self.assertIn('test_diff -arch', s)
            self.assertIn('Hello file', s)

    def test_force(self):
        emit = self.temp_file(mode='r')
        script = self.temp_file(mode='w')
        script.file.write('Hello file\n')
        script.file.write('.emit to file "%s"\n' % emit.name.replace('\\', '/'))
        script.file.flush()

        os.chmod(emit.name, stat.S_IRUSR)

        argv = ['test_force',
                '-arch', script.name,
                '-nopersist']

        self.assertRaises(SystemExit, rsl.main, argv)
        argv.append('-force')
        rsl.main(argv)

        with open(emit.name, 'r') as f:
            s = f.read()
            self.assertIn('Hello file', s)

    def test_dumpsql(self):
        dump = self.temp_file(mode='r')
        schema = self.temp_file(mode='w')
        schema.file.write('CREATE TABLE Person (Name STRING, Age INTEGER);\n')
        schema.file.flush()

        script = self.temp_file(mode='w')
        script.file.write('.create object instance person1 of Person\n')
        script.file.write('.assign person1.Name = "Levi"\n')
        script.file.write('.assign person1.Age = 24\n')
        script.file.flush()

        argv = ['test_dumpsql',
                '-nopersist',
                '-dumpsql', dump.name,
                '-import', schema.name,
                '-arch', script.name]

        self.assertEqual(0, rsl.main(argv))

        with open(dump.name, 'r') as f:
            s = f.read()
            self.assertIn('INSERT INTO Person', s)

if __name__ == "__main__":
    unittest.main()


import asyncio
from typing import Callable, Iterator
from aiogram import Dispatcher  # type: ignore
from aiogram import types
from tgbot.keyboards.user import commands_keyboard
from tgbot.parsing.parse_info import (
    get_bachelor_text,
    get_master_text,
    get_postgraduate_text,
    parse
)

async def start(message: types.Message) -> None:
    photo = open("images/UNI_photo.jpg", "rb")
    name = message.from_user.username
    await message.answer_photo(
        photo,
        caption=f"Привет, {name}, это бот команды ENIGMA",
        reply_markup=commands_keyboard
    )  # текст нужно придумать

async def help(message: types.Message) -> None:
    await message.reply(
        "Привет)",
        reply_markup=commands_keyboard
    )  # текст нужно придумать

async def answer(
        call: types.CallbackQuery,
        url: str,
        get_info: Callable[[str], Iterator[tuple[str, str]]]) -> None:
    await call.message.answer("ПОИСК ИНФОРМАЦИИ НАЧАЛСЯ")

    try:
        texts = await parse(url, get_info)
    except asyncio.TimeoutError as ex:
        await call.message.answer("С сайтом неполадки. Попробуйте снова позже")
        raise ex
    except Exception as ex:
        await call.message.answer("Что-то пошло не так...")
        raise ex

    if not texts:
        await call.message.answer("Информация не найдена")

    for text in texts:
        await call.message.answer(text)

async def bachelor(call: types.CallbackQuery) -> None:
    url = "https://apply.innopolis.university/bachelor/"\
          "?lang=ru&id=12&site=s1&template=university24&landing_mode=edit"
    await answer(call, url, get_bachelor_text)

async def master(call: types.CallbackQuery) -> None:
    url = "https://apply.innopolis.university/master/datascience/"\
          "?lang=ru&id=12&site=s1&template=university24&landing_mode=edit"
    await answer(call, url, get_master_text)

async def postgraduate(call: types.CallbackQuery) -> None:
    url = "https://apply.innopolis.university/postgraduate-study/"\
          "?lang=ru&id=12&site=s1&template=university24&landing_mode=edit"
    await answer(call, url, get_postgraduate_text)

async def trash(message: types.Message) -> None:
    await message.answer(
        "Вы ввели неправильную комманду.\nУзнать условия поступления:",
        reply_markup=commands_keyboard
    )

def register_handlers(dp: Dispatcher) -> None:
    dp.register_message_handler(
        start,
        commands=["start"]
    )
    dp.register_message_handler(
        help,
        commands=["help"]
    )
    dp.register_message_handler(trash)
    dp.register_callback_query_handler(
        bachelor,
        lambda c: c.data == "bachelor"
    )
    dp.register_callback_query_handler(
        master,
        lambda c: c.data == "master"
    )
    dp.register_callback_query_handler(
        postgraduate,
        lambda c: c.data == "postgraduate"
    )


import json
import websocket
import threading
from parlai.core.params import ParlaiParser
from parlai.scripts.interactive_web import WEB_HTML, STYLE_SHEET, FONT_AWESOME
from http.server import SimpleHTTPRequestHandler, BaseHTTPRequestHandler, HTTPServer
from parlai.chat_service.services.browser_chat.paraphraser.infer import get_one_sentence

SHARED = {}

def setup_interactive(ws):
    SHARED['ws'] = ws

new_message = None
message_available = threading.Event()

class BrowserHandler(SimpleHTTPRequestHandler):
    """
    Handle HTTP requests.
    """
    def _interactive_running(self, reply_text):
        data = {}
        data['text'] = reply_text.decode('utf-8')
        if data['text'] == "[DONE]":
            print('[ Closing socket... ]')
            SHARED['ws'].close()
            SHARED['wb'].shutdown()
        json_data = json.dumps(data)
        SHARED['ws'].send(json_data)

    def do_HEAD(self):
        """
        Handle HEAD requests.
        """
        self.send_response(200)
        self.send_header('Content-type', 'text/html')
        self.end_headers()

    def do_GET(self):
        """Serve a GET request."""
        print(self.path)
        if self.path == '/personality-page' or self.path == '/chat-page':
            self.path = '/'
        f = self.send_head()
        if f:
            try:
                self.copyfile(f, self.wfile)
            finally:
                f.close()

    def do_POST(self):
        """
        Handle POST request, especially replying to a chat message.
        """
        if self.path == '/interact':
            content_length = int(self.headers['Content-Length'])
            body = self.rfile.read(content_length)
            self._interactive_running(body)
            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.end_headers()
            model_response = {'id': 'Model', 'episode_done': False}
            message_available.wait()
            # model_response['text'] = new_message
            model_response['text'] = new_message
            model_response['text_trans'] = get_one_sentence(new_message)
            message_available.clear()
            json_str = json.dumps(model_response)
            self.wfile.write(bytes(json_str, 'utf-8'))
        elif self.path == '/reset':
            self._interactive_running(b"[RESET]")
            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.end_headers()
            self.wfile.write(bytes("{}", 'utf-8'))
            message_available.wait()
            message_available.clear()
        else:
            return self._respond({'status': 500})

def on_message(ws, message):
    """
    Prints the incoming message from the server.

    :param ws: a WebSocketApp
    :param message: json with 'text' field to be printed
    """
    incoming_message = json.loads(message)
    global new_message
    new_message = incoming_message['text']
    message_available.set()

def on_error(ws, error):
    """
    Prints an error, if occurs.

    :param ws: WebSocketApp
    :param error: An error
    """
    print(error)

def on_close(ws):
    """
    Cleanup before closing connection.

    :param ws: WebSocketApp
    """
    # Reset color formatting if necessary
    print("Connection closed")

def _run_browser():
    #TODO: npm build the frontend and copy build/ to current directory
    host = opt.get('host', 'localhost')
    serving_port = opt.get('serving_port', 8080)

    httpd = HTTPServer((host, serving_port), BrowserHandler)

    print('Please connect to the link: http://{}:{}/'.format(host, serving_port))

    SHARED['wb'] = httpd

    httpd.serve_forever()

def on_open(ws):
    """
    Starts a new thread that loops, taking user input and sending it to the websocket.

    :param ws: websocket.WebSocketApp that sends messages to a browser_manager
    """
    threading.Thread(target=_run_browser).start()

def setup_args():
    """
    Set up args, specifically for the port number.

    :return: A parser that parses the port from commandline arguments.
    """
    parser = ParlaiParser(False, False)
    parser_grp = parser.add_argument_group('Browser Chat')
    parser_grp.add_argument(
        '--port', default=35496, type=int, help='Port used by the web socket (run.py)'
    )
    parser_grp.add_argument(
        '--host',
        default='localhost',
        type=str,
        help='Host from which allow requests, use 0.0.0.0 to allow all IPs',
    )
    parser_grp.add_argument(
        '--serving_port',
        default=8080,
        type=int,
        help='Port used to configure the server',
    )

    return parser.parse_args()

if __name__ == "__main__":
    opt = setup_args()
    port = opt.get('port', 34596)
    print("Connecting to port: ", port)
    ws = websocket.WebSocketApp(
        "ws://localhost:{}/websocket".format(port),
        on_message=on_message,
        on_error=on_error,
        on_close=on_close,
    )
    ws.on_open = on_open
    setup_interactive(ws)
    ws.run_forever()


#!/usr/bin/env python

class Donor:
    """A class to track a donor's information."""
    def __init__(self, fullname, don_sum=0, don_count=0):
        """Initialize Donor object."""
        self.fullname = fullname
        self.firstname = fullname.split()[0]
        try:
            self.lastname = fullname.split()[1]
        except IndexError:
            self.lastname = ""
        self.donation_sum = don_sum
        self.donation_count = don_count
        self.donation_avg = (self.donation_sum /
                             self.donation_count)

    def __repr__(self):
        """Return repr of Donor object."""
        return f"[{self.fullname}, ${self.donation_sum:03.2f}, " \
               f"{self.donation_count}, ${self.donation_avg:03.2f}]"

    def __str__(self):
        """Return str of Donor object."""
        return "Donor profile for " + self.fullname

    def new_donation(self, amount):
        """Add new donation to Donor object."""
        self.donation_sum += amount
        self.donation_count += 1
        self.donation_avg = (self.donation_sum /
                             self.donation_count)

    def thank_you(self):
        """Return thank_you letter for Donor."""
        return f"\nDear {self.fullname},\n\n" \
               "Thank you for your generosity.\n\n" \
               f"You have donated: ${self.donation_sum:03.2f}" \
               "\n\nWe are very grateful." \
               "\n\nBest,\n\n" \
               "Local Charity\n"

    def report_row(self):
        """Return formatted report_row for Donor object."""
        return ('{:<25}{:^5}${:>14,.2f}{:^5}{:>10}{:^5}${:>14,.2f}'
                ).format(self.fullname, ' ',
                         self.donation_sum, ' ',
                         self.donation_count, ' ',
                         self.donation_avg
                         )

class DonorCollection:
    """A class to store a collection of Donor objects."""
    def __init__(self, *donors):
        """Initialize DonorCollection object."""
        self.names = []
        self.data = []
        self.donors = list(donors)
        for donor in donors:
            self.names.append(donor.fullname)
            self.data.append(repr(donor))

    def __repr__(self):
        """Return repr of DonorCollection object."""
        return "\n".join(self.data)

    def __str__(self):
        """Return str of DonorCollection object."""
        return ("A collection of data for "
                + "the following donors:\n\n"
                + "\n".join(self.names)
                )

    def append(self, donor):
        """Append new Donor object to DonorCollection."""
        self.names.append(donor.fullname)
        self.data.append(repr(donor))
        self.donors.append(donor)

    def report(self):
        """Return formatted report of Donor objects in DonorCollection."""
        h = ['Donor Name', '|', 'Total Given', '|', 'Num Gifts',
             '|', 'Average Gift']
        # create table heading
        report_headers = '{:<25}{:^5}{:<15}{:^5}{:<10}{:^5}{:<15}'.format(*h)
        table_divider = '-' * 80
        return "\n" + report_headers + "\n" \
               + table_divider + "\n" \
               + "\n".join([donor.report_row() for donor in self.donors]) \
               + "\n"


import os
import shutil
import re
import time
import json
import xml.etree.ElementTree as ET

def get_file_name_path(dir_file, format_key):
    jpg_file_name = []
    jpg_file_path = []
    for root, dirs, files in os.walk(dir_file):
        for f in files:
            if format_key in f:
                fp = os.path.join(root, f)
                jpg_file_name.append(f)
                jpg_file_path.append(fp)
    # print("------------")
    # print(jpg_file_name)
    # print(jpg_file_path)
    # print("-------------")

    return jpg_file_name, jpg_file_path

# {'60':'bread','69':'apple','74':'cake','108':'orange','109':'banana','131':'pizza','171':'strawberry','181':'pear','211':'mango','219':'ice','229':'pineapple','256':'sandwich','232':'peach'}

# {'bread','apple','cake','orange','banana','pizza','strawberry','pear','mango','ice','pineapple','sandwich','peach'}

# file_name_dict={'obj365_val_000000354609.jpg':[width,height,total,name,bbox,name,bbox]}

# 怎么保证id/width/height只有一份。

def solve(json_path, saved_xml_folder):
    need_categories_id = [60, 69, 74, 108, 109,
                          131, 171, 181, 211, 219, 229, 256, 232]
    need_name = ['bread', 'apple', 'cake', 'orange', 'banana', 'pizza',
                 'strawberry', 'pear', 'mango', 'ice cream', 'pine apple', 'sandwich', 'peach']
    file_name_dict = {}
    file_name_list = []
    previous_file_name = ""
    change = 0
    # open(saved_xml_list,'r')
    with open(json_path, 'r') as load_f:
        load_dict = json.load(load_f)
    # print(load_dict['images'][0])
    ############
    # annotations: {'image_id': 354609, 'area': 9945.156009311546, 'category_id': 21, 'id': 0, 'iscrowd': 0, 'bbox': [133.8088989172, 178.6294555648, 80.1972656534, 124.0086669824]}
    # categories: {'id': 224, 'name': 'scale'}
    # images: {'id': 354609, 'height': 512, 'file_name': 'obj365_val_000000354609.jpg', 'width': 683}
    # print(load_dict)
        length_annotations = len(load_dict['annotations'])
        length_images = len(load_dict['images'])
        length_categories = len(load_dict['categories'])
        count = 0
        for num in range(length_annotations):

            if load_dict['annotations'][num]['category_id'] in need_categories_id:
                # count = count + 1
                image_id = load_dict['annotations'][num]['image_id']
                category_id = load_dict['annotations'][num]['category_id']
                bbox = load_dict['annotations'][num]['bbox']

                for number_c in range(length_categories):
                    if category_id in load_dict['categories'][number_c].values():
                        name = load_dict['categories'][number_c]['name']
                    # else:
                        # name ='fucking'
                        # print(name)
                # 在images 里查询并获取file_name

                for number in range(length_images):

                    if image_id in load_dict['images'][number].values():
                        file_name = load_dict['images'][number]['file_name']
                        width = load_dict['images'][number]['width']
                        height = load_dict['images'][number]['height']
                        count_tmp = count
                        if file_name not in file_name_dict:
                            count = count + 1
                            # file_name_dict[file_name].append(image_id)
                            # print(type(file_name))
                            file_name_dict.setdefault(
                                file_name, []).append(width)
                            file_name_dict.setdefault(
                                file_name, []).append(height)
                            # file_name_dict.setdefault(file_name，[]).append(0)
                            next_file_name = file_name
                        # else:
                        if previous_file_name == "":
                            previous_file_name = file_name
                        if previous_file_name == file_name and num < length_annotations - 1:
                            change = 0
                        elif previous_file_name != file_name:
                            change = 1
                            anno_file_name = previous_file_name
                            previous_file_name = file_name  # border condition
                        else:  # final
                            change = 1
                            anno_file_name = previous_file_name
                        # file_name_dict[file_name][2] = file_name_dict[file_name][2] + 1
                        file_name_dict.setdefault(file_name, []).append(name)
                        file_name_dict.setdefault(file_name, []).append(bbox)

                        # print(anno_file_name)
                        # file_name_dict[anno_file_name]
                        # if file_name_dict.keys()=='obj365_val_000000646829.jpg':
                        #	print("fucking")
                        if change == 1:  # 将上一个存起来写入xml, 最后一个file_name怎么解决？
                            annotation = ET.Element('annotation')
                            folder = ET.SubElement(annotation, 'folder')
                            folder.text = 'voc2012'
                            filename2 = ET.SubElement(annotation, 'filename')
                            filename2.text = anno_file_name
                            source = ET.SubElement(annotation, 'source')
                            database = ET.SubElement(source, 'database')
                            database.text = 'Unknown'
                            size = ET.SubElement(annotation, 'size')
                            width = ET.SubElement(size, 'width')
                            # copy image

                            width.text = str(file_name_dict[anno_file_name][0])
                            height = ET.SubElement(size, 'height')
                            height.text = str(
                                file_name_dict[anno_file_name][1])
                            depth = ET.SubElement(size, 'depth')
                            depth.text = str(3)
                            segment = ET.SubElement(annotation, 'segment')
                            segment.text = '0'
                            length = len(file_name_dict[anno_file_name]) - 2
                            object_node = file_name_dict[anno_file_name]
                            for object_number in range(int(length/2)):

                                ob = ET.SubElement(annotation, 'object')
                                name = ET.SubElement(ob, 'name')
                                if object_node[2*object_number+2] == "pine apple":
                                    # name.text = object_node[2*object_number+2]
                                    name.text = "pineapple"

                                elif object_node[2*object_number+2] == "ice cream":
                                    name.text = "ice"
                                else:
                                    name.text = object_node[2*object_number+2]
                                pose = ET.SubElement(ob, 'pose')
                                pose.text = 'Unspecified'
                                truncated = ET.SubElement(ob, 'truncated')
                                truncated.text = '0'
                                difficult = ET.SubElement(ob, 'difficult')
                                difficult.text = '0'

                                bndbox = ET.SubElement(ob, 'bndbox')
                                xmin = ET.SubElement(bndbox, 'xmin')
                                # need to revise
                                xmin.text = str(
                                    int(object_node[2*object_number+1+2][0]))
                                ymin = ET.SubElement(bndbox, 'ymin')
                                # need to revise
                                ymin.text = str(
                                    int(object_node[2*object_number+1+2][1]))
                                xmax = ET.SubElement(bndbox, 'xmax')
                                # need to revise
                                xmax.text = str(
                                    int(object_node[2*object_number+1+2][0]+object_node[2*object_number+1+2][2]))
                                ymax = ET.SubElement(bndbox, 'ymax')
                                # need  to revise
                                ymax.text = str(
                                    int(object_node[2*object_number+1+2][1]+object_node[2*object_number+1+2][3]))
                            xml_path = saved_xml_folder + \
                                anno_file_name.split(
                                    '.')[0] + '.xml'  # need to revise
                            tree = ET.ElementTree(annotation)
                            tree.write(xml_path)
                            print("XML_Path:", xml_path)

def copy_file(xml_path, img_path, new_path):
	xml_n, xml_p = get_file_name_path(xml_path, '.xml')
	jpg_n, jpg_p = get_file_name_path(img_path, '.jpg')

	for jp in jpg_p:
		jp_name = re.split('/', jp)
		jp_name = jp_name[-1][:-4] + '.xml'

		if jp_name in xml_n:
			shutil.copy2(jp, new_path)
			print("Copy:", jp, "To:", new_path)
		else:
			print("Not Found:", jp)

#################################
if __name__ == '__main__':
    c = 1
    json_path = "../tmp/object365/train.json"
    saved_xml_folder = "../tmp/object365/use_data/train/annotation/"

    img_path = "../tmp/object365/images/Part/"
    save_img_path = "../tmp/object365/use_data/train/images/"
    if c == 0:
        solve(json_path, saved_xml_folder)
    else:
        copy_file(saved_xml_folder, img_path, save_img_path)

# obj365_val_000000190026.jpg


"""
Views related to the video upload feature
"""
import csv
import logging
from datetime import datetime, timedelta
from uuid import uuid4

import rfc6266
from boto import s3
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_noop
from django.views.decorators.http import require_GET, require_http_methods
from edxval.api import (
    SortDirection,
    VideoSortField,
    create_video,
    get_videos_for_course,
    remove_video_for_course,
    update_video_status
)
from opaque_keys.edx.keys import CourseKey

from contentstore.models import VideoUploadConfig
from contentstore.utils import reverse_course_url
from edxmako.shortcuts import render_to_response
from util.json_request import JsonResponse, expect_json

from .course import get_course_and_check_access

__all__ = ["videos_handler", "video_encodings_download"]

LOGGER = logging.getLogger(__name__)

# Default expiration, in seconds, of one-time URLs used for uploading videos.
KEY_EXPIRATION_IN_SECONDS = 86400

VIDEO_SUPPORTED_FILE_FORMATS = {
    '.mp4': 'video/mp4',
    '.mov': 'video/quicktime',
}

VIDEO_UPLOAD_MAX_FILE_SIZE_GB = 5

# maximum time for video to remain in upload state
MAX_UPLOAD_HOURS = 24

class StatusDisplayStrings(object):
    """
    A class to map status strings as stored in VAL to display strings for the
    video upload page
    """

    # Translators: This is the status of an active video upload
    _UPLOADING = ugettext_noop("Uploading")
    # Translators: This is the status for a video that the servers are currently processing
    _IN_PROGRESS = ugettext_noop("In Progress")
    # Translators: This is the status for a video that the servers have successfully processed
    _COMPLETE = ugettext_noop("Ready")
    # Translators: This is the status for a video that is uploaded completely
    _UPLOAD_COMPLETED = ugettext_noop("Uploaded")
    # Translators: This is the status for a video that the servers have failed to process
    _FAILED = ugettext_noop("Failed")
    # Translators: This is the status for a video that is cancelled during upload by user
    _CANCELLED = ugettext_noop("Cancelled")
    # Translators: This is the status for a video which has failed
    # due to being flagged as a duplicate by an external or internal CMS
    _DUPLICATE = ugettext_noop("Failed Duplicate")
    # Translators: This is the status for a video which has duplicate token for youtube
    _YOUTUBE_DUPLICATE = ugettext_noop("YouTube Duplicate")
    # Translators: This is the status for a video for which an invalid
    # processing token was provided in the course settings
    _INVALID_TOKEN = ugettext_noop("Invalid Token")
    # Translators: This is the status for a video that was included in a course import
    _IMPORTED = ugettext_noop("Imported")
    # Translators: This is the status for a video that is in an unknown state
    _UNKNOWN = ugettext_noop("Unknown")

    _STATUS_MAP = {
        "upload": _UPLOADING,
        "ingest": _IN_PROGRESS,
        "transcode_queue": _IN_PROGRESS,
        "transcode_active": _IN_PROGRESS,
        "file_delivered": _COMPLETE,
        "file_complete": _COMPLETE,
        "upload_completed": _UPLOAD_COMPLETED,
        "file_corrupt": _FAILED,
        "pipeline_error": _FAILED,
        "upload_failed": _FAILED,
        "s3_upload_failed": _FAILED,
        "upload_cancelled": _CANCELLED,
        "duplicate": _DUPLICATE,
        "youtube_duplicate": _YOUTUBE_DUPLICATE,
        "invalid_token": _INVALID_TOKEN,
        "imported": _IMPORTED,
    }

    @staticmethod
    def get(val_status):
        """Map a VAL status string to a localized display string"""
        return _(StatusDisplayStrings._STATUS_MAP.get(val_status, StatusDisplayStrings._UNKNOWN))    # pylint: disable=translation-of-non-string

@expect_json
@login_required
@require_http_methods(("GET", "POST", "DELETE"))
def videos_handler(request, course_key_string, edx_video_id=None):
    """
    The restful handler for video uploads.

    GET
        html: return an HTML page to display previous video uploads and allow
            new ones
        json: return json representing the videos that have been uploaded and
            their statuses
    POST
        json: create a new video upload; the actual files should not be provided
            to this endpoint but rather PUT to the respective upload_url values
            contained in the response
    DELETE
        soft deletes a video for particular course
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    if request.method == "GET":
        if "application/json" in request.META.get("HTTP_ACCEPT", ""):
            return videos_index_json(course)
        else:
            return videos_index_html(course)
    elif request.method == "DELETE":
        remove_video_for_course(course_key_string, edx_video_id)
        return JsonResponse()
    else:
        if is_status_update_request(request.json):
            return send_video_status_update(request.json)

        return videos_post(course, request)

@login_required
@require_GET
def video_encodings_download(request, course_key_string):
    """
    Returns a CSV report containing the encoded video URLs for video uploads
    in the following format:

    Video ID,Name,Status,Profile1 URL,Profile2 URL
    aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa,video.mp4,Complete,http://example.com/prof1.mp4,http://example.com/prof2.mp4
    """
    course = _get_and_validate_course(course_key_string, request.user)

    if not course:
        return HttpResponseNotFound()

    def get_profile_header(profile):
        """Returns the column header string for the given profile's URLs"""
        # Translators: This is the header for a CSV file column
        # containing URLs for video encodings for the named profile
        # (e.g. desktop, mobile high quality, mobile low quality)
        return _("{profile_name} URL").format(profile_name=profile)

    profile_whitelist = VideoUploadConfig.get_profile_whitelist()

    videos = list(_get_videos(course))
    name_col = _("Name")
    duration_col = _("Duration")
    added_col = _("Date Added")
    video_id_col = _("Video ID")
    status_col = _("Status")
    profile_cols = [get_profile_header(profile) for profile in profile_whitelist]

    def make_csv_dict(video):
        """
        Makes a dictionary suitable for writing CSV output. This involves
        extracting the required items from the original video dict and
        converting all keys and values to UTF-8 encoded string objects,
        because the CSV module doesn't play well with unicode objects.
        """
        # Translators: This is listed as the duration for a video that has not
        # yet reached the point in its processing by the servers where its
        # duration is determined.
        duration_val = str(video["duration"]) if video["duration"] > 0 else _("Pending")
        ret = dict(
            [
                (name_col, video["client_video_id"]),
                (duration_col, duration_val),
                (added_col, video["created"].isoformat()),
                (video_id_col, video["edx_video_id"]),
                (status_col, video["status"]),
            ] +
            [
                (get_profile_header(encoded_video["profile"]), encoded_video["url"])
                for encoded_video in video["encoded_videos"]
                if encoded_video["profile"] in profile_whitelist
            ]
        )
        return {
            key.encode("utf-8"): value.encode("utf-8")
            for key, value in ret.items()
        }

    response = HttpResponse(content_type="text/csv")
    # Translators: This is the suggested filename when downloading the URL
    # listing for videos uploaded through Studio
    filename = _("{course}_video_urls").format(course=course.id.course)
    # See https://tools.ietf.org/html/rfc6266#appendix-D
    response["Content-Disposition"] = rfc6266.build_header(
        filename + ".csv",
        filename_compat="video_urls.csv"
    )
    writer = csv.DictWriter(
        response,
        [
            col_name.encode("utf-8")
            for col_name
            in [name_col, duration_col, added_col, video_id_col, status_col] + profile_cols
        ],
        dialect=csv.excel
    )
    writer.writeheader()
    for video in videos:
        writer.writerow(make_csv_dict(video))
    return response

def _get_and_validate_course(course_key_string, user):
    """
    Given a course key, return the course if it exists, the given user has
    access to it, and it is properly configured for video uploads
    """
    course_key = CourseKey.from_string(course_key_string)

    # For now, assume all studio users that have access to the course can upload videos.
    # In the future, we plan to add a new org-level role for video uploaders.
    course = get_course_and_check_access(course_key, user)

    if (
            settings.FEATURES["ENABLE_VIDEO_UPLOAD_PIPELINE"] and
            getattr(settings, "VIDEO_UPLOAD_PIPELINE", None) and
            course and
            course.video_pipeline_configured
    ):
        return course
    else:
        return None

def convert_video_status(video):
    """
    Convert status of a video. Status can be converted to one of the following:

        *   FAILED if video is in `upload` state for more than 24 hours
        *   `YouTube Duplicate` if status is `invalid_token`
        *   user-friendly video status
    """
    now = datetime.now(video['created'].tzinfo)
    if video['status'] == 'upload' and (now - video['created']) > timedelta(hours=MAX_UPLOAD_HOURS):
        new_status = 'upload_failed'
        status = StatusDisplayStrings.get(new_status)
        message = 'Video with id [%s] is still in upload after [%s] hours, setting status to [%s]' % (
            video['edx_video_id'], MAX_UPLOAD_HOURS, new_status
        )
        send_video_status_update([
            {
                'edxVideoId': video['edx_video_id'],
                'status': new_status,
                'message': message
            }
        ])
    elif video['status'] == 'invalid_token':
        status = StatusDisplayStrings.get('youtube_duplicate')
    else:
        status = StatusDisplayStrings.get(video['status'])

    return status

def _get_videos(course):
    """
    Retrieves the list of videos from VAL corresponding to this course.
    """
    videos = list(get_videos_for_course(course.id, VideoSortField.created, SortDirection.desc))

    # convert VAL's status to studio's Video Upload feature status.
    for video in videos:
        video["status"] = convert_video_status(video)

    return videos

def _get_index_videos(course):
    """
    Returns the information about each video upload required for the video list
    """
    return list(
        {
            attr: video[attr]
            for attr in ["edx_video_id", "client_video_id", "created", "duration", "status"]
        }
        for video in _get_videos(course)
    )

def videos_index_html(course):
    """
    Returns an HTML page to display previous video uploads and allow new ones
    """
    return render_to_response(
        "videos_index.html",
        {
            "context_course": course,
            "video_handler_url": reverse_course_url("videos_handler", unicode(course.id)),
            "encodings_download_url": reverse_course_url("video_encodings_download", unicode(course.id)),
            "previous_uploads": _get_index_videos(course),
            "concurrent_upload_limit": settings.VIDEO_UPLOAD_PIPELINE.get("CONCURRENT_UPLOAD_LIMIT", 0),
            "video_supported_file_formats": VIDEO_SUPPORTED_FILE_FORMATS.keys(),
            "video_upload_max_file_size": VIDEO_UPLOAD_MAX_FILE_SIZE_GB
        }
    )

def videos_index_json(course):
    """
    Returns JSON in the following format:
    {
        "videos": [{
            "edx_video_id": "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa",
            "client_video_id": "video.mp4",
            "created": "1970-01-01T00:00:00Z",
            "duration": 42.5,
            "status": "upload"
        }]
    }
    """
    return JsonResponse({"videos": _get_index_videos(course)}, status=200)

def videos_post(course, request):
    """
    Input (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "content_type": "video/mp4"
        }]
    }

    Returns (JSON):
    {
        "files": [{
            "file_name": "video.mp4",
            "upload_url": "http://example.com/put_video"
        }]
    }

    The returned array corresponds exactly to the input array.
    """
    error = None
    if "files" not in request.json:
        error = "Request object is not JSON or does not contain 'files'"
    elif any(
            "file_name" not in file or "content_type" not in file
            for file in request.json["files"]
    ):
        error = "Request 'files' entry does not contain 'file_name' and 'content_type'"
    elif any(
            file['content_type'] not in VIDEO_SUPPORTED_FILE_FORMATS.values()
            for file in request.json["files"]
    ):
        error = "Request 'files' entry contain unsupported content_type"

    if error:
        return JsonResponse({"error": error}, status=400)

    bucket = storage_service_bucket()
    course_video_upload_token = course.video_upload_pipeline["course_video_upload_token"]
    req_files = request.json["files"]
    resp_files = []

    for req_file in req_files:
        file_name = req_file["file_name"]

        try:
            file_name.encode('ascii')
        except UnicodeEncodeError:
            error_msg = 'The file name for %s must contain only ASCII characters.' % file_name
            return JsonResponse({'error': error_msg}, status=400)

        edx_video_id = unicode(uuid4())
        key = storage_service_key(bucket, file_name=edx_video_id)
        for metadata_name, value in [
                ("course_video_upload_token", course_video_upload_token),
                ("client_video_id", file_name),
                ("course_key", unicode(course.id)),
        ]:
            key.set_metadata(metadata_name, value)
        upload_url = key.generate_url(
            KEY_EXPIRATION_IN_SECONDS,
            "PUT",
            headers={"Content-Type": req_file["content_type"]}
        )

        # persist edx_video_id in VAL
        create_video({
            "edx_video_id": edx_video_id,
            "status": "upload",
            "client_video_id": file_name,
            "duration": 0,
            "encoded_videos": [],
            "courses": [course.id]
        })

        resp_files.append({"file_name": file_name, "upload_url": upload_url, "edx_video_id": edx_video_id})

    return JsonResponse({"files": resp_files}, status=200)

def storage_service_bucket():
    """
    Returns an S3 bucket for video uploads.
    """
    conn = s3.connection.S3Connection(
        settings.AWS_ACCESS_KEY_ID,
        settings.AWS_SECRET_ACCESS_KEY
    )
    # We don't need to validate our bucket, it requires a very permissive IAM permission
    # set since behind the scenes it fires a HEAD request that is equivalent to get_all_keys()
    # meaning it would need ListObjects on the whole bucket, not just the path used in each
    # environment (since we share a single bucket for multiple deployments in some configurations)
    return conn.get_bucket(settings.VIDEO_UPLOAD_PIPELINE["BUCKET"], validate=False)

def storage_service_key(bucket, file_name):
    """
    Returns an S3 key to the given file in the given bucket.
    """
    key_name = "{}/{}".format(
        settings.VIDEO_UPLOAD_PIPELINE.get("ROOT_PATH", ""),
        file_name
    )
    return s3.key.Key(bucket, key_name)

def send_video_status_update(updates):
    """
    Update video status in edx-val.
    """
    for update in updates:
        update_video_status(update.get('edxVideoId'), update.get('status'))
        LOGGER.info(
            'VIDEOS: Video status update with id [%s], status [%s] and message [%s]',
            update.get('edxVideoId'),
            update.get('status'),
            update.get('message')
        )

    return JsonResponse()

def is_status_update_request(request_data):
    """
    Returns True if `request_data` contains status update else False.
    """
    return any('status' in update for update in request_data)


import os
import datetime
import random
import zlib
import math

from flask import Flask, render_template, redirect, url_for, request
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.abspath(os.path.dirname(__file__)) + '/db.sqlite'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

class Actor(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(255), unique=True, nullable=False)
    in_session = db.Column(db.Boolean, default=True, nullable=False)

    def __repr__(self):
        return '%s' % self.name

class Event(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    time = db.Column(db.DateTime, default=datetime.datetime.utcnow)

    actor_id = db.Column(db.Integer, db.ForeignKey('actor.id'), nullable=False)
    actor = db.relationship('Actor', backref=db.backref('events', lazy=True))

def get_people():
    return Actor.query.all()

@app.cli.command()
def init_db():
    db.create_all()

@app.cli.command()
def deinit_db():
    db.drop_all()

@app.cli.command()
def reinit_db():
    deinit_db()
    init_db()

@app.route('/', methods=['get'])
def index():
    return render_template('index.html', people=get_people())

@app.route('/add/', methods=['post'])
def add_person_to_session():
    if request.form.get('id') is None:
        na = Actor(name=request.form.get('name'))
        db.session.add(na)
        db.session.commit()
        return redirect(url_for('index', _scheme='https', _external=True))
    else:
        na = Actor.query.filter_by(id=request.form.get('id')).first()
        na.in_session = True
        db.session.commit()
        return '200'

@app.route('/remove/', methods=['post'])
def remove_person_from_session():
    na = Actor.query.filter_by(id=request.form.get('id')).first()
    if na is not None:
        na.in_session = False
        db.session.commit()
    return '200'

@app.route('/choose/', methods=['get'])
def choose_from_session():
    suitable_actors = Actor.query.filter_by(in_session=True).all()
    actor_key_index = {}
    roll_results = {}
    times_to_reroll = 20000
    for index, a in enumerate(suitable_actors):
        actor_key_index[index] = a
        roll_results[index] = 0
    seed = zlib.crc32(str(datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S%f')).encode()) + zlib.crc32(request.headers.get('User-Agent').encode())
    random.seed(seed)
    roll_results_keys_cache = list(roll_results.keys())
    for _ in range(times_to_reroll):
        roll_results[random.choice(roll_results_keys_cache)] += 1
    reverse_sorted_actors_by_win = sorted(roll_results, key=lambda x: roll_results[x])
    ca = actor_key_index[reverse_sorted_actors_by_win[-1]]
    e = Event(actor=ca)
    db.session.add(e)
    db.session.commit()
    result_string = []
    for zp in reversed(reverse_sorted_actors_by_win):
        result_string.append('%s @ %s%% %s/%s' % (actor_key_index[zp].name, round((roll_results[zp]/times_to_reroll)*100, 2), roll_results[zp], times_to_reroll))
    return '%s at %s%% of %s iteraties \n\nbreakdown: \n\n%s' % (ca.name, round((roll_results[reverse_sorted_actors_by_win[-1]]/times_to_reroll)*100, 2), times_to_reroll, '\n'.join(result_string))

if __name__ == '__main__':
    app.run(port=5000)


# import argparse
import enum
import os
import sys
import subprocess

def _pad(x, size):
    chars = str(x)
    pad_len = size - len(chars)
    ret = chars + ' ' * pad_len
    return bytearray(ret, 'utf-8')

GLOBAL_HEADER_SIZE = 8
GLOBAL_HEADER = b"!<arch>\n"
GLOBAL_THIN_HEADER = b"!<thin>\n"
END_OF_FILE_HEADER_MARKER = b"\x60\x0A"
ENTRY_SIZE = 60
SPECIAL_ENTRIES = ("/", "//", "/SYM64/")

COMMON_MODIFICATION_TIME_STAMP = _pad(476064000, 12)
DEFAULT_MODE = _pad(100644, 8)
ZERO_BYTES = _pad(0, 6)

def _modify_buffer(buffer: bytearray):
    """
    Header attribute length in bytes
    + 16 : Name
    + 12 : Timestamp
    + 6  : UserID
    + 6  : GroupID
    + 8  : Mode
    + 10 : Size
    + 2  : End Header
    """
    pos = 0
    # skip the name
    name = buffer[pos:pos+16]
    pos += 16
    # Modify Timestamp
    buffer[pos:pos+12] = COMMON_MODIFICATION_TIME_STAMP
    pos += 12
    # Modify UserID
    buffer[pos:pos+6]  = ZERO_BYTES
    pos += 6
    # Modify GroupID
    buffer[pos:pos+6]  = ZERO_BYTES
    pos += 6
    # Modify Mode
    buffer[pos:pos+8]  = DEFAULT_MODE
    pos += 8
    # Skip Size
    size = buffer[pos:pos+10]
    # Skip End Header
    pos += 10
    end_header = buffer[pos:pos+2]
    # sanity check
    if end_header != END_OF_FILE_HEADER_MARKER:
        raise Exception("invalid file magic")
    return name, int(size)

def scrub(bianry_path: str):
    with open(bianry_path, "r+b") as f:
        header = f.read(GLOBAL_HEADER_SIZE)
        if header not in [GLOBAL_HEADER, GLOBAL_THIN_HEADER]:
            raise Exception("invalid global header")
        thin = header == GLOBAL_THIN_HEADER
        while True:
            pos = f.tell()
            buffer = bytearray(f.read(ENTRY_SIZE))
            if len(buffer) < ENTRY_SIZE:
                break
            name, size = _modify_buffer(buffer)
            name = name.decode().strip()
            f.seek(pos)
            written = f.write(buffer)
            if written != ENTRY_SIZE:
                raise Exception("Not all bytes have been written")
            if not thin or name in SPECIAL_ENTRIES:
                f.seek(size, os.SEEK_CUR)

def main():
    ret = subprocess.call(sys.argv[1:])
    if ret != 0:
        sys.exit(ret)
    scrub(sys.argv[3])

if __name__ == "__main__":
    main()


#!/usr/bin/python3
# This is the tool i'm using to make my video re-encode
import os
from subprocess import run

class FFmpeg:
    encoder = '/usr/bin/ffmpeg'
    HDR = 'YUV444p10le'
    STD = 'yuv444p'

    @classmethod
    def run(cls, *args):
        command = (cls.encoder, *args)
        print('running', *command)
        return run(command, check=True)

    @classmethod
    def to_x264(cls, source, output, *args, quality=25, format='yuv444p'):
        # 'YUV444p10le'
        cmd = (
            '-i', source,
            '-vcodec', 'h264_nvenc',
            '-vf', f'format={format}',
            '-qp', str(quality),
            '-gpu', 'any',
            '-preset', 'fast',
            '-y',
            '-threads', '12',
            '-acodec', 'copy'
        )
        return cls.run(*cmd, *args, output)

    @classmethod
    def to_hevc(cls, source, output, *args, quality=25):
        cmd = (
            '-i', source,
            '-vcodec', 'libx265',
            '-rc', 'vbr_hq',
            '-rc-lookahead', '32',
            '-qp', str(quality),
            '-gpu', 'any',
            '-acodec', 'copy'
        )
        return cls.run(*cmd, *args, output)

    @classmethod
    def to_webm(cls, source, output, *args, quality=25):
        output = '.'.join(output.split('.')[0:-1]) + '.webm'
        cmd = (
            '-i', source,
            '-threads', '12',
            '-vcodec', 'libvpx-vp9',
            '-b:v', '1M',
            '-q:v', str(quality),
            '-f', 'webm'
        )
        return cls.run(*cmd, *args, output)

    @classmethod
    def convert(cls, source_dir, dest_dir, cvt_func=None, *args, **kwargs):
        if not os.path.isdir(source_dir):
            raise FileNotFoundError(source_dir)
        if cvt_func is None:
            cvt_func = cls.to_x264
        len_src = len(source_dir)
        for pwd, _, files in os.walk(source_dir):
            for file in sorted(files):
                relative = pwd[len_src:]
                source = os.path.join(pwd, file)
                dest = os.path.join(dest_dir, relative, file)
                os.makedirs(os.path.join(dest_dir, relative), exist_ok=True)
                cvt_func(source, dest, *args, **kwargs)


import requests
from bs4 import BeautifulSoup
from selenium import webdriver
import time
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException

CHROME_DRIVER_PATH = "/Users/akshat/repo/chromedriver/chromedriver"
ZILLOW_URL = 'https://www.zillow.com/homes/for_rent/1-_beds/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3Anull%2C%22mapBounds%22%3A%7B%22west%22%3A-122.56276167822266%2C%22east%22%3A-122.30389632177734%2C%22south%22%3A37.69261345230467%2C%22north%22%3A37.857877098316834%7D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22fr%22%3A%7B%22value%22%3Atrue%7D%2C%22fsba%22%3A%7B%22value%22%3Afalse%7D%2C%22fsbo%22%3A%7B%22value%22%3Afalse%7D%2C%22nc%22%3A%7B%22value%22%3Afalse%7D%2C%22cmsn%22%3A%7B%22value%22%3Afalse%7D%2C%22auc%22%3A%7B%22value%22%3Afalse%7D%2C%22fore%22%3A%7B%22value%22%3Afalse%7D%2C%22pmf%22%3A%7B%22value%22%3Afalse%7D%2C%22pf%22%3A%7B%22value%22%3Afalse%7D%2C%22mp%22%3A%7B%22max%22%3A3000%7D%2C%22price%22%3A%7B%22max%22%3A872627%7D%2C%22beds%22%3A%7B%22min%22%3A1%7D%7D%2C%22isListVisible%22%3Atrue%2C%22mapZoom%22%3A12%7D'
FORMS_URL = 'https://docs.google.com/forms/d/e/1FAIpQLSfrQ14K59TciFvTVC6a9CaXQN3qGNhsow4wwHLEW6uzWoWkLw/viewform?usp=sf_link'

def get_static_contents(file_name: str) -> str:
    with open(file_name, 'r') as file_handle:
        return file_handle.read()

def store_static_contents(file_name: str, contents: str):
    with open(file_name, 'w') as file_handle:
        file_handle.write(contents)

def get_zillow_data():
    """
    Use beautiful soup to scrape data from the Zillow website.
    For each apartment the Price, Address and Link will be scraped and
    added to a list object.
    The list object will be returned
    :return:
    list[str]: List of dictionary will be returned
    """
    # Adding headers to avoid captcha
    headers = {
        "User-Agent": "en-US,en;q=0.5",
        "Accept-Language": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:84.0) Gecko/20100101 Firefox/84.0"
    }

    # Get html contents dynamically
    contents = requests.get(url=ZILLOW_URL, headers=headers).text

    # Get html contents statically - For testing
    # contents: str = get_static_contents('data.html')

    soup = BeautifulSoup(contents, "html.parser")
    data = []
    # print(soup.prettify())

    all_prices = soup.find_all(class_='list-card-price')
    all_addresses = soup.find_all(class_='list-card-addr')
    all_links = soup.find_all(class_="list-card-link")

    ZILLOW_TEXT = 'https://www.zillow.com'

    for counter in range(len(all_prices)):
        link_str = all_links[counter]['href']
        if ZILLOW_TEXT not in link_str:
            link_str = ZILLOW_TEXT + link_str
        data.append({
            'address': all_addresses[counter].getText(),
            'price': all_prices[counter].getText().split(' ')[0].replace('/mo', '').replace('+', ''),
            'link': link_str
        })
    # print(data)
    return data

def fill_google_form(data):
    """
    Open the google form URL.
    for each property as part of list, input the data into the form.
    :param data: List data that is used to fill google form
    :return:
    """
    driver = webdriver.Chrome(executable_path=CHROME_DRIVER_PATH)
    driver.get(url=FORMS_URL)
    time.sleep(2)
    for row in data:
        # print(row['price'], row['address'], row['link'])
        try:
            address_input = driver.find_element_by_xpath(
                '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[1]/div/div/div[2]/div/div[1]/div/div[1]/input')
            address_input.send_keys(row['address'])
            price_input = driver.find_element_by_xpath(
                '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[2]/div/div/div[2]/div/div[1]/div/div[1]/input')
            price_input.send_keys(row['price'])
            link_input = driver.find_element_by_xpath(
                '//*[@id="mG61Hd"]/div[2]/div/div[2]/div[3]/div/div/div[2]/div/div[1]/div/div[1]/input')
            link_input.send_keys(row['link'])
            submit_btn = driver.find_element_by_xpath('//*[@id="mG61Hd"]/div[2]/div/div[3]/div[1]/div/div/span/span')
            submit_btn.click()
            time.sleep(1)

            submit_another = driver.find_element_by_link_text('Submit another response')
            submit_another.click()
            print(row['price'], row['address'], row['link'])
            time.sleep(1)
        except NoSuchElementException:
            print('Exception occurred...')

    driver.quit()

rental_data = get_zillow_data()
fill_google_form(rental_data)


# -*- coding: utf-8 - vim: tw=80
r"""
Monodromy matrices
"""

# Copyright 2017, 2018, 2019 Marc Mezzarobba
# Copyright 2017, 2018, 2019 Centre national de la recherche scientifique
# Copyright 2017, 2018 Université Pierre et Marie Curie
# Copyright 2019 Sorbonne Université
#
# Distributed under the terms of the GNU General Public License (GPL) either
# version 2, or (at your option) any later version
#
# http://www.gnu.org/licenses/

import collections
import logging

import sage.matrix.special as matrix

from sage.graphs.graph import Graph
from sage.matrix.matrix_complex_ball_dense import Matrix_complex_ball_dense
from sage.misc.cachefunc import cached_method
from sage.misc.misc_c import prod
from sage.rings.all import (CC, CBF, ComplexBallField, QQ, QQbar,
        QuadraticField, RBF)
from sage.functions.all import exp
from sage.symbolic.all import pi, SR

from . import analytic_continuation as ancont, local_solutions, path, utilities

from .differential_operator import DifferentialOperator
from .local_solutions import LocalBasisMapper, log_series

logger = logging.getLogger(__name__)

QQi = QuadraticField(-1, 'i')

##############################################################################
# Formal monodromy
##############################################################################

def formal_monodromy(dop, sing, ring=SR):
    r"""
    Compute the formal monodromy matrix of a system of fundamental solutions at
    a given singular point.

    INPUT:

    - ``dop`` - a differential operator,
    - ``sing`` - one of its singular points (typically given as an element of
      ``QQbar`` or of a parent with a coercion to ``QQbar``).

    OUTPUT:

    The formal monodromy matrix around ``sing`` in the basis given by
    ``dop.local_basis_expansions(sing)``.

    EXAMPLES::

        sage: from ore_algebra import *
        sage: from ore_algebra.analytic.monodromy import formal_monodromy
        sage: Dops, x, Dx = DifferentialOperators()

        sage: dop = Dx*x*Dx
        sage: dop.local_basis_expansions(0)
        [log(x), 1]
        sage: formal_monodromy(dop, 0)
        [       1      0]
        [(2*I*pi)      1]

        sage: mon = formal_monodromy(dop, 0, CBF)
        sage: tm = dop.numerical_transition_matrix([0,1])
        sage: tm*mon*~tm
        [ 1.0000... [6.2831853071795...]*I]
        [         0      1.000000000000000]
        sage: dop.numerical_transition_matrix([1, i, -1, -i, 1])
        [ [1.0000...] + [+/- ...]*I [+/- ...] + [6.2831853071795...]*I]
        [   [+/- ...] + [+/- ...]*I          [1.0000...] + [+/- ...]*I]

        sage: dop = ((x*Dx)^2 - 1/4)^2*((x*Dx)-3/2)
        sage: dop.local_basis_expansions(0)
        [x^(-1/2)*log(x), x^(-1/2), x^(1/2)*log(x), x^(1/2), x^(3/2)]
        sage: formal_monodromy(dop, 0)
        [       -1         0         0         0         0]
        [-(2*I*pi)        -1         0         0         0]
        [        0         0        -1         0         0]
        [        0         0 -(2*I*pi)        -1         0]
        [        0         0         0         0        -1]

        sage: dop = (x*Dx-1/2+x^2)^2*(x*Dx-5/2)
        sage: dop.local_basis_expansions(0)
        [x^(1/2)*log(x) + 1/2*x^(5/2)*log(x)^2 - 1/2*x^(5/2)*log(x) - 1/8*x^(9/2)*log(x) + 1/8*x^(9/2),
        x^(1/2) + x^(5/2)*log(x) - 1/8*x^(9/2),
        x^(5/2)]
        sage: formal_monodromy(dop, 0)
        [               -1                 0                 0]
        [        -(2*I*pi)                -1                 0]
        [-(-I*pi - 2*pi^2)         -(2*I*pi)                -1]

    TESTS::

        sage: formal_monodromy(Dx*x*Dx, 1)
        [1 0]
        [0 1]
        sage: formal_monodromy((x*Dx)^2 - 2, 0)
        [e^(-1.414213562373095?*(2*I*pi))                                0]
        [                               0  e^(1.414213562373095?*(2*I*pi))]

        sage: from ore_algebra.analytic.monodromy import _test_formal_monodromy
        sage: _test_formal_monodromy(Dx*x*Dx)
        sage: _test_formal_monodromy((x*Dx-1/2+x^2)^2*(x*Dx-5/2))
        sage: _test_formal_monodromy((x*Dx)^2 - 2)
    """
    dop = DifferentialOperator(dop)
    sing = path.Point(sing, dop)
    ldop = dop.shift(sing)
    # XXX should use binary splitting when the indicial polynomial has large
    # dispersion
    mon, _ = _formal_monodromy_naive(ldop, ring)
    return mon

def _formal_monodromy_naive(dop, ring):
    crit = _critical_monomials(dop)
    mor = dop.base_ring().base_ring().hom(ring)
    return _formal_monodromy_from_critical_monomials(crit, mor)

def _critical_monomials(dop):

    class Mapper(LocalBasisMapper):
        def fun(self, ini):
            # XXX should share algebraic part with Galois conjugates
            order = max(s for s, _ in self.shifts) + 1
            ser = log_series(ini, self.shifted_bwrec, order)
            return {s: ser[s] for s, _ in self.shifts}

    return Mapper(dop).run()

def _formal_monodromy_from_critical_monomials(critical_monomials, mor):
    r"""
    Compute the formal monodromy matrix of the canonical system of fundamental
    solutions at the origin.

    INPUT:

    - ``critical_monomials``: list of ``FundamentalSolution`` objects ``sol``
      such that, if ``sol = z^(λ+n)·(1 + Õ(z)`` where ``λ`` is the leftmost
      valuation of a group of solutions and ``s`` is another shift of ``λ``
      appearing in the basis, then ``sol.value[s]`` contains the list of
      coefficients of ``z^(λ+s)·log(z)^k/k!``, ``k = 0, 1, ...`` in ``sol``

    - ``mor``: a morphism from the parent of critical monomials to a ring
      suitable for representing the entries of the formal monodromy matrix
      (typically ``CBF`` or ``SR``)

    OUTPUT:

    - the formal monodromy matrix, with coefficients in the codomain of ``mor``

    - a boolean flag indicating whether the local monodromy is scalar (useful
      when the target is an inexact ring!)
    """

    ring = mor.codomain()
    mat = matrix.matrix(ring, len(critical_monomials))
    twopii = 2*pi*QQbar(QQi.gen())
    expo0 = critical_monomials[0].leftmost
    scalar = True

    for j, jsol in enumerate(critical_monomials):

        for i, isol in enumerate(critical_monomials):
            if isol.leftmost != jsol.leftmost:
                continue
            for k, c in enumerate(jsol.value[isol.shift]):
                delta = k - isol.log_power
                if c.is_zero():
                    continue
                if delta >= 0:
                    # explicit conversion sometimes necessary (Sage bug #31551)
                    mat[i,j] += mor(c)*twopii**delta/delta.factorial()
                if delta >= 1:
                    scalar = False

        expo = jsol.leftmost
        if expo != expo0:
            scalar = False
        if expo.parent() is QQ:
            eigv = ring(QQbar.zeta(expo.denominator())**expo.numerator())
        else:
            # conversion via QQbar seems necessary with some number fields
            # XXX We should actually follow expo along mor, but this is not easy
            # to do with the current code structure, see comment in
            # _monodromy_matrices().
            eigv = twopii.mul(QQbar(expo), hold=True).exp(hold=True)
        eigv = ring(eigv)
        if ring is SR:
            _rescale_col_hold_nontrivial(mat, j, eigv)
        else:
            mat.rescale_col(j, eigv)

    return mat, scalar

def _rescale_col_hold_nontrivial(mat, j, c):
    for i in range(mat.nrows()):
        if mat[i,j].is_zero():
            pass
        elif mat[i,j].is_one():
            mat[i,j] = c
        else:
            mat[i,j] = mat[i,j].mul(c, hold=True)

def _test_formal_monodromy(dop):
    i = QQi.gen()
    ref = dop.numerical_transition_matrix([1, i, -1, -i, 1])
    tmat = dop.numerical_transition_matrix([0,1])
    fmon = formal_monodromy(dop, 0, CBF)
    mon = tmat*fmon*~tmat
    assert all(c.contains_zero() and c.rad() < 1e-10 for c in (ref - mon).list())

##############################################################################
# Generators of the monodromy group
##############################################################################

# XXX we use assume_analytic to bypass singularities in cases where the
# transition matrix may not actually be analytic but we do not care which path
# is taken

def _local_monodromy_loop(dop, x, eps):
    polygon = path.polygon_around(x)
    n = len(polygon)
    mats = []
    for i in range(n):
        step = [polygon[i], polygon[(i+1)%n]]
        logger.debug("center = %s, step = %s", x, step)
        step[0].options['keep_value'] = False # XXX bugware
        step[1].options['keep_value'] = True
        mat = x.dop.numerical_transition_matrix(step, eps, assume_analytic=True)
        prec = utilities.prec_from_eps(eps)
        assert all(c.accuracy() >= prec//2 or c.above_abs()**2 <= eps
                   for c in mat.list())
        mats.append(mat)
    return polygon, mats

class TodoItem():

    def __init__(self, alg, dop, *, want_self=False, want_conj=False):
        self.alg = alg
        self._dop = dop
        self.want_self: bool = want_self
        self.want_conj: bool = want_conj
        self.local_monodromy = None
        self.polygon = None
        self.done: bool = False

    def __repr__(self):
        return repr(self.alg)

    @cached_method
    def point(self):
        return path.Point(self.alg, self._dop)

    def __eq__(self, other):
        return self is other

    def __hash__(self):
        return id(self)

    def __lt__(self, other):
        # Sage graphs require vertices to be comparable
        return id(self) < id(other)

def _merge_conjugate_singularities(dop, sing, base, todo):
    need_conjugates = False
    sgn = 1 if base.alg.imag() >= 0 else -1
    for x in sing:
        if sgn*x.imag() < 0:
            need_conjugates = True
            del todo[x]
            xconj = x.conjugate()
            item = todo.get(xconj) # dict queries on elts of QQbar are slow
            if item is None:
                todo[xconj] = item = TodoItem(xconj, dop)
            item.want_conj = True
    return need_conjugates

def _spanning_tree(base, verts):
    graph = Graph([list(verts), lambda x, y: x is not y])
    def length(edge):
        x, y, _ = edge
        return abs(CC(x.alg) - CC(y.alg))
    tree = graph.min_spanning_tree(length)
    tree = Graph(tree)
    tree.add_vertex(base)
    return tree

def _closest_unsafe(lst, x):
    x = CC(x.alg)
    return min(enumerate(lst), key=lambda y: abs(CC(y[1].value) - x))

def _extend_path_mat(dop, path_mat, x, y, eps, matprod):
    anchor_index_x, anchor_x = _closest_unsafe(x.polygon, y)
    anchor_index_y, anchor_y = _closest_unsafe(y.polygon, x)
    bypass_mat_x = matprod(x.local_monodromy[:anchor_index_x])
    bypass_mat_y = matprod(y.local_monodromy[anchor_index_y:]
                           if anchor_index_y > 0
                           else [])
    anchor_x.options["keep_value"] = False # XXX bugware
    anchor_y.options["keep_value"] = True
    edge_mat = dop.numerical_transition_matrix([anchor_x, anchor_y],
                                                eps, assume_analytic=True)
    new_path_mat = bypass_mat_y*edge_mat*bypass_mat_x*path_mat
    assert isinstance(new_path_mat, Matrix_complex_ball_dense)
    return new_path_mat

LocalMonodromyData = collections.namedtuple("LocalMonodromyData",
        ["point", "monodromy", "is_scalar"])

def _monodromy_matrices(dop, base, eps=1e-16, sing=None):
    r"""
    EXAMPLES::

        sage: from ore_algebra import *
        sage: from ore_algebra.analytic.monodromy import _monodromy_matrices
        sage: Dops, x, Dx = DifferentialOperators()
        sage: rat = 1/(x^2-1)
        sage: dop = (rat*Dx - rat.derivative()).lclm(Dx*x*Dx)
        sage: [rec.point for rec in _monodromy_matrices(dop, 0) if not rec.is_scalar]
        [0]

    TESTS::

        sage: from ore_algebra.examples import fcc
        sage: mon = list(_monodromy_matrices(fcc.dop5, -1, 2**(-2**7))) # long time (2.3 s)
        sage: [rec.monodromy[0][0] for rec in mon if rec.point == -5/3] # long time
        [[1.01088578589319884254557667137848...]]

    Thanks to Alexandre Goyer for this example::

        sage: L1 = ((x^5 - x^4 + x^3)*Dx^3 + (27/8*x^4 - 25/9*x^3 + 8*x^2)*Dx^2
        ....:      + (37/24*x^3 - 25/9*x^2 + 14*x)*Dx - 2*x^2 - 3/4*x + 4)
        sage: L2 = ((x^5 - 9/4*x^4 + x^3)*Dx^3 + (11/6*x^4 - 31/4*x^3 + 7*x^2)*Dx^2
        ....:      + (7/30*x^3 - 101/20*x^2 + 10*x)*Dx + 4/5*x^2 + 5/6*x + 2)
        sage: L = L1*L2
        sage: L = L.parent()(L.annihilator_of_composition(x+1))
        sage: mon = list(_monodromy_matrices(L, 0, eps=1e-30)) # long time (1.3-1.7 s)
        sage: mon[-1][0], mon[-1][1][0][0] # long time
        (0.6403882032022075?,
        [1.15462187280628880820271...] + [-0.018967673022432256251718...]*I)
    """
    dop = DifferentialOperator(dop)
    base = QQbar.coerce(base)
    eps = RBF(eps)
    if sing is None:
        sing = dop._singularities(QQbar)
    else:
        sing = [QQbar.coerce(s) for s in sing]

    todo = {x: TodoItem(x, dop, want_self=True, want_conj=False)
            for x in sing}
    base = todo.setdefault(base, TodoItem(base, dop))
    if not base.point().is_regular():
        raise ValueError("irregular singular base point")
    # If the coefficients are rational, reduce to handling singularities in the
    # same half-plane as the base point, and share some computations between
    # Galois conjugates.
    need_conjugates = False
    crit_cache = None
    if all(c in QQ for pol in dop for c in pol):
        need_conjugates = _merge_conjugate_singularities(dop, sing, base, todo)
        # TODO: do something like that even over number fields?
        # XXX this is actually a bit costly: do it only after checking that the
        # monodromy is not scalar?
        # XXX keep the cache from one run to the next when increasing prec?
        crit_cache = {}

    Scalars = ComplexBallField(utilities.prec_from_eps(eps))
    id_mat = matrix.identity_matrix(Scalars, dop.order())
    def matprod(elts):
        return prod(reversed(elts), id_mat)

    for key, todoitem in list(todo.items()):
        point = todoitem.point()
        # We could call _local_monodromy_loop() if point is irregular, but
        # delaying it may allow us to start returning results earlier.
        if point.is_regular():
            if crit_cache is None or point.algdeg() == 1:
                crit = _critical_monomials(dop.shift(point))
                emb = point.value.parent().hom(Scalars)
            else:
                mpol = point.value.minpoly()
                try:
                    NF, crit = crit_cache[mpol]
                except KeyError:
                    NF = point.value.parent()
                    crit = _critical_monomials(dop.shift(point))
                    # Only store the critical monomials for reusing when all
                    # local exponents are rational. We need to restrict to this
                    # case because we do not have the technology in place to
                    # follow algebraic exponents along the embedding of NF in ℂ.
                    # (They are represented as elements of "new" number fields
                    # given by as_embedded_number_field_element(), even when
                    # they actually lie in NF itself as opposed to a further
                    # algebraic extension. XXX: Ideally, LocalBasisMapper should
                    # give us access to the tower of extensions in which the
                    # exponents "naturally" live.)
                    if all(sol.leftmost.parent() is QQ for sol in crit):
                        crit_cache[mpol] = NF, crit
                emb = NF.hom([Scalars(point.value.parent().gen())], check=False)
            mon, scalar = _formal_monodromy_from_critical_monomials(crit, emb)
            if scalar:
                # No need to compute the connection matrices then!
                # XXX When we do need them, though, it would be better to get
                # the formal monodromy as a byproduct of their computation.
                if todoitem.want_self:
                    yield LocalMonodromyData(key, mon, True)
                if todoitem.want_conj:
                    conj = key.conjugate()
                    logger.info("Computing local monodromy around %s by "
                                "complex conjugation", conj)
                    conj_mat = ~mon.conjugate()
                    yield LocalMonodromyData(conj, conj_mat, True)
                if todoitem is not base:
                    del todo[key]
                    continue
            todoitem.local_monodromy = [mon]
            todoitem.polygon = [point]

    if need_conjugates:
        base_conj_mat = dop.numerical_transition_matrix(
                            [base.point(), base.point().conjugate()],
                            eps, assume_analytic=True)
        def conjugate_monodromy(mat):
            return ~base_conj_mat*~mat.conjugate()*base_conj_mat

    tree = _spanning_tree(base, todo.values())

    def dfs(x, path, path_mat):

        logger.info("Computing local monodromy around %s via %s", x, path)

        local_mat = matprod(x.local_monodromy)
        based_mat = (~path_mat)*local_mat*path_mat

        if x.want_self:
            yield LocalMonodromyData(x.alg, based_mat, False)
        if x.want_conj:
            conj = x.alg.conjugate()
            logger.info("Computing local monodromy around %s by complex "
                        "conjugation", conj)
            conj_mat = conjugate_monodromy(based_mat)
            yield LocalMonodromyData(conj, conj_mat, False)

        x.done = True

        for y in tree.neighbors(x):
            if y.done:
                continue
            if y.local_monodromy is None:
                y.polygon, y.local_monodromy = _local_monodromy_loop(dop,
                                                                 y.point(), eps)
            new_path_mat = _extend_path_mat(dop, path_mat, x, y, eps, matprod)
            yield from dfs(y, path + [y], new_path_mat)

    yield from dfs(base, [base], id_mat)

def monodromy_matrices(dop, base, eps=1e-16, sing=None):
    r"""
    Compute generators of the monodromy group of ``dop`` with base point
    ``base``.

    OUTPUT:

    A list of matrices, each encoding the analytic continuation of solutions
    along a simple positive loop based in ``base`` around a singular point
    of ``dop`` (with no other singular point inside the loop). The precise
    choice of elements of the fundamental group corresponding to each matrix
    (position with respect to the other singular points, order) is unspecified.

    EXAMPLES::

        sage: from ore_algebra import *
        sage: from ore_algebra.analytic.monodromy import monodromy_matrices
        sage: Dops, x, Dx = DifferentialOperators()

        sage: monodromy_matrices(Dx*x*Dx, 1, 1e-30)
        [
        [  1.0000...  [6.2831853071795864769252867665...]*I]
        [          0     1.00000000000000000000000000000...]
        ]

        sage: monodromy_matrices(Dx*x*Dx, 1/2)
        [
        [ 1.0000...  [+/- ...] + [3.1415926535897...]*I]
        [         0                           1.0000...]
        ]

        sage: dop = (x^2 + 1)*Dx^2 + 2*x*Dx
        sage: mon = monodromy_matrices(dop, 0)
        sage: mon[0]
        [ [1.000000000000...] + [+/- ...]*I [3.141592653589...] + [+/- ...]*I]
        [           [+/- ...] + [+/- ...]*I [1.000000000000...] + [+/- ...]*I]
        sage: mon[1]
        [ [1.000000000000...] + [+/- ...]*I [-3.14159265358...] + [+/- ...]*I]
        [           [+/- ...] + [+/- ...]*I [1.000000000000...] + [+/- ...]*I]

        sage: dop = (x**2*Dx + 3)*((x-3)*Dx + 4*x**5) # B. Salvy
        sage: dop.leading_coefficient().factor()
        (x - 3) * x^2
        sage: mon = monodromy_matrices(dop, 1)
        sage: mon[0].is_one() # around 3
        True
        sage: mon[1] # around 0
        [[1.000000000...] + [-2.8675932949...]*I          [+/- ...] + [1.4337966474...]*I]
        [       [+/- ...] + [-5.7351865899...]*I  [1.0000000000...] + [2.8675932949...]*I]

    The base point can be a singular point::

        sage: monodromy_matrices(Dx*x*Dx, 0)
        [
        [     1.0000000000000000                    0]
        [[6.28318530717958...]*I   1.0000000000000000]
        ]

    ...but currently not an irregular singular one::

        sage: monodromy_matrices(x^2*Dx - 1, 0)
        Traceback (most recent call last):
        ...
        ValueError: irregular singular base point

    One can limit the computation to a subset of the monodromy matrices::

        sage: from ore_algebra.examples import fcc
        sage: mon = monodromy_matrices(fcc.dop5, 0, sing=[0, 1])
        sage: mon[0]
        [                  1.00...                         0            0           0         0        0]
        [              [6.28...]*I                   1.00...            0           0         0        0]
        [               [-19.7...]               [6.28...]*I      1.00...           0         0        0]
        [             [-41.3...]*I                [-19.7...]  [6.28...]*I      1.00...        0        0]
        [                [64.9...]              [-41.3...]*I   [-19.7...]  [6.28...]*I  1.00...        0]
        [[-2.96...] + [-7.14...]*I  [-2.96...] + [0.94...]*I  [0.94...]*I            0        0  1.00...]
        sage: mon[1]
        [ [1.27...] + [+/-...]*I [-0.97...] + [+/-...]*I [-1.31...] + [+/-...]*I [-0.86...] + [+/-...]*I [-0.30...] + [+/-...]*I [-1.80...] + [+/-...]*I]
        [[-1.01...] + [+/-...]*I  [4.60...] + [+/-...]*I  [4.86...] + [+/-...]*I  [3.19...] + [+/-...]*I  [1.13...] + [+/-...]*I  [6.65...] + [+/-...]*I]
        [ [1.42...] + [+/-...]*I [-5.04...] + [+/-...]*I [-5.80...] + [+/-...]*I [-4.46...] + [+/-...]*I [-1.58...] + [+/-...]*I [-9.31...] + [+/-...]*I]
        [[-0.63...] + [+/-...]*I  [2.24...] + [+/-...]*I  [3.02...] + [+/-...]*I  [2.98...] + [+/-...]*I  [0.70...] + [+/-...]*I  [4.14...] + [+/-...]*I]
        [[-0.24...] + [+/-...]*I  [0.87...] + [+/-...]*I  [1.18...] + [+/-...]*I  [0.77...] + [+/-...]*I  [1.27...] + [+/-...]*I  [1.61...] + [+/-...]*I]
        [ [0.20...] + [+/-...]*I [-0.73...] + [+/-...]*I [-0.98...] + [+/-...]*I [-0.64...] + [+/-...]*I [-0.23...] + [+/-...]*I [-0.34...] + [+/-...]*I]

    """
    return list(mat for _, mat, _ in _monodromy_matrices(dop, base, eps, sing))

def _test_monodromy_matrices():
    r"""
    TESTS::

        sage: from ore_algebra.analytic.monodromy import _test_monodromy_matrices
        sage: _test_monodromy_matrices()
    """
    from sage.all import matrix
    from ore_algebra import DifferentialOperators
    Dops, x, Dx = DifferentialOperators()

    h = QQ(1)/2
    i = QQi.gen()

    def norm(m):
        return sum(c.abs()**2 for c in m.list()).sqrtpos()

    mon = monodromy_matrices((x**2+1)*Dx-1, QQ(1000000))
    assert norm(mon[0] - CBF(pi).exp()) < RBF(1e-10)
    assert norm(mon[1] - CBF(-pi).exp()) < RBF(1e-10)

    mon = monodromy_matrices((x**2-1)*Dx-1, QQ(0))
    assert all(m == -1 for m in mon)

    dop = (x**2 + 1)*Dx**2 + 2*x*Dx
    mon = monodromy_matrices(dop, QQbar(i+1)) # mon[0] <--> i
    assert norm(mon[0] - matrix(CBF, [[1,pi*(1+2*i)], [0,1]])) < RBF(1e-10)
    assert norm(mon[1] - matrix(CBF, [[1,-pi*(1+2*i)], [0,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(-i+1)) # mon[0] <--> -i
    assert norm(mon[0] - matrix(CBF, [[1,pi*(-1+2*i)], [0,1]])) < RBF(1e-10)
    assert norm(mon[1] - matrix(CBF, [[1,pi*(1-2*i)], [0,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(i)) # mon[0] <--> i
    assert norm(mon[0] - matrix(CBF, [[1,0], [2*pi*i,1]])) < RBF(1e-10)
    assert norm(mon[1] - matrix(CBF, [[1,0], [-2*pi*i,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(i), sing=[QQbar(i)])
    assert len(mon) == 1
    assert norm(mon[0] - matrix(CBF, [[1,0], [2*pi*i,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(i), sing=[QQbar(-i)])
    assert len(mon) == 1
    assert norm(mon[0] - matrix(CBF, [[1,0], [-2*pi*i,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(-i), sing=[QQbar(i)])
    assert len(mon) == 1
    assert norm(mon[0] - matrix(CBF, [[1,0], [-2*pi*i,1]])) < RBF(1e-10)
    mon = monodromy_matrices(dop, QQbar(i), sing=[])
    assert mon == []

    dop = (x**2+1)*(x**2-1)*Dx**2 + 1
    mon = monodromy_matrices(dop, QQ(0), sing=[QQ(1),QQbar(i)])
    m0 = dop.numerical_transition_matrix([0,i+1,2*i,i-1,0])
    assert norm(m0 - mon[0]) < RBF(1e-10)
    m1 = dop.numerical_transition_matrix([0,1-i,2,1+i,0])
    assert norm(m1 - mon[1]) < RBF(1e-10)

    dop = x*(x-3)*(x-4)*(x**2 - 6*x + 10)*Dx**2 - 1
    mon = monodromy_matrices(dop, QQ(-1))
    m0 = dop.numerical_transition_matrix([-1,-i,1,i,-1])
    assert norm(m0 - mon[0]) < RBF(1e-10)
    m1 = dop.numerical_transition_matrix([-1,i/2,3-i/2,3+h,3+i/2,i/2,-1])
    assert norm(m1 - mon[1]) < RBF(1e-10)
    m2 = dop.numerical_transition_matrix([-1,i/2,3+i/2,4-i/2,4+h,4+i/2,i/2,-1])
    assert norm(m2 - mon[2]) < RBF(1e-10)
    m3 = dop.numerical_transition_matrix([-1,3+i+h,3+2*i,-1])
    assert norm(m3 - mon[3]) < RBF(1e-10)
    m4 = dop.numerical_transition_matrix([-1,3-2*i,3-i+h,3-i/2,-1])
    assert norm(m4 - mon[4]) < RBF(1e-10)

    dop = (x-i)**2*(x+i)*Dx - 1
    mon = monodromy_matrices(dop, 0)
    assert norm(mon[0] + i) < RBF(1e-10)
    assert norm(mon[1] - i) < RBF(1e-10)

    dop = (x-i)**2*(x+i)*Dx**2 - 1
    mon = monodromy_matrices(dop, 0)
    m0 = dop.numerical_transition_matrix([0,i+1,2*i,i-1,0])
    assert norm(m0 - mon[0]) < RBF(1e-10)
    m1 = dop.numerical_transition_matrix([0,-i-1,-2*i,-i+1,0])
    assert norm(m1 - mon[1]) < RBF(1e-10)