import os
from pathlib import Path
from pytools import memoize_method
from onir import util, datasets, indices
from onir.interfaces import trec, car, plaintext

_SOURCES = {
    'corpus': 'http://trec-car.cs.unh.edu/datareleases/v1.5/paragraphcorpus-v1.5.tar.xz',
    'train': 'http://trec-car.cs.unh.edu/datareleases/v1.5/train-v1.5.tar.xz',
    'test': 'http://trec-car.cs.unh.edu/datareleases/v1.5/benchmarkY1test.public-v1.5.tar.xz',
    'test-qrels': 'http://trec-car.cs.unh.edu/datareleases/v1.5/trec-car-2017-qrels.tar.gz',
    'test200': 'http://trec-car.cs.unh.edu/datareleases/v1.5/test200-v1.5.tar.xz',
}

@datasets.register('car')
class CarV1Dataset(datasets.IndexBackedDataset):
    """
    An interface to the TREC Complex Answer Retrieval (CAR) dataset.
     > Laura Dietz, Ben Gamari, Jeff Dalton, and Nick Craswell. 2017. TREC Complex Answer Retrieval
     > Overview. In TREC.
    """
    DUA = """Will begin downloading CAR dataset.
Please confirm you agree to the authors' data usage stipulations found at
http://trec-car.cs.unh.edu/"""

    @staticmethod
    def default_config():
        result = datasets.IndexBackedDataset.default_config()
        result.update({
            'subset': 'train-f0',
            'rankfn': 'bm25',
            'ranktopk': 100,
            'rel': 'auto'
        })
        return result

    def __init__(self, config, logger, vocab):
        super().__init__(config, logger, vocab)
        base_path = util.path_dataset(self)
        self.index = indices.AnseriniIndex(os.path.join(base_path, 'anserini'), stemmer='none')
        self.index_stem = indices.AnseriniIndex(os.path.join(base_path, 'anserini.porter'), stemmer='porter')
        self.doc_store = indices.SqliteDocstore(os.path.join(base_path, 'docs.sqlite'))

    def path_segment(self):
        result = '{name}_{subset}_{rankfn}.{ranktopk}'.format(name=self.name, **self.config)
        if self.config['rel'] != 'auto':
            result += '_{rel}'.format(**self.config)
        return result

    def _get_index(self, record):
        return self.index

    def _get_docstore(self):
        return self.doc_store

    def _get_index_for_batchsearch(self):
        return self.index_stem

    def qrels(self, fmt='dict'):
        return self._load_qrels(self.config['subset'], fmt)

    @memoize_method
    def _load_qrels(self, subset, fmt):
        with self.logger.duration('loading qrels'):
            path = util.path_dataset(self)
            rel = self.config['rel']
            qrel_file = os.path.join(path, f'{subset}.{rel}.qrels')
            return trec.read_qrels_fmt(qrel_file, fmt)

    @memoize_method
    def _load_queries_base(self, subset):
        path = os.path.join(util.path_dataset(self), f'{subset}.queries.tsv')
        result = {}
        for cols in plaintext.read_tsv(path):
            result[cols[0]] = ' '.join(cols[1:])
        return result

    def init(self, force=False):
        base_path = util.path_dataset(self)
        base = Path(base_path)

        # DOCUMENT COLLECTION
        idx = [self.index, self.index_stem, self.doc_store]
        self._init_indices_parallel(idx, self._init_doc_iter(), force)

        # TRAIN

        files = {}
        files.update({
            base / f'train-f{f}.auto.qrels': f'train/train.fold{f}.cbor.hierarchical.qrels' for f in range(5)
        })
        files.update({
            base / f'train-f{f}.queries.tsv': f'train/train.fold{f}.cbor.outlines' for f in range(5)
        })
        if force or not all(f.exists() for f in files) and self._confirm_dua():
            with util.download_tmp(_SOURCES['train'], tarf=True) as f:
                for member in f:
                    for f_out, f_in in files.items():
                        if member.name == f_in:
                            if f_out.suffix == '.qrels':
                                self._init_file_copy(f.extractfile(member), f_out, force)
                            elif f_out.suffix == '.tsv':
                                self._init_queryfile(f.extractfile(member), f_out, force)

        # TEST

        files = {
            base / 'test.queries.tsv': 'benchmarkY1test.public/test.benchmarkY1test.cbor.outlines'
        }
        if force or not all(f.exists() for f in files) and self._confirm_dua():
            with util.download_tmp(_SOURCES['test'], tarf=True) as f:
                for f_out, f_in in files.items():
                    self._init_queryfile(f.extractfile(f_in), f_out, force)

        files = {
            base / 'test.auto.qrels': 'TREC_CAR_2017_qrels/automatic.benchmarkY1test.cbor.hierarchical.qrels',
            base /'test.manual.qrels': 'TREC_CAR_2017_qrels/manual.benchmarkY1test.cbor.hierarchical.qrels',
        }
        if force or not all(f.exists() for f in files) and self._confirm_dua():
            with util.download_tmp(_SOURCES['test-qrels'], tarf=True) as f:
                for f_out, f_in in files.items():
                    self._init_file_copy(f.extractfile(f_in), f_out, force)

        # TEST200

        files = {
            base / 'test200.auto.qrels': 'test200/train.test200.cbor.hierarchical.qrels',
            base / 'test200.queries.tsv': 'test200/train.test200.cbor.outlines',
        }
        if force or not all(f.exists() for f in files) and self._confirm_dua():
            with util.download_tmp(_SOURCES['test200'], tarf=True) as f:
                for f_out, f_in in files.items():
                    if f_out.suffix == '.qrels':
                        self._init_file_copy(f.extractfile(f_in), f_out, force)
                    elif f_out.suffix == '.tsv':
                        self._init_queryfile(f.extractfile(f_in), f_out, force)

    def _init_doc_iter(self):
        with util.download_tmp(_SOURCES['corpus'], tarf=True) as f:
            cbor_file = f.extractfile('paragraphcorpus/paragraphcorpus.cbor')
            for did, text in self.logger.pbar(car.iter_paras(cbor_file), desc='documents'):
                yield indices.RawDoc(did, text)

    def _init_queryfile(self, in_stream, out_path, force=False):
        if force or not os.path.exists(out_path):
            with util.finialized_file(out_path, 'wt') as out:
                with self.logger.duration(f'extracting to {out_path}'):
                    for qid, headings in car.iter_queries(in_stream):
                        plaintext.write_tsv(out, [(qid, *headings)])

    def _init_file_copy(self, f_in, f_out, force=False):
        if force or not os.path.exists(f_out):
            with util.finialized_file(f_out, 'wb') as f:
                with self.logger.duration(f'extracting to {f_out}'):
                    for block in iter(lambda: f_in.read(2048), b''):
                        f.write(block)


# vim: ai ts=4 sts=4 et sw=4 encoding=utf-8
from string import lower
import time

from django.utils.unittest.case import SkipTest
from nose.plugins.attrib import attr

from framework.base_test import HeadlessRunnerTest
from framework.utils.data_fetcher import fetch_, from_
from pages.questionnairetabpage.questionnaire_tab_page import MANDATORY_FIELD_ERROR_MESSAGE
from pages.dashboardpage.dashboard_page import DashboardPage
from pages.globalnavigationpage.global_navigation_page import GlobalNavigationPage
from pages.loginpage.login_page import LoginPage
from pages.previewnavigationpage.preview_navigation_page import PreviewNavigationPage
from pages.projectspage.projects_page import ProjectsPage
from pages.questionnairetabpage.questionnaire_tab_page import SUCCESS_PROJECT_SAVE_MESSAGE, DUPLICATE_QUESTIONNAIRE_CODE_MESSAGE
from pages.warningdialog.questionnaire_modified_dialog import QuestionnaireModifiedDialog
from pages.warningdialog.redistribute_questionnaire_dialog import RedistributeQuestionnaireDialog
from testdata.test_data import DATA_WINNER_LOGIN_PAGE, DATA_WINNER_SMS_TESTER_PAGE, url, DATA_WINNER_ALL_PROJECTS_PAGE, ACCOUNT_USERS, DATA_WINNER_DASHBOARD_PAGE, LOGOUT
from tests.logintests.login_data import VALID_CREDENTIALS
from tests.projects.questionnairetests.project_questionnaire_data import *
from pages.smstesterpage.sms_tester_page import SMSTesterPage
from pages.warningdialog.warning_dialog import WarningDialog
from framework.utils.common_utils import by_id, by_css, random_string, by_xpath
from tests.submissionlogtests.submission_log_tests import send_sms_with
from tests.editusertests.edit_user_data import SMS_TO_TEST_PERMISSION, SMS
from tests.dashboardtests.dashboard_tests_data import USER_RASITEFA_CREDENTIALS
from tests.alldatasenderstests.all_data_sender_data import *
from time import sleep

CLOSE_WARNING_DIALOGUE_LINK = by_css(
    'div.ui-dialog[style*="block"] > div.ui-dialog-titlebar>a.ui-dialog-titlebar-close')

def verify_on_edit_project_page(verify_edit_page_functionality):
    project_overview_page = verify_edit_page_functionality()
    return project_overview_page.navigate_to_edit_project_page()

class TestProjectQuestionnaire(HeadlessRunnerTest):

    def setUp(self):
        self.driver.go_to(DATA_WINNER_LOGIN_PAGE)
        login_page = LoginPage(self.driver)
        self.driver.create_screenshot("debug-setup-test-project-questionnaire")
        self.global_navigation = login_page.do_successful_login_with(VALID_CREDENTIALS)
        project_questionnaire_data = EDIT_PROJECT_QUESTIONNAIRE_DATA.copy()
        project_questionnaire_data[QUESTIONS][8][NEW_UNIQUE_ID_TYPE] = 'new type'+random_number(3)
        self.project_name, self.questionnaire_code = self._create_project(EDIT_PROJECT_DATA, project_questionnaire_data)

    def tearDown(self):
        self.global_navigation.sign_out()

    @classmethod
    def _create_project(cls, project_data, questionnaire_data):
        global_navigation = GlobalNavigationPage(cls.driver)
        dashboard_page = global_navigation.navigate_to_dashboard_page()
        create_questionnaire_options_page = dashboard_page.navigate_to_create_project_page()
        cls.create_questionnaire_page = create_questionnaire_options_page.select_blank_questionnaire_creation_option()
        cls.create_questionnaire_page.create_questionnaire_with(project_data, questionnaire_data)
        questionnaire_code = cls.create_questionnaire_page.get_questionnaire_code()
        overview_page = cls.create_questionnaire_page.save_and_create_project_successfully()
        cls.questionnaire_tab_page = overview_page.navigate_to_questionnaire_tab()
        return overview_page.get_project_title(), questionnaire_code

    @attr('functional_test')
    def test_editing_existing_questionnaire(self):
        self.global_navigation.navigate_to_view_all_project_page()
        ProjectsPage(self.driver).navigate_to_project_overview_page(self.project_name)\
                                 .navigate_to_questionnaire_tab()
        questionnaire_tab_page = self.questionnaire_tab_page
        self.assertEqual(questionnaire_tab_page.get_select_or_edit_question_message(),
                         "Add or edit a question",
                         "No question should be selected by default")
        self.assertIn(EDIT_PROJECT_DATA[PROJECT_NAME], questionnaire_tab_page.get_questionnaire_heading())
        self.assertEqual(questionnaire_tab_page.get_existing_questions_count(),
                         len(EDIT_PROJECT_QUESTIONNAIRE_DATA[QUESTIONS]), "Question count does not match")
        expected_existing_questions = [question[QUESTION] for question in EDIT_PROJECT_QUESTIONNAIRE_DATA[QUESTIONS]]
        self.assertEqual(questionnaire_tab_page.get_existing_question_list(), expected_existing_questions,
                         "Mismatch in question list")
        questionnaire_tab_page.submit_questionnaire()
        self.assertEqual(questionnaire_tab_page.get_success_message(), SUCCESS_PROJECT_SAVE_MESSAGE, "Saving of unaltered questionnaire failed")

    @attr('functional_test')
    def test_add_question_with_invalid_selections(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.refresh()
        self._validate_word_answer_type()
        self._validate_number_answer_type()
        self._validate_multiple_choice_type()
        self._validate_unique_id_type()

    @attr('functional_test')
    def test_adding_questions_with_valid_answer_types(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.refresh()
        project_title, questionnaire_code = self._create_project(QUESTIONNAIRE_TAB_PROJECT_DATA, QUESTIONNAIRE_TAB_QUESTIONNAIRE_DATA)
        self.do_sms_submission(questionnaire_code, QUESTIONNAIRE_TAB_SUBMISSION_SMS, project_title)
        questionnaire_tab_page.select_question_link(1)
        questionnaire_tab_page.change_answer_type_to_choice_question()
        self.assertEqual("default", questionnaire_tab_page.get_nth_option_choice_text(1), "Default text should be present for 1st choice") #Bug - 3152
        questionnaire_tab_page.delete_question(1)
        self._expect_delete_question_dialog_to_be_shown(proceed=False)
        self.assertEqual(questionnaire_tab_page.get_existing_questions_count(), 1, "Question should not be deleted.")
        questionnaire_tab_page.delete_question(1)
        self._expect_delete_question_dialog_to_be_shown(proceed=True)
        questionnaire_tab_page.submit_questionnaire()
        self._expect_empty_questionnaire_dialog_to_be_shown()
        questionnaire_tab_page.add_questions(ADDITIONAL_TAB_QUESTIONNAIRE_DATA)
        questionnaire_tab_page.set_questionnaire_code(self.questionnaire_code)
        self.driver.find_element_by_name("question_title")
        questionnaire_tab_page.submit_questionnaire()
        self.assertEqual(questionnaire_tab_page.get_error_message(), DUPLICATE_QUESTIONNAIRE_CODE_MESSAGE,"Duplicate error message not shown")
        questionnaire_tab_page.set_questionnaire_code(questionnaire_code)
        questionnaire_tab_page.submit_questionnaire()
        self._expect_redistribute_dialog_to_be_shown()
        self.assertEqual(questionnaire_tab_page.get_success_message(), SUCCESS_PROJECT_SAVE_MESSAGE, "Saving of questionnaire failed")
        self.assertEqual(questionnaire_tab_page.get_existing_questions_count(), 8, "Question count of updated questionnaire does not match")
        self._verify_users_added_to_project()

    @attr('functional_test')
    def test_should_show_warning_popup_when_exiting_a_modified_questionnaire_qre(self):
        modified_warning_dialog = QuestionnaireModifiedDialog(self.driver)
        sleep(2)
        self._verify_edit_dialog_cancel(modified_warning_dialog)
        self._verify_edit_dialog_ignore_changes(modified_warning_dialog)
        self._verify_edit_dialog_save_changes(modified_warning_dialog)

    def _verify_edit_dialog_cancel(self, modified_warning_dialog):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.click_add_question_link()
        questionnaire_tab_page.set_question_title("some question")
        questionnaire_tab_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[0])
        self.assertTrue(modified_warning_dialog.is_hidden())
        self.global_navigation.navigate_to_all_data_page()
        self.assertTrue(modified_warning_dialog.is_visible(), "Should show modified warning dialog")
        modified_warning_dialog.cancel()
        self.assertEqual(questionnaire_tab_page.get_questionnaire_heading(), self.project_name)

    def _verify_edit_dialog_ignore_changes(self, modified_warning_dialog):
        questionnaire_tab_page = self.questionnaire_tab_page
        all_projects_page = self.global_navigation.navigate_to_view_all_project_page()
        self.assertTrue(modified_warning_dialog.is_visible(), "Should show modified warning dialog")
        modified_warning_dialog.ignore_changes()
        all_projects_page.navigate_to_project_overview_page(self.project_name).navigate_to_questionnaire_tab()
        self.assertNotIn("some question", questionnaire_tab_page.get_existing_question_list(), "Newly added question should not have been saved")

    def _verify_edit_dialog_save_changes(self, modified_warning_dialog):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.click_add_question_link()
        questionnaire_tab_page.set_question_title("some question")
        questionnaire_tab_page.change_question_type(WATERPOINT_QUESTIONNAIRE_DATA[QUESTIONS][4])
        all_projects_page = self.global_navigation.navigate_to_view_all_project_page()
        self.assertTrue(modified_warning_dialog.is_visible(), "Should show modified warning dialog");
        modified_warning_dialog.save_changes()
        self._expect_redistribute_dialog_to_be_shown()
        all_projects_page.wait_for_page_to_load()
        all_projects_page.navigate_to_project_overview_page(self.project_name).navigate_to_questionnaire_tab()
        self.assertIn("some question", questionnaire_tab_page.get_existing_question_list(), "Newly added question should be saved")

    def do_sms_submission(self, questionnaire_code, sms_data, project_title):
        self.driver.go_to(DATA_WINNER_SMS_TESTER_PAGE)
        sms_tester_page = SMSTesterPage(self.driver)
        sms_data[SMS] = sms_data[SMS] % questionnaire_code
        sms_tester_page.send_sms_with(sms_data)
        message = sms_tester_page.get_response_message()
        self.assertTrue(fetch_(SUCCESS_MESSAGE, sms_data) in message, "message:" + message)
        self.driver.go_to(DATA_WINNER_ALL_PROJECTS_PAGE)
        ProjectsPage(self.driver).navigate_to_project_overview_page(project_title)\
                                 .navigate_to_questionnaire_tab()

    def _validate_multiple_choice_type(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.click_add_question_link()
        questionnaire_tab_page.set_question_title("Choice question")
        questionnaire_tab_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[3])
        questionnaire_tab_page.click_add_question_link()
        self._validate_errored_choice_inputs()
        self._validate_valid_choice_inputs()
        #cleaning up state
        questionnaire_tab_page.delete_question(11)
        questionnaire_tab_page.delete_question(10)

    def _validate_valid_choice_inputs(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.change_nth_option_of_choice(1, "choice1")
        questionnaire_tab_page.change_nth_option_of_choice(2, "choice2")
        questionnaire_tab_page.change_nth_option_of_choice(3, "choice3")
        questionnaire_tab_page.click_add_question_link()
        self.assertEqual(questionnaire_tab_page.get_existing_questions_count(), 11,
                         "Choice question should not have validation errors")

    def _validate_errored_choice_inputs(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        is_visible, message = questionnaire_tab_page.get_choice_error_message(index=1)
        self.assertTrue(is_visible, "Choice1 error message message not displayed")
        self.assertEqual(message, MANDATORY_FIELD_ERROR_MESSAGE, "Error message is incorrect for choice1")
        is_visible, message = questionnaire_tab_page.get_choice_error_message(index=2)
        self.assertTrue(is_visible, "Choice2 error message message not displayed")
        self.assertEqual(message, MANDATORY_FIELD_ERROR_MESSAGE, "Error message is incorrect for choice2")
        is_visible, message = questionnaire_tab_page.get_choice_error_message(index=3)
        self.assertTrue(is_visible, "Choice3 error message message not displayed")
        self.assertEqual(message, MANDATORY_FIELD_ERROR_MESSAGE, "Error message is incorrect for choice3")
        questionnaire_tab_page.delete_option_for_multiple_choice_question(3)
        questionnaire_tab_page.delete_option_for_multiple_choice_question(2)
        questionnaire_tab_page.add_choice_option_to_selected_question()
        is_visible, message = questionnaire_tab_page.get_choice_error_message(index=2)
        self.assertFalse(is_visible, "Newly added question should not have error message")
        questionnaire_tab_page.add_choice_option_to_selected_question()
        is_visible, message = questionnaire_tab_page.get_choice_error_message(index=3)
        self.assertFalse(is_visible, "Newly added question should not have error message")

    def _validate_number_answer_type(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.click_add_question_link()
        questionnaire_tab_page.set_question_title("Some title")
        self._validate_range_with_non_number_input()
        self._validate_range_for_max_less_than_min_input()
        self._validate_range_for_valid_input()
        #cleaning up state
        questionnaire_tab_page.delete_question(11)
        questionnaire_tab_page.delete_question(10)

    def _validate_range_for_valid_input(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.set_min_range_limit(5)
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_min_range_error_message()
        self.assertFalse(is_visible, "Min range error message should not displayed for valid input")
        is_visible, message = questionnaire_tab_page.get_max_range_error_message()
        self.assertFalse(is_visible, "Max range error message should not displayed for valid input")

    def _validate_range_for_max_less_than_min_input(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.change_number_question_limit(max_value=10, min_value=100)
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_max_range_error_message()
        self.assertTrue(is_visible, "Max range error message not displayed")
        self.assertEqual(message, "Max should be greater than min.", "Error message is incorrect")
        is_visible, message = questionnaire_tab_page.get_min_range_error_message()
        self.assertFalse(is_visible, "Min range error message should not displayed for valid input")

    def _validate_range_with_non_number_input(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[1])
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_min_range_error_message()
        self.assertTrue(is_visible, "Min range error message not displayed")
        self.assertEqual(message, "Please insert a valid number.", "Error message is incorrect")
        is_visible, message = questionnaire_tab_page.get_max_range_error_message()
        self.assertTrue(is_visible, "Max range error message not displayed")
        self.assertEqual(message, "Please insert a valid number.", "Error message is incorrect")

    def _validate_word_answer_type(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.click_add_question_link()
        questionnaire_tab_page.set_question_title("Some title")
        self._validate_max_length_for_invalid_entry()
        self._validate_max_length_for_mandatory_entry()
        self._validate_max_length_for_valid_entry()
        #cleaning up state
        questionnaire_tab_page.delete_question(11)
        questionnaire_tab_page.delete_question(10)

    def _validate_max_length_for_invalid_entry(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[0])
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_max_length_error_message()
        self.assertTrue(is_visible, "Max length error message not displayed")
        self.assertEqual(message, "Please insert a valid number.", "Error message is incorrect")

    def _validate_max_length_for_mandatory_entry(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.set_word_question_max_length("")
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_max_length_error_message()
        self.assertTrue(is_visible, "Max length error message not displayed")
        self.assertEqual(message, "This field is required.", "Error message is incorrect")

    def _validate_max_length_for_valid_entry(self):
        questionnaire_tab_page = self.questionnaire_tab_page
        questionnaire_tab_page.set_word_question_max_length(10)
        questionnaire_tab_page.click_add_question_link()
        is_visible, message = questionnaire_tab_page.get_max_length_error_message()
        self.assertFalse(is_visible, "Max length error message should not be displayed for valid length")

    def test_successful_questionnaire_editing(self):
        """
        Function to test the successful editing of a Questionnaire with given details
        """
        create_questionnaire_page = self.create_or_navigate_to_project_questionnaire_page()

        new_questionnaire_code = "code" + random_string(3)
        self.verify_questions(create_questionnaire_page)

        def verify_warning_for_deleting_an_option_in_multi_choice_question():
            create_questionnaire_page.select_question_link(5)
            create_questionnaire_page.delete_option_for_multiple_choice_question(2)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_adding_an_option_to_multi_choice_question():
            create_questionnaire_page.select_question_link(5)
            create_questionnaire_page.add_option_to_a_multiple_choice_question("new option")
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_numeric_range_modification():
            create_questionnaire_page.select_question_link(2)
            create_questionnaire_page.change_number_question_limit(max_value=30, min_value=5)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_word_type_for_character_length_change():
            create_questionnaire_page.select_question_link(1)
            create_questionnaire_page.set_word_question_max_length(25)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_date_format_change():
            create_questionnaire_page.select_question_link(3)
            create_questionnaire_page.change_date_type_question(MM_DD_YYYY)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_addintion_of_new_question():
            new_question = {"question": "newly added question", "code": "grades", "type": "number",
                            "min": "1", "max": "100"}
            time.sleep(2)
            create_questionnaire_page.add_question(new_question)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_change_in_questionnaire_code():
            create_questionnaire_page.set_questionnaire_code(new_questionnaire_code)
            #to get the focus out
            create_questionnaire_page.select_question_link(3)
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        def verify_warning_for_change_in_question_text():
            create_questionnaire_page.change_question_text(3, "question number 3")
            return self._expect_redistribute_dialog_to_be_shown(create_questionnaire_page)

        verify_on_edit_project_page(verify_warning_for_adding_an_option_to_multi_choice_question)
        verify_on_edit_project_page(verify_warning_for_deleting_an_option_in_multi_choice_question)
        verify_on_edit_project_page(verify_warning_for_numeric_range_modification)
        verify_on_edit_project_page(verify_warning_for_word_type_for_character_length_change)
        verify_on_edit_project_page(verify_warning_for_date_format_change)
        verify_on_edit_project_page(verify_warning_for_addintion_of_new_question)
        verify_on_edit_project_page(verify_warning_for_change_in_questionnaire_code)
        verify_on_edit_project_page(verify_warning_for_change_in_question_text)

        self.sms_preview_of_questionnaire_on_the_questionnaire_tab(self.project_name)
        self.web_preview_of_questionnaire_on_the_questionnaire_tab()
        self.smart_phone_preview_of_questionnaire_on_the_questionnaire_tab()

        self.verify_sms_submission_after_edit(new_questionnaire_code)

    def verify_questions(self, create_questionnaire_page):
        questions = fetch_(QUESTIONS, from_(QUESTIONNAIRE_DATA_CLINIC_PROJECT))
        #create_questionnaire_page.select_question_link(2)
        #self.driver.find(by_css("#question_title")).click()
        #assert self.driver.find_visible_element(by_id("periode_green_message"))
        #create_questionnaire_page.select_question_link(3)
        #tip_message_element = self.driver.find(by_id('periode_green_message'))
        #self.assertFalse(tip_message_element.is_displayed())
        # create_questionnaire_page.select_question_link(1)
        self.assertEqual(questions[0], create_questionnaire_page.get_word_type_question())
        create_questionnaire_page.select_question_link(2)
        self.assertEqual(questions[1], create_questionnaire_page.get_number_type_question())
        create_questionnaire_page.select_question_link(3)
        self.assertEqual(questions[2], create_questionnaire_page.get_date_type_question())
        create_questionnaire_page.select_question_link(4)
        self.assertEqual(questions[3], create_questionnaire_page.get_list_of_choices_type_question())
        create_questionnaire_page.select_question_link(5)
        self.assertEqual(questions[4], create_questionnaire_page.get_list_of_choices_type_question())

    def sms_preview_of_questionnaire_on_the_questionnaire_tab(self, project_name):
        preview_navigation_page = PreviewNavigationPage(self.driver)
        sms_questionnaire_preview_page = preview_navigation_page.sms_questionnaire_preview()

        self.assertIsNotNone(sms_questionnaire_preview_page.sms_questionnaire())
        self.assertEqual(sms_questionnaire_preview_page.get_project_name(), project_name)
        self.assertIsNotNone(sms_questionnaire_preview_page.get_sms_instruction())

        sms_questionnaire_preview_page.close_preview()
        self.assertFalse(sms_questionnaire_preview_page.sms_questionnaire_exist())

    def web_preview_of_questionnaire_on_the_questionnaire_tab(self):
        preview_navigation_page = PreviewNavigationPage(self.driver)
        web_questionnaire_preview_page = preview_navigation_page.web_questionnaire_preview()

        self.assertIsNotNone(web_questionnaire_preview_page.get_web_instruction())
        web_questionnaire_preview_page.close_preview()

    def smart_phone_preview_of_questionnaire_on_the_questionnaire_tab(self):
        preview_navigation_page = PreviewNavigationPage(self.driver)
        smart_phone_preview_page = preview_navigation_page.smart_phone_preview()
        self.assertIsNotNone(smart_phone_preview_page.get_smart_phone_instruction())
        smart_phone_preview_page.close_preview()

    def add_question_to_project(self, create_questionnaire_page):
        question_name = "how many grades did you get last year?"
        new_question = {"question": question_name, "code": "grades", "type": "number",
                        "min": "1", "max": "100"}
        create_questionnaire_page.add_question(new_question)
        create_questionnaire_page.save_and_create_project_successfully()

        return question_name

    def confirm_warning_dialog(self, element_id):
        confirm_locator = by_id(element_id)
        warning_dialog = WarningDialog(self.driver, confirm_link=confirm_locator)
        warning_dialog.confirm()

    def _expect_redistribute_dialog_to_be_shown(self):
        warning_dialog = RedistributeQuestionnaireDialog(self.driver)
        message = warning_dialog.get_message()
        self.assertEqual(message, REDISTRIBUTE_QUESTIONNAIRE_MSG)
        warning_dialog.confirm()

    def _expect_delete_question_dialog_to_be_shown(self, proceed=False):
        warning_dialog = WarningDialog(self.driver)
        message = warning_dialog.get_message()
        self.assertEqual(message, DELETE_QUESTION_MSG)
        if proceed:
            warning_dialog.confirm()
        else:
            warning_dialog.cancel()

    def _expect_empty_questionnaire_dialog_to_be_shown(self):
        warning_dialog = WarningDialog(self.driver, confirm_link=by_xpath(".//*[@id='no_questions_exists']/div/a"))
        message = warning_dialog.get_message()
        self.assertEqual(message, EMPTY_QUESTIONNAIRE_MSG)
        warning_dialog.confirm()

    def create_or_navigate_to_project_questionnaire_page(self):
        project_overview_page = self.create_new_project()
        return project_overview_page.navigate_to_questionnaire_tab()

    @classmethod
    def create_new_project(cls):
        if cls.project_name:
            all_project_page = cls.global_navigation.navigate_to_view_all_project_page()
            overview_page = all_project_page.navigate_to_project_overview_page(cls.project_name)
            return overview_page

        dashboard_page = cls.global_navigation.navigate_to_dashboard_page()
        questionnaire_creation_options_page = dashboard_page.navigate_to_create_project_page()
        create_questionnaire_page = questionnaire_creation_options_page.select_blank_questionnaire_creation_option()
        create_questionnaire_page = create_questionnaire_page.create_questionnaire_with(CLINIC_PROJECT_DATA,
                                                                                QUESTIONNAIRE_DATA_CLINIC_PROJECT)

        overview_page = create_questionnaire_page.save_and_create_project_successfully()
        cls.project_name = overview_page.get_project_title()
        return overview_page

    def verify_sms_submission_after_edit(self, new_questionnaire_code):
        self.driver.go_to(DATA_WINNER_SMS_TESTER_PAGE)
        sms_tester_page = SMSTesterPage(self.driver)
        VALID_SMS[SMS] = new_questionnaire_code + ' mino 25 05.12.2010 a d -18.1324,27.6547 45'
        sms_tester_page.send_sms_with(VALID_SMS)
        message = sms_tester_page.get_response_message()
        self.assertTrue(fetch_(SUCCESS_MESSAGE, VALID_SMS) in message, "message:" + message)

    def goto_dashboard(self):
        self.driver.go_to(url("/dashboard/"))
        return DashboardPage(self.driver)

    def _validate_unique_id_type(self):
        create_questionnaire_page = self.create_questionnaire_page
        create_questionnaire_page.click_add_question_link()
        create_questionnaire_page.set_question_title("Unique Id question")
        create_questionnaire_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[6])
        create_questionnaire_page.click_add_question_link()
        self._validate_errored_unique_id_input(create_questionnaire_page)

        create_questionnaire_page.delete_question(10)
        create_questionnaire_page.click_add_question_link()
        create_questionnaire_page.set_question_title("Unique Id question")
        create_questionnaire_page.change_question_type(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[7])
        self._validate_duplicate_unique_id(create_questionnaire_page)
        #cleaning up state
        create_questionnaire_page.delete_question(10)
        self.driver.refresh()

    def _validate_errored_unique_id_input(self, create_questionnaire_page):
        is_visible, message = create_questionnaire_page.get_unique_id_error_msg()
        self.assertTrue(is_visible, "Mandatory validaton error msg for unique id not visible")
        self.assertEqual(message, MANDATORY_FIELD_ERROR_MESSAGE, "Error message is incorrect for choice1")

    def _validate_duplicate_unique_id(self, create_questionnaire_page):
        new_type_name = fetch_(EXISTING_UNIQUE_ID_TYPE, from_(QUESTIONS_WITH_INVALID_ANSWER_DETAILS[7]))

        create_questionnaire_page.add_new_unique_id_type(new_type_name)
        is_visible, message = create_questionnaire_page.get_new_unique_id_error_msg()
        self.assertTrue(is_visible)
        self.assertEqual(message, '%s already exists.' % new_type_name.capitalize())

    def _verify_users_added_to_project(self):
        self.driver.find(by_xpath("//a[text()='My Data Senders']")).click()
        registered_ds_names = [element.text for element in self.driver.find_elements_(by_xpath("//*[@id='datasender_table']/tbody/tr/td[2]"))]
        self.driver.go_to(ACCOUNT_USERS)
        user_names = [element.text for element in self.driver.find_elements_(by_xpath("//*[@id='users_list']/table/tbody/tr/td[2]"))]

    @attr('functional_test')
    def test_pm_should_not_have_ds_permission_to_newly_created_questionnaire(self):
        dashboard_page = self.global_navigation.navigate_to_dashboard_page()
        questionnaire_creation_options_page = dashboard_page.navigate_to_create_project_page()
        create_questionnaire_page = questionnaire_creation_options_page.select_blank_questionnaire_creation_option()
        create_questionnaire_page = create_questionnaire_page.create_questionnaire_with(CLINIC_PROJECT_DATA,
                                                                                QUESTIONNAIRE_DATA_CLINIC_PROJECT)
        overview_page = create_questionnaire_page.save_and_create_project_successfully()
        my_ds_page = overview_page.navigate_to_datasenders_page()
        questionnaire_code = overview_page.get_questionnaire_code()
        SMS_TO_TEST_PERMISSION[SMS] = CLINIC_SUCCESS_SMS % questionnaire_code
        response = send_sms_with(SMS_TO_TEST_PERMISSION)
        self.assertEqual(response, SUCCESS_MESSAGE_MSG)

        my_ds_page.open_setting_popup()
        my_ds_page.set_setting_value('restricted')
        my_ds_page.save_setting()

        response = send_sms_with(SMS_TO_TEST_PERMISSION)
        self.assertEqual(response, ERROR_MESSAGE_MSG)

    @attr('functional_test')
    def test_should_check_if_deleted_questionnaires_and_poll_not_be_in_questionnaires_list(self):
        self.driver.go_to(LOGOUT)
        self.driver.go_to(DATA_WINNER_LOGIN_PAGE)
        login_page = LoginPage(self.driver)
        login_page.do_successful_login_with(USER_RASITEFA_CREDENTIALS)
        self.driver.go_to(DATA_WINNER_DASHBOARD_PAGE)
        dashboard_page = DashboardPage(self.driver)
        self.assertEqual(len(dashboard_page.get_projects_list()), 3)
        create_questionnaire_options_page = dashboard_page.navigate_to_create_project_page()

        create_questionnaire_page = create_questionnaire_options_page.select_blank_questionnaire_creation_option()
        create_questionnaire_page.create_questionnaire_with(NEW_PROJECT, QUESTIONNAIRE_DATA)
        global_navigation = GlobalNavigationPage(self.driver)
        project_overview_page = create_questionnaire_page.save_and_create_project_successfully()
        project_name = project_overview_page.get_project_title()
        all_projects_page = global_navigation.navigate_to_view_all_project_page()
        all_projects_page.delete_project(project_name)
        self.driver.go_to(DATA_WINNER_DASHBOARD_PAGE)
        dashboard_page = DashboardPage(self.driver)
        all_projects_page = ProjectsPage(self.driver)
        self.assertEqual(len(dashboard_page.get_projects_list()), 3)
        self.assertFalse(all_projects_page.is_project_present(project_name))


# REPLAY MEMORY
import lmdb
import json
import numpy as np
import psutil
import os
import time
from utils import show_RAM_usage, show_image
from collections import deque, OrderedDict
from IPython import display

print('initialized with priority experience replay memory')

class SumTree:
    # little modified from https://github.com/jaromiru/AI-blog/blob/master/SumTree.py
    def __init__(self, capacity):
        self.capacity = capacity    # N, the size of replay buffer, so as to the number of sum tree's leaves
        self.tree = np.zeros(2 * capacity - 1)  # equation, to calculate the number of nodes in a sum tree
        self.transitions = np.empty(capacity, dtype=object)
        self.next_idx = 0

    @property
    def total_p(self):
        return self.tree[0]

    def add(self, priority, transition):
        idx = self.next_idx + self.capacity - 1
        self.transitions[self.next_idx] = transition
        self.update(idx, priority)
        self.next_idx = (self.next_idx + 1) % self.capacity

    def update(self, idx, priority):
        change = priority - self.tree[idx]
        self.tree[idx] = priority
        self._propagate(idx, change)    # O(logn)

    def _propagate(self, idx, change):
        parent = (idx - 1) // 2
        self.tree[parent] += change
        if parent != 0:
            self._propagate(parent, change)

    def get_leaf(self, s):
        idx = self._retrieve(0, s)   # from root
        trans_idx = idx - self.capacity + 1
        return idx, self.tree[idx], self.transitions[trans_idx]

    def _retrieve(self, idx, s):
        left = 2 * idx + 1
        right = left + 1
        if left >= len(self.tree):
            return idx
        if s <= self.tree[left]:
            return self._retrieve(left, s)
        else:
            return self._retrieve(right, s - self.tree[left])

class Memory():
    def __init__(self, maxlen, env_name, version, render=False):
        self.map_size = 104857600
        self.path = 'memory/' + env_name + version
        self.env = None
        self.env_name = env_name
        self.state_store = OrderedDict()
        self.state_store_disc = None
        self.memory_store = None
        self.parameter_store = None
        self.memory = None
        self.render = render
        self.beta=0.4
        self.beta_increment_per_sample=0.001
        self.maxlen=maxlen
        self.first = False
        self.keycount = 0
        self.first_frame = None
        self.memory_counter = 0

    def setup(self):
        env = lmdb.open(path = self.path, map_size=self.map_size,  max_dbs=3, writemap=True)
        self.env = env
        self.memory = SumTree(capacity=self.maxlen)
        self.parameter_store = self.env.open_db(b'agent_parameters')
        self.first = True

    def generate_key(self):
        self.keycount += 1
        key = ''.join([self.env_name, str(self.keycount).zfill(7)])
        return key

    def save_state(self, state):
        key = self.generate_key()
        print('saving key', key)

        self.state_store[key] = state
        if len(self.state_store) > self.maxlen+4:
            self.state_store.popitem(False)
        return key

    def get_state(self, state_keys):
        states = []
        for key in state_keys:
            if key == 'first_frame':
                states.append(self.first_frame)
            else:
                states.append(self.state_store[key])
        return states

    def save_memory(self, memory):
        start = time.time()
        print('saving memory',memory['nextState'])

        if self.first:
            priority = 1
            self.first = False
        else:
            priority = np.max(self.memory.tree[-self.memory.capacity:])
            if self.render:
                print('saving memory with priority {}'.format(priority), '\n')
                print(memory)
                #time.sleep(2)
        self.memory.add(priority, memory)
        self.memory_counter += 1

    def load_minibatch(self, batchsize):
        #print('loading minibatch from replayMemory...')
        start = time.time()
        minibatch = []

        self.beta = min(1., self.beta + self.beta_increment_per_sample)
        # calculate max_weight
        segment = self.memory.total_p / batchsize
        min_prob = 0.01     # for later calculate ISweight
        for i in range(batchsize):
            s = np.random.uniform(segment * i, segment * (i + 1))
            idx, prio, batch = self.memory.get_leaf(s)
            probability = prio / self.memory.total_p
            ISWeight  = np.power(probability/min_prob, -self.beta)

            # get idx and weight
            batch_ready = {}
            batch_ready['state'] = self.get_state(batch['state'])
            batch_ready['nextState']  = self.state_store[batch['nextState']]
            batch_ready['action'] = batch['action']
            batch_ready['reward'] = batch['reward']
            batch_ready['done'] = batch['done']
            batch_ready['idx'] = idx
            batch_ready['weigth'] = np.array([ISWeight])
            minibatch.append(batch_ready.copy())

            if self.render:
                print('loading minibatch ', prio, ' weight: ', ISWeight)
                print('currentState')
                print('keys: ', batch['state'], '\nnext', batch['nextState'])
                for frame in batch_ready['state']:
                    show_image(frame)
                print('nextState', 'action: ', batch_ready['action'])
                show_image(batch_ready['nextState'])
                #time.sleep(2)
                display.clear_output(wait=True)

        return minibatch

    def save_parameters(self, agent):

        parameters = {'epsilon': agent.epsilon,
                      'global_steps': agent.global_steps,
                      'training_steps': agent.training_steps,
                      'mean_rewards': agent.mean_rewards,
                      'global_episodes': agent.global_episodes}

        if agent.render:
            print('saving agent parameters\n', parameters)
        value_encoded = json.dumps(parameters)
        value_encoded = value_encoded.encode()
        key = agent.version.encode()
        with self.env.begin(db=self.parameter_store, write=True) as txn:
            txn.put(key=key, value=value_encoded, dupdata=False)

    def load_parameters(self, version):
        key = version.encode()
        with self.env.begin(db=self.parameter_store) as txn:
            cursor = txn.cursor()
            value = cursor.get(key)
        parameters = json.loads(value.decode())
        return parameters

    """
    def save_experience(self):
        with self.env.begin(write=True) as txn:
            txn.drop(db=self.state_store_disc)
        with self.env.begin(db=self.state_store_disc, write=True) as txn:
            for key, state in self.state_store.items():
                value_encoded = json.dumps(state.tolist()).encode()
                txn.put(key=key.encode(), value=value_encoded, dupdata=False)

    def load_experience(self):
        print('loading experience from state store on disc... ')
        counter = 0
        with self.env.begin(db=self.state_store_disc) as txn:
            cursor = txn.cursor()
            while cursor.next():
                counter += 1
                if counter % 1000 == 0:
                    print('loaded {} states from disc'.format(counter))
                self.state_store[cursor.key().decode()] = np.array(json.loads(cursor.value().decode()))
    """


"""Define the top-level cli command."""
import asyncio
import logging
import sys

import click
from loguru import logger

from scrapd.cli.base import AbstractCommand
from scrapd.core import apd
from scrapd.core.formatter import Formatter
from scrapd.core.version import detect_from_metadata

# Set the project name.
APP_NAME = 'scrapd'

# Retrieve the project version from packaging.
__version__ = detect_from_metadata(APP_NAME)

# pylint: disable=unused-argument
#   The arguments are used via the `self.args` dict of the `AbstractCommand` class.
@click.version_option(version=__version__)
@click.command()
@click.option('-a', '--attempts', type=click.INT, default=3, help='number of attempts per report', show_default=True)
@click.option('-b', '--backoff', type=click.INT, default=3, help='initial backoff time (second)', show_default=True)
@click.option('--dump', is_flag=True, help='dump reports with parsing issues', show_default=True)
@click.option(
    '-f',
    '--format',
    'format_',
    type=click.Choice(sorted(Formatter.formatters)),
    default='json',
    help='specify output format',
    show_default=True,
)
@click.option('--from', 'from_', help='start date')
@click.option('--pages', default=-1, help='number pages to process')
@click.option('--to', help='end date')
@click.option('-v', '--verbose', count=True, help='adjust the log level')
@click.pass_context
def cli(ctx, attempts, backoff, dump, format_, from_, pages, to, verbose):  # noqa: D403
    """Retrieve APD's traffic fatality reports."""
    ctx.obj = {**ctx.params}
    ctx.auto_envvar_prefix = 'VZ'

    # Configure logger.
    INITIAL_LOG_LEVEL = logging.WARNING
    LOG_FORMAT_COMPACT = "<level>{message}</level>"
    LOG_FORMAT_VERBOSE = "<level>{time:YYYY-MM-DDTHH:mm:ssZZ} {name}:{line:<4} {message}</level>"
    log_level = max(INITIAL_LOG_LEVEL - verbose * 10, 0)
    log_format = LOG_FORMAT_VERBOSE if log_level < logging.INFO else LOG_FORMAT_COMPACT

    # Remove any predefined logger.
    logger.remove()

    # Set the log colors.
    logger.level('ERROR', color='<red><bold>')
    logger.level('WARNING', color='<yellow>')
    logger.level('SUCCESS', color='<green>')
    logger.level('INFO', color='<cyan>')
    logger.level('DEBUG', color='<blue>')
    logger.level('TRACE', color='<magenta>')

    # Add the logger.
    logger.add(sys.stderr, format=log_format, level=log_level, colorize=True)

    # Prepare the command.
    command = Retrieve(ctx.params, ctx.obj)
    command.execute()

class Retrieve(AbstractCommand):
    """Retrieve APD's traffic fatality reports."""

    def _execute(self):
        """Define the internal execution of the command."""
        # Collect the results.
        results, _ = asyncio.run(
            apd.async_retrieve(
                self.args['pages'],
                self.args['from_'],
                self.args['to'],
                self.args['attempts'],
                self.args['backoff'],
                self.args['dump'],
            ))
        result_count = len(results)
        logger.info(f'Total: {result_count}')

        # Get the format and print the results.
        format_ = self.args['format_'].lower()
        formatter = Formatter(format_)
        formatter.print(results)


import logging
import os
import time

from logging.handlers import TimedRotatingFileHandler

from buildbot.status.base import StatusReceiverMultiService

class StatusEventLogger(StatusReceiverMultiService):
  """Logs all status events to a file on disk. Uses a rotating file log."""
  def __init__(self, logfile='status.log'):
    """Create a StatusEventLogger.

    Args:
      logfile: base filename for events to be written to.
    """
    self.logfile = logfile
    # Will be initialized in startService
    self.logger = None
    self.status = None
    self._active = False
    self._last_checked_active = 0
    # Can't use super because StatusReceiverMultiService is an old-style class.
    StatusReceiverMultiService.__init__(self)

  @property
  def active(self):
    now = time.time()
    # Cache the value for self._active for one minute.
    if now - self._last_checked_active > 60:
      self._active = os.path.isfile(
          os.path.join(self.parent.basedir, '.logstatus'))
      self._last_checked_active = now
    return self._active

  def startService(self):
    """Start the service and subscribe for updates."""
    logger = logging.getLogger(__name__)
    logger.propagate = False
    logger.setLevel(logging.INFO)
    # %(bbEvent)19s because builderChangedState is 19 characters long
    formatter = logging.Formatter('%(asctime)s - %(bbEvent)19s - %(message)s')
    # Use delay=True so we don't open an empty file while self.active=False.
    handler = TimedRotatingFileHandler(
        os.path.join(self.parent.basedir, self.logfile),
        when='H', interval=1, delay=True)
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    self.logger = logger

    StatusReceiverMultiService.startService(self)
    self.status = self.parent.getStatus()
    self.status.subscribe(self)

  def log(self, event, message, *args):
    """Simple wrapper for log. Passes string formatting args through."""
    if self.active:
      self.logger.info(message, *args, extra={'bbEvent': event})

  def requestSubmitted(self, request):
    builderName = request.getBuilderName()
    self.log('requestSubmitted', '%s, %r', builderName, request)

  def requestCancelled(self, builder, request):
    builderName = builder.getName()
    self.log('requestCancelled', '%s, %r', builderName, request)

  def buildsetSubmitted(self, buildset):
    reason = buildset.getReason()
    self.log('buildsetSubmitted', '%r, %s', buildset, reason)

  def builderAdded(self, builderName, builder):
    # Use slavenames rather than getSlaves() to just get strings.
    slaves = builder.slavenames
    self.log('builderAdded', '%s, %r', builderName, slaves)
    # Must return self in order to subscribe to builderChangedState and
    # buildStarted/Finished events.
    return self

  def builderChangedState(self, builderName, state):
    self.log('builderChangedState', '%s, %r', builderName, state)

  def buildStarted(self, builderName, build):
    build_number = build.getNumber()
    slave = build.getSlavename()
    self.log('buildStarted', '%s, %d, %s', builderName, build_number, slave)
    # Must return self in order to subscribe to stepStarted/Finished events.
    return self

  def buildETAUpdate(self, build, ETA):
    # We don't actually care about ETA updates; they happen on a periodic clock.
    pass

  def changeAdded(self, change):
    self.log('changeAdded', '%r', change)

  def stepStarted(self, build, step):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    self.log('stepStarted', '%s, %d, %s', build_name, build_number, step_name)
    # Must return self in order to subscribe to logStarted/Finished events.
    return self

  def stepTextChanged(self, build, step, text):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    self.log('stepTextChanged', '%s, %d, %s, %s',
             build_name, build_number, step_name, text)

  def stepText2Changed(self, build, step, text2):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    self.log('stepText2Changed', '%s, %d, %s, %s',
             build_name, build_number, step_name, text2)

  def stepETAUpdate(self, build, step, ETA, expectations):
    # We don't actually care about ETA updates; they happen on a periodic clock.
    pass

  def logStarted(self, build, step, log):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    log_name = log.getName()
    log_file = log.filename
    self.log('logStarted', '%s, %d, %s, %s, %s',
             build_name, build_number, step_name, log_name, log_file)
    # Create an attr on the stateful log object to count its chunks.
    log.__num_chunks = 0
    # Must return self in order to subscribe to logChunk events.
    return self

  def logChunk(self, build, step, log, channel, text):
    # Like the NSA, we only want to process metadata.
    log.__num_chunks += 1

  def logFinished(self, build, step, log):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    log_name = log.getName()
    log_file = log.filename
    log_size = log.length
    # Access to protected member __num_chunks. pylint: disable=W0212
    log_chunks = log.__num_chunks
    self.log('logFinished', '%s, %d, %s, %s, %s, %d, %d',
             build_name, build_number, step_name,
             log_name, log_file, log_size, log_chunks)

  def stepFinished(self, build, step, results):
    build_name = build.getBuilder().name
    build_number = build.getNumber()
    step_name = step.getName()
    self.log('stepFinished', '%s, %d, %s, %r',
             build_name, build_number, step_name, results)

  def buildFinished(self, builderName, build, results):
    build_number = build.getNumber()
    slave = build.getSlavename()
    self.log('buildFinished', '%s, %d, %s, %r',
             builderName, build_number, slave, results)

  def builderRemoved(self, builderName):
    self.log('builderRemoved', '%s', builderName)

  def slaveConnected(self, slaveName):
    self.log('slaveConnected', '%s', slaveName)

  def slaveDisconnected(self, slaveName):
    self.log('slaveDisconnected', '%s', slaveName)


from django.conf import settings
from django.http import Http404, HttpResponse
from django.shortcuts import render, redirect
from django.contrib.auth.decorators import login_required
from django.contrib.admin.views.decorators import staff_member_required

from celery.result import AsyncResult

from .models import Guest, GuestNameChange

common_context = {
    "event_name": "King's Affair 2017",
    "site_name": "Online Ticketing System"
}

@login_required
def tickets(request):
    tickets = Guest.objects.filter(owner=request.user).order_by('id')
    primary = tickets.get(parent__isnull=True)

    ctx = {
        "user": request.user,
        "primary": primary,
        "tickets": tickets,
        'hidereturn': True,
        'pending_nc': GuestNameChange.has_pending_namechange(request.user),
        'total_nc_cost': GuestNameChange.namechange_total_cost(request.user)
    }

    ctx.update(common_context)

    return render(request, 'original/ticket_details.tpl', context=ctx)

@login_required
def user(request):
    return render(request, 'original/user.tpl')

def faq(request):
    ctx = {
        'page_name': 'Frequently Asked Questions'
    }

    ctx.update(common_context)

    return render(request, 'original/faq.tpl', context=ctx)

@staff_member_required(login_url=None)
def admin(request):
    return render(request, 'original/admin.tpl', context=common_context)

def terms(request):
    return render(request, 'original/terms.tpl', context=common_context)

def extract_nc(post_data, i):
    names = ['tid', 'change', 'fname', 'lname']
    out = {}

    for name in names:
        key = 'g%d_%s' % (i, name)
        if key not in post_data:
            return None

        out[name] = post_data[key]

    if len(out) != len(names):
        return None

    return out

@staff_member_required
def ticket_pdf_status(request):
    if 'jid' not in request.GET:
        raise Http404()

    jid = request.GET['jid']

    result = AsyncResult(jid)

    out = '{"state":"%s", "progress": %s}' % (result.state, str(result.result or result.state))

    return HttpResponse(out)

@staff_member_required
def ticket_pdf(request):
    f = open(settings.PDF_OUTPUT_LOCATION, 'rb')

    response = HttpResponse(content=f)
    response['Content-Type'] = 'application/pdf'
    return response

@login_required
def namechange(request):
    if request.method == 'POST':
        count = int(request.POST['g_count'])

        for i in range(count):
            details = extract_nc(request.POST, i)

            if details is None or not details['change']:
                continue

            # create name change for the user
            # check that the user is the owner of this ticket
            ticket = Guest.objects.get(id=details['tid'])

            # print(ticket.owner.id, details.id)
            if ticket.owner != request.user:
                # hack hack hack
                continue

            ticket.create_namechange(details['fname'], details['lname'])

        return redirect('tickets')
    else:
        tickets = Guest.objects.filter(owner=request.user).order_by('id')
        guests = tickets.exclude(parent__isnull=True)

        ctx = {
            "guests": guests
        }

        ctx.update(common_context)

        return render(request, 'original/name_change_request.tpl', context=ctx)


import networkx as nx
import symdump

class SympdumpSorter():
    def __init__(self) -> None:
        self.graph = nx.DiGraph()

        sortable_definitions = [x for x in symdump.SymFile().type_definitions.values() if type(x) is not symdump.symbols.ArraySymbol and not x.is_fake]
        for definition in sortable_definitions:
            self.graph.add_node(definition)
            if definition.children is not None:
                for child in definition.children:
                    if type(child.symbol) is symdump.symbols.ArraySymbol:
                        if child.symbol.tag is not None \
                            and symdump.SymFile().type_definitions.get(child.symbol.tag) is not None \
                            and not child.symbol.is_fake \
                            and child.symbol.tag != type_def.name \
                            and ('pointer', '*{}') not in child.symbol.type_modifiers:
                            self.graph.add_edge(symdump.SymFile().type_definitions[child.symbol.tag], definition)

        self.sorted_graph = nx.topological_sort(self.graph)


# 3.

# print("Hello World \n" * 5 + "I love Python \n" * 5)

# 4.

num = input("Please enter a number: ")
print(int(num))
for i in range(1, 5):
    num = int(num * i) + int(num)
    print(num)

Assignments and Daily Challeneges/Week 4/Day 1/XP Gold/xp_gold.py


import numpy as np
from .RandomVariable import RandomVariable
from .ReportPlots import plot_confusion_matrix, CurvePlotCrossValidation
from .Curves import make_cross_validation_roc_curve, make_cross_validation_precision_recall_curve
from .EvaluationBaseClasses import BinaryEvaluation, MultiClassEvaluation

class CrossValidationEvaluation(object):
    def __init__(self, array_of_evaluations):

        self.class_names = array_of_evaluations[0].class_names
        accuracy, precision, recall, f1score, support, confusion_matrix = [], [], [], [], [], []

        for evaluation in array_of_evaluations:
            accuracy.append(evaluation.accuracy)
            precision.append(evaluation.precision)
            recall.append(evaluation.recall)
            f1score.append(evaluation.f1score)
            support.append(evaluation.support)
            confusion_matrix.append(evaluation.confusion_matrix)

        self.accuracy = RandomVariable(accuracy)
        self.precision = [RandomVariable(r) for r in np.array(precision).transpose()]
        self.recall = [RandomVariable(r) for r in np.array(recall).transpose()]
        self.f1score = [RandomVariable(r) for r in np.array(f1score).transpose()]
        self.support = [RandomVariable(r) for r in np.array(support).transpose()]
        self.confusion_matrix = RandomVariable(confusion_matrix)

        self.CurvePlot = CurvePlotCrossValidation

    def plot_confusion_matrix(self):

        normalized_cm = RandomVariable(np.apply_along_axis(func1d=lambda x: x.astype('float')/x.sum(),
                                                           axis=2, arr=self.confusion_matrix.array) * 100)

        cm_string = self.confusion_matrix.to_string(0) + '\n(' + normalized_cm.to_string(1) + '%)'

        plot_confusion_matrix(self.confusion_matrix.mean, cm_string, self.class_names)

class CrossValidationEvaluationBinary(CrossValidationEvaluation, BinaryEvaluation):
    def __init__(self, array_of_evaluations):
        super(CrossValidationEvaluationBinary, self).__init__(array_of_evaluations)

        self.roc_curve = make_cross_validation_roc_curve(
            [evaluation.roc_curve for evaluation in array_of_evaluations])
        self.precision_recall_curve = make_cross_validation_precision_recall_curve(
            [evaluation.precision_recall_curve for evaluation in array_of_evaluations])

class CrossValidationEvaluationMultiClass(CrossValidationEvaluation, MultiClassEvaluation):
    def __init__(self, array_of_evaluations):
        super(CrossValidationEvaluationMultiClass, self).__init__(array_of_evaluations)
        keys = self.class_names + ['micro', 'macro']
        self.roc_curve = {}
        self.precision_recall_curve = {}
        for key in keys:
            self.roc_curve[key] = make_cross_validation_roc_curve(
                [evaluation.roc_curve[key] for evaluation in array_of_evaluations])
            self.precision_recall_curve[key] = make_cross_validation_precision_recall_curve(
                [evaluation.precision_recall_curve[key] for evaluation in array_of_evaluations])


import os

from celery import Celery
from celery.schedules import crontab

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djangodocker.settings")

app = Celery("djangodocker")

app.config_from_object("django.conf:settings", namespace="CELERY")

app.autodiscover_tasks()

@app.task(bind=True)
def debug_task(self):
    print(f"Request: {self.request!r}")

app.conf.beat_schedule = {
    "populate_driver_data": {
        "task": "drivers.tasks.populate_driver_data",
        "schedule": crontab(minute=0, hour=0),
    },
    "populate_driver_metric_data": {
        "task": "drivers.tasks.log_driver_twitter_metric",
        "schedule": crontab(minute="*/60"),
    },
}


"""Bridging transforms between known spaces."""
from functools import lru_cache
from typing import Dict, Iterable, Iterator, Optional, Set, Tuple

import networkx as nx
import numpy as np

from .base import Transform, TransformSequence
from .util import SpaceRef, chain_or, dim_intersection, window

def split_sequence(seq: TransformSequence) -> Iterator[Transform]:
    """Split a TransformSequence into Transforms with spaces defined.

    If a component Transform has its spaces defined,
    it will be yielded as-is.
    A chain of Transforms without spaces defined are yielded as a TransformSequence.

    Parameters
    ----------
    seq : TransformSequence

    Yields
    -------
    Transform
    """
    this_seq = []
    for t in seq.transforms:
        if t.source_space is not None and t.target_space is not None:
            yield t
            continue

        this_seq.append(t)
        if t.target_space is not None:
            yield TransformSequence(this_seq)
            this_seq = []

class TransformGraph:
    def __init__(self):
        """Transform between any number of arbitrary spaces.

        Finds the shortest path for transforming one space
        into another, via some intermediate spaces.

        Populate with `my_transform_graph.add_transforms(my_transforms)`.

        """
        self.graph = nx.OrderedDiGraph()
        self.ndim: Optional[Set[int]] = None

    def add_transforms(self, transforms: Iterable[Transform]):
        """
        Parameters
        ----------
        transforms : Iterable[Transform]
            Transforms which must have a source and target space defined.
            TransformSequences are split out if their inner transforms'
            spaces are defined.

        Raises
        ------
        ValueError
            Undefined source and target spaces.
        """
        # TODO: weighting of split-out sequences could be problematic
        edges: Dict[Tuple[SpaceRef, SpaceRef], Transform] = dict()
        self.get_sequence.cache_clear()

        ndim = self.ndim

        for t in transforms:
            ndim = dim_intersection(ndim, t.ndim)
            if ndim is not None and len(ndim) == 0:
                raise ValueError("This TransformGraph supports no dimensionality")

            if isinstance(t, TransformSequence):
                ts = list(split_sequence(t))
            else:
                ts = [t]

            for t2 in ts:
                if chain_or(t.source_space, t.target_space, default=None) is None:
                    raise ValueError(
                        "All transforms in a graph "
                        "need explicit source and target spaces"
                    )
                edges[(t2.source_space, t2.target_space)] = t2

        self.ndim = ndim

        count = 0

        for (src, tgt), t in edges.items():
            self.graph.add_edge(src, tgt, transform=t)
            count += 1
            if (tgt, src) not in edges:
                try:
                    self.graph.add_edge(tgt, src, transform=~t)
                    count += 1
                except NotImplementedError:
                    pass

        return count

    @lru_cache()
    def get_sequence(
        self, source_space: SpaceRef, target_space: SpaceRef
    ) -> TransformSequence:
        """Get the shortest TransformSequence for transforming between two spaces.

        Parameters
        ----------
        source_space : SpaceRef
        target_space : SpaceRef

        Returns
        -------
        TransformSequence
        """
        path = nx.shortest_path(self.graph, source_space, target_space)
        if len(path) <= 1:
            transforms = []
        else:
            transforms = [
                self.graph.edges[src, tgt]["transform"] for src, tgt in window(path, 2)
            ]
        return TransformSequence(
            transforms,
            spaces=(source_space, target_space),
        )

    def transform(
        self, source_space: SpaceRef, target_space: SpaceRef, coords: np.ndarray
    ) -> np.ndarray:
        """Transform coordinates from one space to another,
        possibly via intermediates.

        Parameters
        ----------
        source_space : SpaceRef
        target_space : SpaceRef
        coords : np.ndarray

        Returns
        -------
        np.ndarray
        """
        t = self.get_sequence(source_space, target_space)
        return t.apply(coords)

    def __iter__(self) -> Iterator[Transform]:
        """Iterate through the transforms present in the graph.

        Includes inferred reverse transforms.

        N.B. `networkx.Graph.__iter__` iterates through nodes,
        where this effectively iterates through edges.

        Yields
        -------
        Transform

        Examples
        --------
        Create a new transform graph using another

        >>> new_tgraph = TransformGraph([extra_transform, *old_tgraph])

        """
        for _, _, t in self.graph.edges.data("transform"):
            yield t


# coding: utf-8

# ## 데이터프레임의 열 삭제하기

# In[1]:

import pandas as pd

scientists=pd.read_csv('./../data/scientists.csv')
scientists

# In[2]:

scientists.columns

# In[3]:

scientists_dropped=scientists.drop(['Age'],axis=1)
scientists_dropped.columns


import anndata as ad
import numpy as np
from scipy.sparse import issparse
from multiprocessing import cpu_count
from sklearn.neighbors import NearestNeighbors
from tqdm import tqdm

def transitions_graph(adata: ad.AnnData,
                      x: str,
                      transitions: str,
                      n_edges: int = 10,
                      timestamps: str = None,
                      out: str = 'graph',
                      n_jobs: int = -1,
                      verbose: bool = False,
                      copy: bool = False):
    """Transitions-based graph

    Before constructing the graph, you must perform a nearest neighbors search
    in the embedding space by running :class:`ocelli.pp.neighbors`.

    This function computes a transitions-based graph using transition probabilities
    between cells (e.g., RNA velocity), stored in `adata.uns[transitions]`
    and, optionally, cell timestamps, stored in `adata.obs[timestamps]`.
    Transitions should be stored as a square matrix, `n_cells` by `n_cells`.
    Each graph node has `n_edges` edges coming out.
    They correspond to the cells' nearest neighbors
    with the highest cell transition probabilities.
    For example, if you calculated 50 nearest neighbors and set `n_edges=10`,
    the resulting graph will connect a cell to 10 cells among its
    50 nearest neighbors with the highest transition probabilities.
    If in a cell's neighborhood, there are less than ``n_edges``
    cells with non-zero transitions, the remaining edges are connected
    to nearest neighbors.

    Assume that cell is from timestamp `t`. If you specify `timestamps`,
    the nearest neighbors in the last step of the graph construction are selected among cells from timestamp `t+1`.
    Cells from the final timestamp are exceptions - neighbors are selected from the same timestamp.

    Parameters
    ----------
    adata
        The annotated data matrix.
    x
        `adata.obsm` key for which nearest neighbors were precomputed.
    transitions
        `adata.uns` key storing a cell transition probability square matrix (of shape `(n_cells, n_cells)`).
    n_edges
        Number of edges coming out of each node. (default: 10)
    timestamps
        `adata.obs` key storing cell timestamps. Timestamps must bu numerical. (default: :obj:`None`)
    out
        `adata.obsm` key where graph is saved. (default: `graph`)
    n_jobs
        The number of parallel jobs. If the number is larger than the number of CPUs, it is changed to -1.
        -1 means all processors are used. (default: -1)
    verbose
        Print progress notifications. (default: `False`)
    copy
        Return a copy of :class:`anndata.AnnData`. (default: `False`)

    Returns
    -------
    :obj:`None`
        By default (`copy=False`), updates `adata` with the following fields:
        `adata.obsm[out]` (graph).
    :class:`anndata.AnnData`
        When `copy=True` is set, a copy of `adata` with those fields is returned.
    """

    if timestamps is None:
        df = list()
        for i, neighbors in enumerate(adata.obsm['neighbors_{}'.format(x)]):
            velocities = adata.uns[transitions][i, neighbors]

            if issparse(velocities):
                velocities = velocities.toarray()

            velocities = velocities.flatten()

            thr = n_edges if np.count_nonzero(velocities) > n_edges else np.count_nonzero(velocities)

            selected = list() if thr == 0 else list(neighbors[np.argpartition(velocities, kth=-thr)[-thr:]])

            if len(selected) != n_edges:
                for _ in range(n_edges - thr):
                    for idx in neighbors:
                        if idx not in selected:
                            selected.append(idx)
                            break
            df.append(selected)

        adata.obsm[out] = np.asarray(df)

    else:
        n_jobs = cpu_count() if n_jobs == -1 else min([n_jobs, cpu_count()])

        if timestamps not in adata.obs:
            raise (KeyError('No timestamps found in adata.obs["{}"].'.format(timestamps)))

        if x not in adata.obsm:
            raise (KeyError('No embedding found in adata.obsm["{}"].'.format(x)))

        transition_activity = list()

        graph = [[] for _ in range(len(adata.obs.index))]

        for cell_id, cell_nn in enumerate(adata.obsm['neighbors_{}'.format(x)]):
            cell_transitions = adata.uns[transitions][cell_id, cell_nn]
            cell_transitions = cell_transitions.toarray().flatten() if issparse(cell_transitions) else cell_transitions.flatten()

            thr = n_edges if np.count_nonzero(cell_transitions) >= n_edges else np.count_nonzero(cell_transitions)

            selected = list() if thr == 0 else list(cell_nn[np.argpartition(cell_transitions, kth=-thr)[-thr:]])

            transition_activity.append(thr)
            graph[cell_id] += selected

        timestamps_unique = np.unique(adata.obs[timestamps])
        n_timestamps = timestamps_unique.shape[0]
        adata.obs['temp'] = [i for i in range(adata.shape[0])]

        for tstamp_id, tstamp in tqdm(enumerate(timestamps_unique)):
            if tstamp_id < n_timestamps - 1:
                t0 = adata[adata.obs[timestamps] == timestamps_unique[tstamp_id]]
                t1 = adata[adata.obs[timestamps] == timestamps_unique[tstamp_id + 1]]

                neigh = NearestNeighbors(n_neighbors=n_edges + 1, n_jobs=n_jobs)
                neigh.fit(t1.obsm[x])

                for cell_id, cell_nn in enumerate(neigh.kneighbors(t0.obsm[x])[1][:, 1:]):
                    graph[t0.obs['temp'][cell_id]] += list(t1.obs['temp'][cell_nn])[:n_edges - transition_activity[t0.obs['temp'][cell_id]]]

            else:
                for cell_id, cell_nn in enumerate(neigh.kneighbors(t1.obsm[x])[1][:, 1:]):
                    graph[t1.obs['temp'][cell_id]] += list(t1.obs['temp'][cell_nn])[:n_edges - transition_activity[t1.obs['temp'][cell_id]]]

        adata.obsm[out] = np.asarray(graph)

        adata.obs.pop('temp')

    if verbose:
        print('Transitions-based graph constructed.')

    return adata if copy else None


from main import data_path

DATA = data_path(__file__)

def puzzle1():
    with open(DATA, "r") as f:
        lines = f.readlines()

    programs = [chr(i + ord("a")) for i in range(16)]

    # lines = ['s1,x3/4,pe/b']
    # programs = list('abcde')

    commands = lines[0].split(",")

    for command in commands[:50]:
        cmd, rest = command[0], command[1:]
        if cmd == "s":
            spin = int(rest)
            programs = programs[-spin:] + programs[:-spin]
        elif cmd == "x":
            a, b = tuple(map(int, rest.split("/")))
            programs[a], programs[b] = programs[b], programs[a]
        elif cmd == "p":
            pa, pb = rest.split("/")
            a = programs.index(pa)
            b = programs.index(pb)
            programs[a], programs[b] = programs[b], programs[a]

        print(command, "".join(programs))

def puzzle2():
    with open(DATA, "r") as f:
        lines = f.readlines()

    programs = [chr(i + ord("a")) for i in range(16)]
    starting = list(programs)

    # lines = ['s1,x3/4,pe/b']
    # programs = list('abcde')

    commands = lines[0].split(",")

    def spin(x):
        nonlocal programs
        programs = programs[-x:] + programs[:-x]

    def exchange(pos1, pos2):
        nonlocal programs
        programs[pos1], programs[pos2] = programs[pos2], programs[pos1]

    def partner(prog1, prog2):
        nonlocal programs
        exchange(programs.index(prog1), programs.index(prog2))

    cmd_list = []
    for command in commands:
        cmd, rest = command[0], command[1:]
        if cmd == "s":
            x = int(rest)
            cmd_list.append((spin, x))
        elif cmd == "x":
            a, b = tuple(map(int, rest.split("/")))
            cmd_list.append((exchange, a, b))
        elif cmd == "p":
            pa, pb = rest.split("/")
            cmd_list.append((partner, pa, pb))

    total = 1000000000
    rounds = -1
    for i in range(total):
        for cmd in cmd_list:
            cmd[0](*cmd[1:])
        if programs == starting:
            rounds = i + 1
            break

    if rounds != -1:
        rest = total % rounds
        for i in range(rest):
            for cmd in cmd_list:
                cmd[0](*cmd[1:])

    print("".join(programs))


from bs4 import BeautifulSoup
import requests
# from aiogram import Bot, Dispatcher, types
# from aiogram.utils import executor
#url = 'https://www.sberbank.ru/ru/person/credits/homenew'
URL = 'https://yandex.ru'
HEADERS = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', 'Accept': '*/*'}
#headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'}
#headers = {'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.134 YaBrowser/22.7.0.1925 Yowser/2.5 Safari/537.36', 'accept': '*/*'}

def get_html(url, params=None):
    r = requests.get(url, headers=HEADERS, params=params)
    return r

def get_content(html):
    soup = BeautifulSoup(html, 'html.parser')
    items = soup.find_all('a', class_="news__item-inner")
    #'kitt-col kitt-col_xs_12 kitt-col_md_6 product-catalog__card-block')
    print(items)

def parse():
    html = get_html(URL)
    if html.status_code == 200:
        #print(html.text)
        get_content(html.text)
    else:
        print('Error')

parse()

# request = requests.get(url)
#
# teme = soup.find_all('td', class_='paragraph')
# print(soup)
# for temes in teme:
#     temes = temes.find('a', {'class':'kitt-text kitt-text_size_m product-catalog__description product-catalog__description_mobile'})
#     if temes is not None and 'Ставка' in str(temes):
#         sublink = temes.get('href')
#         print(str(temes.text) + ' ' + str(sublink))


from scipy.integrate import solve_ivp
import numpy as np
import matplotlib.pyplot as plt
from math import floor, ceil
from timeit import default_timer as timer

def model_no_measures(_, y):
    (S, E, I, R) = y

    N = 37.741 * 10**6
    alpha = 1.0/8.0
    beta = 0.9
    gamma = 0.06
    mu = 0.01/365

    dSdt = mu*N - mu*S - (beta/N)*I*S
    dEdt = (beta/N)*I*S - alpha*E - mu*E
    dIdt = alpha*E - gamma*I - mu*I
    dRdt = gamma*I - mu*R
    return (dSdt, dEdt, dIdt, dRdt)

def root_25000(_, y):
    E = y[1]
    return E - 25000
root_25000.terminal = True

def model_with_measures(_, y):
    (S, E, I, R) = y

    N = 37.741* (10**6)
    alpha = 1.0/8.0
    beta = 0.005
    gamma = 0.06
    mu = 0.01/365

    dSdt = mu*N - mu*S - (beta/N)*I*S
    dEdt = (beta/N)*I*S - alpha*E - mu*E
    dIdt = alpha*E - gamma*I - mu*I
    dRdt = gamma*I - mu*R

    return (dSdt, dEdt, dIdt, dRdt)

def root_10000(_, y):
    E = y[1]
    return E - 10000
root_10000.terminal = True

def experiment(method):
    N = 37.741 * (10**6)
    E0 = 103
    I0 = 1
    R0 = 0
    S0 = N - (E0 + I0 + R0)
    y0 = (S0, E0, I0, R0)
    t0 = 0

    res = np.array([[], [], [], []])
    times = np.array([])
    t_initial = t0
    y_initial = y0
    measures_implemented = False
    count_number_evaluations = 0
    time_elapsed = 0
    while t_initial < 180:
        tspan = [t_initial, 180]
        sol, start, end = None, None, None
        if (measures_implemented):
            start = timer()
            sol = solve_ivp(model_with_measures, tspan, y_initial, events=root_10000, dense_output=True, method=method)
            end = timer()
            measures_implemented = False
        else:
            start = timer()
            sol = solve_ivp(model_no_measures, tspan, y_initial, events=root_25000, dense_output=True, method=method)
            end = timer()
            measures_implemented = True
        t_event = 180 if len(sol.t_events[0]) == 0 else sol.t_events[0][0]
        t_calc = np.arange(t_initial, t_event, 1)
        y_cal = sol.sol(t_calc)
        res = np.concatenate((res, y_cal), axis=1)
        times = np.concatenate((times, t_calc))
        t_initial = t_event
        last_index = len(sol.y[0]) - 1
        y_initial = [sol.y[0][last_index], sol.y[1][last_index], sol.y[2][last_index], sol.y[3][last_index]]
        count_number_evaluations += sol.nfev
        time_elapsed += (end - start)
    return (times, res, count_number_evaluations, time_elapsed)

(times_lsoda, res_lsoda, nfev_lsoda, elapsed_lsoda) = experiment("LSODA")
(times_rk45, res_rk45, nfev_rk45, elapsed_rk45) = experiment('RK45')
(times_bdf, res_bdf, nfev_bdf, elapsed_bdf) = experiment('BDF')
(times_radau, res_radau, nfev_radau, elapsed_radau) = experiment('Radau')
(times_dop853, res_dop853, nfev_dop853, elapsed_dop853) = experiment('DOP853')
(times_rk23, res_rk23, nfev_rk23, elapsed_rk23) = experiment('RK23')

plt.plot(times_lsoda, res_lsoda[1])
plt.plot(times_bdf, res_bdf[1])
plt.plot(times_radau, res_radau[1])
plt.plot(times_rk45, res_rk45[1])
plt.plot(times_dop853, res_dop853[1])
plt.plot(times_rk23, res_rk23[1])
plt.ylabel("E(t)")
plt.xlabel('time')
plt.legend(['lsoda', 'bdf', 'radau', 'rk45', 'dop853', 'rk23'], shadow=True)
plt.show()

print(f"lsoda & {nfev_lsoda} & {elapsed_lsoda}")
print(f"bdf & {nfev_bdf} & {elapsed_bdf}")
print(f"radau & {nfev_radau} & {elapsed_radau}")
print(f"rk45 & {nfev_rk45} & {elapsed_rk45}")
print(f"dop853 & {nfev_dop853} & {elapsed_dop853}")
print(f"rk23 & {nfev_rk23} & {elapsed_rk23}")


#coding=utf8
from django.db import models
from django.contrib.auth.models import User, Group, Permission
from django.db.models.signals import post_save
from django.dispatch import receiver

class Country(models.Model):
	name = models.CharField(verbose_name='Страна', max_length=65, unique=True)
	class Meta:
		ordering = ('name', )
		verbose_name = 'Страна'
		verbose_name_plural = 'Страны'
	def __str__(self):
		return self.name

class Manufacturer(models.Model):
    name = models.CharField(verbose_name='Изготовитель', max_length=100)
    contact_address = models.CharField(verbose_name='Контактный адрес', max_length=250)
    manufacturers_address = models.CharField(
        verbose_name='Адрес производства',
    	max_length = 250
    )
    country = models.ForeignKey(
    	verbose_name='Страна',
    	to=Country,
    	null=True,
    	on_delete=models.CASCADE)
    def __str__(self):
    	return self.name
    class Meta:
        ordering = ('country', )
        verbose_name = 'Изготовитель'
        verbose_name_plural = 'Изготовители'

class TypeOfSweet(models.Model):
	name = models.CharField(verbose_name='Тип сладости', max_length=50, unique=True)
	class Meta:
		ordering = ('name', )
		verbose_name = 'Тип сладости'
		verbose_name_plural = 'Типы сладостей'
	def __str__(self):
		return self.name

class Sweet(models.Model):
    title = models.CharField(verbose_name='Название', max_length=100)
    price_for_one = models.FloatField(verbose_name='Цена за одну штуку', null=True)
    price_weight = models.FloatField(verbose_name='Цена за 100 грамм', blank=True, null=True)
    structure = models.CharField(verbose_name='Состав', max_length=201)
    description = models.CharField(verbose_name='Описание', max_length=200)
    manufacturer = models.ForeignKey(
    	verbose_name='Изготовитель',
    	to=Manufacturer,
    	null=True,
    	on_delete=models.CASCADE
    )
    typeOfSweet = models.ForeignKey(
    	verbose_name='Тип сладости',
    	to=TypeOfSweet,
    	on_delete=models.CASCADE,
    	null=True
    )

    class Meta:
        ordering = ('title', 'manufacturer', )
        verbose_name = 'Сладость'
        verbose_name_plural = 'Сладости'
        permissions = (
        	('buy_sweet' , 'Купить сладость'),
        )
    def __str__(self):
    	return self.title

class Profile(models.Model):
	user = models.OneToOneField(
		User,
		unique=True,
		on_delete=models.CASCADE
	)
	bio = models.TextField(max_length=500, blank=True, null=True)
	vk_page = models.CharField(verbose_name='Страница Вконтакте',max_length=30,blank=True, null=True)
	balance = models.FloatField(verbose_name='Баланс', default=100.0)
	timezone = models.CharField(verbose_name='Часовой пояс',max_length=50, default='Europe/Moscow')

	def __str__(self):
		return self.user.username

@receiver(post_save, sender=User)
def create_user_profile(sender, instance, created, **kwargs):
    if created:
        Profile.objects.create(user=instance)

@receiver(post_save, sender=User)
def save_user_profile(sender, instance, **kwargs):
    instance.profile.save()

@receiver(post_save, sender=User, dispatch_uid='myproject.myapp.models.user_post_save_handler')
def user_post_save(sender, instance, created, **kwargs):
    if created:
        instance.groups.add(Group.objects.get(name='Пользователи'))


from os import listdir
import os
import torch
from keras_preprocessing.sequence import pad_sequences
from torch.utils.data import TensorDataset, SequentialSampler, DataLoader
from tqdm import tqdm

sep = os.sep

def make_mask(batch_ids):
    batch_mask = []
    for ids in batch_ids:
        mask = [int(token_id > 0) for token_id in ids]
        batch_mask.append(mask)
    return torch.tensor(batch_mask)

def read_data(path_raw, classes=[]):
    content = []
    labels = []
    for folder in listdir(path_raw):
        for file in listdir(path_raw + sep + folder):
            with open(path_raw + sep + folder + sep + file, 'r', encoding="utf-8") as f:
                print("read file: " + path_raw + sep + folder + sep + file)
                all_of_it = f.read()
                sentences = all_of_it.split('\n')
                for str in sentences:
                    content.append(str)
                for _ in sentences:
                    labels.append(classes.index(folder))
                del all_of_it, sentences
    return content, labels

def dataloader_from_text(texts=[], labels=[], tokenizer=None, max_len=256, batch_size=16, infer=False):
    print(texts[0])
    print(labels[0])
    # text to id
    ids = []
    for text in tqdm(texts):
        encoded_sent = tokenizer.encode(text)
        ids.append(encoded_sent)
    del texts
    # padding sentences 256 length
    ids_padded = pad_sequences(ids, maxlen=max_len, dtype="long", value=0, truncating="post", padding="post")
    del ids
    print(ids_padded)

    print("CONVERT TO TORCH TENSOR")
    ids_inputs = torch.tensor(ids_padded)
    del ids_padded

    if not infer:
        labels = torch.tensor(labels)

    print("CREATE DATALOADER")
    if infer:
        input_data = TensorDataset(ids_inputs)
    else:
        input_data = TensorDataset(ids_inputs, labels)
    input_sampler = SequentialSampler(input_data)

    data_loader = DataLoader(input_data, sampler=input_sampler, batch_size=batch_size)

    print("len data_loader:", len(data_loader))
    print("LOAD DATA ALL DONE")
    return data_loader


# -*- coding: utf-8 -*-

from openerp import models, fields, api, exceptions
import urllib
import pytz
import uuid
from openerp.http import request
from datetime import datetime

class ResUsers(models.Model):
    _inherit = 'res.users'

    screen_width = fields.Char(
    	string='Screen Width',
    	readonly=True
    )
    screen_height = fields.Char(
    	string='Screen Height',
    	readonly=True
    )
    screen_depth = fields.Char(
    	string='Screen Color Depth',
    	readonly=True
    )
    screen_scale = fields.Char(
    	string='Screen Scalling Factor',
    	readonly=True
    )
    user_agent = fields.Char(
        string='Browser User Agent',
        readonly=True
    )
    browser_plugin = fields.Char(
        string='Browser Plugins',
        readonly=True
    )
    browser_dnt = fields.Boolean(
        string='Browser DNT',
        readonly=True
    )
    client_ip = fields.Char(
        string='Client Ip',
    )
    # client_factor_ref = fields.Char(
    #     string='Client Factor Reference',
    #     readonly=True
    # )
    client_device_id = fields.Char(
        string='Client Device ID'
    )
    client_ip_address = fields.Char(
        string='Client IP Address'
    )
    client_timezone = fields.Char(
        string='Client Timezone'
    )

    def get_user_uuid4(self, cr, uid, ids, context=None):
        return str(uuid.uuid4())

    def _get_users_headers(self):
        return {
            "Gov-client-screens" : 'width={screen_width}&height={screen_height}&scaling-factor={screen_scale}&colour-depth={screen_depth}'.format(screen_width=self.screen_width, screen_height=self.screen_height, screen_scale=self.screen_scale, screen_depth=self.screen_depth),
            "Gov-client-window-size" : 'width={screen_width}&height={screen_height}'.format(screen_width=self.screen_width, screen_height=self.screen_height),
            "Gov-Client-Browser-Plugins" : '{browserplugin}'.format(browserplugin= self.browser_plugin),
            "Gov-Client-Browser-JS-User-Agent" : '{useragent}'.format(useragent=self.user_agent),
            "Gov-Client-Browser-Do-Not-Track" : '{browserdnt}'.format(browserdnt=self.browser_dnt).lower(),
            "Gov-Client-Device-Id": '{uuid1}'.format(uuid1=self.client_device_id),
            "Gov-Client-Timezone": self.client_timezone,
            "Gov-client-local-ips": self.client_ip_address,
        }

    def _get_vendor_headers(self, vendor_ip):
        mtd_module = self.env['ir.module.module'].search([('name','=','account_mtd')], limit=1)
        return {
            "Gov-vendor-version":'Odoo%20SA=13.0&moduleversion={version}'.format(version=mtd_module.installed_version),
            # "Gov-vendor-license-ids": urllib.urlencode({'Odoo':'13.0-20201006-community-edition'}),
            "Gov-vendor-public-ip": '{vendorip}'.format(vendorip=vendor_ip),
            "Gov-vendor-forwarded":'by={vendorip}&for={clientip}'.format(vendorip=vendor_ip, clientip=self.client_ip),
            "Gov-Vendor-Product-Name": 'Product%20Odoo',
        }

    def _get_client_headers(self, company):
        iptime_now = str(datetime.now(pytz.timezone('UTC')).strftime('%Y-%m-%dT%H:%M:%S.%f'))[:-3]
        if not company.hmrc_username:
            raise exceptions.Warning('HMRC UserID Mandatory in Company')

        # multi_factor_now = str(datetime.now(pytz.timezone('UTC')).strftime('%Y-%m-%dT%H:%MZ'))
        # if not self.client_factor_ref:
        #     self.write({'client_factor_ref': uuid.uuid4().hex[:13]})
        # multi_factor = {
        #     'type': 'TOTP',
        #     'timestamp': str(multi_factor_now),
        #     'unique-reference': self.client_factor_ref,
        # }
        return {
            "Gov-Client-Connection-Method": 'WEB_APP_VIA_SERVER',
            "Gov-Client-Public-IP": self.client_ip,
            "Gov-Client-Public-Port": '{remoteport}'.format(remoteport=request.httprequest.environ['REMOTE_PORT']),
            "Gov-Client-User-Ids": 'Odoo=opusvl{username}'.format(username=company.hmrc_username),
            "Gov-Client-Local-IPs-Timestamp": str(iptime_now)+ 'Z',
            "Gov-Client-Public-IP-Timestamp": str(iptime_now)+ 'Z',
            # "Gov-client-multi-factor": urllib.urlencode(multi_factor),
        }


import faust

from simple_settings import settings

app = faust.App(
    id=1,
    debug=settings.DEBUG,
    autodiscover=["page_views"],
    broker=settings.KAFKA_BOOTSTRAP_SERVER,
    store=settings.STORE_URI,
    logging_config=settings.LOGGING,
    topic_allow_declare=settings.TOPIC_ALLOW_DECLARE,
    topic_disable_leader=settings.TOPIC_DISABLE_LEADER,
    broker_credentials=settings.SSL_CONTEXT,
)


import numpy as np
import pandas as pd

data1 = pd.DataFrame({'姓名': ['张三', '李四', '王五', '李四', '张三', '张三'], '出手次数1': np.arange(6)})
data2 = pd.DataFrame({'姓名': ['赵六', '王五', '李四'], '出手次数2': [1, 2, 3]})
# 内连接
data3 = pd.merge(data1, data2)
print(f'data1:\n{data1}\ndata2:\n{data2}')
print(f'内连接:\n', data3)
# data4=pd.merge(data1,data2,on='姓名',how='inner')
# print(f'data4:\n',data4)
# 左连接
data5 = pd.merge(data1, data2, on='姓名', how='left')
print(f'左连接:\n', data5)
data6 = pd.merge(data1, data2, on='姓名', how='right')
print(f'右连接:\n', data6)
data7 = pd.merge(data1, data2, on='姓名', how='outer')
print(f'全连接:\n', data7)
data8 = pd.DataFrame({'姓名': ['张三', '张三', '王五'], '班级': ['1班', '2班', '1班'], '分数': [10, 20, 30]})
data9 = pd.DataFrame(
    {'姓名': ['张三', '张三', '王五', '王五'], '班级': ['1班', '1班', '1班', '2班'], '分数': [40, 50, 60, 70]})
data10 = pd.merge(data8, data9, on=['姓名', '班级'])  # 内连接
data11 = pd.merge(data8, data9, on=['姓名', '班级'], how='outer')  # 外连接
print(f'内连接:\n{data10}\n外连接:{data11}')
data12 = pd.merge(data8, data9, on='姓名')  # 内连接
print(f'data12:\n', data12)
data13 = pd.merge(data8, data9, on='姓名', suffixes=('_数据1', '_数据2'))
print(f'data13:\n', data13)
data14 = pd.DataFrame({'姓名': ['张三', '李四', '王五', '张三', '李四'], '次数': range(5)})
data15 = pd.DataFrame({'数据': [10, 20]}, index=['张三', '李四'])
data16 = pd.merge(data14, data15, left_on='姓名', right_index=True)
print('data16\n', data16)
data17 = pd.merge(data14, data15, left_on='姓名', right_index=True, how='outer')
print('data17\n', data17)