"""Data utilities for index preparation."""
import ast
import html
import json
import os
import re
import requests
import openai
import re
import tempfile
import time
from abc import ABC, abstractmethod
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass
from functools import partial
from typing import Callable, List, Dict, Optional, Generator, Tuple, Union

import markdown
import tiktoken
from azure.identity import DefaultAzureCredential
from azure.ai.formrecognizer import DocumentAnalysisClient
from azure.core.credentials import AzureKeyCredential
from azure.storage.blob import ContainerClient
from bs4 import BeautifulSoup
from langchain.text_splitter import TextSplitter, MarkdownTextSplitter, RecursiveCharacterTextSplitter, PythonCodeTextSplitter
from tqdm import tqdm
from typing import Any

FILE_FORMAT_DICT = {
        "md": "markdown",
        "txt": "text",
        "html": "html",
        "shtml": "html",
        "htm": "html",
        "py": "python",
        "pdf": "pdf"
    }

RETRY_COUNT = 5

SENTENCE_ENDINGS = [".", "!", "?"]
WORDS_BREAKS = list(reversed([",", ";", ":", " ", "(", ")", "[", "]", "{", "}", "\t", "\n"]))

HTML_TABLE_TAGS = {"table_open": "
", "table_close": "
", "row_open":"
"}

PDF_HEADERS = {
    "title": "h1",
    "sectionHeading": "h2"
}

class TokenEstimator(object):
    GPT2_TOKENIZER = tiktoken.get_encoding("gpt2")

    def estimate_tokens(self, text: Union[str, List]) -> int:

        return len(self.GPT2_TOKENIZER.encode(text, allowed_special="all"))

    def construct_tokens_with_size(self, tokens: str, numofTokens: int) -> str:
        newTokens = self.GPT2_TOKENIZER.decode(
            self.GPT2_TOKENIZER.encode(tokens, allowed_special="all")[:numofTokens]
        )
        return newTokens

TOKEN_ESTIMATOR = TokenEstimator()

class PdfTextSplitter(TextSplitter):
    def __init__(self, length_function: Callable[[str], int] =TOKEN_ESTIMATOR.estimate_tokens, separator: str = "\n\n", **kwargs: Any):
        """Create a new TextSplitter for htmls from extracted pdfs."""
        super().__init__(**kwargs)
        self._table_tags = HTML_TABLE_TAGS
        self._separators = separator or ["\n\n", "\n", " ", ""]
        self._length_function = length_function
        self._noise = 50 # tokens to accommodate differences in token calculation, we don't want the chunking-on-the-fly to inadvertently chunk anything due to token calc mismatch

    def extract_caption(self, text):
        separator = self._separators[-1]
        for _s in self._separators:
            if _s == "":
                separator = _s
                break
            if _s in text:
                separator = _s
                break

        # Now that we have the separator, split the text
        if separator:
            lines = text.split(separator)
        else:
            lines = list(text)

        # remove empty lines
        lines = [line for line in lines if line!='']
        caption = ""

        if len(text.split(f"<{PDF_HEADERS['title']}>"))>1:
            caption +=  text.split(f"<{PDF_HEADERS['title']}>")[-1].split(f"</{PDF_HEADERS['title']}>")[0]
        if len(text.split(f"<{PDF_HEADERS['sectionHeading']}>"))>1:
            caption +=  text.split(f"<{PDF_HEADERS['sectionHeading']}>")[-1].split(f"</{PDF_HEADERS['sectionHeading']}>")[0]

        caption += "\n"+ lines[-1].strip()

        return caption

    def split_text(self, text: str) -> List[str]:
        start_tag = self._table_tags["table_open"]
        end_tag = self._table_tags["table_close"]
        splits = text.split(start_tag)

        final_chunks = self.chunk_rest(splits[0]) # the first split is before the first table tag so it is regular text

        table_caption_prefix = ""
        if len(final_chunks)>0:
            table_caption_prefix += self.extract_caption(final_chunks[-1]) # extracted from the last chunk before the table
        for part in splits[1:]:
            table, rest = part.split(end_tag)
            table = start_tag + table + end_tag
            minitables = self.chunk_table(table, table_caption_prefix)
            final_chunks.extend(minitables)

            if rest.strip()!="":
                text_minichunks = self.chunk_rest(rest)
                final_chunks.extend(text_minichunks)
                table_caption_prefix = self.extract_caption(text_minichunks[-1])
            else:
                table_caption_prefix = ""

        final_final_chunks = [chunk for chunk, chunk_size in merge_chunks_serially(final_chunks, self._chunk_size)]

        return final_final_chunks

    def chunk_rest(self, item):
        separator = self._separators[-1]
        for _s in self._separators:
            if _s == "":
                separator = _s
                break
            if _s in item:
                separator = _s
                break
        chunks = []
        if separator:
            splits = item.split(separator)
        else:
            splits = list(item)
        _good_splits = []
        for s in splits:
            if self._length_function(s) < self._chunk_size - self._noise:
                _good_splits.append(s)
            else:
                if _good_splits:
                    merged_text = self._merge_splits(_good_splits, separator)
                    chunks.extend(merged_text)
                    _good_splits = []
                other_info = self.chunk_rest(s)
                chunks.extend(other_info)
        if _good_splits:
            merged_text = self._merge_splits(_good_splits, separator)
            chunks.extend(merged_text)
        return chunks

    def chunk_table(self, table, caption):
        if self._length_function("\n".join([caption, table])) < self._chunk_size - self._noise:
            return ["\n".join([caption, table])]
        else:
            headers = ""
            if re.search(".*	", table):
                headers += re.search(".*	", table).group() # extract the header out. Opening tag may contain rowspan/colspan
            splits = table.split(self._table_tags["row_open"]) #split by row tag
            tables = []
            current_table = caption + "\n"
            for part in splits:
                if len(part)>0:
                    if self._length_function(current_table + self._table_tags["row_open"] + part) < self._chunk_size: # if current table length is within permissible limit, keep adding rows
                        if part not in [self._table_tags["table_open"], self._table_tags["table_close"]]: # need add the separator (row tag) when the part is not a table tag
                            current_table += self._table_tags["row_open"]
                        current_table += part

                    else:

                        # if current table size is beyond the permissible limit, complete this as a mini-table and add to final mini-tables list
                        current_table += self._table_tags["table_close"]
                        tables.append(current_table)

                        # start a new table
                        current_table = "\n".join([caption, self._table_tags["table_open"], headers])
                        if part not in [self._table_tags["table_open"], self._table_tags["table_close"]]:
                            current_table += self._table_tags["row_open"]
                        current_table += part

            # TO DO: fix the case where the last mini table only contain tags

            if not current_table.endswith(self._table_tags["table_close"]):

                tables.append(current_table + self._table_tags["table_close"])
            else:
                tables.append(current_table)
            return tables

@dataclass
class Document(object):
    """A data class for storing documents

    Attributes:
        content (str): The content of the document.
        id (Optional[str]): The id of the document.
        title (Optional[str]): The title of the document.
        filepath (Optional[str]): The filepath of the document.
        url (Optional[str]): The url of the document.
        metadata (Optional[Dict]): The metadata of the document.
    """

    content: str
    id: Optional[str] = None
    title: Optional[str] = None
    filepath: Optional[str] = None
    url: Optional[str] = None
    metadata: Optional[Dict] = None
    contentVector: Optional[List[float]] = None

def cleanup_content(content: str) -> str:
    """Cleans up the given content using regexes
    Args:
        content (str): The content to clean up.
    Returns:
        str: The cleaned up content.
    """
    output = re.sub(r"\n{2,}", "\n", content)
    output = re.sub(r"[^\S\n]{2,}", " ", output)
    output = re.sub(r"-{2,}", "--", output)

    return output.strip()

class BaseParser(ABC):
    """A parser parses content to produce a document."""

    @abstractmethod
    def parse(self, content: str, file_name: Optional[str] = None) -> Document:
        """Parses the given content.
        Args:
            content (str): The content to parse.
            file_name (str): The file name associated with the content.
        Returns:
            Document: The parsed document.
        """
        pass

    def parse_file(self, file_path: str) -> Document:
        """Parses the given file.
        Args:
            file_path (str): The file to parse.
        Returns:
            Document: The parsed document.
        """
        with open(file_path, "r") as f:
            return self.parse(f.read(), os.path.basename(file_path))

    def parse_directory(self, directory_path: str) -> List[Document]:
        """Parses the given directory.
        Args:
            directory_path (str): The directory to parse.
        Returns:
            List[Document]: List of parsed documents.
        """
        documents = []
        for file_name in os.listdir(directory_path):
            file_path = os.path.join(directory_path, file_name)
            if os.path.isfile(file_path):
                documents.append(self.parse_file(file_path))
        return documents

class MarkdownParser(BaseParser):
    """Parses Markdown content."""

    def __init__(self) -> None:
        super().__init__()
        self._html_parser = HTMLParser()

    def parse(self, content: str, file_name: Optional[str] = None) -> Document:
        """Parses the given content.
        Args:
            content (str): The content to parse.
            file_name (str): The file name associated with the content.
        Returns:
            Document: The parsed document.
        """
        html_content = markdown.markdown(content, extensions=['fenced_code', 'toc', 'tables', 'sane_lists'])

        return self._html_parser.parse(html_content, file_name)

class HTMLParser(BaseParser):
    """Parses HTML content."""
    TITLE_MAX_TOKENS = 128
    NEWLINE_TEMPL = "<NEWLINE_TEXT>"

    def __init__(self) -> None:
        super().__init__()
        self.token_estimator = TokenEstimator()

    def parse(self, content: str, file_name: Optional[str] = None) -> Document:
        """Parses the given content.
        Args:
            content (str): The content to parse.
            file_name (str): The file name associated with the content.
        Returns:
            Document: The parsed document.
        """
        soup = BeautifulSoup(content, 'html.parser')

        # Extract the title
        title = ''
        if soup.title and soup.title.string:
            title = soup.title.string
        else:
            # Try to find the first
 tag
            h1_tag = soup.find('h1')
            if h1_tag:
                title = h1_tag.get_text(strip=True)
            else:
                h2_tag = soup.find('h2')
                if h2_tag:
                    title = h2_tag.get_text(strip=True)
        if title is None or title == '':
            # if title is still not found, guess using the next string
            try:
                title = next(soup.stripped_strings)
                title = self.token_estimator.construct_tokens_with_size(title, self.TITLE_MAX_TOKENS)

            except StopIteration:
                title = file_name

                # Helper function to process text nodes

        # Parse the content as it is without any formatting changes
        result = content
        if title is None:
            title = '' # ensure no 'None' type title

        return Document(content=cleanup_content(result), title=str(title))

class TextParser(BaseParser):
    """Parses text content."""

    def __init__(self) -> None:
        super().__init__()

    def _get_first_alphanum_line(self, content: str) -> Optional[str]:
        title = None
        for line in content.splitlines():
            if any([c.isalnum() for c in line]):
                title = line.strip()
                break
        return title

    def _get_first_line_with_property(
        self, content: str, property: str = "title: "
    ) -> Optional[str]:
        title = None
        for line in content.splitlines():
            if line.startswith(property):
                title = line[len(property) :].strip()
                break
        return title

    def parse(self, content: str, file_name: Optional[str] = None) -> Document:
        """Parses the given content.
        Args:
            content (str): The content to parse.
            file_name (str): The file name associated with the content.
        Returns:
            Document: The parsed document.
        """
        title = self._get_first_line_with_property(
            content
        ) or self._get_first_alphanum_line(content)

        return Document(content=cleanup_content(content), title=title or file_name)

class PythonParser(BaseParser):
    def _get_topdocstring(self, text):
        tree = ast.parse(text)
        docstring = ast.get_docstring(tree)  # returns top docstring
        return docstring

    def parse(self, content: str, file_name: Optional[str] = None) -> Document:
        """Parses the given content.
        Args:
            content (str): The content to parse.
            file_name (str): The file name associated with the content.
        Returns:
            Document: The parsed document.
        """
        docstring = self._get_topdocstring(content)
        if docstring:
            title = f"{file_name}: {docstring}"
        else:
            title = file_name
        return Document(content=content, title=title)

    def __init__(self) -> None:
        super().__init__()

class ParserFactory:
    def __init__(self):
        self._parsers = {
            "html": HTMLParser(),
            "text": TextParser(),
            "markdown": MarkdownParser(),
            "python": PythonParser()
        }

    @property
    def supported_formats(self) -> List[str]:
        "Returns a list of supported formats"
        return list(self._parsers.keys())

    def __call__(self, file_format: str) -> BaseParser:
        parser = self._parsers.get(file_format, None)
        if parser is None:
            raise UnsupportedFormatError(f"{file_format} is not supported")

        return parser

parser_factory = ParserFactory()

class UnsupportedFormatError(Exception):
    """Exception raised when a format is not supported by a parser."""

    pass

@dataclass
class ChunkingResult:
    """Data model for chunking result

    Attributes:
        chunks (List[Document]): List of chunks.
        total_files (int): Total number of files.
        num_unsupported_format_files (int): Number of files with unsupported format.
        num_files_with_errors (int): Number of files with errors.
        skipped_chunks (int): Number of chunks skipped.
    """
    chunks: List[Document]
    total_files: int
    num_unsupported_format_files: int = 0
    num_files_with_errors: int = 0
    # some chunks might be skipped to small number of tokens
    skipped_chunks: int = 0

def extractStorageDetailsFromUrl(url):
    matches = re.fullmatch(r'https:\/\/([^\/.]*)\.blob\.core\.windows\.net\/([^\/]*)\/(.*)', url)
    if not matches:
        raise Exception(f"Not a valid blob storage URL: {url}")
    return (matches.group(1), matches.group(2), matches.group(3))

def downloadBlobUrlToLocalFolder(blob_url, local_folder, credential):
    (storage_account, container_name, path) = extractStorageDetailsFromUrl(blob_url)
    container_url = f'https://{storage_account}.blob.core.windows.net/{container_name}'
    container_client = ContainerClient.from_container_url(container_url, credential=credential)
    if path and not path.endswith('/'):
        path = path + '/'

    last_destination_folder = None
    for blob in container_client.list_blobs(name_starts_with=path):
        relative_path = blob.name[len(path):]
        destination_path = os.path.join(local_folder, relative_path)
        destination_folder = os.path.dirname(destination_path)
        if destination_folder != last_destination_folder:
            os.makedirs(destination_folder, exist_ok=True)
            last_destination_folder = destination_folder
        blob_client = container_client.get_blob_client(blob.name)
        with open(file=destination_path, mode='wb') as local_file:
            stream = blob_client.download_blob()
            local_file.write(stream.readall())

def get_files_recursively(directory_path: str) -> List[str]:
    """Gets all files in the given directory recursively.
    Args:
        directory_path (str): The directory to get files from.
    Returns:
        List[str]: List of file paths.
    """
    file_paths = []
    for dirpath, _, files in os.walk(directory_path):
        for file_name in files:
            file_path = os.path.join(dirpath, file_name)
            file_paths.append(file_path)
    return file_paths

def convert_escaped_to_posix(escaped_path):
    windows_path = escaped_path.replace("\\\\", "\\")
    posix_path = windows_path.replace("\\", "/")
    return posix_path

def _get_file_format(file_name: str, extensions_to_process: List[str]) -> Optional[str]:
    """Gets the file format from the file name.
    Returns None if the file format is not supported.
    Args:
        file_name (str): The file name.
        extensions_to_process (List[str]): List of extensions to process.
    Returns:
        str: The file format.
    """

    # in case the caller gives us a file path
    file_name = os.path.basename(file_name)
    file_extension = file_name.split(".")[-1]
    if file_extension not in extensions_to_process:
        return None
    return FILE_FORMAT_DICT.get(file_extension, None)

def table_to_html(table):
    table_html = "
"
    rows = [sorted([cell for cell in table.cells if cell.row_index == i], key=lambda cell: cell.column_index) for i in range(table.row_count)]
    for row_cells in rows:
        table_html += "
"
        for cell in row_cells:
            tag = "th" if (cell.kind == "columnHeader" or cell.kind == "rowHeader") else "td"
            cell_spans = ""
            if cell.column_span > 1: cell_spans += f" colSpan={cell.column_span}"
            if cell.row_span > 1: cell_spans += f" rowSpan={cell.row_span}"
            table_html += f"<{tag}{cell_spans}>{html.escape(cell.content)}</{tag}>"
        table_html +="
"
    table_html += "
"
    return table_html

def extract_pdf_content(file_path, form_recognizer_client, use_layout=False):
    offset = 0
    page_map = []
    model = "prebuilt-layout" if use_layout else "prebuilt-read"
    with open(file_path, "rb") as f:
        poller = form_recognizer_client.begin_analyze_document(model, document = f)
    form_recognizer_results = poller.result()

    # (if using layout) mark all the positions of headers
    roles_start = {}
    roles_end = {}
    for paragraph in form_recognizer_results.paragraphs:
        if paragraph.role!=None:
            para_start = paragraph.spans[0].offset
            para_end = paragraph.spans[0].offset + paragraph.spans[0].length
            roles_start[para_start] = paragraph.role
            roles_end[para_end] = paragraph.role

    for page_num, page in enumerate(form_recognizer_results.pages):
        tables_on_page = [table for table in form_recognizer_results.tables if table.bounding_regions[0].page_number == page_num + 1]

        # (if using layout) mark all positions of the table spans in the page
        page_offset = page.spans[0].offset
        page_length = page.spans[0].length
        table_chars = [-1]*page_length
        for table_id, table in enumerate(tables_on_page):
            for span in table.spans:
                # replace all table spans with "table_id" in table_chars array
                for i in range(span.length):
                    idx = span.offset - page_offset + i
                    if idx >=0 and idx < page_length:
                        table_chars[idx] = table_id

        # build page text by replacing charcters in table spans with table html and replace the characters corresponding to headers with html headers, if using layout
        page_text = ""
        added_tables = set()
        for idx, table_id in enumerate(table_chars):
            if table_id == -1:
                position = page_offset + idx
                if position in roles_start.keys():
                    role = roles_start[position]
                    if role in PDF_HEADERS:
                        page_text += f"<{PDF_HEADERS[role]}>"
                if position in roles_end.keys():
                    role = roles_end[position]
                    if role in PDF_HEADERS:
                        page_text += f"</{PDF_HEADERS[role]}>"

                page_text += form_recognizer_results.content[page_offset + idx]

            elif not table_id in added_tables:
                page_text += table_to_html(tables_on_page[table_id])
                added_tables.add(table_id)

        page_text += " "
        page_map.append((page_num, offset, page_text))
        offset += len(page_text)

    full_text = "".join([page_text for _, _, page_text in page_map])
    return full_text

def merge_chunks_serially(chunked_content_list: List[str], num_tokens: int) -> Generator[Tuple[str, int], None, None]:
    # TODO: solve for token overlap
    current_chunk = ""
    total_size = 0
    for chunked_content in chunked_content_list:
        chunk_size = TOKEN_ESTIMATOR.estimate_tokens(chunked_content)
        if total_size > 0:
            new_size = total_size + chunk_size
            if new_size > num_tokens:
                yield current_chunk, total_size
                current_chunk = ""
                total_size = 0
        total_size += chunk_size
        current_chunk += chunked_content
    if total_size > 0:
        yield current_chunk, total_size

def get_embedding(text, embedding_model_endpoint=None, embedding_model_key=None, azure_credential=None):
    endpoint = embedding_model_endpoint if embedding_model_endpoint else os.environ.get("EMBEDDING_MODEL_ENDPOINT")
    key = embedding_model_key if embedding_model_key else os.environ.get("EMBEDDING_MODEL_KEY")

    if azure_credential is None and (endpoint is None or key is None):
        raise Exception("EMBEDDING_MODEL_ENDPOINT and EMBEDDING_MODEL_KEY are required for embedding")

    try:
        endpoint_parts = endpoint.split("/openai/deployments/")
        base_url = endpoint_parts[0]
        deployment_id = endpoint_parts[1].split("/embeddings")[0]

        openai.api_version = '2023-05-15'
        openai.api_base = base_url

        if azure_credential is not None:
            openai.api_key = azure_credential.get_token("https://cognitiveservices.azure.com/.default").token
            openai.api_type = "azure_ad"
        else:
            openai.api_type = 'azure'
            openai.api_key = key

        embeddings = openai.Embedding.create(deployment_id=deployment_id, input=text)
        return embeddings['data'][0]["embedding"]

    except Exception as e:
        raise Exception(f"Error getting embeddings with endpoint={endpoint} with error={e}")

def chunk_content_helper(
        content: str, file_format: str, file_name: Optional[str],
        token_overlap: int,
        num_tokens: int = 256
) -> Generator[Tuple[str, int, Document], None, None]:
    if num_tokens is None:
        num_tokens = 1000000000

    parser = parser_factory(file_format.split("_pdf")[0]) # to handle cracked pdf converted to html
    doc = parser.parse(content, file_name=file_name)
    # if the original doc after parsing is < num_tokens return as it is
    doc_content_size = TOKEN_ESTIMATOR.estimate_tokens(doc.content)
    if doc_content_size < num_tokens:
        yield doc.content, doc_content_size, doc
    else:
        if file_format == "markdown":
            splitter = MarkdownTextSplitter.from_tiktoken_encoder(
                chunk_size=num_tokens, chunk_overlap=token_overlap)
            chunked_content_list = splitter.split_text(
                content)  # chunk the original content
            for chunked_content, chunk_size in merge_chunks_serially(chunked_content_list, num_tokens):
                chunk_doc = parser.parse(chunked_content, file_name=file_name)
                chunk_doc.title = doc.title
                yield chunk_doc.content, chunk_size, chunk_doc
        else:
            if file_format == "python":
                splitter = PythonCodeTextSplitter.from_tiktoken_encoder(
                    chunk_size=num_tokens, chunk_overlap=token_overlap)
            else:
                if file_format == "html_pdf": # cracked pdf converted to html
                    splitter = PdfTextSplitter(separator=SENTENCE_ENDINGS + WORDS_BREAKS, chunk_size=num_tokens, chunk_overlap=token_overlap)
                else:
                    splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
                            separators=SENTENCE_ENDINGS + WORDS_BREAKS,
                            chunk_size=num_tokens, chunk_overlap=token_overlap)
            chunked_content_list = splitter.split_text(doc.content)
            for chunked_content in chunked_content_list:
                chunk_size = TOKEN_ESTIMATOR.estimate_tokens(chunked_content)
                yield chunked_content, chunk_size, doc

def chunk_content(
    content: str,
    file_name: Optional[str] = None,
    url: Optional[str] = None,
    ignore_errors: bool = True,
    num_tokens: int = 256,
    min_chunk_size: int = 10,
    token_overlap: int = 0,
    extensions_to_process = FILE_FORMAT_DICT.keys(),
    cracked_pdf = False,
    use_layout = False,
    add_embeddings = False,
    azure_credential = None,
    embedding_endpoint = None
) -> ChunkingResult:
    """Chunks the given content. If ignore_errors is true, returns None
        in case of an error
    Args:
        content (str): The content to chunk.
        file_name (str): The file name. used for title, file format detection.
        url (str): The url. used for title.
        ignore_errors (bool): If true, ignores errors and returns None.
        num_tokens (int): The number of tokens in each chunk.
        min_chunk_size (int): The minimum chunk size below which chunks will be filtered.
        token_overlap (int): The number of tokens to overlap between chunks.
    Returns:
        List[Document]: List of chunked documents.
    """

    try:
        if file_name is None or (cracked_pdf and not use_layout):
            file_format = "text"
        elif cracked_pdf:
            file_format = "html_pdf" # differentiate it from native html
        else:
            file_format = _get_file_format(file_name, extensions_to_process)
            if file_format is None:
                raise Exception(
                    f"{file_name} is not supported")

        chunked_context = chunk_content_helper(
            content=content,
            file_name=file_name,
            file_format=file_format,
            num_tokens=num_tokens,
            token_overlap=token_overlap
        )
        chunks = []
        skipped_chunks = 0
        for chunk, chunk_size, doc in chunked_context:
            if chunk_size >= min_chunk_size:
                if add_embeddings:
                    for _ in range(RETRY_COUNT):
                        try:
                            doc.contentVector = get_embedding(chunk, azure_credential=azure_credential, embedding_model_endpoint=embedding_endpoint)
                            break
                        except:
                            time.sleep(30)
                    if doc.contentVector is None:
                        raise Exception(f"Error getting embedding for chunk={chunk}")

                chunks.append(
                    Document(
                        content=chunk,
                        title=doc.title,
                        url=url,
                        contentVector=doc.contentVector
                    )
                )
            else:
                skipped_chunks += 1

    except UnsupportedFormatError as e:
        if ignore_errors:
            return ChunkingResult(
                chunks=[], total_files=1, num_unsupported_format_files=1
            )
        else:
            raise e
    except Exception as e:
        if ignore_errors:
            return ChunkingResult(chunks=[], total_files=1, num_files_with_errors=1)
        else:
            raise e
    return ChunkingResult(
        chunks=chunks,
        total_files=1,
        skipped_chunks=skipped_chunks,
    )

def chunk_file(
    file_path: str,
    ignore_errors: bool = True,
    num_tokens=256,
    min_chunk_size=10,
    url = None,
    token_overlap: int = 0,
    extensions_to_process = FILE_FORMAT_DICT.keys(),
    form_recognizer_client = None,
    use_layout = False,
    add_embeddings=False,
    azure_credential = None,
    embedding_endpoint = None
) -> ChunkingResult:
    """Chunks the given file.
    Args:
        file_path (str): The file to chunk.
    Returns:
        List[Document]: List of chunked documents.
    """
    file_name = os.path.basename(file_path)
    file_format = _get_file_format(file_name, extensions_to_process)
    if not file_format:
        if ignore_errors:
            return ChunkingResult(
                chunks=[], total_files=1, num_unsupported_format_files=1
            )
        else:
            raise UnsupportedFormatError(f"{file_name} is not supported")

    cracked_pdf = False
    if file_format == "pdf":
        if form_recognizer_client is None:
            raise UnsupportedFormatError("form_recognizer_client is required for pdf files")
        content = extract_pdf_content(file_path, form_recognizer_client, use_layout=use_layout)
        cracked_pdf = True
    else:
        try:
            with open(file_path, "r", encoding="utf8") as f:
                content = f.read()
        except UnicodeDecodeError:
            from chardet import detect
            with open(file_path, "rb") as f:
                binary_content = f.read()
                encoding = detect(binary_content).get('encoding', 'utf8')
                content = binary_content.decode(encoding)

    return chunk_content(
        content=content,
        file_name=file_name,
        ignore_errors=ignore_errors,
        num_tokens=num_tokens,
        min_chunk_size=min_chunk_size,
        url=url,
        token_overlap=max(0, token_overlap),
        extensions_to_process=extensions_to_process,
        cracked_pdf=cracked_pdf,
        use_layout=use_layout,
        add_embeddings=add_embeddings,
        azure_credential=azure_credential,
        embedding_endpoint=embedding_endpoint
    )

def process_file(
        file_path: str, # !IMP: Please keep this as the first argument
        directory_path: str,
        ignore_errors: bool = True,
        num_tokens: int = 1024,
        min_chunk_size: int = 10,
        url_prefix = None,
        token_overlap: int = 0,
        extensions_to_process: List[str] = FILE_FORMAT_DICT.keys(),
        form_recognizer_client = None,
        use_layout = False,
        add_embeddings = False,
        azure_credential = None,
        embedding_endpoint = None
    ):

    if not form_recognizer_client:
        form_recognizer_client = SingletonFormRecognizerClient()

    is_error = False
    try:
        url_path = None
        rel_file_path = os.path.relpath(file_path, directory_path)
        if url_prefix:
            url_path = url_prefix + rel_file_path
            url_path = convert_escaped_to_posix(url_path)

        result = chunk_file(
            file_path,
            ignore_errors=ignore_errors,
            num_tokens=num_tokens,
            min_chunk_size=min_chunk_size,
            url=url_path,
            token_overlap=token_overlap,
            extensions_to_process=extensions_to_process,
            form_recognizer_client=form_recognizer_client,
            use_layout=use_layout,
            add_embeddings=add_embeddings,
            azure_credential=azure_credential,
            embedding_endpoint=embedding_endpoint
        )
        for chunk_idx, chunk_doc in enumerate(result.chunks):
            chunk_doc.filepath = rel_file_path
            chunk_doc.metadata = json.dumps({"chunk_id": str(chunk_idx)})
    except Exception as e:
        print(e)
        if not ignore_errors:
            raise
        print(f"File ({file_path}) failed with ", e)
        is_error = True
        result =None
    return result, is_error

def chunk_blob_container(
        blob_url: str,
        credential,
        ignore_errors: bool = True,
        num_tokens: int = 1024,
        min_chunk_size: int = 10,
        url_prefix = None,
        token_overlap: int = 0,
        extensions_to_process: List[str] = list(FILE_FORMAT_DICT.keys()),
        form_recognizer_client = None,
        use_layout = False,
        njobs=4,
        add_embeddings = False,
        azure_credential = None,
        embedding_endpoint = None
):
    with tempfile.TemporaryDirectory() as local_data_folder:
        print(f'Downloading {blob_url} to local folder')
        downloadBlobUrlToLocalFolder(blob_url, local_data_folder, credential)
        print(f'Downloaded.')

        result = chunk_directory(
            local_data_folder,
            ignore_errors=ignore_errors,
            num_tokens=num_tokens,
            min_chunk_size=min_chunk_size,
            url_prefix=url_prefix,
            token_overlap=token_overlap,
            extensions_to_process=extensions_to_process,
            form_recognizer_client=form_recognizer_client,
            use_layout=use_layout,
            njobs=njobs,
            add_embeddings=add_embeddings,
            azure_credential=azure_credential,
            embedding_endpoint=embedding_endpoint
        )

    return result

def chunk_directory(
        directory_path: str,
        ignore_errors: bool = True,
        num_tokens: int = 1024,
        min_chunk_size: int = 10,
        url_prefix = None,
        token_overlap: int = 0,
        extensions_to_process: List[str] = list(FILE_FORMAT_DICT.keys()),
        form_recognizer_client = None,
        use_layout = False,
        njobs=4,
        add_embeddings = False,
        azure_credential = None,
        embedding_endpoint = None
):
    """
    Chunks the given directory recursively
    Args:
        directory_path (str): The directory to chunk.
        ignore_errors (bool): If true, ignores errors and returns None.
        num_tokens (int): The number of tokens to use for chunking.
        min_chunk_size (int): The minimum chunk size.
        url_prefix (str): The url prefix to use for the files. If None, the url will be None. If not None, the url will be url_prefix + relpath.
                            For example, if the directory path is /home/user/data and the url_prefix is https://example.com/data,
                            then the url for the file /home/user/data/file1.txt will be https://example.com/data/file1.txt
        token_overlap (int): The number of tokens to overlap between chunks.
        extensions_to_process (List[str]): The list of extensions to process.
        form_recognizer_client: Optional form recognizer client to use for pdf files.
        use_layout (bool): If true, uses Layout model for pdf files. Otherwise, uses Read.
        add_embeddings (bool): If true, adds a vector embedding to each chunk using the embedding model endpoint and key.

    Returns:
        List[Document]: List of chunked documents.
    """
    chunks = []
    total_files = 0
    num_unsupported_format_files = 0
    num_files_with_errors = 0
    skipped_chunks = 0

    all_files_directory = get_files_recursively(directory_path)
    files_to_process = [file_path for file_path in all_files_directory if os.path.isfile(file_path)]
    print(f"Total files to process={len(files_to_process)} out of total directory size={len(all_files_directory)}")

    if njobs==1:
        print("Single process to chunk and parse the files. --njobs > 1 can help performance.")
        for file_path in tqdm(files_to_process):
            total_files += 1
            result, is_error = process_file(file_path=file_path,directory_path=directory_path, ignore_errors=ignore_errors,
                                       num_tokens=num_tokens,
                                       min_chunk_size=min_chunk_size, url_prefix=url_prefix,
                                       token_overlap=token_overlap,
                                       extensions_to_process=extensions_to_process,
                                       form_recognizer_client=form_recognizer_client, use_layout=use_layout, add_embeddings=add_embeddings,
                                       azure_credential=azure_credential, embedding_endpoint=embedding_endpoint)
            if is_error:
                num_files_with_errors += 1
                continue
            chunks.extend(result.chunks)
            num_unsupported_format_files += result.num_unsupported_format_files
            num_files_with_errors += result.num_files_with_errors
            skipped_chunks += result.skipped_chunks
    elif njobs > 1:
        print(f"Multiprocessing with njobs={njobs}")
        process_file_partial = partial(process_file, directory_path=directory_path, ignore_errors=ignore_errors,
                                       num_tokens=num_tokens,
                                       min_chunk_size=min_chunk_size, url_prefix=url_prefix,
                                       token_overlap=token_overlap,
                                       extensions_to_process=extensions_to_process,
                                       form_recognizer_client=None, use_layout=use_layout, add_embeddings=add_embeddings,
                                       azure_credential=azure_credential, embedding_endpoint=embedding_endpoint)
        with ProcessPoolExecutor(max_workers=njobs) as executor:
            futures = list(tqdm(executor.map(process_file_partial, files_to_process), total=len(files_to_process)))
            for result, is_error in futures:
                total_files += 1
                if is_error:
                    num_files_with_errors += 1
                    continue
                chunks.extend(result.chunks)
                num_unsupported_format_files += result.num_unsupported_format_files
                num_files_with_errors += result.num_files_with_errors
                skipped_chunks += result.skipped_chunks

    return ChunkingResult(
            chunks=chunks,
            total_files=total_files,
            num_unsupported_format_files=num_unsupported_format_files,
            num_files_with_errors=num_files_with_errors,
            skipped_chunks=skipped_chunks,
        )

class SingletonFormRecognizerClient:
    instance = None
    def __new__(cls, *args, **kwargs):
        if not cls.instance:
            print("SingletonFormRecognizerClient: Creating instance of Form recognizer per process")
            url = os.getenv("FORM_RECOGNIZER_ENDPOINT")
            key = os.getenv("FORM_RECOGNIZER_KEY")
            if url and key:
                cls.instance = DocumentAnalysisClient(endpoint=url, credential=AzureKeyCredential(key))
            else:
                print("SingletonFormRecognizerClient: Skipping since credentials not provided. Assuming NO form recognizer extensions(like .pdf) in directory")
                cls.instance = object() # dummy object
        return cls.instance

    def __getstate__(self):
        return self.url, self.key

    def __setstate__(self, state):
        url, key = state
        self.instance = DocumentAnalysisClient(endpoint=url, credential=AzureKeyCredential(key))


from easydict import EasyDict as edit

config = edit()
config.input_size = 640    # 网络输入尺寸
config.input_channel = 3    #
config.test_img_shape = 640

config.anchor_scales = [[12, 16],
                        [19, 36],
                        [40, 28],
                        [36, 75],
                        [76, 55],
                        [72, 146],
                        [142, 110],
                        [192, 243],
                        [459, 401]]
config.max_box = 150 # 数量
config.num_classes = 4
config.out_channels = (config.num_classes + 5) * 3  # 定值
config.jitter = 0.3
config.hue = 0.015
config.saturation = 1.5
config.value = 0.4   # 不解其意
config.label_smooth = 0
config.label_smooth_factor = 0.1
config.ignore_threshold = 0.7
config.momentum = 0.9
config.weight_decay = 0.0005
config.loss_scale = 1024

# 根据不同的version选取响应的input_shape
config.input_shape=[[3, 32, 64, 128, 256, 512, 1],
                    [3, 48, 96, 192, 384, 768, 2],
                    [3, 64, 128, 256, 512, 1024, 3],
                    [3, 80, 160, 320, 640, 1280, 4]]

# train
config.log_interval = 100
config.batch_size = 8   # train batchsize
config.lr_scheduler = "cosine_annealing"
config.max_epoch = 300
config.T_max = 300     #??
config.lr_epochs = "220,250"
config.image_path = "obs://4in1/4in1s/images/"   # 桶地址
config.anno_path = "obs://4in1/4in1s/labels/"
config.train_ids = "obs://4in1/4in1s/train.txt"
config.val_ids = "obs://4in1/4in1s/val.txt"
config.yolo_version = 'yolov5s'
config.output_dir = "obs://4in1/models/"

# test
config.test_ignore_threshold = 0.001
config.labels = ['person', 'calling', 'play_phone', 'smoke']
config.eval_nms_thresh = 0.3


import datetime as dt
import decimal
import pricehistory

class portfolio:

    def __init__ (self, equities=[], cash = 0.0):
        "equities must be a list of tuples of (symbol, quantity)"
        self.cash = decimal.Decimal(cash).quantize(decimal.Decimal('0.01'))
        self.quantities = {}
        for e in equities:
            if e[1] != 0:
                self.quantities[e[0]] = e[1]

    def __repr__ (self):
        return 'portfolio(%s, %s)' % ([(s,self.quantities[s]) for s in sorted(self.quantities.keys())], self.cash)

    def dump (self, dumper, writer):
        dumper(dict(equities=self.quantities, cash=str(self.cash)), writer, indent=5, sort_keys=True)

    def load (self, loader, reader):
        d = loader(reader)
        if 'cash' in d:
            self.cash = decimal.Decimal(d['cash'])
        else:
            self.cash = decimal.Decimal('0.00')
        if 'equities' in d:
            self.quantities = d['equities']
            for symbol,quantity in self.quantities.items():
                if not isinstance(quantity, int):
                    raise TypeError('Non-integer equity quantity in portfolio serialization')
        else:
            self.quantities = {}

    def __add__ (self, other):
        result = {}

        for symbol in self.quantities:
            result[symbol] = self.quantities[symbol]

        for symbol in other.quantities:
            if symbol in result:
                result[symbol] += other.quantities[symbol]
            else:
                result[symbol] = other.quantities[symbol]
            if result[symbol] == 0:
                del result[symbol]

        return portfolio([(s,result[s]) for s in result], self.cash+other.cash)

    def __sub__ (self, other):
        result = {}

        for symbol in self.quantities:
            result[symbol] = self.quantities[symbol]

        for symbol in other.quantities:
            if symbol in result:
                result[symbol] -= other.quantities[symbol]
            else:
                result[symbol] = - other.quantities[symbol]
            if result[symbol] == 0:
                    del result[symbol]

        return portfolio([(s,result[s]) for s in result], self.cash-other.cash)

    def values(self, price_source, end_date=dt.date.today()):
        gh = pricehistory.GroupHistory(self.quantities.keys())
        print('gh', gh)
        gh.load_to_date(price_source, 1, end_date=end_date)
        result = {}
#        print(gh.symbols)
        print('\nUsing following prices:')
        for s in sorted(gh.symbols):
            print('{0}@{1}'.format(s, gh[s].last_price()))

        for s in sorted(gh.symbols):
            result[s] = self.quantities[s]*gh[s].last_price()
        return result

    def value(self, price_source, end_date=dt.date.today()):
        result = self.cash
        values = self.values(price_source, end_date)
        print('\nAllocation by value:')
        for s in sorted(values.keys()):
            print("{0:>5s}: {1:10n}".format(s,values[s]))
            result += values[s]
        print(" cash: {0:10n}".format(self.cash))
        print("total: {0:10n}".format(result))
        return result

    def allocation(self, price_source):
        "Return a list of tuples of (symbol, percent_of_portfolio_value)"
        v = float(self.value(price_source))
        values = self.values(price_source)
        return [(s,float(values[s]/v)) for s in self.quantities.keys()]

    @classmethod
    def from_allocation(cls, price_source, allocation, cash_available, end_date=dt.date.today()):
        "Return a portfolio from an allocation"
        symbols = [a[0] for a in allocation]
        ph = pricehistory.GroupHistory(symbols)
        ph.load_to_date(price_source, 1, end_date=end_date)
        percent_allocated = sum([a[1] for a in allocation])
        equities = []
        cash_left = decimal.Decimal(cash_available)
        for a in allocation:
            quantity = int(round(a[1]*float(cash_available)/float(ph[a[0]].last_price())))
            equities.append((a[0],quantity))
            cash_left -= quantity * ph[a[0]].last_price()
        return portfolio(equities, cash_left)

    def symbols(self):
        return sorted(self.quantities.keys())


from aws_cdk import (
    Duration,
    Stack,
    Tags,
    aws_events as events,
    aws_cloudwatch_actions as cw_actions,
    aws_events_targets as targets,
    aws_sns as sns,
    aws_cloudwatch as cloudwatch,
    aws_logs as logs
   )
import aws_cdk
from aws_cdk import App, CfnOutput, NestedStack, NestedStackProps, Stack
from cdklab.config.stack_config import AWSPipeline, service_prefix
from constructs import Construct

"""
Stack that creates CloudWatch metrics, Alarm and Events Rule with a Target

Creates a Metric from an existing Metric CPUUtilization, creates an alarm and sends notification to a SNS Topic
Creates an Event Rule that monitors for Guard duty alerts and sends notification to a SNS Topic with a formatted message
"""
class cloudwatch_stack(NestedStack):

    def __init__(self, scope: Construct, construct_id: str, stage, **kwargs) -> None:
        super().__init__(scope, construct_id, **kwargs)

        acct_info = AWSPipeline[stage]
        region = acct_info.region
        stage = acct_info.stage
        account= acct_info.accountID

        Tags.of(self).add("Name",service_prefix+"_cloudwatch_alerts_"+stage)
        Tags.of(self).add("Stage", stage)
        Tags.of(self).add("Owner:Name", "SecurityTeam")
        Tags.of(self).add("Owner:ContactEmail", "SecurityTeam@email.com")
        Tags.of(self).add("Owner:BusinessUnit", "IT Security")
        Tags.of(self).add("Owner:CostCenter", "111")
        Tags.of(self).add("DataClassification", "Highly Confidential")
        Tags.of(self).add("Severity", "1")
        Tags.of(self).add("Compliance", "PCI")

        cpu_metric =  cloudwatch.Metric(
            namespace= "AWS/EC2",
            metric_name= 'CPUUtilization',
            dimensions_map= {  "stat": "Average"  }
            )

        cpualarm = cloudwatch.Alarm(self,"cpu_alarm",
            metric=cpu_metric,
            alarm_name=service_prefix+" CPU High Alarm",
            threshold= 70,
            evaluation_periods= 1,
            datapoints_to_alarm= 1,
            # period=aws_cdk.Duration.minutes(5),
            alarm_description="A Alarm to send an sns message if CPU goes above 70 percent"
        )

        sns_topic = sns.Topic.from_topic_arn(self, "getsnstopic","arn:aws:sns:"+region+":"+account+":medium_alert")

        cpualarm.add_alarm_action(cw_actions.SnsAction(sns_topic) )

        #Transform for message text to be added to the alert
        ruleTargetText = "The alert in ${EventField.fromPath('$.detail.accountId')}' in region ${EventField.fromPath('$.detail.awsRegion')} for alert ${EventField.fromPath('$.detail.eventName')} " \

        #create rules with Targets
        rule1 = events.Rule(self, 'AWSGuardDutyEventReporting',
        description='Creates a CloudWatch Event and SNS Push Notification for events triggered by AWS GuardDuty',
        rule_name='AWSGuardDutyEventReporting',
        enabled= True,
        event_pattern=events.EventPattern(
            source=['aws.guardduty'],
            detail_type= ['GuardDuty Finding']
        )
        )

        rule1.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))

        rule2 = events.Rule(self, 'DetectNetworkChangeEvents',
        description= 'Publishes formatted network change events to an Alert',
        rule_name= 'DetectNetworkChangeEvents',
        enabled= True,
            event_pattern=events.EventPattern(
            source=['ec2'],
            detail_type=['AWS API Call via CloudTrail'],
            detail={
                "eventSource": ['ec2.amazonaws.com'],
                "eventName": ['AttachInternetGateway',
                            'AssociateRouteTable',
                            'CreateCustomerGateway',
                            'CreateInternetGateway',
                            'CreateRoute',
                            'CreateRouteTable',
                            'DeleteCustomerGateway',
                            'DeleteInternetGateway',
                            'DeleteRoute',
                            'DeleteRouteTable',
                            'DeleteDhcpOptions',
                            'DetachInternetGateway',
                            'DisassociateRouteTable',
                            'ReplaceRoute',
                            'ReplaceRouteTableAssociation']
            }

        ))

        rule2.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))

        rule3 = events.Rule(self, 'DetectCloudTrailChanges',
        description= 'Publishes formatted CloudTrail change events to an Alert',
        rule_name= 'DetectCloudTrailChanges',
        enabled=True,
        event_pattern=events.EventPattern(
            source= ['ec2'],
            detail_type=['AWS API Call via CloudTrail'],
            detail= {
            "eventSource": ['ec2.amazonaws.com'],
            "eventName": [ 'CreateTrail',
                            'UpdateTrail',
                            'StopLogging',
                            'DeleteTrail',]
            }
        ))

        rule3.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))

        rule4 = events.Rule(self, 'DetectNetworkAclChanges',
        description= 'Publishes formatted network ACL change events to an alert',
        rule_name= 'DetectNetworkAclChanges',
        enabled= True,
        event_pattern=events.EventPattern(
        source= ['ec2'],
        detail_type=['AWS API Call via CloudTrail'],
        detail= {
            "eventSource": ['ec2.amazonaws.com'],
            "eventName": [ 'CreateNetworkAcl',
                        'CreateNetworkAclEntry',
                        'DeleteNetworkAcl',
                        'DeleteNetworkAclEntry',
                        'ReplaceNetworkAclEntry',
                        'ReplaceNetworkAclAssociation',]
            }
        ))

        rule4.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))

        rule5 = events.Rule(self, 'DetectCloudWatchLogsChanges',
            description= 'Changes to Amazon CloudWatch',
            rule_name= 'DetectCloudWatchLogsChanges',
            enabled= True,
            event_pattern=events.EventPattern(
            source= ['aws.logs'],
            detail_type=['AWS API Call via CloudTrail'],
            detail= {
                "eventsource": ['logs.amazonaws.com'],
                "eventName": [ 'DeleteDestination',
                            'DeleteLogGroup',
                            'DeleteLogStream',
                            'DeleteRetentionPolicy',]
            }
        ))

        rule5.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))

        rule6 = events.Rule(self, 'DetectSecurityGroupChanges',
            description= 'Publishes formatted security group change events to an Alert',
            rule_name= 'DetectSecurityGroupChanges',
            enabled= True,
            event_pattern=events.EventPattern(
            source= ['ec2'],
            detail_type=['AWS API Call via CloudTrail'],
            detail= {
                "eventsource":['ec2.amazonaws.com'],
                "eventName":  [ 'AuthorizeSecurityGroupIngress',
                            'AuthorizeSecurityGroupEgress',
                            'RevokeSecurityGroupIngress',
                            'RevokeSecurityGroupEgress',
                            'CreateSecurityGroup',
                            'DeleteSecurityGroup',]
            }
        ))

        rule6.add_target(target=targets.SnsTopic(sns_topic, message=events.RuleTargetInput.from_text(ruleTargetText)))


""" Twelve days - Solution """

from typing import List

ORDINALS = [
    'first',
    'second',
    'third',
    'fourth',
    'fifth',
    'sixth',
    'seventh',
    'eighth',
    'ninth',
    'tenth',
    'eleventh',
    'twelfth'
]
GIFTS = [
    "twelve Drummers Drumming, ",
    "eleven Pipers Piping, ",
    "ten Lords-a-Leaping, ",
    "nine Ladies Dancing, ",
    "eight Maids-a-Milking, ",
    "seven Swans-a-Swimming, ",
    "six Geese-a-Laying, ",
    "five Gold Rings, ",
    "four Calling Birds, ",
    "three French Hens, ",
    "two Turtle Doves, ",
    "and a Partridge in a Pear Tree.",
]

def recite(start_verse: int, end_verse: int) -> List[str]:
    """ Build twelve days music verses """
    response = []
    for verse in range(start_verse, end_verse + 1):
        first_part = f'On the {ORDINALS[verse - 1]} day of Christmas my true love gave to me: '
        if verse == 1:
            second_part = GIFTS[-1][4:]
        else:
            second_part = ''.join(GIFTS[verse * -1:])
        response.append(f'{first_part}{second_part}')
    return response


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn import tree
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import accuracy_score, roc_curve, auc

# import numpy as np
# import seaborn as sns
# import graphviz
# import matplotlib.pyplot as plt

class TitanicSinkingModel:
    def __init__(self, filename):
        self.df = pd.read_csv(filename)
        self.df_normalized = None
        self.data_train = None
        self.data_test = None
        self.target_train = None
        self.target_test = None
        self.clf = None

    def preprocessing(self):
        # this can be done because the target value is either 0 or 1
        # thus apply the normalization have no effect on them
        scaler = MinMaxScaler()
        self.df_normalized = pd.DataFrame(scaler.fit_transform(self.df.values),
                                          columns=self.df.columns, index=self.df.index)
        self.split_dataframe()

    def split_dataframe(self):
        data = self.df_normalized[
            [col for col in self.df_normalized.columns if col != 'Survived']]
        target = self.df_normalized['Survived']
        # split the test and train data set
        self.data_train, self.data_test, self.target_train, self.target_test = train_test_split(data, target,
                                                                                test_size=0.3,
                                                                                shuffle=False)

    def fit_decision_tree(self):
        self.clf = tree.DecisionTreeClassifier()
        self.clf = self.clf.fit(self.data_train, self.target_train)

        # dot_data = tree.export_graphviz(clf, out_file=None)
        # graph = graphviz.Source(dot_data)
        # graph.format = 'jpg'
        # graph.render("decision_tree_plot_orignial")
        labels_test = self.clf.predict(self.data_test)
        acc = accuracy_score(labels_test, self.target_test)
        print("acc for test set is : " + str(acc))
        labels_test2 = self.clf.predict(self.data_train)
        acc2 = accuracy_score(labels_test2, self.target_train)
        print("acc for train set is : " + str(acc2))

    def find_optimal_decision_tree(self):
        # min_samples_leaf the minmum number of leaves a split
        # can happen according to the value of entropy
        auc_tain = {}
        auc_test = {}
        for i in range(2, 21):
            descion_tree = tree.DecisionTreeClassifier(min_samples_leaf=i)
            descion_tree.fit(self.data_train, self.target_train)
            false_positive_rate, true_positive_rate, thresholds = roc_curve(self.target_train, descion_tree.predict(self.data_train))
            auc_tain[i] = auc(false_positive_rate, true_positive_rate)
            false_positive_rate, true_positive_rate, thresholds = roc_curve(self.target_test, descion_tree.predict(self.data_test))
            auc_test[i] = auc(false_positive_rate, true_positive_rate)
            # fig = plt.figure()
            # d = {'min_samples_leaf': np.array(list(auc_tain)), 'AUC score': np.array(list(auc_tain.values()))}
            # pd_plot = pd.DataFrame(d)
            # sns.lineplot(x='min_samples_leaf', y='AUC score', data=pd_plot)
            # plt.show()
            # fig.savefig('plot_train.png')
        self.clf = tree.DecisionTreeClassifier(min_samples_leaf=6)
        self.clf.fit(self.data_train, self.target_train)
        # dot_data = tree.export_graphviz(self.clf, out_file=None)
        # graph = graphviz.Source(dot_data)
        # graph.format = 'jpg'
        # graph.render("decision_tree_plot_optimal")

    def part_D_calculation(self):
        print("female and survived: ")
        print(self.df.query('Sex == 1  & Survived ==1'))
        print("all female: ")
        print(self.df.query('Sex == 1'))
        print("first class and survived: ")
        print(self.df.query('Pclass == 1  & Survived ==1'))
        print("all first class: ")
        print(self.df.query('Pclass == 1'))

def print_df_normalized(self):
    print(self.df_normalized.head().to_string())

if __name__ == '__main__':
    titianic_sinking_model = TitanicSinkingModel("titanic.csv")
    titianic_sinking_model.preprocessing()
    # titianic_sinking_model.print_df_normalized()
    # print(titianic_sinking_model.data_train.to_string())
    titianic_sinking_model.fit_decision_tree()
    titianic_sinking_model.find_optimal_decision_tree()
    titianic_sinking_model.part_D_calculation()


import json
from tkinter import *
from tkinter import Radiobutton
from tkinter import messagebox as mb

root = Tk()

root.state('zoomed')
root.title('Quizz')
root.config(bg='#fff')
root.iconbitmap("./img/logo.ico")

font = ('san-serif', 16, 'bold')

with open('data.json') as f:
    data = json.load(f)

question = (data['question'])
options = (data['options'])
answer = (data['answer'])

que_no = 0
opt_selected = IntVar()
correct = 0
data_size = len(question)

def display_question():
    q_no = Label(root, text=question[que_no], width=60,
                 font=(font, 16), anchor='w', bg='white')
    q_no.place(x=300, y=150)

def display_options():
    opts = radio_buttons()
    val = 0
    opt_selected.set(0)
    for option in options[que_no]:
        opts[val]['text'] = option
        val += 1

def radio_buttons():
    q_list = []
    y_pos = 250
    while len(q_list) < 4:
        radio_btn = Radiobutton(root, variable=opt_selected, value=len(
            q_list) + 1, font=(font, 16), bg='white')
        q_list.append(radio_btn)
        radio_btn.place(x=300, y=y_pos)
        y_pos += 40
    return q_list

def next_btn():
    global correct
    global que_no
    if check_ans(que_no):
        correct += 1
    que_no += 1
    if que_no == data_size:
        btn_Next.destroy()
        mb.showwarning("Warning", "All Question are completed")
        btn_Result = Button(image=btnResult, border=0,
                            bg='white', borderwidth=0, command=display_result)
        btn_Result.pack(side=BOTTOM, pady=90)

    else:
        display_question()
        display_options()

def start_btn():
    label1.destroy()
    title.destroy()
    btn_Start.destroy()
    display_question()
    display_options()

def check_ans(q_no):
    global correct
    global que_no
    if opt_selected.get() == answer[q_no]:
        return True
    if not opt_selected.get():
        mb.showwarning("Warning", "Please select a option")
        que_no -= 1

def display_result():
    wrong_count = data_size - correct
    correct_count = int(correct)
    score = int(correct_count / data_size * 100)
    label2 = Label(image=thanks, border=0)
    label2.place(x=0, y=0)
    correct_label = Label(text='Your Score', font=(font, 30), bg="#FDBC00")
    correct_label.place(x=1000, y=200)
    correct_label = Label(
        text=f'Correct: {correct}', font=(font, 30), bg="#FDBC00")
    correct_label.place(x=1000, y=300)
    wrong_label = Label(
        text=f'Wrong: {wrong_count}', font=(font, 30), bg="#FDBC00")
    wrong_label.place(x=1000, y=400)
    result_label = Label(
        text=f'Score: {score}%', font=(font, 30), bg="#FDBC00")
    result_label.place(x=1000, y=500)

btnResult = PhotoImage(file="img/Result.png")
thanks = PhotoImage(file="./img/image 2.png")
bg = PhotoImage(file="./img/MacBook Air - 4.png")
label1 = Label(image=bg, bg='white', border=0)
label1.place(x=0, y=0)

title_img = PhotoImage(file="./img/Group 6.png")
title = Label(image=title_img, bg='white', border=0)
title.pack(pady=50)

quizz_img = PhotoImage(file="img/image 1.png")
label1 = Label(image=quizz_img, border=0, bg='white')
label1.pack()

btnStart = PhotoImage(file="img/Start.png")
btn_Start = Button(image=btnStart, border=0, bg='white',
                   borderwidth=0, command=start_btn)
btn_Start.pack(pady=80)

btnNext = PhotoImage(file="img/Next.png")
btn_Next = Button(image=btnNext, border=0, bg='white',
                  borderwidth=0, command=next_btn)
btn_Next.pack(side=BOTTOM, pady=90)

root.mainloop()


n = int(input())
len = 2**n
num = []
se = set()
num.append("0"*n)
se.add(num[0])
for i in range(1, len):
    prefix = num[i - 1][1::]
    if (not((prefix + "1") in se)):
        num.append(prefix + "1")
        se.add(num[i])
    else:
        num.append(prefix + "0")
        se.add(num[i])
print(*num, sep = "\n")


# FAÇA UM PROGRAMA QUE LEIA 3 NPUMEROS E MOSTRE QUAL O MAIOR E QUAL O MENOR

a = int(input('Digite um número: '))
b = int(input('Digite um número: '))
c = int(input('Digite um número: '))
# VERIFICANDO QUEM É O MENOR
if a < b and a < c:
    menor = a
if c < b and c < a:
    menor = c
if b < a and b < c:
    menor = b
# VERIFICANDO QUEM É O MAIOR
if a > b and a > c:
    maior = a
if c > b and c > a:
    maior = c
if b > a and b > c:
    maior = b
print('O menor valor digital foi {}.'.format(menor))
print('O maior valor digital foi {}.'.format(maior))


import cv2 as cv
from matplotlib import pyplot as plt
import numpy as np

# Smoothing (Image Filtering)
# kernel, convolution matrix, or mask is a small matrix. It is used for blurring, sharpening, embossing, edge detection, and more.
# This is accomplished by doing a convolution between a kernel and an image.

# As in one-dimensional signals, images also can be filtered with various low-pass filters (LPF),
# high-pass filters (HPF), etc. LPF helps in removing noise, blurring images, etc. HPF filters help
# in finding edges in images.

# Homogeneous filters ==> each pixel is the mean of its kernel neighbours

# Image blurring is achieved by convolving the image with a low-pass filter kernel. It is useful
# for removing noise. It actually removes high frequency content (eg: noise, edges) from the image.
# So edges are blurred a little bit in this operation

# 1-Averaging ==> This is done by convolving an image with a normalized box filter.
# It simply takes the average of all the pixels under the kernel area and replaces the central element

# 2-Gaussian filter ==> using diff-weighted-pixels, for sides weights are low weighted pixels, while for center is high weighted pixels

# 3-Median Blurring ==> takes median of all the pixels under the kernel area and the central element is
# replaced with this median value, This is highly effective against salt-and-pepper noise in an image.
# the central element is always replaced by some pixel value in the image. It reduces the noise effectively.
# Its kernel size should be a positive odd integer.

# 4-Bilateral Filtering ==> highly effective in noise removal while keeping edges sharp,
# But the operation is slower compared to other filters, Bilateral filtering also takes a Gaussian filter
# in space, but one more Gaussian filter which is a function of pixel difference.

# Depth Combinations ==> CV_8U	        -1/CV_16S/CV_32F/CV_64F
#                        CV_16U/CV_16S	-1/CV_32F/CV_64F
#                        CV_32F	        -1/CV_32F/CV_64F
#                        CV_64F	        -1/CV_64F

img = cv.imread('opencv-logo.png')
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)

# Generating [5,5] white matrix of |det| = 1/25
kernel = np.ones((5,5),np.float32)/25

dst = cv.filter2D(img,-1,kernel)
blur = cv.blur(img, (2,2))
gausBlur = cv.GaussianBlur(img, (5,5),0)
mediBlur = cv.medianBlur(img, 5)
bf = cv.bilateralFilter(img, 5, 75,75)

#blur = cv.boxFilter(img,2 ,(2,2))

titles = ['Original', '2D Convolution', 'Blur', 'GaussianBlur', 'MedianBlur', 'Bilateral']
images = [img,dst,blur,gausBlur,mediBlur, bf]

rows = 2
columns = 3
iterations = len(images)
for ite in range(iterations):
    plt.subplot(rows,columns, ite+1 ), plt.imshow(images[ite])
    plt.title(titles[ite])

plt.show()
cv.waitKey(0)
cv.destroyAllWindows()


"""empty message

Revision ID: f61413c0517d
Revises: 490bdb384e28
Create Date: 2021-08-23 10:00:27.399097

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'f61413c0517d'
down_revision = '490bdb384e28'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('FOIRawRequests', sa.Column('assignedgroup', sa.String(length=250), nullable=True))
    op.add_column('FOIMinistryRequests', sa.Column('assignedgroup', sa.String(length=250), nullable=True))
    op.alter_column('FOIMinistryRequests', 'assignedto', nullable=True)
    #ALTER TABLE "FOIMinistryRequests" ALTER COLUMN "assignedto" DROP NOT NULL;
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('FOIRawRequests', 'assignedgroup')
    op.drop_column('FOIMinistryRequests', 'assignedgroup')
    op.alter_column('FOIMinistryRequests', 'assignedto', nullable=False)
    # ### end Alembic commands ###


#Qiita API v2対応
import sys, json, requests, re
from QiitaAPI import *

class QiitaAPIv2(QiitaAPI):

	#v2 APIのベースURL
	BASEURL='https://qiita.com/api/v2/'
	#V2 APIの要素定義
	API_PROP_TITLE='title'
	API_PROP_ITEMID='id'
	API_PROP_URL='url'
	API_PROP_TAGS='tags'
	API_PROP_USER='user'
	API_PROP_VIEW='page_views_count'
	API_PROP_LIKE='likes_count'
	API_PROP_HTML='rendered_body'
	API_PROP_MARKDOWN='body'
	API_PROP_CREATE_TIME='created_at'
	API_PROP_UPDATE_TIME='updated_at'
	API_PROP_PAGE='page'
	API_PROP_PER_PAGE='per_page'
	#stockはstockersの要素数をカウントする
	API_PROP_STOCK='stock'
	#stockはcommentsの要素数をカウントする
	API_PROP_COMMENT='comment'

	#1回の取得データ最大数の制限
	DEFAULT_RESULT_MAX=5000

	#内部で使うタグ
	#conf設定。ここ以外の各要素はoutputに使うので、QiitaAPIと合わせる
	CONF_PROP_SHOW='show'
	CONF_PROP_ITEM='item'
	CONF_PROP_USER=QiitaAPI.COMMON_USER
	CONF_PROP_TOKEN='access_token'
	#内部情報管理用
	MNG_PROP_SHOW='show'
	MNG_PROP_HAVE_LIST='have_list'
	MNG_PROP_API_PROP='property'
	#http header parse
	HTTP_PROP_HEADER='header'
	HTTP_PROP_LINK='Link'
	HTTP_PROP_COUNT='Total-Count'
	HTTP_PROP_BODY='body'

	#private
	def _parse_setting(self, data):
		#デフォルトを設定
		self._set_default()

		#認証設定
		if self.CONF_PROP_TOKEN in data:
			token=data[self.CONF_PROP_TOKEN]
			self._headers['Authorization']=f'Bearer {token}'

		#ユーザー名設定
		if QiitaAPI.COMMON_USER in data:
			self._user=data[QiitaAPI.COMMON_USER]

		#最大数設定
		if QiitaAPI.COMMON_MAX in data:
			self._result_max=data[QiitaAPI.COMMON_MAX]

		#show関連のデータがないならデフォルトをそのまま使う
		if not self.MNG_PROP_SHOW in data:
			return

		#item関連の設定更新
		self._set_item_config(data[self.MNG_PROP_SHOW])

		#user関連の設定更新
		self._set_user_config(data[self.MNG_PROP_SHOW])

	#デフォルト設定を行う
	def _set_default(self):
		#デフォルトを設定
		self._headers={}
		self._result_max=self.DEFAULT_RESULT_MAX

		#item関連
		self._item_config={
				self.API_PROP_PAGE:1,
				self.API_PROP_PER_PAGE:100,
				QiitaAPI.ITEM_RAW:False,
				#itemは{responseキー:{表示するか, 一覧取得で取得可能か, IDの対応}}
				self.CONF_PROP_ITEM:{
					self.API_PROP_TITLE:	{self.MNG_PROP_SHOW:True , self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_TITLE},
					self.API_PROP_URL:		{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_URL},
					self.API_PROP_TAGS:		{self.MNG_PROP_SHOW:True , self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_TAGS},
					self.API_PROP_MARKDOWN :{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_MARKDOWN_DATA},
					self.API_PROP_HTML:		{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_HTML_DATA},
					self.API_PROP_CREATE_TIME:	{self.MNG_PROP_SHOW:False , self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_CREATED_AT},
					self.API_PROP_UPDATE_TIME:	{self.MNG_PROP_SHOW:False , self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_UPDATED_AT},
					self.API_PROP_UPDATE_TIME:	{self.MNG_PROP_SHOW:False , self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_UPDATED_AT},
					self.API_PROP_VIEW:		{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:False, self.MNG_PROP_API_PROP:QiitaAPI.ITEM_VIEW},
					self.API_PROP_USER:		{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:True, self.MNG_PROP_API_PROP:QiitaAPI.ITEM_USER},
					self.API_PROP_LIKE:		{self.MNG_PROP_SHOW:True, self.MNG_PROP_HAVE_LIST:True , self.MNG_PROP_API_PROP:QiitaAPI.ITEM_LIKE},
					self.API_PROP_STOCK:	{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:False, self.MNG_PROP_API_PROP:QiitaAPI.ITEM_STOCK},
					self.API_PROP_COMMENT:	{self.MNG_PROP_SHOW:False, self.MNG_PROP_HAVE_LIST:False, self.MNG_PROP_API_PROP:QiitaAPI.ITEM_COMMENT}
				}
		}

		#user関連
		self._user_config={
				self.API_PROP_PAGE:1,
				self.API_PROP_PER_PAGE:100,
				QiitaAPI.ITEM_RAW:False,
				self.CONF_PROP_USER:{
					#itemは{responseキー:{表示するか, IDの対応}}
					self.API_PROP_ITEMID:	{self.MNG_PROP_SHOW:True, self.MNG_PROP_API_PROP:QiitaAPI.USER_ID}
				}
		}

	#item周りの設定更新を行う
	def _set_item_config(self, show_data):
		#itemがないならデフォルトで
		if not self.CONF_PROP_ITEM in show_data:
			return

		for key, value in show_data[self.CONF_PROP_ITEM].items():
			#item要素以外はそのまま代入
			if key in self._item_config:
				if key != self.CONF_PROP_ITEM:
					self._item_config[key]=value
			#それ以外はconfigテーブルの表示に関して更新する
			else:
				self._update_item_config(key, value)

	#item_configの更新を行う
	def _update_item_config(self, key, value):
		self._update_show_config(key, value, self._item_config[self.CONF_PROP_ITEM])

	#user周りの更新を行う
	def _set_user_config(self, show_data):
		#userがないならデフォルトで
		if not self.CONF_PROP_USER in show_data:
			return

		for key, value in show_data[self.CONF_PROP_USER].items():
			#item要素はそのまま代入
			if key in self._user_config:
				if key != self.CONF_PROP_USER:
					self._user_config[key]=value
			#それ以外はconfigテーブルの表示に関して更新する
			else:
				self._update_user_config(key, value)

	#user_configの更新を行う
	def _update_user_config(self, key, value):
		self._update_show_config(key, value, self._user_config[self.CONF_PROP_USER])

	#item_configの更新を行う
	def _update_show_config(self, key, value, config):
		#正しいkey, valueじゃないなら無視
		if type(value) is not bool:
			return

		for index, tables in config.items():
			if tables[self.MNG_PROP_API_PROP] == key:
				#アップデート
				config[index][self.MNG_PROP_SHOW]=value
				break

	#get response body
	def _get_api_response_body(self, extraurl):
		res=self._callapi(extraurl)
		res_body=json.loads(res.text)
		return res_body

	#get response all
	def _get_api_response_all(self, extraurl, has_baseurl):
		if has_baseurl:
			res=self._send_get_req(extraurl)
		else:
			res=self._callapi(extraurl)
		res_all={self.HTTP_PROP_HEADER:res.headers}
		res_all[self.HTTP_PROP_BODY]=json.loads(res.text)
		return res_all

	def _get_next_link(self, header):
		#Linkタグがないならreturn
		if not self.HTTP_PROP_LINK in header:
			return ""

		#format:
		#{'Link': '<url>; rel="first",<url>; rel="prev" , <url>; rel="next", <url>; rel="last"'}
		linklist=re.split(r',', header[self.HTTP_PROP_LINK])
		#<url>; rel="xxx"で分割
		for link_raw in linklist:
			#無駄文字を削除後;で分割
			link_split=re.split(r';', re.sub(r"[<>\" ]", "", link_raw))
			# rel="next"を採用
			if link_split[1] == 'rel=next':
				return link_split[0]

		#nextが無かった
		return ""

	#update response, parse header, and return next
	def _update_response_and_get_next_link(self, url, response_all):
		#responseを取得
		response=self._get_api_response_all(url, True)

		#最大数を超えてしまいそうならそこでstop
		merged_count=len(response[self.HTTP_PROP_BODY])+len(response_all)
		if self._result_max < merged_count:
			return ""

		#responseを追加。listなのでextendだけでOK
		response_all.extend(response[self.HTTP_PROP_BODY])

		#最大値に行ったら終了
		if self._result_max == merged_count:
			return ""
		else:
			#次のURLを返却
			return self._get_next_link(response[self.HTTP_PROP_HEADER])

	#get response with link 1st
	def _get_api_response_with_link(self, extraurl, response):
		next_url=self.BASEURL+extraurl
		#nextがなくなるまでgetの繰り返し
		while len(next_url) is not 0:
			next_url=self._update_response_and_get_next_link(next_url, response)

		return response

	#qiita api call
	def _callapi(self, extraurl):
		url=self.BASEURL+extraurl
		return self._send_get_req(url)

	#get request direct
	def _send_get_req(self, url):
		try:
			#print(url)
			res=requests.get(url, headers=self._headers)
			#正しい結果か？
			if not self._is_valid_response(res):
				sys.exit()

			return res
		except:
			print(f'Failed to get {url}')
			traceback.print_exc()
			sys.exit()

	#check response data
	def _is_valid_response(self, res):
		return res.status_code is 200

	def _get_page_query(self):
		return f'page={self._item_config[self.API_PROP_PAGE]}&per_page={self._item_config[self.API_PROP_PER_PAGE]}'

	#get user raw result
	def _get_user_items_by_api(self):
		query=self._get_page_query()
		if hasattr(self, '_user'):
			url=f'users/{self._user}/items?{query}'
		else :
			url=f'authenticated_user/items?{query}'
		#user itemsはlink系
		response=[]
		self._get_api_response_with_link(url, response)
		return response

	#get items raw result
	def _get_items_by_api(self):
		query=self._get_page_query()
		response=[]
		self._get_api_response_with_link(f'items?{query}', response)
		return response

	#get_itemの生データ取得
	def _get_item_by_api(self, item):
		return self._get_api_response_body(f'items/{item}')

	#viewはitemsから取得するしかない
	def _get_views(self, item):
		#ヘッダにtokenが設定されていないと意味ないのでチェック
		if not 'Authorization' in self._headers:
			return None

		#viewはitemsの情報内から取得可能
		res=self._get_api_response_body(f'items/{item}')
		return res[self.API_PROP_VIEW]

	def _get_stock(self, item):
		#stockはstockersから
		#stockのqueryはユーザー情報に依存(いらないかも)
		query=f'page={self._user_config[self.API_PROP_PAGE]}&per_page={self._user_config[self.API_PROP_PER_PAGE]}'
		res=self._get_api_response_all(f'items/{item}/stockers?${query}', False)
		return res[self.HTTP_PROP_HEADER][self.HTTP_PROP_COUNT]

	def _get_comment(self, item):
		#commentはcommentsから
		#commentのqueryはユーザー情報に依存(いらないかも)
		query=f'page={self._user_config[self.API_PROP_PAGE]}&per_page={self._user_config[self.API_PROP_PER_PAGE]}'
		res=self._get_api_response_all(f'items/{item}/comments?${query}', False)
		return res[self.HTTP_PROP_HEADER][self.HTTP_PROP_COUNT]

	def _get_extra_item_data(self, item_config, item):
		if item_config[self.MNG_PROP_API_PROP] == QiitaAPI.ITEM_VIEW:
			return self._get_views(item)
		elif item_config[self.MNG_PROP_API_PROP] == QiitaAPI.ITEM_STOCK:
			return self._get_stock(item)
		elif item_config[self.MNG_PROP_API_PROP] == QiitaAPI.ITEM_COMMENT:
			return self._get_comment(item)
		else:
			return None

	def _parse_result(self, item_config, raw_result):
		if item_config[self.MNG_PROP_API_PROP] == QiitaAPI.ITEM_USER:
			return self._parse_raw_user(raw_result)
		else:
			return raw_result

	def _does_show_item(self, key, config):
		if not key in config:
			return False

		return config[key][self.MNG_PROP_SHOW]

	def _parse_raw_item(self, itemid, itemdetail):
		#stock情報を追加しておく
		if self._item_config[self.CONF_PROP_ITEM][self.API_PROP_STOCK][self.MNG_PROP_SHOW]:
			itemdetail[self.API_PROP_STOCK]=None
		#comment情報を追加しておく
		if self._item_config[self.CONF_PROP_ITEM][self.API_PROP_COMMENT][self.MNG_PROP_SHOW]:
			itemdetail[self.API_PROP_COMMENT]=None

		response={}
		for key, value in itemdetail.items():
			#非表示データはスキップ
			if not self._does_show_item(key, self._item_config[self.CONF_PROP_ITEM]):
				continue

			this_item_config=self._item_config[self.CONF_PROP_ITEM][key]
			#user一覧にある情報はそのままparse
			if this_item_config[self.MNG_PROP_HAVE_LIST]:
				response[this_item_config[self.MNG_PROP_API_PROP]]=self._parse_result(this_item_config, value)
			#それ以外は取得しなおし
			else:
				#Noneなら取り直し、それ以外はそのまま
				if value == None:
					response[this_item_config[self.MNG_PROP_API_PROP]]=self._get_extra_item_data(this_item_config, itemid)
				else:
					response[this_item_config[self.MNG_PROP_API_PROP]]=value
		return response

	def _parse_raw_user(self, raw_data):
		if self._user_config[QiitaAPI.ITEM_RAW]:
			return raw_data

		response={}
		for key, value in raw_data.items():
			#非表示データはスキップ
			if not self._does_show_item(key, self._user_config[self.CONF_PROP_USER]):
				continue

			#今は特にデータ編集もいらないのでそのまま代入
			this_config=self._user_config[self.CONF_PROP_USER][key]
			response[this_config[self.MNG_PROP_API_PROP]]=value

		return response

	#parse raw items
	def _parse_raw_items(self, raw_data):
		#データを加工して返却
		response={}
		for itemdetail in raw_data:
			itemid=itemdetail[self.API_PROP_ITEMID]
			response[itemid]=self._parse_raw_item(itemid, itemdetail)
		return response

	#public
	# itemsを取得する。
	# @ret dict of {itemid:{'titlle', other(related to conf}}
	# @note __init__データにでユーザー指定がある場合は、そのユーザー情報を利用します
	def get_items(self):
		#u記事一覧の取得
		raw_data = self._get_items_by_api()
		#raw指定ならそのまま返却
		if self._item_config[QiitaAPI.ITEM_RAW]:
			return raw_data
		#データを加工して返却
		else:
			return self._parse_raw_items(raw_data)

	# アカウントの全情報を取得する
	# @ret dict of {itemid:{'titlle', other(related to conf}}
	def get_user_items(self):
		#userの記事一覧の取得
		raw_data = self._get_user_items_by_api()
		#raw指定ならそのまま返却
		if self._item_config[QiitaAPI.ITEM_RAW]:
			return raw_data
		#データを加工して返却
		else:
			return self._parse_raw_items(raw_data)

	# item情報を取得する
	# @ret dict of {itemid:{'titlle', other(related to conf}
	def get_item(self, item):
		#userの記事一覧の取得
		raw_data = self._get_item_by_api(item)
		#raw指定ならそのまま返却
		if self._item_config[QiitaAPI.ITEM_RAW]:
			return raw_data
		#データを加工して返却
		else:
			return self._parse_raw_item(item, raw_data)


import json
import logging
import os
import pickle
import shutil
import time

import numpy as np
import tensorflow as tf
import yaml
from sklearn.model_selection import train_test_split

from conf import preprocess_yaml, root_dir
from data_helper.load_data import load_data, batch_iter
from model.model_v1_cnn.multi_cnn_model import MultiCnnModel

CHECKPOINT_PATH = "./checkpoints/"
SUMMARY_PATH = "./summary/"

logging.getLogger().setLevel(logging.INFO)

FLAGS=tf.app.flags.FLAGS
tf.app.flags.DEFINE_float('learning_rate', 0.003, 'learning_rate')
tf.app.flags.DEFINE_integer('batch_size', 128, 'batch size for training/evaluating.')
tf.app.flags.DEFINE_integer('decay_steps', 1000, 'how many steps before decay learning_rate.')
tf.app.flags.DEFINE_float('decay_rate', 0.9, 'Rate of decay for learning rate.')
tf.app.flags.DEFINE_boolean('is_training', True, 'is_training.true: training, false:testing')
tf.app.flags.DEFINE_integer('num_epochs', 80, 'num of epochs to run.')
tf.app.flags.DEFINE_integer('validate_step', 100, 'how many step to validate.')

tf.app.flags.DEFINE_integer('sequences_count', 4, 'maximum of the number of sequences in one column')
tf.app.flags.DEFINE_integer('sequence_length', 18, 'sequence length')
tf.app.flags.DEFINE_integer('word_embedding_size', 64, 'dimension of word embedding')
tf.app.flags.DEFINE_string('conv_filter_sizes', '2,3', 'convolution filter size')
tf.app.flags.DEFINE_integer('num_filters', 32, 'output channels of each filters')

tf.app.flags.DEFINE_integer('hidden_size', 8, 'hidden size of output dense layer')

tf.app.flags.DEFINE_float('dropout_keep_prob', 0.6, 'dropout')
tf.app.flags.DEFINE_float('l2_lambda', 0.0005, 'l2_regularization')
tf.app.flags.DEFINE_float('clip_gradients', 3.0, 'the normalized gradients')

tf.app.flags.DEFINE_integer('max_step', 9999999, 'the most step of training')

data_path = os.path.join(root_dir, 'data/dataset/train_data_part1_simple.pickle')

def train(data_path, pretrain_path=None):
    # 生成训练数据
    data_x, data_y, word_embedding, vocabulary, vocabulary_inv,\
        sequences_count, sequence_length, columns_sequences_count_dict\
        = load_data(data_path,
                    word_embedding_size=FLAGS.word_embedding_size,
                    max_count=FLAGS.sequences_count,
                    max_length=FLAGS.sequence_length,
                    shuffle=True,
                    pretrain_embedding_path=pretrain_path)

    # 词向量矩阵
    word_embedding_mat = [word_embedding[word] for index, word in enumerate(vocabulary_inv)]
    word_embedding_mat = np.array(word_embedding_mat, dtype=np.float32)

    # split the original dataset into train set and test set
    data_x, data_x_test, data_y, data_y_test = train_test_split(
        data_x, data_y, test_size=0.1)

    # split the train set into train set and dev set
    data_x_train, data_x_dev, data_y_train, data_y_dev = train_test_split(
        data_x, data_y, test_size=0.1)

    logging.info('data_train: {}, data_dev: {}, data_test: {}'.format(len(data_x_train), len(data_x_dev), len(data_x_test)))

    timestamp = str(int(time.time()))
    trained_dir = './trained_result/trained_results_' + timestamp + '/'
    if os.path.exists(trained_dir):
        shutil.rmtree(trained_dir)
    os.makedirs(trained_dir)
    logging.critical('The trained result is saved in {}'.format(trained_dir))

    params = dict()
    params['learning_rate'] = FLAGS.learning_rate
    params['batch_size'] = FLAGS.batch_size
    params['decay_steps'] = FLAGS.decay_steps
    params['decay_rate'] = FLAGS.decay_rate
    params['num_epochs'] = FLAGS.num_epochs

    params['max_count'] = FLAGS.sequences_count
    params['sequences_count'] = sequences_count
    params['sequence_length'] = sequence_length
    params['word_embedding_size'] = FLAGS.word_embedding_size
    params['conv_filter_sizes'] = FLAGS.conv_filter_sizes
    params['num_filters'] = FLAGS.num_filters
    params['hidden_size'] = FLAGS.hidden_size

    params['dropout_keep_prob'] = FLAGS.dropout_keep_prob
    params['l2_lambda'] = FLAGS.l2_lambda
    params['clip_gradients'] = FLAGS.clip_gradients

    with open(trained_dir + 'trained_parameters.json', 'w') as outfile:
        json.dump(params, outfile, indent=4, sort_keys=True, ensure_ascii=False)

    with open(trained_dir + 'preprocess.json', 'w') as outfile:
        json.dump(preprocess_yaml, outfile, indent=4, sort_keys=True, ensure_ascii=False)

    with open(trained_dir + 'columns_sequences_count.json', 'w') as outfile:
        json.dump(columns_sequences_count_dict, outfile, indent=4, ensure_ascii=False)

    # 初始化Model
    model = MultiCnnModel(
        # batch_size
        batch_size=FLAGS.batch_size,
        # sequence数量
        sequences_count=sequences_count,
        # sequence长度
        sequence_length=sequence_length,
        # 词向量维度
        word_embedding_size=FLAGS.word_embedding_size,
        # 词向量矩阵
        word_embedding_mat=word_embedding_mat,
        # 词典size
        vocabulary_size=len(vocabulary),
        # 卷积层filter_size
        conv_filter_sizes=list(map(int, FLAGS.conv_filter_sizes.split(','))),
        # 卷积层输出通道数
        num_filters=FLAGS.num_filters,
        # 输出dense层维度
        hidden_size=FLAGS.hidden_size,
        # 输出维度
        output_size=data_y.shape[1],
        # dropout
        dropout_keep_prob=FLAGS.dropout_keep_prob,
        # 是否训练
        is_training=FLAGS.is_training,
        # 衰减步数
        decay_steps=FLAGS.decay_steps,
        # 衰减速率
        decay_rate=FLAGS.decay_rate,
        # 学习速率
        learning_rate=FLAGS.learning_rate,
        # 梯度裁剪
        clip_gradients=FLAGS.clip_gradients,
        # 正二范数权值
        l2_lambda=FLAGS.l2_lambda,
    )

    # summary_dir = SUMMARY_PATH + 'summary_' + timestamp + '/'
    # if os.path.exists(summary_dir):
    #     shutil.rmtree(summary_dir)
    # os.makedirs(summary_dir)
    # logging.critical('The summary information is saved in {}'.format(summary_dir))
    # train_writer = tf.summary.FileWriter(summary_dir + 'train', model.sess.graph)

    if not os.path.exists(CHECKPOINT_PATH):
        os.makedirs(CHECKPOINT_PATH)

    checkpoint_dir = CHECKPOINT_PATH + 'check_points_' + timestamp + '/'
    if os.path.exists(checkpoint_dir):
        shutil.rmtree(checkpoint_dir)
    os.makedirs(checkpoint_dir)
    checkpoint_prefix = os.path.join(checkpoint_dir, 'model')

    init = tf.global_variables_initializer()
    model.sess.run(init)

    saver = tf.train.Saver(tf.global_variables())

    # Train start here
    total_size = np.ceil(len(data_x_train) / FLAGS.batch_size) * FLAGS.num_epochs

    best_loss = 999
    best_step = 0
    batch_count = 0

    train_batches = batch_iter(list(zip(data_x_train, data_y_train)), FLAGS.batch_size, FLAGS.num_epochs)

    for train_batch in train_batches:
        batch_count += 1
        x_train_batch, y_train_batch = zip(*train_batch)
        train_loss = model.train_step(x_train_batch, y_train_batch)
        current_step = tf.train.global_step(model.sess, model.global_step)

        #Evaluate the model
        if current_step % FLAGS.validate_step == 0:
            progress = ('%.2f' % (batch_count / total_size * 100))
            dev_batches = batch_iter(list(zip(data_x_dev, data_y_dev)), FLAGS.batch_size, 1)
            dev_loss, dev_batch_num = 0.0, 0
            for dev_batch in dev_batches:
                dev_batch_num += 1
                x_dev_batch, y_dev_batch = zip(*dev_batch)
                dev_loss_batch = model.dev_step(x_dev_batch, y_dev_batch)
                dev_loss += dev_loss_batch

            dev_loss /= dev_batch_num
            logging.info('Progress at : {}% examples, Train Loss is {}, Dev Loss is {}'.format(progress, train_loss, dev_loss))

            if dev_loss < best_loss:
                best_loss, best_step = dev_loss, current_step
                path = saver.save(model.sess, checkpoint_prefix, global_step=current_step)
                logging.critical(
                    'Best loss {} at step {}'.format(best_loss, best_step))

        # if current step exceeds max step, stop training
        if current_step >= FLAGS.max_step:
            break

    logging.critical('Training is complete, testing the best model on x_test and y_test')

    # close summary writer
    # train_writer.close()

    # evaluate x_test and y_test
    saver.restore(model.sess, checkpoint_prefix + '-' + str(best_step))

    #Save the model files to trained_dir
    saver.save(model.sess, trained_dir + 'best_model.ckpt')

    test_batches = batch_iter(list(zip(data_x_test, data_y_test)), FLAGS.batch_size, 1)
    test_loss, test_batch_num = 0.0, 0
    for test_batch in test_batches:
        test_batch_num += 1
        x_test_batch, y_test_batch = zip(*test_batch)
        test_loss_batch = model.dev_step(x_test_batch, y_test_batch)
        test_loss += test_loss_batch

    test_loss /= test_batch_num
    logging.critical('Loss on test set is {}'.format(test_loss))

    with open(trained_dir + 'vocabulary.json', 'w') as outfile:
        json.dump(vocabulary, outfile, indent=4, ensure_ascii=False)

    word_embedding_mat = model.sess.run(model.word_embedding_mat)
    with open(trained_dir + 'word_embedding.pickle', 'wb') as outfile:
        pickle.dump(word_embedding_mat, outfile, pickle.HIGHEST_PROTOCOL)

if __name__ == '__main__':
    train()


#!/usr/bin/env python
# Filename: Behavior.py
# Project Github: http://github.com/super3/PyTrack
# Author: Shawn Wilkinson <me@super3.org>
# Author Website: http://super3.org/
# License: GPLv3 <http://gplv3.fsf.org/>

# Imports
import math

class AbTrack:
	"""Tracks the abdomen's movement."""
	def __init__(self, thres):
		self.body = (0,0)
		self.tail = (0,0)
		self.thres = thres
		self.dir = "Unknown"
	def compare(self, boundCenter, targetCentroid):
		self.body = boundCenter
		self.tail = targetCentroid
		if abs(self.body[0] - self.tail[0]) <= self.thres:
			self.dir = "Center"
		elif (self.body[0] - self.tail[0]) > 0:
			self.dir = "Left"
		else:
			self.dir = "Right"
	def getDir(self):
		return self.dir

class CentroidTrack:
	"""Tracks centroid accuracy."""
	def __init__(self, thres):
		self.count = 0
		self.decount = 0
		self.thres = thres
		self.previousCentroid = (0,0)
	def add(self, centroid):
		x1 = self.previousCentroid[0]
		x2 = centroid[0]
		y1 = self.previousCentroid[1]
		y2 = centroid[1]
		dist = math.hypot(x1-x2, y1-y2)
		if dist <= self.thres:
			self.count += 1
			self.decount = 0
		else:
			self.decount += 1
			self.count = 0
		self.previousCentroid = centroid
	def printInfo(self):
		info = "[Abdomen Track] "
		if self.count < 3:
			info += "Poor"
		elif self.count >= 3 and self.count <= 7:
			info += "Fair"
		else:
			info += "Good"
		info += str(self.count)
		info += "\n[Legs Deployed] "
		if self.decount <= 3:
			info += "No"
		else:
			info += "Yes!"
		info += str(self.decount)
		print(info)


import collections
import random
MaskedLmInstance = collections.namedtuple("MaskedLmInstance",
                                          ["index", "label"])
def create_masked_lm_predictions(tokens, masked_lm_prob,
                                 max_predictions_per_seq, vocab_words, rng):
  """Creates the predictions for the masked LM objective."""

  cand_indexes = []
  for (i, token) in enumerate(tokens):
    if token == "[CLS]" or token == "[SEP]":
      continue
    # Whole Word Masking means that if we mask all of the wordpieces
    # corresponding to an original word. When a word has been split into
    # WordPieces, the first token does not have any marker and any subsequence
    # tokens are prefixed with ##. So whenever we see the ## token, we
    # append it to the previous set of word indexes.
    #
    # Note that Whole Word Masking does *not* change the training code
    # at all -- we still predict each WordPiece independently, softmaxed
    # over the entire vocabulary.
    if (0 and len(cand_indexes) >= 1 and
        token.startswith("##")):
      cand_indexes[-1].append(i)
    else:
      cand_indexes.append([i])

  rng.shuffle(cand_indexes)

  output_tokens = list(tokens)

  num_to_predict = min(max_predictions_per_seq,
                       max(1, int(round(len(tokens) * masked_lm_prob))))

  masked_lms = []
  covered_indexes = set()
  for index_set in cand_indexes:
    if len(masked_lms) >= num_to_predict:
      break
    # If adding a whole-word mask would exceed the maximum number of
    # predictions, then just skip this candidate.
    if len(masked_lms) + len(index_set) > num_to_predict:
      continue
    is_any_index_covered = False
    for index in index_set:
      if index in covered_indexes:
        is_any_index_covered = True
        break
    if is_any_index_covered:
      continue
    for index in index_set:
      covered_indexes.add(index)

      masked_token = None
      # 80% of the time, replace with [MASK]
      if rng.random() < 0.8:
        masked_token = "[MASK]"
      else:
        # 10% of the time, keep original
        if rng.random() < 0.5:
          masked_token = tokens[index]
        # 10% of the time, replace with random word
        else:
          masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)]

      output_tokens[index] = masked_token

      masked_lms.append(MaskedLmInstance(index=index, label=tokens[index]))
  assert len(masked_lms) <= num_to_predict
  masked_lms = sorted(masked_lms, key=lambda x: x.index)

  masked_lm_positions = []
  masked_lm_labels = []
  for p in masked_lms:
    masked_lm_positions.append(p.index)
    masked_lm_labels.append(p.label)

  return (output_tokens, masked_lm_positions, masked_lm_labels)
print(create_masked_lm_predictions(['0','2','3', '4', '3', '1'], 0.15, 5, ['0', '1', '2', '3','4'] , random.Random(0.5)))