import re
import numpy as np

print("Program started")

###initialization
# compile all punctuations
punc = re.compile("[^"+"|".join(list('!"#$%&\'()*+,--./:;<=>?@\[\\]^_`{|}~'))+"]")

###Functions
def preprocess(file):
    """ Read and process lines into a list
    :param file: (str) path to and the name of the file
    :return: (list) a list of lines
    """
    # read doc
    with open(file, 'r', encoding='UTF-8', errors='ignore') as f:
        doc = f.readlines()
        lines = []
        for x in doc:
            lines.append(x.lower().strip(""))
        del doc

    # process all punctuations and extra white space
    lines_np = ["".join(re.findall(punc, x)) for x in lines]
    lines_np_one = [x.strip().replace("  ", " ") for x in lines_np]

    # process the digits/ranks with digits out
    lines_np_nd = []
    for x in lines_np_one:
        if len(re.findall("\d+", x)) > 0:
            new = x
            for digits in re.findall(r"\b(\d+(th|st|nd)?)\b", x):
                new = re.sub(re.compile(r"\b" + digits[0] + r"\b"), "NUMBER", new)
            lines_np_nd.append(new)
        else:
            lines_np_nd.append(x)

    return lines_np_nd

def ngrams_words(lst, returnwords = True, n = 2):
    """process lines of sentences into ngrams"""
    # n-grams init
    ngrams = []
    # if each word is needed
    if returnwords:
        words_all = []
    else:
        words_all = None
    #create n-grams
    for sent in lst:
        words = sent.split(' ')
        if returnwords:
            words_all += words
        length = len(words)
        for i in range(0, length - n + 1):
            n_gram = ' '.join(words[i : i + n])
            ngrams.append(n_gram)
    return ngrams, words_all

def ngrams_process(ngrams, words_all, normal=True, addone=True, turing=True):
    """process bigrams into probabilities"""
    # vocabulary
    vocab = np.unique(words_all).tolist()
    vocab.remove('')
    vocab.remove(']')

    # word dict
    vocab_dic = dict(zip(vocab, range(len(vocab))))
    vocab_dic_r = dict(zip(range(len(vocab)), vocab))

    # count table
    ngram_cnt = np.zeros((len(vocab), len(vocab)), dtype="int")
    for ngram in ngrams:
        words = ngram.split(" ")
        try:
            ngram_cnt[vocab_dic[words[0]], vocab_dic[words[1]]] += 1
        except:
            next

    # prob table
    if normal:
        ngram_p = ngram_cnt / ngram_cnt.sum()
    if addone:
        ngram_cnt_t = ngram_cnt + 1
        ngram_p_ao = ngram_cnt_t / ngram_cnt_t.sum()
    if turing:
        ngram_cnt_dict = {}
        u = np.unique(ngram_cnt)
        for h in u:
            ngram_cnt_dict[h] = len(np.where(ngram_cnt == h)[0])

    return vocab, vocab_dic, vocab_dic_r, ngram_cnt, ngram_p, ngram_p_ao, ngram_cnt_dict

###Read training & test files
# english train
eng_np_nd = preprocess("LangId.train.English")
# italian train
it_np_nd = preprocess("LangId.train.Italian")
# french train
fr_np_nd = preprocess("LangId.train.French")
# test
test_np_nd = preprocess("LangId.test")

### words n-gram Training

# English

ngram_eng, words_eng = ngrams_words(eng_np_nd)

vocab_eng, vocab_eng_dic, vocab_eng_dic_r, ngram_cnt, ngram_eng_p, ngram_eng_ao, ngram_eng_gt = ngrams_process(
    ngram_eng, words_eng)

# Italian

ngram_it, words_it = ngrams_words(it_np_nd)

vocab_it, vocab_it_dic, vocab_it_dic_r, ngram_it_cnt, ngram_it_p, ngram_it_ao, ngram_it_gt = ngrams_process(
    ngram_it, words_it)

# French

ngram_fr, words_fr = ngrams_words(fr_np_nd)

vocab_fr, vocab_fr_dic, vocab_fr_dic_r, ngram_fr_cnt, ngram_fr_p, ngram_fr_ao, ngram_fr_gt = ngrams_process(
    ngram_fr, words_fr)

print("Trained letter bigram model")

### Words bi-gram Test

def keepLines(lst, n=2):
    """process as ngrams with lines preserved"""
    # n-grams init
    lines = {}

    for l in range(len(lst)):
        words = lst[l].split(" ")
        n_grams = []
        length = len(words)
        for i in range(0, length - n + 1):
            n_gram = ' '.join(words[i: i + n])
            n_grams.append(n_gram)

        lines[l] = n_grams

    return lines

# test processing
test_dict = keepLines(test_np_nd)

ngram_test, words_test = ngrams_words(test_np_nd)

# prep dictionaries for processing
dicts = dict(zip(["english","french","italian"],
                 [[vocab_eng_dic, ngram_eng_p, ngram_eng_ao, ngram_eng_gt, ngram_cnt],
                  [vocab_fr_dic, ngram_fr_p, ngram_fr_ao, ngram_fr_gt, ngram_fr_cnt],
                  [vocab_it_dic, ngram_it_p, ngram_it_ao, ngram_it_gt, ngram_it_cnt]]))

def likeliness(ngram, lg = "english", smoothing=None):
    """bigram likehood"""
    words = ngram.split(" ")
    lead = words[0]
    follow = words[1]
    vocab_dic = dicts[lg][0]
    #nothing or add-one only (good-turing seperated)
    if smoothing == None:
        ngram_p = dicts[lg][1]
    if smoothing == "add-one":
        ngram_p = dicts[lg][2]
    # check p
    if lead in vocab_dic.keys():
        if follow in vocab_dic.keys():
            p_sub = ngram_p[vocab_dic[lead], vocab_dic[follow]]
            if p_sub == 0:
                p_sub = 1e-100
        else:
            p_sub = 1e-100
    else:
        p_sub = 1e-100
    return np.log(p_sub)

def whatLg(test_dict, smoothing=None):
    """total probability for each line"""
    p_lines = {}
    for k, v in test_dict.items():
        # english
        eng_prob = 1e-100
        for ngram in v:
            if smoothing == "good-turing":
                eng_prob += turing_likeliness(ngram)
            else:
                eng_prob += likeliness(ngram, smoothing=smoothing)

        # french
        fr_prob = 1e-100
        for ngram in v:
            if smoothing == "good-turing":
                fr_prob += turing_likeliness(ngram, "french")
            else:
                fr_prob += likeliness(ngram, "french", smoothing=smoothing)

        # italian
        it_prob = 1e-100
        for ngram in v:
            if smoothing == "good-turing":
                it_prob += turing_likeliness(ngram, "italian")
            else:
                it_prob += likeliness(ngram, "italian", smoothing=smoothing)

        players = {'English': eng_prob, 'French': fr_prob, 'Italian': it_prob}
        winner = max(players, key=players.get)
        p_lines[k] = winner

    return p_lines

# run add-one smoothing
ao_lines = whatLg(test_dict, smoothing="add-one")

print("Prediction made")

# write out solution
with open("wordLangId.out", "w") as f:
    for k, v in ao_lines.items():
        f.writelines(" ".join([str(k+1), v, "\n"]))
    f.close()

print("Find result in wordLangId.out")


# 아이디어 : 1. 약수의 개수 구하기(제곱근)
#           2. 도중에 개수가 limit를 넘어가면 power만큼 추가하기
# 알고리즘 : number theory - divisor

import math

def solution(number, limit, power):
    divisor = [1] # 1의 약수는 무조건 1개

    for num in range(2, number + 1) :
        cnt = 0 # 약수의 개수
        for i in range(1, int(math.sqrt(num)) + 1) :
            # num가 i로 나누어 떨어진다면
            if num % i == 0 :
                # num가 제곱수라면
                if num == i ** 2 :
                    cnt += 1
                else :
                    cnt += 2

        if cnt > limit :
            divisor.append(power)
        else :
            divisor.append(cnt)

    return sum(divisor)

프로그래머스/unrated/136798. 기사단원의 무기/기사단원의 무기.py


"""Chain for question-answering against a vector database."""
from __future__ import annotations

import inspect
import warnings
from abc import abstractmethod
from typing import Any, Dict, List, Optional

from langchain_core.documents import Document
from langchain_core.language_models import BaseLanguageModel
from langchain_core.prompts import PromptTemplate
from langchain_core.pydantic_v1 import Extra, Field, root_validator
from langchain_core.retrievers import BaseRetriever
from langchain_core.vectorstores import VectorStore

from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain.chains.base import Chain
from langchain.chains.combine_documents.base import BaseCombineDocumentsChain
from langchain.chains.combine_documents.stuff import StuffDocumentsChain
from langchain.chains.llm import LLMChain
from langchain.chains.question_answering import load_qa_chain
from langchain.chains.question_answering.stuff_prompt import PROMPT_SELECTOR

class BaseRetrievalQA(Chain):
    """Base class for question-answering chains."""

    combine_documents_chain: BaseCombineDocumentsChain
    """Chain to use to combine the documents."""
    input_key: str = "query"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_source_documents: bool = False
    """Return the source documents or not."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True
        allow_population_by_field_name = True

    @property
    def input_keys(self) -> List[str]:
        """Input keys.

        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Output keys.

        :meta private:
        """
        _output_keys = [self.output_key]
        if self.return_source_documents:
            _output_keys = _output_keys + ["source_documents"]
        return _output_keys

    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: Optional[PromptTemplate] = None,
        callbacks: Callbacks = None,
        llm_chain_kwargs: Optional[dict] = None,
        **kwargs: Any,
    ) -> BaseRetrievalQA:
        """Initialize from LLM."""
        _prompt = prompt or PROMPT_SELECTOR.get_prompt(llm)
        llm_chain = LLMChain(
            llm=llm, prompt=_prompt, callbacks=callbacks, **(llm_chain_kwargs or {})
        )
        document_prompt = PromptTemplate(
            input_variables=["page_content"], template="Context:\n{page_content}"
        )
        combine_documents_chain = StuffDocumentsChain(
            llm_chain=llm_chain,
            document_variable_name="context",
            document_prompt=document_prompt,
            callbacks=callbacks,
        )

        return cls(
            combine_documents_chain=combine_documents_chain,
            callbacks=callbacks,
            **kwargs,
        )

    @classmethod
    def from_chain_type(
        cls,
        llm: BaseLanguageModel,
        chain_type: str = "stuff",
        chain_type_kwargs: Optional[dict] = None,
        **kwargs: Any,
    ) -> BaseRetrievalQA:
        """Load chain from chain type."""
        _chain_type_kwargs = chain_type_kwargs or {}
        combine_documents_chain = load_qa_chain(
            llm, chain_type=chain_type, **_chain_type_kwargs
        )
        return cls(combine_documents_chain=combine_documents_chain, **kwargs)

    @abstractmethod
    def _get_docs(
        self,
        question: str,
        *,
        run_manager: CallbackManagerForChainRun,
    ) -> List[Document]:
        """Get documents to do question answering over."""

    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run get_relevant_text and llm on input query.

        If chain has 'return_source_documents' as 'True', returns
        the retrieved documents as well under the key 'source_documents'.

        Example:
        .. code-block:: python

        res = indexqa({'query': 'This is my query'})
        answer, docs = res['result'], res['source_documents']
        """
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        question = inputs[self.input_key]
        accepts_run_manager = (
            "run_manager" in inspect.signature(self._get_docs).parameters
        )
        if accepts_run_manager:
            docs = self._get_docs(question, run_manager=_run_manager)
        else:
            docs = self._get_docs(question)  # type: ignore[call-arg]
        answer = self.combine_documents_chain.run(
            input_documents=docs, question=question, callbacks=_run_manager.get_child()
        )

        if self.return_source_documents:
            return {self.output_key: answer, "source_documents": docs}
        else:
            return {self.output_key: answer}

    @abstractmethod
    async def _aget_docs(
        self,
        question: str,
        *,
        run_manager: AsyncCallbackManagerForChainRun,
    ) -> List[Document]:
        """Get documents to do question answering over."""

    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run get_relevant_text and llm on input query.

        If chain has 'return_source_documents' as 'True', returns
        the retrieved documents as well under the key 'source_documents'.

        Example:
        .. code-block:: python

        res = indexqa({'query': 'This is my query'})
        answer, docs = res['result'], res['source_documents']
        """
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        question = inputs[self.input_key]
        accepts_run_manager = (
            "run_manager" in inspect.signature(self._aget_docs).parameters
        )
        if accepts_run_manager:
            docs = await self._aget_docs(question, run_manager=_run_manager)
        else:
            docs = await self._aget_docs(question)  # type: ignore[call-arg]
        answer = await self.combine_documents_chain.arun(
            input_documents=docs, question=question, callbacks=_run_manager.get_child()
        )

        if self.return_source_documents:
            return {self.output_key: answer, "source_documents": docs}
        else:
            return {self.output_key: answer}

class RetrievalQA(BaseRetrievalQA):
    """Chain for question-answering against an index.

    Example:
        .. code-block:: python

            from langchain.llms import OpenAI
            from langchain.chains import RetrievalQA
            from langchain.vectorstores import FAISS
            from langchain_core.vectorstores import VectorStoreRetriever
            retriever = VectorStoreRetriever(vectorstore=FAISS(...))
            retrievalQA = RetrievalQA.from_llm(llm=OpenAI(), retriever=retriever)

    """

    retriever: BaseRetriever = Field(exclude=True)

    def _get_docs(
        self,
        question: str,
        *,
        run_manager: CallbackManagerForChainRun,
    ) -> List[Document]:
        """Get docs."""
        return self.retriever.get_relevant_documents(
            question, callbacks=run_manager.get_child()
        )

    async def _aget_docs(
        self,
        question: str,
        *,
        run_manager: AsyncCallbackManagerForChainRun,
    ) -> List[Document]:
        """Get docs."""
        return await self.retriever.aget_relevant_documents(
            question, callbacks=run_manager.get_child()
        )

    @property
    def _chain_type(self) -> str:
        """Return the chain type."""
        return "retrieval_qa"

class VectorDBQA(BaseRetrievalQA):
    """Chain for question-answering against a vector database."""

    vectorstore: VectorStore = Field(exclude=True, alias="vectorstore")
    """Vector Database to connect to."""
    k: int = 4
    """Number of documents to query for."""
    search_type: str = "similarity"
    """Search type to use over vectorstore. `similarity` or `mmr`."""
    search_kwargs: Dict[str, Any] = Field(default_factory=dict)
    """Extra search args."""

    @root_validator()
    def raise_deprecation(cls, values: Dict) -> Dict:
        warnings.warn(
            "`VectorDBQA` is deprecated - "
            "please use `from langchain.chains import RetrievalQA`"
        )
        return values

    @root_validator()
    def validate_search_type(cls, values: Dict) -> Dict:
        """Validate search type."""
        if "search_type" in values:
            search_type = values["search_type"]
            if search_type not in ("similarity", "mmr"):
                raise ValueError(f"search_type of {search_type} not allowed.")
        return values

    def _get_docs(
        self,
        question: str,
        *,
        run_manager: CallbackManagerForChainRun,
    ) -> List[Document]:
        """Get docs."""
        if self.search_type == "similarity":
            docs = self.vectorstore.similarity_search(
                question, k=self.k, **self.search_kwargs
            )
        elif self.search_type == "mmr":
            docs = self.vectorstore.max_marginal_relevance_search(
                question, k=self.k, **self.search_kwargs
            )
        else:
            raise ValueError(f"search_type of {self.search_type} not allowed.")
        return docs

    async def _aget_docs(
        self,
        question: str,
        *,
        run_manager: AsyncCallbackManagerForChainRun,
    ) -> List[Document]:
        """Get docs."""
        raise NotImplementedError("VectorDBQA does not support async")

    @property
    def _chain_type(self) -> str:
        """Return the chain type."""
        return "vector_db_qa"


import asyncio

# import MpesaApiWrapper from Routes
from Routes.mpesa_api_wrapper import MpesaApiWrapper
from Config.config import settings

async def other_task():
    for i in range(10):
        print(f"Running other tasks...{i}")
        await asyncio.sleep(0.1)
    return "Finished Other Task !!"

def test_config_env_file():
    print(f"authorization_key = {settings.authorization_key}")
    print(f"base_url = {settings.base_url}")
    print(f"confirmation_url = {settings.confirmation_url}")
    print(f"short_code = {settings.short_code}")
    print(f"response_type = {settings.response_type}")
    print(f"validation_url = {settings.validation_url}")

async def get_mpesa_access_token():
    mpesa_api_wrapper = MpesaApiWrapper()
    return await mpesa_api_wrapper.get_access_token()

async def demo_async_funcs():
    access_token_response = asyncio.create_task(get_mpesa_access_token())
    other_task_response = asyncio.create_task(other_task())

    print(f"\n<----RECEIVED MPESA API RESPONSE... {await access_token_response}\n")
    print(f"\n{await other_task_response}\n")

asyncio.run(demo_async_funcs())


import numpy as np
import pylab as plt
import time
import random
from scipy.sparse.linalg import eigs,eigsh
from scipy.sparse.linalg import LinearOperator

# def Sx(phi1,i1):
#     k=np.zeros([2**(l-1-i1),2**(i1+1)])
#     k[:,:]=np.reshape(phi1,[2**(l-1-i1),2**(i1+1)],'F')[:,:]
#     for i in np.arange(0,2**(i1+1),2):
#         k[:,[i,i+1]]=k[:,[i+1,i]]
#     k=(1)*np.reshape(k,2**l,'F')
#     return  k

def Sx(phi1,i1):
    k=phi1.reshape(2**(i1),2,2**(l-1-i1))
    k=k[:,[1,0],:]
    return k.reshape(-1)

def Sy(phi1,i1):
    # k=np.zeros(2**l,complex)
    # k[:]=complex(0,Sxnew(Sznew(phi1,i1),i1)-Sznew(Sxnew(phi1,i1),i1))
    return Sx(Sz(phi1,i1),i1)*1j

def Sz(phi1,i1):
    return -2*(s[:,i1]-0.5)*phi1[:]

def H(phi1):
    k=np.zeros(2**l,complex)
    for i in range(l):
        k[:]+=(-1)**f*(Sx(Sx(phi1,np.mod(i+1,l)),i)[:]+Sy(Sy(phi1,np.mod(i+1,l)),i)[:]+Sz(Sz(phi1,np.mod(i+1,l)),i)[:])
    return k

ll=np.arange(4,5)
t=[]
f=1
for l in ll:
    tic=time.time()

    s=np.zeros([2**l,l],int)
    for i in range(2**l):
        for j in range(l):
            s[i,j]=format(i,"0"+str(l)+"b")[j]

    m=4
    E,V=eigsh(LinearOperator((2**l,2**l),matvec=H),k=m,which='SA')

# G=np.zeros([5,5])
# for i in range(5):
#     for j in range(5):
#         G[i,j]=np.dot(V1[:,i],V[:,j])

# c=np.zeros(2**l)
# c=V1[:,0]
# for i in range(5):
#     c[:]-=np.dot(V1[:,0],V[:,i])*V[:,i]


from pytube import YouTube

url = "https://youtu.be/6GaYbeHYLqA"
my_video = YouTube(url)

# resolving MacOS error: run the following in PyCharm terminal
# open /Applications/Python\ 3.7/Install\ Certificates.command

my_video = my_video.streams.get_highest_resolution()
print("done1")
my_video.download()
print("done2")


from cgitb import html
from script import get_html, algorithm, replace

def main(link, text):
    content = get_html.page_content(link)
    try:
        coords = algorithm.coordinates(text)
        replaced_content = replace.replace_html(content, coords)
    except Exception as e:
        print(e)
        return content
    return replaced_content

'''
it imports all the functions from the directory and calls them
1. first it gets the html page via "get_html.page_content"
2. it gets all the coords for the text via "algorithm.coordinates"
3. it replaces all the coords with the programmatically generated coordinates
4. then it gives that coordinates to the "replace.replace_html" to get the new html
5. then it returns that html file as text
'''


import unittest
import functools

from scoring_api import api, store

def cases(cases):
    def decorator(f):
        @functools.wraps(f)
        def wrapper(*args):
            for c in cases:
                new_args = args + (c if isinstance(c, tuple) else (c,))
                f(*new_args)
        return wrapper
    return decorator

class TestFields(unittest.TestCase):
    def get_response(self, request):
        return api.method_handler({"body": request, "headers": self.headers}, self.context, self.store)

    @cases([
        (True, True, None),
    ])
    def test_basic_field(self, required, nullable, value):
        field = api.Field(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 123),
        (False, False, [1, 2]),
        (False, True, None),
    ])
    def test_char_field(self, required, nullable, value):
        field = api.CharField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 123),
        (False, False, '123'),
        (False, False, None),
    ])
    def test_arguments_field(self, required, nullable, value):
        field = api.ArgumentsField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 'foo.gmail.com'),
        (False, False, 123),
    ])
    def test_email_field(self, required, nullable, value):
        field = api.EmailField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 7999),
        (False, False, '7999'),
        (False, False, '89990123456'),
        (False, False, '+79990123456'),
    ])
    def test_phone_field(self, required, nullable, value):
        field = api.PhoneField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 20000101),
        (False, False, '01-01-2000'),
        (False, False, '01/01/2000'),
        (False, False, 'foo-bar'),
    ])
    def test_date_field(self, required, nullable, value):
        field = api.DateField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, '01-01-2000'),
        (False, False, '01.01.1900'),
    ])
    def test_birthday_field(self, required, nullable, value):
        field = api.BirthDayField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (False, False, 123),
        (False, False, '1'),
        (False, False, -999),
    ])
    def test_gender_field(self, required, nullable, value):
        field = api.GenderField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

    @cases([
        (True, True, 123),
        (True, True, '123'),
        (True, True, ['123', '321']),
        (True, True, []),
        (True, True, None),
    ])
    def test_client_ids_field(self, required, nullable, value):
        field = api.ClientIDsField(required, nullable)
        with self.assertRaises(ValueError):
            field.validate(value)

class TestStorage(unittest.TestCase):
    def setUp(self):
        self.storage = store.DBConnector()

    def cache_set(self):
        self.assertEqual(self.storage.cache_get('key_does_not_exist'), None)
        self.assertEqual(self.storage.cache_set('key_does_not_exist', 'foo', 10), 'foo')

    def test_cache_get(self):
        self.assertEqual(self.storage.cache_get('key_does_not_exist'), None)

    def test_get(self):
        self.assertEqual(self.storage.get('key_does_not_exist'), None)

if __name__ == "__main__":
    unittest.main()


def cal_array_w_v_pair(w_v_pair, hight, now_weight, now_value,
                       limit, return_array):
    # pythonの配列は参照渡しであるので、return_arrayに加えることで値が追加される
    if hight < limit:
        cal_array_w_v_pair(w_v_pair, hight + 1, now_weight, now_value,
                           limit, return_array)
        now_weight += w_v_pair[hight][0]
        now_value += w_v_pair[hight][1]
        cal_array_w_v_pair(w_v_pair, hight + 1, now_weight, now_value,
                           limit, return_array)
    else:
        return_array.append((now_weight, now_value))

def cal_max_value(front_w_v_pair, back_w_v_pair, limit_weight):
    max_value = 0
    left = 0
    right = len(back_w_v_pair) - 1
    while right >= 0 and left < len(front_w_v_pair):
        now_weight = front_w_v_pair[left][0] + back_w_v_pair[right][0]
        now_value = front_w_v_pair[left][1] + back_w_v_pair[right][1]
        if now_weight <= limit_weight:
            # weightの合計がlimit_weight以下の時、weightを大きくできる、解の候補でもある
            max_value = max(max_value, now_value)
            left += 1
        else:
            # weightの合計がlimit_weightより大きい時、weightは小さくしなければならない
            right -= 1

    return max_value

def check(array):
    increase_array = []
    max_value = -1
    for i in range(len(array)):
        if max_value < array[i][1]:
            # weightが増えた時価値が上がる場合
            max_value = array[i][1]
            increase_array.append(array[i])
    return increase_array

def main():
    # 入力
    N, limit_weight = map(int, input().split())
    w_v_pair = []
    for i in range(N):
        v, w = map(int, input().split())
        w_v_pair.append((w, v))
    front_w_v_pair = []
    back_w_v_pair = []
    half = int(N / 2)

    # 半分全列挙を使ってそれぞれの組み合わせをだしておく
    cal_array_w_v_pair(w_v_pair, 0, 0, 0, half, front_w_v_pair)
    cal_array_w_v_pair(w_v_pair, half, 0, 0, N, back_w_v_pair)
    # 尺取り法を用いるためsortしておく
    front_w_v_pair.sort()
    back_w_v_pair.sort()

    # weightが増えたときに、価値が上がらないと尺取り法は使えないので、そのような組み合わせを弾く
    increase_front_w_v_pair = check(front_w_v_pair)
    increase_back_w_v_pair = check(back_w_v_pair)

    max_value = cal_max_value(increase_front_w_v_pair,
                              increase_back_w_v_pair, limit_weight)

    print(max_value)

if __name__ == "__main__":
    main()


import cv2
import numpy as np
import torch
from skimage.morphology import skeletonize

from mmocr.models.builder import POSTPROCESSOR
from .base_postprocessor import BasePostprocessor
from .utils import centralize, fill_hole, merge_disks

@POSTPROCESSOR.register_module()
class TextSnakePostprocessor(BasePostprocessor):
    """Decoding predictions of TextSnake to instances. This was partially
    adapted from https://github.com/princewang1994/TextSnake.pytorch.

    Args:
        text_repr_type (str): The boundary encoding type 'poly' or 'quad'.
        min_text_region_confidence (float): The confidence threshold of text
            region in TextSnake.
        min_center_region_confidence (float): The confidence threshold of text
            center region in TextSnake.
        min_center_area (int): The minimal text center region area.
        disk_overlap_thr (float): The radius overlap threshold for merging
            disks.
        radius_shrink_ratio (float): The shrink ratio of ordered disks radii.
    """

    def __init__(self,
                 text_repr_type='poly',
                 min_text_region_confidence=0.6,
                 min_center_region_confidence=0.2,
                 min_center_area=30,
                 disk_overlap_thr=0.03,
                 radius_shrink_ratio=1.03,
                 **kwargs):
        """ 已看过，TextSnake的后处理初始化函数
        Args:
            text_repr_type: 输出匡选匡的方式，这里预设会是多边形
            min_text_region_confidence: 一个文字团的平均置信度阈值
            min_center_region_confidence: 一个文字团中心地区的平均置信度阈值
            min_center_area: 一个文字团的中心地区最小面积
            disk_overlap_thr: 圆盘重叠的阈值
            radius_shrink_ratio: 半径缩小率
        """
        # 继承自BasePostprocessor，对继承对象进行初始化
        super().__init__(text_repr_type)
        assert text_repr_type == 'poly'
        # 保存传入的参数
        self.min_text_region_confidence = min_text_region_confidence
        self.min_center_region_confidence = min_center_region_confidence
        self.min_center_area = min_center_area
        self.disk_overlap_thr = disk_overlap_thr
        self.radius_shrink_ratio = radius_shrink_ratio

    def __call__(self, preds):
        """
        Args:
            preds (Tensor): Prediction map with shape :math:`(C, H, W)`.

        Returns:
            list[list[float]]: The instance boundary and its confidence.
        """
        # 已看过，获取预测出来的标注匡资讯
        # preds = 预测图，tensor shape [channel=5, height, width]

        # 检查传入的preds是否合法
        assert preds.dim() == 3

        # channel=0是预测是否为文字的置信度图，channel=1是预测文字中心区的置信度图，所以这里需要经过sigmoid将值控制在[0, 1]之间
        preds[:2, :, :] = torch.sigmoid(preds[:2, :, :])
        # 将preds从tensor转成ndarray型态
        preds = preds.detach().cpu().numpy()

        # 取出preds[0]部分，这个会是预测是否为文字的置信度图，ndarray shape [height, width]
        pred_text_score = preds[0]
        # 构建文字的mask，当置信度分数小于设定阈值时该地方会是False，其他地方会是True
        pred_text_mask = pred_text_score > self.min_text_region_confidence
        # 将文字中心置信度分数提取出来，这里会将text_score与其相乘
        pred_center_score = preds[1] * pred_text_score
        # 获取中心mask，当center的置信度小于阈值时该地方会是False，其他地方会是True
        pred_center_mask = \
            pred_center_score > self.min_center_region_confidence
        # 获取sin的预测图
        pred_sin = preds[2]
        # 获取cos的预测图
        pred_cos = preds[3]
        # 获取radius的预测图
        pred_radius = preds[4]
        # 获取mask的shape = [height, width]
        mask_sz = pred_text_mask.shape

        # 获取sin与cos的放大倍率，这里与训练时操作相同
        scale = np.sqrt(1.0 / (pred_sin**2 + pred_cos**2 + 1e-8))
        # 将sin与cos乘上放大倍率获取最后的sin与cos
        pred_sin = pred_sin * scale
        pred_cos = pred_cos * scale

        # 获取中心部分的mask，ndarray shape [height, width]，在中心部分会是1非中心部分会是0
        pred_center_mask = fill_hole(pred_center_mask).astype(np.uint8)
        # 透过findContours获取轮廓，center_contours会是tuple(ndarray) ndarray shape [points, 1, 2]
        # 表示文字中心的轮廓的多个点的座标
        center_contours, _ = cv2.findContours(pred_center_mask, cv2.RETR_TREE,
                                              cv2.CHAIN_APPROX_SIMPLE)

        # 最终边匡的座标点
        boundaries = []
        # 遍历文字中心的轮廓匡
        for contour in center_contours:
            if cv2.contourArea(contour) < self.min_center_area:
                # 如果匡出的面积小于设定的阈值就会跳过该中心轮廓
                continue
            # 获取全为0且shape与mask_sz相同的ndarray，shape [height, width]
            instance_center_mask = np.zeros(mask_sz, dtype=np.uint8)
            # 使用cv2的drawContours画出中心文字部分
            # (要被标注的图像, 标注轮廓, -1=画出所有传入标注匡, 填充的值, -1=使用填充方式，将轮廓中的范围都填充起来)
            # 当前遍历到的文字中心区块会被填充为1
            cv2.drawContours(instance_center_mask, [contour], -1, 1, -1)
            # 将填充好的instance_center_mask放入到skeletonize当中
            # skeletonize = 骨架提取，二值图像细致化。这种算法能将一个连通区域细化成一个像素的宽度，用于特征提取和目标拓扑表示。
            # 如果对于结果有保持疑问可以画出一个矩形看结果会是如何
            skeleton = skeletonize(instance_center_mask)
            # 获取skeleton哪些座标的值大于0，shape ndarray [points, 2]
            skeleton_yx = np.argwhere(skeleton > 0)
            # 将(x, y)提取出来，row会是y，col会是x，ndarray shape [points]
            y, x = skeleton_yx[:, 0], skeleton_yx[:, 1]
            # 获取指定座标的预测cos与sin与radius的值，ndarray shape [points, 1]
            cos = pred_cos[y, x].reshape((-1, 1))
            sin = pred_sin[y, x].reshape((-1, 1))
            radius = pred_radius[y, x].reshape((-1, 1))

            # 获取中心线，ndarray shape [points, 2]
            # 将骨架点网上顶到合法点的最上限与骨架点顶到最下限合法点的平均值，就会是最后的中心线
            center_line_yx = centralize(skeleton_yx, cos, -sin, radius,
                                        instance_center_mask)
            # 将x与y分别提取出来
            y, x = center_line_yx[:, 0], center_line_yx[:, 1]
            # 将指定点的预测radius取出来并且乘上radius_shrink_ratio值，ndarray shape [points, 1]
            radius = (pred_radius[y, x] * self.radius_shrink_ratio).reshape(
                (-1, 1))
            # 将指定点的预测文本中心置信度分数提取出来，ndarray shape [points, 1]
            score = pred_center_score[y, x].reshape((-1, 1))
            # np.fliplr = [[1, 2], [3, 4]] -> [[2, 1], [4, 3]]
            # instance_disks = ndarray shape [points, 4]
            instance_disks = np.hstack(
                [np.fliplr(center_line_yx), radius, score])
            # 通过merge_disks进行disk融合，会将比较接近的disk进行融合，这样最后结果不会有一堆圆圈
            instance_disks = merge_disks(instance_disks, self.disk_overlap_thr)

            # 构建一个全为0的ndarray shape [height, width]
            instance_mask = np.zeros(mask_sz, dtype=np.uint8)
            # 遍历所有的圆盘资讯
            for x, y, radius, score in instance_disks:
                if radius > 1:
                    # 当半径大于1就会到这里，将指定的位置填充为1
                    cv2.circle(instance_mask, (int(x), int(y)), int(radius), 1,
                               -1)
            # contours = 图像当中连通块的最小外接矩形的点，ndarray [points, 1, 2]
            contours, _ = cv2.findContours(instance_mask, cv2.RETR_TREE,
                                           cv2.CHAIN_APPROX_SIMPLE)

            # 获取有mask到的地方的置信度总和之后再算出平均置信度值
            score = np.sum(instance_mask * pred_text_score) / (
                np.sum(instance_mask) + 1e-8)
            if (len(contours) > 0 and cv2.contourArea(contours[0]) > 0
                    and contours[0].size > 8):
                # 如果面积大于0且点数超过8就会进来
                # boundary shape = [points * 2]
                boundary = contours[0].flatten().tolist()
                # 最后在后端加上预测的平均置信度分数就可以了
                boundaries.append(boundary + [score])

        return boundaries


from PIL import Image
import numpy as np

def learn(img):
    img_arr = img.reshape(48, 48, 1)
    x = []
    x.append(img_arr)
    x = np.array(x)
    result = model.predict(x)
    label = ['angry  ','disgust','fear   ','happy  ','neutral','sad    ','surprise']

    return label, result


from datetime import timedelta, datetime
from decimal import Decimal
import json
from urllib.parse import urlencode
from chalicelib import dynamo, constants
import requests

def is_valid_entry(item, expected_entries, date):
    """Function to remove traversal time entries which do not have data for each leg of the trip."""
    if item["entries"] < expected_entries:
        print(f"No speed value for ({date}): Insufficient data.")
        return False
    return True

def get_agg_tt_api_requests(stops, current_date, delta):
    """Create API requests from parameters"""
    api_requests = []
    for stop_pair in stops:
        params = {
            "from_stop": stop_pair[0],
            "to_stop": stop_pair[1],
            "start_date": datetime.strftime(current_date, constants.DATE_FORMAT_BACKEND),
            "end_date": datetime.strftime(current_date + delta - timedelta(days=1), constants.DATE_FORMAT_BACKEND),
        }
        url = constants.DD_URL_AGG_TT.format(parameters=urlencode(params, doseq=True))
        api_requests.append(url)
    return api_requests

def send_requests(api_requests):
    """Send API requests to Datadashboard backend."""
    speed_object = {}
    for request in api_requests:
        response = requests.get(request)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            print(response.content.decode("utf-8"))
            raise
        data = json.loads(response.content.decode("utf-8"), parse_float=Decimal, parse_int=Decimal)
        for item in data:
            if item["service_date"] in speed_object:
                speed_object[item["service_date"]]["median"] += item["50%"]
                speed_object[item["service_date"]]["mean"] += item["mean"]
                speed_object[item["service_date"]]["count"] += item["count"] / 2
                speed_object[item["service_date"]]["entries"] += 1
            else:
                speed_object[item["service_date"]] = {
                    "median": item["50%"] if item["50%"] else 0,
                    "count": item["count"] / 2 if item["count"] else 0,
                    "mean": item["mean"] if item["mean"] else 0,
                    "entries": 1,
                }
    return speed_object

def format_tt_objects(speed_objects, route_metadata, expected_num_entries, date_range):
    """Remove invalid entries and format for Dynamo."""
    formatted_speed_objects = []
    route_name = (
        f"{route_metadata['line']}-{route_metadata['route']}" if route_metadata["route"] else route_metadata["line"]
    )
    for current_date in date_range:
        metrics = speed_objects.get(current_date)
        new_speed_object = {
            "route": route_name,
            "line": route_metadata["line"],
            "date": current_date,
            "count": None,
        }
        if metrics:
            new_speed_object["count"] = metrics["count"]
        if metrics and is_valid_entry(metrics, expected_num_entries, current_date):
            new_speed_object["median"] = metrics["median"]
            new_speed_object["mean"] = round(metrics["mean"], 1)
            new_speed_object["miles_covered"] = metrics["count"] * Decimal(route_metadata["length"])
            new_speed_object["track_mileage"] = Decimal(route_metadata["length"])
            new_speed_object["total_time"] = round(metrics["mean"], 1) * metrics["count"]

        formatted_speed_objects.append(new_speed_object)
    return formatted_speed_objects

def get_date_range_strings(start_date, end_date):
    date_range = []
    current_date = start_date
    while current_date <= end_date:
        date_range.append(current_date.strftime("%Y-%m-%d"))
        current_date += timedelta(days=1)
    return date_range

def populate_daily_table(start_date, end_date, line, route):
    """Populate DeliveredTripMetrics table. Calculates median TTs and trip counts for all days between start and end dates."""
    print(f"populating DeliveredTripMetrics for Line/Route: {line}/{route if route else '(no-route)'}")
    current_date = start_date
    delta = timedelta(days=180)
    speed_objects = []
    while current_date < end_date:
        route_metadata = constants.get_route_metadata(line, current_date, route)
        print(f"Calculating daily values for 180 day chunk starting at: {current_date}")
        API_requests = get_agg_tt_api_requests(route_metadata["stops"], current_date, delta)
        curr_speed_object = send_requests(API_requests)
        date_range = get_date_range_strings(current_date, current_date + delta - timedelta(days=1))
        formatted_speed_object = format_tt_objects(curr_speed_object, route_metadata, len(API_requests), date_range)
        speed_objects.extend(formatted_speed_object)
        if (
            line == "line-green"
            and current_date < constants.GLX_EXTENSION_DATE
            and current_date + delta >= constants.GLX_EXTENSION_DATE
        ):
            current_date = constants.GLX_EXTENSION_DATE
        else:
            current_date += delta
        dynamo.dynamo_batch_write(speed_objects, "DeliveredTripMetrics")
        speed_objects = []
        print("Writing objects to DeliveredTripMetrics table")
        print("Done")

def update_daily_table(date):
    """Update DailySpeed table"""
    speed_objects = []
    for route in constants.ALL_ROUTES:
        route_metadata = constants.get_route_metadata(route[0], date, route[1])
        delta = timedelta(days=1)
        date_string = datetime.strftime(date, constants.DATE_FORMAT_BACKEND)
        print(f"Calculating update on [{route[0]}/{route[1] if route[1] else '(no-route)'}] for date: {date_string}")
        API_requests = get_agg_tt_api_requests(route_metadata["stops"], date, delta)
        speed_object = send_requests(API_requests)
        formatted_speed_object = format_tt_objects(speed_object, route_metadata, len(API_requests), [date_string])
        if len(formatted_speed_object) == 0:
            print("No data for date {date_string}")
            continue
        speed_objects.extend(formatted_speed_object)
    print(f"Writing values: {speed_objects}")
    dynamo.dynamo_batch_write(speed_objects, "DeliveredTripMetrics")
    print("Complete.")


from threading import Timer
import time

def func(a, b):
    print("Called function")
    return a * b

# Schedule a timer for 5 seconds
# We pass arguments 3 and 4
t = Timer(5.0, func, [3, 4])

start_time = time.time()
print(start_time)
# Start the timer
t.start()

end_time = time.time()

while end_time - start_time < 5.0:
    time.sleep(1)
    end_time = time.time()
    print("Timer will wait for sometime before calling the function")
    # print(end_time)


import ast
import random
from time import time

def bubblesort(a):
    n = len(a)
    f = False
    while not f:
        f = True
        for i in range(n - 1):
            if a[i] > a[i + 1]:
                a[i], a[i + 1] = a[i + 1], a[i]
                f = False

# a = [random.randint(-100_000, 100_000) for k in range(100)]
# print(a, end='\n\n')

resf = open("../results/bubblesort.txt", "w+")

resf.write("rnd:    ")
print("rnd")

for i in range(5000, 50001, 5000):
    randfile = open("../arrs/" + str(i) + "/random.txt")
    randstr = str(randfile.read())
    randlist = list(ast.literal_eval(randstr))

    t0 = time()
    bubblesort(randlist)
    t1 = time() - t0

    resf.write("{:.2f}".format(t1) + ";")
    print(i)

print("rev")
resf.write("\nrev:    ")

for i in range(5000, 50001, 5000):
    revlist = list(range(i, 0, -1))

    t0 = time()
    bubblesort(revlist)
    t2 = time() - t0

    resf.write("{:.2f}".format(t2) + ";")
    print(i)


import numpy as np
import cv2
from src.utils import *

class MOSSE:

    def __init__(self, lr = 1e-2, gauss_sig = 2, n_train = 100):

        """
        lr : is the learning rate for the updating of A and B
        gauss_sig: is the standard deviation used for the response map in the distance calculation
        n_train: number of times to train on the first image using random affine transformations
        """

        self.lr = lr
        self.gauss_sig = gauss_sig
        self.n_train = n_train

    def initialization(self, frame1, bbox):

        """
        To be called on the first frame where the bbox is available.
        The tracker is trained on the first frame n_train times to optimal
        values for both A and B.
        """
        if len(frame1.shape) == 3:
            frame1 = cv2.cvtColor(frame1,cv2.COLOR_BGR2GRAY)
        frame1 = frame1.astype(np.float32)

        self.w, self.h = bbox[2], bbox[3]
        self.x0, self.y0 = bbox[0], bbox[1]

        self.x1_clip, self.y1_clip, self.x2_clip, self.y2_clip = clip_bbox(self.x0,
                                                                           self.y0,
                                                                           self.w,
                                                                           self.h,
                                                                           frame1.shape)

        self.cosine_window = get_cosine_window((self.w, self.h))

        g = self._get_gaussian_map()
        G = fft2(g)

        f = frame1[self.y1_clip:self.y2_clip,
                   self.x1_clip:self.x2_clip]
        F = self._preprocess_frame(f)

        self.A = G * np.conjugate(F)
        self.B = F * np.conjugate(F)

        self._train(f, G)

        self.A = self.lr * self.A
        self.B = self.lr * self.B

    def track(self, frame):

        """
        Based on the previous frame's bbox position, we predict the position
        of the next bbox.
        """

        if len(frame.shape) == 3:
            frame = cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
        frame = frame.astype(np.float32)
        H = self.A/self.B

        f = frame[self.y1_clip:self.y2_clip, self.x1_clip:self.x2_clip]
        F = self._preprocess_frame(f)

        G = H * F

        g = scale_pixels(ifft2(G))
        max_yx = np.where(g == np.max(g))

        # updating the position of the bbox
        dy = int(np.mean(max_yx[0]) - g.shape[0] / 2)
        dx = int(np.mean(max_yx[1]) - g.shape[1] / 2)

        self.x0 += dx
        self.y0 += dy

        # make sure the values are within the frame
        self.x1_clip, self.y1_clip, self.x2_clip, self.y2_clip = clip_bbox(self.x0,
                                                                           self.y0,
                                                                           self.w,
                                                                           self.h,
                                                                           frame.shape)
        # Get the current frame with new bbox
        f = frame[self.y1_clip:self.y2_clip,
                  self.x1_clip:self.x2_clip]
        F = self._preprocess_frame(f)

        # update A and B
        self.A = self.lr * (G * np.conjugate(F)) + (1 - self.lr) * self.A
        self.B = self.lr * (F * np.conjugate(F)) + (1 - self.lr) * self.B

        return self.x0, self.y0, self.x0+self.w, self.y0+self.h

    def _train(self, frame, G):

        """
        The objective is to initialize A and B through training on various images
        which went through dome affine transformation.

        The filters were initialized by applying random small affine perturbations
        to the tracking window for the first frame of the video
         - Visual Object Tracking using Adaptive Correlation Filters
        """

        for i in range(self.n_train):

            F = self._preprocess_frame(get_affine_transform(frame))
            self.A = self.A + G * np.conjugate(F)
            self.B = self.B + F * np.conjugate(F)

    def _preprocess_frame(self, frame):

        frame = cv2.resize(frame, (self.w, self.h))
        frame = log_transform(frame)
        frame = normalize_pixels(frame)

        return fft2(self.cosine_window * frame)

    def _get_gaussian_map(self):
        """
        "In this case, gi is generated from ground truth such that
        it has a compact (σ = 2.0) 2D Gaussian shaped peak centered
        on the target in training image fi."
        - Visual Object Tracking using Adaptive Correlation Filters
        """

        """h,w = frame.shape
        xx, yy = np.meshgrid(np.arange(self.w), np.arange(self.h))"""

        # getting the distance from the center from the center
        # high values in the center of the gaussian map vs low values
        # the further away you are

        xx,yy = np.meshgrid(np.arange(self.w)-self.w//2, np.arange(self.h)-self.h//2)
        dist = (xx**2+yy**2) / (2*self.gauss_sig**2)

        # response_map

        response = np.exp(-0.5*dist)
        response = scale_pixels(response)

        return response


# 移除数组中重复的数，然后返回长度，不能使用额外空间

def remove_duplicates(arr):
    left, right = 1, 1

    while right < len(arr):
        # left到right的数都是一样的
        if arr[left - 1] != arr[right]:
            arr[left] = arr[right]
            left += 1
        right += 1

    return left

def remove_duplicates2(arr):
    left, right = 1, 1
    while right < len(arr):
        if arr[left - 1] != arr[right]:
            arr[left] = arr[right]
            left += 1
        right += 1
    return left

def main():
    print(remove_duplicates([2, 3, 3, 3, 6, 9, 9]))
    print(remove_duplicates([2, 2, 2, 11]))

    print(remove_duplicates2([2, 3, 3, 3, 6, 9, 9]))
    print(remove_duplicates2([2, 2, 2, 11]))

main()


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 15 09:07:33 2018

@author: gatesk
"""

import os
import csv
import pandas as pd
import pyodbc
import matplotlib.pyplot as plt, mpld3
import sys
import collections
from valid_H1000_conversion import *

outfile = open(r'C:\Users\gatesk\Documents\SURFACE_SAMPLE_ASSAY_VALUE_ERRORS\test_H1000.txt','w')

def line_generator(filepath):
    with open(filepath) as fin:
        for line in fin:
            yield line

def detect_delimter(fin):
    '''
    Input: a filepath
    Output: the file's delimiter
    '''
    try:
        with open(fin, 'r') as f:
            dialect = csv.Sniffer().sniff(f.readline())
            if dialect.delimiter == ',':
                delimiter = ','
                return delimiter
            if dialect.delimiter == '\t':
                delimiter = '\t'
                return delimiter
            else:
                delimiter = 'unk'
                return delimiter

    except Exception as e:
        print (e, 'on %s' %fin)
        delimiter = 'error'
        return delimiter

def get_headers(line,delimiter,headers):
    '''
    Input: an open file, delimiter of the file, key header list
    Output: list of headers within the file
    '''
#    for line in fin:
    if line.startswith('H'):
        headers.add(line.split(delimiter)[0])
    return headers

def generate_H1000lk():
    H1000lk = dict()
    with open(r"C:\Users\gatesk\Documents\SURFACE_SAMPLE_ASSAY_VALUE_ERRORS\H1000set_CONVERSIONS.csv",'r') as fin:
        for line in fin:
            items = line.split(',')
            H1000lk[items[0]] = items[1:]
    return H1000lk

H1000lk = generate_H1000lk()

def fix_H1000(H1000s,H1000fix,H1000lk):
    for i in H1000s:
        H1000fix.append((H1000lk[i.strip()][0],H1000lk[i.strip()][1],H1000lk[i.strip()][2].strip()))
    return H1000fix

def H1000f(line,headers,delimiter,H1000):
    if 'H1000' in headers:
        if line.startswith('H1000'):
            try:
                for i in line.split(delimiter):
                    H1000.append(i)
            except:
                pass
        return H1000

def H1001f(line,headers,delimiter,H1001):
    if 'H1000' in headers:
        if 'H1001' in headers:
            if line.startswith('H1001'):
                try:
                    for i in line.split(delimiter):
                        if len(i) == '':
                            H1001.append('unk')
                        elif len(i) != '':
                            H1001.append(i.strip())
                except:
                    pass
        return H1001

def H1002f(line,headers,delimiter,H1002):
    if 'H1002' in headers:
        if line.startswith('H1002'):
            try:
                for i in line.split(delimiter):
                    H1002.append(i.strip())
            except:
                pass
        return H1002

def H1003f(line,headers,delimiter,H1003):
    if 'H1003' in headers:
        if line.startswith('H1003'):
            try:
                for i in line.split(delimiter):
                    H1003.append(i.strip())
            except:
                pass
        return H1003

def main():
    path = "X:\_SS_RELOAD\PENDING"
    folders = os.listdir(path)

    for folder in folders:
        print (folder)
        rinfiles = os.listdir(os.path.join(path,folder))
        for file in rinfiles:

            filepath = os.path.join(os.path.join(path,folder),file)
            delimiter = detect_delimter(filepath)

            if delimiter == 'unk' or delimiter == 'error' :
                (print("Delimiter is not tab or csv - Manually examine the file"))
                sys.exit()

            H1000 = []
            H1000fix = []
            H1001 = []
            H1002 = []
            headers = set()

            for line in line_generator(filepath):
                headers = get_headers(line,delimiter,headers)

                H1000s = H1000f(line,headers,delimiter,H1000)
                H1001s = H1001f(line,headers,delimiter,H1001)
                H1002s = H1002f(line,headers,delimiter,H1002)

            check_H1000lk(H1000s,H1000lk)

            if 'H1000' in headers:
                H1000fixed = fix_H1000(H1000s,H1000fix,H1000lk)

            if 'H1000' not in headers:
                print ('No H1000 field')
                #sys.exit()

            if 'H1001' in headers and 'H1002' in headers:
                headersdict = dict()
                for idx, column in enumerate(H1000):
                    headersdict[column] = H1000fixed[idx],H1001[idx],H1002[idx]
                    if H1001[idx] == None:
                        H1001[idx] = 'unk'
                    if H1002[idx] == None:
                        H1002[idx] = 'unk'

            if 'H1001' in headers and 'H1002' not in headers:
                for idx, column in enumerate(H1000):
                    headersdict[column] = H1000fixed[idx],H1001[idx],'unk'
                    if H1001[idx] == None:
                        H1001[idx] = 'unk'
                    if H1002[idx] == None:
                        H1002[idx] = 'unk'

            if 'H1001' not in headers and 'H1002' not in headers:
                for idx, column in enumerate(H1000):
                    headersdict[column] = H1000fixed[idx],'unk','unk'

            H1000convert = []
            for item in H1000fixed:
                H1000convert.append(item[0].strip())
                H1000convertline =  str(delimiter.join(H1000convert))

            H1001new = []
            for item in H1000s:
                H1001new.append(headersdict[item][0][1])
            H1001new[0] = 'H1001'
            H1001new[1] = 'UNITS'
            H1001newline =  str(delimiter.join(H1001new))

            H1002new = []
            for item in H1000s:
                H1002new.append(headersdict[item][0][2])
            H1002new[0] = 'H1002'
            H1002new[1] = 'ASSAY_CODE'
            H1002newline =  str(delimiter.join(H1002new))

            H1000s_line = delimiter.join(H1000s)
            H1001s_line = delimiter.join(H1001s)

            if 'H1002' in headers:
                H1002s_line = delimiter.join(H1002s)
            else:
                pass

            outfile.write(H1000s_line)
            outfile.write(H1000convertline+'\n')

            if "H1001" in headers:
                outfile.write(H1001s_line+'\n')
            else:
                pass

            outfile.write(H1001newline+'\n')

            if 'H1002' in headers:
                outfile.write(H1002s_line+'\n')
            else:
                pass

            outfile.write(H1002newline+'\n')

main()


import os.path
from Day_1.database import user_data, get_csv_filenames, demography_to_db

demography_base_folder = os.path.join("..", "Task2", "Task2", "trainDemography")
csv_filenames = get_csv_filenames(demography_base_folder)

print("Extracting data from files: \n {}".format("\n".join(csv_filenames)))

for csv_file in csv_filenames:
    print("Opening demography file {}".format(csv_file))
    demography_to_db(user_data, csv_file)


#!/usr/bin/env python
# coding: utf-8

#1- ReflectanceImagesFolder
#2- panel corners
#3- band index

import sys, time, os, datetime, glob
import micasense.metadata as metadata
import micasense.capture as capture
import micasense.imageset as imageset
import micasense.utils as msutils
import micasense.plotutils as plotutils
import micasense.panel as panel
import micasense.dls as dls
from micasense.image import Image
import cv2
from PIL import Image as Img
import numpy as np
import subprocess
import math
from ipywidgets import FloatProgress, Layout
from IPython.display import display
import pandas as pd
import matplotlib.pyplot as plt

from platform import python_version

print(f"(Sys version) :|: {sys.version} :|:")
os.system("which python")
print(f"(Python version) :#: {python_version()} :#:")

## This progress widget is used for display of the long-running process
f = FloatProgress(min=0, max=1, layout=Layout(width='100%'), description="Loading")

PlaqueIMG="IMG_0071_"
OutputFolder="OUT"

# RRDP panelCorners to panel_corners
panel_corners = [[[610, 540], [600, 540], [600, 530], [610, 530]]]

# The location of the files to be recorded will be determined
#Linux filepath
# ReflectanceImagesFolder = os.path.join('.','data','RedEdge3','OUT')
#Windows filepath
# ReflectanceImagesFolder = os.path.join('r:\\','proc_field','RedEdge3', OutputFolder)
ReflectanceImagesFolder = os.path.join('r:\\','proc_field', OutputFolder)

# Path part of raw images will be written in this part
#Linux filepath
#images_path = os.path.join('.','data','RedEdge3')
#Windows filepath
images_path = os.path.join('r:\\','proc_field','RedEdge3')
# images_path_B = os.path.join('r:\\','proc_field','RedEdge3')

#Linux filepath
# image_names = os.path.join('.','data','RedEdge3','OUT')
#Windows filepath
# image_names = glob.glob(os.path.join(images_path, 'IMG_0063_*.tif'))
image_names = glob.glob(os.path.join(images_path,PlaqueIMG+'*.tif'))

# panelNames
#Linux filepath
# image_names = os.path.join('.','data','RedEdge3','OUT')
#Windows filepath
# img = Image(os.path.join('r:\\','proc_field','RedEdge3','IMG_0063_1.tif'))
img = Image(os.path.join('r:\\','proc_field','RedEdge3',PlaqueIMG+'1.tif'))

#Linux filepath
# image_names = os.path.join('.','data','RedEdge3','OUT')
#Windows filepath
# imgPNB = glob.glob(os.path.join(imgNB,'IMG_0063_4.tif'))[0]
imgPNB = glob.glob(os.path.join(images_path,PlaqueIMG+'4.tif'))[0]

cap = capture.Capture.from_filelist(image_names)
cap.plot_raw(fig_size=(9,6.75),num=1)

print('images_path or imgNB: {0}'.format(images_path))
print(ReflectanceImagesFolder)
print(images_path)
print(image_names)

print(cap)
print('CAP: {0}'.format(cap))

print(panel_corners)
print()
print(panel_corners[0])
print()
pnl = panel.Panel(img,panelCorners = panel_corners[0])
print("Panel found: {}".format(pnl.panel_detected()))
print("Panel serial: {}".format(pnl.serial))
print("QR Code Corners:\n{}".format(pnl.qr_corners()))
mean, std, count, saturated_count = pnl.raw()
print("Panel mean raw pixel value: {}".format(mean))
print("Panel raw pixel standard deviation: {}".format(std))
print("Panel region pixel count: {}".format(count))
print("Panel region saturated pixel count: {}".format(count))

imgB = Image(imgPNB)
imgB.plot_all(figsize=(9,6.75),num=2)

pnl.plot(figsize=(9,6.75),num=3)

# RRDP
# cap.set_panelCorners(panelCorners)
cap.set_panel_corners(panel_corners)

# DLS2 - ground seviyesinde
dls_irradiances = []
center_wavelengths = []
for img in cap.images:
    dls_irr = img.horizontal_irradiance
    dls_irradiances.append(dls_irr)
    center_wavelengths.append(img.center_wavelength)

plt.figure(figsize=(9,6.75),num=4)
plt.scatter(center_wavelengths, dls_irradiances)
plt.xlabel('Wavelength (nm)')
plt.ylabel('Irradiance ($W/m^2/nm$)')
plt.show()
plt.close()

#cap.plot_undistorted_reflectance(dls_irradiances)
cap.plot_undistorted_reflectance(dls_irradiances,fig_size=(9,8),num=5)

# panel_reflectance_by_band = [0.519, 0.521, 0.52, 0.518, 0.519]  # Altum band_index
panel_reflectance_by_band = [0.57, 0.57, 0.56, 0.50, 0.55]  # RedEdge3 band_index

panel_radiances = np.array(cap.panel_radiance())
irr_from_panel = math.pi * panel_radiances / panel_reflectance_by_band
dls_correction = irr_from_panel / dls_irradiances
cap.plot_undistorted_reflectance(dls_irradiances * dls_correction,fig_size=(9,8),num=6)

nbins = 1024
vmin = 0
vmax = 2**16
bins = range(vmin,vmax, int(vmax/nbins))
hist = cv2.calcHist([imgB.raw().ravel()],[0],None,[nbins],[vmin,vmax])
plt.figure(figsize=(9,6.75),num=7)
plt.plot(bins,hist)
plt.xlim(imgB.raw().min(),imgB.raw().max())
plt.ylim(0,hist.max())
plt.xlabel('Pixel Value')
plt.ylabel('Frequency')
plt.show()
plt.close()

## This progress widget is used for display of the long-running process
f = FloatProgress(min=0, max=1, layout=Layout(width='100%'), description="Loading")
display(f)
def update_f(val):
    if (val - f.value) > 0.005 or val == 1: #reduces cpu usage from updating the progressbar by 10x
        f.value=val

imgset = imageset.ImageSet.from_directory(images_path, progress_callback=update_f)

data, columns = imgset.as_nested_lists()
print("Columns: {}".format(columns))
df = pd.DataFrame.from_records(data, index='timestamp', columns=columns)

# 'b' as blue |'g' as green | 'r' as red | 'c' as cyan | 'm' as magenta | 'y' as yellow | 'k' as black | 'w' as white
ax=df.plot(y=columns[3:], subplots=True, figsize=(15,6.75), style=['g','c','y','k','b','g','r','k','m'])
for a in ax:
    a.legend(loc='right', bbox_to_anchor=(1.1, 0.5), ncol=1, fancybox=True, shadow=True)
plt.show()
plt.close()

plt.figure(num=8)
# plot the histogram of the altitude data
df.altitude.hist()
# find the altitude above which the flight images occur
cutoff_altitude = df.altitude.mean()-3.0*df.altitude.std()
plt.axvline(x=cutoff_altitude,c='r')
plt.xlabel('Capture altitude (m)')
plt.ylabel('Number of occurances')
plt.show()
plt.close()

channels = ['*.tif']

def band_adi(image):
    meta = metadata.Metadata(image)
    band = meta.get_item('XMP:BandName')
    return band

dls_correction_blue = dls_correction[0]
dls_correction_green = dls_correction[1]
dls_correction_red = dls_correction[2]
dls_correction_nir = dls_correction[3]
dls_correction_redEdge = dls_correction[4]

center_wavelengths=[]

for channel in channels:
    files= glob.glob(os.path.join(images_path,'*'+ channel + '*'))
    for file in files:
        img = Image(file)
        band = band_adi(file)
        cap = capture.Capture.from_file(file)
        dls_irr = img.horizontal_irradiance
        meta = metadata.Metadata(file)
        center_wavelengths.append(img.center_wavelength)

        if file.endswith(".tif"):
            figname = file[:-4] + '_reflectance.tif'
            print("figname", figname)

        path, filename = os.path.split(file)

        if band == "Blue":
            Reflectance = img.undistorted_reflectance(dls_irr* dls_correction_blue)
            #plotutils.plotwithcolorbar(Reflectance, 'Reflektans görüntüsü')
            print("Band ok1-Blue")

        if band == "Green":
            Reflectance = img.undistorted_reflectance(dls_irr* dls_correction_green)
            #plotutils.plotwithcolorbar(Reflectance, 'Reflektans görüntüsü')
            print("Band ok2-Green")

        if band == "Red":
            Reflectance = img.undistorted_reflectance(dls_irr* dls_correction_red)
            #plotutils.plotwithcolorbar(Reflectance, 'Reflektans görüntüsü')
            print("Band ok3-Red")

        if band == "NIR":
            Reflectance = img.undistorted_reflectance(dls_irr* dls_correction_nir)
            #plotutils.plotwithcolorbar(Reflectance, 'Reflektans görüntüsü')
            print("Band ok4-NIR")

        if band == "Red edge":
            Reflectance = img.undistorted_reflectance(dls_irr* dls_correction_redEdge)
            #plotutils.plotwithcolorbar(Reflectance, 'Reflektans görüntüsü')
            print("Band ok5-RedEdge")

        outfile = os.path.join(ReflectanceImagesFolder,filename)
        print(outfile)
        im = Img.fromarray(Reflectance)
        with open(outfile, 'w') as img: #create CSV
            im.save(os.path.join(outfile),format= 'tiff')##123
            print("Save image")

    if os.environ.get('exiftoolpath') is not None:
        exiftool_cmd = os.path.normpath(os.environ.get('exiftoolpath'))
    else:
        exiftool_cmd = 'exiftool'

    cmd = 'exiftool -tagsFromFile "{}" -ALL -XMP {}'.format(file, figname)
    print(cmd)
#RRDP
#subprocess.check_call(cmd)
    # if(subprocess.check_call(cmd) == 0):
    #     print("Successfully updated stack metadata")

    #     subprocess.run(['exiftool', '-tagsFromFile', file, '-ALL', '-XMP', figname])
    #     print("Exiftool 1")

    #     subprocess.run(['exiftool', '-delete_original!', figname])
    #     print("Exiftool OK")


# *_*coding:utf-8 *_*
# @Author : YueMengRui
from configs.prompt_template import kb_qa_prompt_template, tableQA_prompt_template
from info.mysql_models import App, KBData, KBDataDetail, KnowledgeBase
from info.utils.kb.MultiQueryRetriever import multiquery_retriever

def get_final_prompt(req, app_id, mysql_db):
    prompt = req.prompt
    related_docs = []
    msg = {}

    if 'tableQA' in req.custom:
        prompt = tableQA_prompt_template.format(query=prompt, context=req.custom['tableQA']['table_content'])
    else:
        app = mysql_db.query(App).get(app_id)

        data_chunk_map = {}
        if app and app.kb_id:
            text_hash_list = []
            embedding_model = None
            kb = mysql_db.query(KnowledgeBase).get(app.kb_id)
            if kb:
                embedding_model = kb.embedding_model
                kb_data_list = mysql_db.query(KBData).filter(KBData.kb_id == app.kb_id, KBData.is_delete == False).all()
                if kb_data_list:
                    for kb_data in kb_data_list:
                        data_chunks = mysql_db.query(KBDataDetail).filter(KBDataDetail.data_id == kb_data.id).all()
                        if data_chunks:
                            for c in data_chunks:
                                if c.content_hash is not None:
                                    data_chunk_map.update({c.content_hash: kb_data.to_dict()})
                                    text_hash_list.append(c.content_hash)

            if all([text_hash_list, embedding_model]):
                if app.is_multiQueryRetriever_enabled:
                    queries, docs, context = multiquery_retriever(prompt, app.llm_name, embedding_model,
                                                                  list(set(text_hash_list)))
                    temp_docs = {}
                    for d in docs:
                        kb_dat = data_chunk_map[d['text_hash']]
                        if kb_dat['file_hash'] in temp_docs:
                            temp_docs[kb_dat['file_hash']]['related_texts'].append(d)
                        else:
                            kb_dat.update({'related_texts': [d]})
                            temp_docs.update({kb_dat['file_hash']: kb_dat})

                    related_docs = list(temp_docs.values())
                    msg.update({"MultiQueryRetriever": {"queries": queries}})
                    prompt = kb_qa_prompt_template.format(query=prompt, context=context)

    return prompt, related_docs, msg


"""Module for filtering and smoothing joint data"""

import numpy as np
import pykalman
from post_processing.src import game_tools
from post_processing.src import hampel_filter
from pose.src import joints

def get_times(hdf5_file, games, game_idx, time_name):
    """Construct important times for games relative to video in an HDF File

    Args:
        hdf5_file: The HDF5 file with the video data
        games: A dictionary of information about the games with start and finish information
        game_idx: The game to get informaiton about
        time_name: The name of the time field in the hdf5 file to work with
    Returns: Dictionary with the the start time, finish time, and start and finish indeces in
             the video time field for the requested game
    """
    times = {}
    times['start_t'] = games['start'][game_idx]
    times['finish_t'] = games['finish'][game_idx]
    times['start_idx'] = np.argmax(hdf5_file[time_name][:] > times['start_t'])
    times['finish_idx'] = np.argmax(
        hdf5_file[time_name][:] > times['finish_t'])
    return times

def filter_file(hdf5_file):
    """For each game recorded in the HDF5 file, gather filtered joint values and smoothed full
    state, as well as covariances (see filter_ss and filter_tt).

    Args:
        hdf5_file: The HDF5 file with data to process (generated by pose/src/process_hdf5.py)
    Return: array of dictionaries of results of processing
    """
    games = game_tools.parse_games(hdf5_file)
    results = [None]*len(games['type'])
    for game_idx, _ in enumerate(games['type']):
        print(f'working on game {game_idx}')
        results[game_idx] = {}
        game_type = games['type'][game_idx]
        results[game_idx]['game_type'] = game_type
        if game_type == 'simon_says':
            time_name = 'vid/lower/color/time'
            keypoint_name = 'vid/lower/pose/openpose:25B/keypoints-median5/3d-realsense-raw'
            pixel_keypoint_name = 'vid/lower/pose/openpose:25B/keypoints-median5/color'
            times = get_times(hdf5_file, games, game_idx, time_name)
            # it is possible that little to none of the game was recorded. handle that
            if (times['finish_idx']-times['start_idx']) < 100:
                print('game with little to no recording, passing')
                continue
            game_time = hdf5_file[time_name][times['start_idx']                                             :times['finish_idx']]
            keypoint_data = hdf5_file[keypoint_name][times['start_idx']                                                     :times['finish_idx']]
            pixel_data = hdf5_file[pixel_keypoint_name][times['start_idx']                                                        :times['finish_idx']]
            results[game_idx]['state'] = filter_ss(
                keypoint_data, pixel_data, game_time)
        if game_type == 'target_touch':
            time_name = 'vid/upper/color/time'
            results[game_idx]['state'] = {}
            for hand in ['right', 'left']:
                keypoint_name = f'vid/upper/pose/mp-hands/{hand}/keypoints-median5/3d-realsense-raw'
                times = get_times(hdf5_file, games, game_idx, time_name)
                # it is possible that little to none of the game was recorded. handle that
                if (times['finish_idx']-times['start_idx']) < 100:
                    print('game with little to no recording, passing')
                    continue
                game_time = hdf5_file[time_name][times['start_idx']:times['finish_idx']]
                keypoint_data = hdf5_file[keypoint_name][times['start_idx']:times['finish_idx']]
                results[game_idx]['state'][hand] = filter_tt(
                    keypoint_data,  game_time)

    return results

def filter_tt(keypoint_data,  game_time):
    """Filter and determine state for the arms in target touch activity.
    For the passed in arm, gathers the raw, filtered, and smoothed values as well as covariances
    for the wrist.

    Args:
        keypoint_data: The 3D keypoint data provided by mediapipe-hands + depth
        game_time: The time of the keypoint and pixel data capture
    """
    print('filtering a target touch game')
    filtered_state = {}
    filtered_state['time'] = game_time
    wrist_idx = joints.get_mediapipe_joint('WRIST')
    wrist_pose = keypoint_data[:, wrist_idx, :]
    if np.all(np.isnan(wrist_pose)):
        return None
    wrist_poses_filtered, wrist_state, wrist_state_covariance = filter_wrist(
        wrist_pose, game_time, 10)
    filtered_state['smooth'] = wrist_state
    filtered_state['filtered'] = wrist_poses_filtered
    filtered_state['raw'] = wrist_pose
    filtered_state['covariance'] = wrist_state_covariance
    return filtered_state

def filter_ss(keypoint_data, pixel_data, game_time):
    """Filter and determine state for the arms in full upper extremity activity.
    For each arm, gathers the raw, filtered, and smoothed values as well as covariances
    for the shoulder, elbow, and wrist.

    Args:
        keypoint_data: The 3D keypoint data provided by openpose + depth
        pixel_data: The pixel positions provided by openpose
        game_time: The time of the keypoint and pixel data capture
    """
    #pylint: disable=too-many-locals
    print('filtering a simon says game')
    filtered_state = {}
    filtered_state['time'] = game_time
    # start with wrists
    filtered_state['smooth'] = {}
    filtered_state['raw'] = {}
    filtered_state['filtered'] = {}
    filtered_state['covariance'] = {}
    for side in ['R', 'L']:
        wrist_name = f'{side}Wrist'
        print(f'working on {wrist_name}')
        wrist_idx = joints.get_openpose_joint(wrist_name)
        wrist_pose = keypoint_data[:, wrist_idx, :]
        if np.all(np.isnan(wrist_pose)):
            continue
        wrist_poses_filtered, wrist_state, wrist_state_covariance = filter_wrist(
            wrist_pose, game_time)
        filtered_state['smooth'][wrist_name] = wrist_state
        filtered_state['filtered'][wrist_name] = wrist_poses_filtered
        filtered_state['raw'][wrist_name] = wrist_pose
        filtered_state['covariance'][wrist_name] = wrist_state_covariance
        # process elbows
        elbow_name = f'{side}Elbow'
        print(f'working on {elbow_name}')
        elbow_idx = joints.get_openpose_joint(elbow_name)
        elbow_pose = keypoint_data[:, elbow_idx, :]
        elbow_pixels = pixel_data[:, elbow_idx, :2]
        wrist_pixels = pixel_data[:, wrist_idx, :2]
        if np.all(np.isnan(elbow_pose)):
            continue
        elbow_poses_filtered, elbow_state, elbow_state_covariance = filter_elbow(
            elbow_pose, wrist_state[:, 2], game_time, elbow_pixels, wrist_pixels)
        filtered_state['smooth'][elbow_name] = elbow_state
        filtered_state['filtered'][elbow_name] = elbow_poses_filtered
        filtered_state['raw'][elbow_name] = elbow_pose
        filtered_state['covariance'][elbow_name] = elbow_state_covariance
        # process shoulders
        shoulder_name = f'{side}Shoulder'
        print(f'working on {shoulder_name}')
        shoulder_idx = joints.get_openpose_joint(shoulder_name)
        shoulder_pose = keypoint_data[:, shoulder_idx, :]
        shoulder_pixels = pixel_data[:, shoulder_idx, :2]
        if np.all(np.isnan(shoulder_pose)):
            continue
        shoulder_poses_filtered, shoulder_state, shoulder_state_covariance = filter_shoulder(
            shoulder_pose, wrist_state[:, 2], game_time,
            shoulder_pixels, elbow_pixels, wrist_pixels)
        filtered_state['smooth'][shoulder_name] = shoulder_state
        filtered_state['filtered'][shoulder_name] = shoulder_poses_filtered
        filtered_state['raw'][shoulder_name] = shoulder_pose
        filtered_state['covariance'][shoulder_name] = shoulder_state_covariance
    return filtered_state

def filter_shoulder(game_pose, wrist_depth, game_time, shoulder_pixels, elbow_pixels, wrist_pixels):
    """Filter the shoulder by first eliminating values where the forearm occludes the shoulder,
    then a hampel filter, and finally a kalman smoother.

    Unlike for the wrist and hand, we treat the shoulder as having a constant velocity (expected
    to be zero)

    Args:
        game_pose: an array with the x,y,z poses from the wrist
        wrist_depth: The depth of the wrist points
        game_time: the times at which the data in game_pose were captured
        shoulder_pixels: the pixel positions of the shoulder.
        elbow_pixels: The pixel positions in color image of elbow
        wrist_pixels: The pixel positions of the wrist
    Return (masked game_pose values generated by hampel filter,
    state: (x,y,z,dx,dy,dz), covariance of state)
    """
    #pylint: disable=too-many-locals
    #pylint: disable=too-many-arguments

    # for more insights on how this is being done, refer to notebooks/simon_says_filtering.ipynb

    # Measurement function (we measure x, y, and z positions)
    H = np.array([[1, 0, 0, 0, 0, 0, ],
                  [0, 1, 0, 0, 0, 0, ],
                  [0, 0, 1, 0, 0, 0, ],
                  ])

    # Observation covariance
    # https://stackoverflow.com/a/9537830/5274985
    clean_depth = game_pose[:, 2]
    mask = np.isnan(clean_depth)
    clean_depth[mask] = np.interp(np.flatnonzero(
        mask), np.flatnonzero(~mask), clean_depth[~mask])
    R = [[[(0.0215*z)**2,             0,           0],
          [0,             (0.0215*z)**2,           0],
          [0,                         0, (0.04*z)**2]]
         for z in clean_depth]

    # Transistion covariance - assume constant velocity model with only slight changes
    # in velocity.
    Q = [np.array([[0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 1, 0, 0],
                   [0, 0, 0, 0, 1, 0],
                   [0, 0, 0, 0, 0, 1]
                   ])*((10*dt)**2) for dt in np.diff(game_time)]
    # State transistion matrix
    Fs = [np.array(
        [[1, 0, 0, dt,  0,  0],
         [0, 1, 0,  0, dt,  0],
         [0, 0, 1,  0,  0, dt],
         [0, 0, 0,  1,  0,  0],
         [0, 0, 0,  0,  1,  0],
         [0, 0, 0,  0,  0,  1]
         ]) for dt in np.diff(game_time)]

    # don't use values where the hand is in front of the elbow.
    # if x and y for wrist and elbow are at the same point, then mask that shit
    # let's say +/- 75 mm
    px_size = wrist_depth*1.375/1920
    margin_error = 75/px_size

    # check if points inside box defined by wrist elbow +/- margin error
    x_high = np.max([wrist_pixels[:, 0], elbow_pixels[:, 0]],
                    axis=0)+margin_error
    x_low = np.min([wrist_pixels[:, 0], elbow_pixels[:, 0]],
                   axis=0)-margin_error
    y_high = np.max([wrist_pixels[:, 1], elbow_pixels[:, 1]],
                    axis=0)+margin_error
    y_low = np.min([wrist_pixels[:, 1], elbow_pixels[:, 1]],
                   axis=0)-margin_error
    in_box = (shoulder_pixels[:, 0] > x_low) & (shoulder_pixels[:, 0] < x_high) & (
        shoulder_pixels[:, 1] > y_low) & (shoulder_pixels[:, 1] < y_high)

    x_1 = wrist_pixels[:, 0]
    y_1 = wrist_pixels[:, 1]
    x_2 = elbow_pixels[:, 0]
    y_2 = elbow_pixels[:, 0]

    x_0 = shoulder_pixels[:, 0]
    y_0 = shoulder_pixels[:, 1]

    # hacky way to prevent division by zero
    same = (x_1 == x_2) & (y_1 == y_2)
    x_1[same] = x_1[same]+1
    dist = np.abs((x_2-x_1)*(y_1-y_0)-(x_1-x_0)*(y_2-y_1)) / \
        np.sqrt((x_2-x_1)**2 + (y_2-y_1)**2)

    game_pose[(dist < margin_error) & in_box, :] = np.nan

    print('Filtering with Hampel Filter')
    bad_vals = np.unique(np.concatenate([hampel_filter.hampel_filter_v(
        game_pose[:, axis], 50, 5) for axis in range(3)], axis=0))
    game_pose[bad_vals, :] = np.nan
    print('Smoothing with Kalman Smoother')
    kalman_filter = pykalman.KalmanFilter(
        transition_matrices=Fs, observation_matrices=H,
        observation_covariance=R, transition_covariance=Q)
    game_pose_masked = np.ma.array(game_pose)
    game_pose_masked[np.isnan(game_pose_masked)] = np.ma.masked
    # Initial vals
    init_meds = [np.ma.median(gp[~gp.mask][:20]) for gp in game_pose_masked.T]
    kalman_filter.initial_state_mean = np.array(
        [init_meds[0], init_meds[1], init_meds[2], 0, 0, 0])
    computed_state, state_covariance = kalman_filter.smooth(game_pose_masked)
    return (game_pose_masked, computed_state, state_covariance)

def filter_elbow(game_pose, wrist_depth, game_time, elbow_pixels, wrist_pixels):
    """Filter data from the elbow by first eliminating values which the wrist
    occludes (wrist radius of +/- 75mm),
    followed by a hampel filter, followed by a kalman smoother

    Args:
        game_pose: an array with the x,y,z poses from the wrist
        wrist_depth: The depth of the wrist points
        game_time: the times at which the data in game_pose were captured
        elbow_pixels: The pixel positions in color image of elbow
        wrist_pixels: The pixel positions of the wrist
    Return (masked game_pose values generated by hampel filter,
    state: (x,y,z,dx,dy,dz,ddx,ddy,ddz), covariance of state)
    """
    #pylint: disable=too-many-locals

    # Measurement function (we measure x, y, and z positions)
    H = np.array([[1, 0, 0, 0, 0, 0, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0, 0, 0],
                  [0, 0, 1, 0, 0, 0, 0, 0, 0],
                  ])

    # Observation covariance
    clean_depth = game_pose[:, 2]
    mask = np.isnan(clean_depth)
    clean_depth[mask] = np.interp(np.flatnonzero(
        mask), np.flatnonzero(~mask), clean_depth[~mask])
    R = [[[(0.0215*z)**2,             0,           0],
          [0,             (0.0215*z)**2,           0],
          [0,                         0, (0.04*z)**2]] for z in clean_depth]

    # Transistion covariance - assume velocity only changes due to acceleration
    # and position only changes due to velocity
    # probably elbow accceleration changes way slower than wrist?
    Q = [np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 1, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 1, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 1]
                   ])*((10000*dt)**2) for dt in np.diff(game_time)]
    # State transistion matrix
    Fs = [np.array(
        [[1, 0, 0, dt,  0,  0, 1/2*(dt**2),           0,           0],
         [0, 1, 0,  0, dt,  0,           0, 1/2*(dt**2),           0],
         [0, 0, 1,  0,  0, dt,           0,           0, 1/2*(dt**2)],
         [0, 0, 0,  1,  0,  0,          dt,           0,           0],
         [0, 0, 0,  0,  1,  0,           0,          dt,           0],
         [0, 0, 0,  0,  0,  1,           0,           0,          dt],
         [0, 0, 0,  0,  0,  0,           1,           0,           0],
         [0, 0, 0,  0,  0,  0,           0,           1,           0],
         [0, 0, 0,  0,  0,  0,           0,           0,           1]
         ]) for dt in np.diff(game_time)]

    # don't use values where the hand is in front of the elbow.
    # if x and y for wrist and elbow are at the same point, then mask that shit
    # let's say +/- 75 mm
    print('Eliminating wrist occlusions')
    px_size = wrist_depth*1.375/1920
    margin_error = 75/px_size
    wrist_elbow_dist = np.sqrt(
        np.sum(np.power(elbow_pixels-wrist_pixels, 2), axis=1))
    game_pose[wrist_elbow_dist < margin_error, :] = np.nan

    print('Filtering with Hampel Filter')
    bad_vals = np.unique(np.concatenate([hampel_filter.hampel_filter_v(
        game_pose[:, axis], 50, 5) for axis in range(3)], axis=0))
    game_pose[bad_vals, :] = np.nan
    print('Smoothing with Kalman Smoother')
    kalman_filter = pykalman.KalmanFilter(
        transition_matrices=Fs, observation_matrices=H,
        observation_covariance=R, transition_covariance=Q)
    game_pose_masked = np.ma.array(game_pose)
    game_pose_masked[np.isnan(game_pose_masked)] = np.ma.masked
    # Initial vals
    init_meds = [np.ma.median(gp[~gp.mask][:20]) for gp in game_pose_masked.T]
    kalman_filter.initial_state_mean = np.array(
        [init_meds[0], init_meds[1], init_meds[2], 0, 0, 0, 0, 0, 0])
    computed_state, state_covariance = kalman_filter.smooth(game_pose_masked)
    return (game_pose_masked, computed_state, state_covariance)

def filter_wrist(game_pose, game_time, threshold=5):
    """Filter data from the wrist using a hampel filter followed by a kalman smoother

    The observation covariance is based on the raw z values provided by game_pose.

    The transistion covariance only has variance for ddx,ddy,ddz which is 100,000mm/s^3*dt

    For insights on the origin of tuning, see notebooks/simon_says_filtering.ipynb

    Args:
        game_pose: an array with the x,y,z poses from the wrist
        game_time: the times at which the data in game_pose were captured
    Return (masked game_pose values generated by hampel filter,
    state: (x,y,z,dx,dy,dz,ddx,ddy,ddz), covariance of state)
    """
    # for more insights on how this is being done, refer to notebooks/simon_says_filtering.ipynb

    # Measurement function (we measure x, y, and z positions)
    H = np.array([[1, 0, 0, 0, 0, 0, 0, 0, 0],
                  [0, 1, 0, 0, 0, 0, 0, 0, 0],
                  [0, 0, 1, 0, 0, 0, 0, 0, 0],
                  ])

    # Observation covariance
    # For x and y:
    #   tand(69/2)*2=1.375
    #   1.375/1920*15 = 0.0107
    #   x2 = 0.0215 (double for good measure, to handle z interaction)
    # For z:
    #   2*2%=0.04
    clean_depth = game_pose[:, 2]
    mask = np.isnan(clean_depth)
    clean_depth[mask] = np.interp(np.flatnonzero(
        mask), np.flatnonzero(~mask), clean_depth[~mask])
    R = [[[(0.0215*z)**2,             0,           0],
          [0,             (0.0215*z)**2,           0],
          [0,                         0, (0.04*z)**2]] for z in clean_depth]

    # Transistion covariance - assume velocity only changes due to acceleration
    # and position only changes due to velocity
    Q = [np.array([[0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 1, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 1, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 1]
                   ])*((100000*dt)**2) for dt in np.diff(game_time)]
    # State transistion matrix
    Fs = [np.array(
        [[1, 0, 0, dt,  0,  0, 1/2*(dt**2),           0,           0],
         [0, 1, 0,  0, dt,  0,           0, 1/2*(dt**2),           0],
         [0, 0, 1,  0,  0, dt,           0,           0, 1/2*(dt**2)],
         [0, 0, 0,  1,  0,  0,          dt,           0,           0],
         [0, 0, 0,  0,  1,  0,           0,          dt,           0],
         [0, 0, 0,  0,  0,  1,           0,           0,          dt],
         [0, 0, 0,  0,  0,  0,           1,           0,           0],
         [0, 0, 0,  0,  0,  0,           0,           1,           0],
         [0, 0, 0,  0,  0,  0,           0,           0,           1]
         ]) for dt in np.diff(game_time)]
    print('Filtering with Hampel Filter')
    bad_vals = np.unique(np.concatenate([hampel_filter.hampel_filter_v(
        game_pose[:, axis], 50, threshold) for axis in range(3)], axis=0))
    game_pose[bad_vals, :] = np.nan
    print('Smoothing with Kalman Smoother')
    kalman_filter = pykalman.KalmanFilter(
        transition_matrices=Fs, observation_matrices=H,
        observation_covariance=R, transition_covariance=Q)
    game_pose_masked = np.ma.array(game_pose)
    game_pose_masked[np.isnan(game_pose_masked)] = np.ma.masked
    # Initial vals
    init_meds = [np.ma.median(gp[~gp.mask][:20]) for gp in game_pose_masked.T]
    kalman_filter.initial_state_mean = np.array(
        [init_meds[0], init_meds[1], init_meds[2], 0, 0, 0, 0, 0, 0])
    computed_state, state_covariance = kalman_filter.smooth(game_pose_masked)
    return (game_pose_masked, computed_state, state_covariance)


import streamlit as st
import joblib
import numpy as np
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()

def main():
    html_temp = """

Bank Customer Churn Prediction Using ML

"""

    st.markdown(html_temp, unsafe_allow_html=True)
    model = joblib.load('churn_predict_model')
    p1 = st.number_input("CreditScore")
    p2 = st.number_input("Age", 18, 90)
    p3 = st.slider("Tenure",0,15)
    p4 = st.number_input("Balance")
    p5 = st.slider("NumOfProducts",1,4)
    p6 = st.slider("HasCrCard",0,1)
    p7 = st.slider("IsActiveMember",0,1)
    p8 = st.number_input("EstimatedSalary")
    # Initialize default values
    Geography_Germany = 0
    Geography_Spain = 0
    Geography_France = 0
    p9 = st.slider("Geography",1,3)
    if p9 == 1:
        Geography_Germany = 1
        Geography_Spain = 0
        Geography_France = 0
    elif p9 == 2:
        Geography_Germany = 0
        Geography_Spain = 1
        Geography_France = 0
    elif p9 == 3:
        Geography_Germany = 0
        Geography_Spain = 0
        Geography_France = 1
    p10 = st.slider("Gender : ", 0, 1)

    if st.button('Predict'):
        input_data = np.array([[p1, p2, p3, p4, p5, p6, p7, p8, Geography_Germany, Geography_Spain, p10]])
        input_data_scaled = sc.fit_transform(input_data)
        pred = model.predict(input_data_scaled)

        st.balloons()
        if pred == 0:
            st.write("
No Churn
", unsafe_allow_html=True)
        else:
            st.write("
Churn Detected
", unsafe_allow_html=True)

        st.success('Thank You')

if __name__ == '__main__':
    main()


import numpy as np
from tqdm import tqdm
import infomap
import json
import time
from multiprocessing.dummy import Pool as Threadpool
from multiprocessing import Pool
import multiprocessing as mp
import os
import shutil
from tools.utils import Timer, mkdir_if_no_exist, l2norm, intdict2ndarray, read_meta
from evaluation import evaluate, accuracy

class knn():
    def __init__(self, feats, k, index_path='', verbose=True):
        pass

    def filter_by_th(self, i):
        th_nbrs = []
        th_dists = []
        nbrs, dists = self.knns[i]
        for n, dist in zip(nbrs, dists):
            if 1 - dist < self.th:
                continue
            th_nbrs.append(n)
            th_dists.append(dist)
        th_nbrs = np.array(th_nbrs)
        th_dists = np.array(th_dists)
        return th_nbrs, th_dists

    def get_knns(self, th=None):
        if th is None or th <= 0.:
            return self.knns
        # TODO: optimize the filtering process by numpy
        # nproc = mp.cpu_count()
        nproc = 1
        with Timer('filter edges by th {} (CPU={})'.format(th, nproc),
                   self.verbose):
            self.th = th
            self.th_knns = []
            tot = len(self.knns)
            if nproc > 1:
                pool = mp.Pool(nproc)
                th_knns = list(
                    tqdm(pool.imap(self.filter_by_th, range(tot)), total=tot))
                pool.close()
            else:
                th_knns = [self.filter_by_th(i) for i in range(tot)]
            return th_knns

class knn_faiss(knn):
    """
    内积暴力循环
    归一化特征的内积等价于余弦相似度
    """
    def __init__(self,
                 feats,
                 k,
                 index_path='',
                 knn_method='faiss-cpu',
                 verbose=True):
        import faiss
        with Timer('[{}] build index {}'.format(knn_method, k), verbose):
            knn_ofn = index_path + '.npz'
            if os.path.exists(knn_ofn):
                print('[{}] read knns from {}'.format(knn_method, knn_ofn))
                self.knns = np.load(knn_ofn)['data']
            else:
                feats = feats.astype('float32')
                size, dim = feats.shape
                if knn_method == 'faiss-gpu':
                    import math
                    i = math.ceil(size/1000000)
                    if i > 1:
                        i = (i-1)*4
                    res = faiss.StandardGpuResources()
                    res.setTempMemory(i * 1024 * 1024 * 1024)
                    index = faiss.GpuIndexFlatIP(res, dim)
                else:
                    index = faiss.IndexFlatIP(dim)
                index.add(feats)
        with Timer('[{}] query topk {}'.format(knn_method, k), verbose):
            knn_ofn = index_path + '.npz'
            if os.path.exists(knn_ofn):
                pass
            else:
                sims, nbrs = index.search(feats, k=k)
                # torch.cuda.empty_cache()
                self.knns = [(np.array(nbr, dtype=np.int32),
                              1 - np.array(sim, dtype=np.float32))
                             for nbr, sim in zip(nbrs, sims)]

def knns2ordered_nbrs(knns, sort=True):
    if isinstance(knns, list):
        knns = np.array(knns)
    nbrs = knns[:, 0, :].astype(np.int32)
    dists = knns[:, 1, :]
    if sort:
        # sort dists from low to high
        nb_idx = np.argsort(dists, axis=1)
        idxs = np.arange(nb_idx.shape[0]).reshape(-1, 1)
        dists = dists[idxs, nb_idx]
        nbrs = nbrs[idxs, nb_idx]
    return dists, nbrs

# 构造边
def get_links(single, links, nbrs, dists, args):
    for i in tqdm(range(nbrs.shape[0])):
        count = 0
        for j in range(1, len(nbrs[i])):
            # 排除本身节点
            if i == nbrs[i][j]:
                pass
            elif dists[i][j] <= 1 - args.min_sim:
                count += 1
                links[(i, nbrs[i][j])] = float(1 - dists[i][j])
            else:
                break
        # 统计孤立点
        if count == 0:
            single.append(i)
    return single, links

def cluster_by_infomap(nbrs, dists, args):
    """
    基于infomap的聚类
    :param nbrs:
    :param dists:
    :param args:
    :return:
    """
    single = []
    links = {}
    with Timer('get links', verbose=True):
        single, links = get_links(single=single, links=links, nbrs=nbrs, dists=dists, args=args)

    infomapWrapper = infomap.Infomap("--two-level --directed")
    for (i, j), sim in tqdm(links.items()):
        _ = infomapWrapper.addLink(int(i), int(j), sim)

    # 聚类运算
    infomapWrapper.run()

    label2idx = {}
    idx2label = {}

    # 聚类结果统计
    for node in infomapWrapper.iterTree():
        # node.physicalId 特征向量的编号
        # node.moduleIndex() 聚类的编号
        idx2label[node.physicalId] = node.moduleIndex()
        if node.moduleIndex() not in label2idx:
            label2idx[node.moduleIndex()] = []
        label2idx[node.moduleIndex()].append(node.physicalId)

    node_count = 0
    for k, v in label2idx.items():
        if k == 0:
            node_count += len(v[2:])
            label2idx[k] = v[2:]
            # print(k, v[0:])
        else:
            node_count += len(v[1:])
            label2idx[k] = v[1:]
            # print(k, v[0:])

    # 孤立点个数
    print("=> Single cluster:{}".format(len(single)))

    keys_len = len(list(label2idx.keys()))
    # print(keys_len)

    # 孤立点放入到结果中
    for single_node in single:
        idx2label[single_node] = keys_len
        label2idx[keys_len] = [single_node]
        keys_len += 1

    print("=> Total clusters:{}".format(keys_len))

    idx_len = len(list(idx2label.keys()))
    print("=> Total nodes:{}".format(idx_len))

    # 保存聚类结果
    if eval(args.save_result):
        with open('pred_label_path.txt', 'w') as of:
            for idx in range(idx_len):
                of.write(str(idx2label[idx]) + '\n')

    # 评价聚类结果
    if eval(args.is_evaluate) and args.label_path is not None:
        pred_labels = intdict2ndarray(idx2label)
        true_lb2idxs, true_idx2lb = read_meta(args.label_path)
        gt_labels = intdict2ndarray(true_idx2lb)
        for metric in args.metrics:
            evaluate(gt_labels, pred_labels, metric)

    # 归档图片
    if args.output_picture_path is not None:
        print("=> Start copy pictures to the output path {} ......".format(args.output_picture_path))
        with open('./tools/pic_path', 'r') as f:
            content = f.read()
            picture_path_dict = json.loads(content)
        mkdir_if_no_exist(args.output_picture_path)
        shutil.rmtree(args.output_picture_path)
        os.mkdir(args.output_picture_path)
        tmp_pth = 'data/input_pictures/alldata'
        for label, idxs in label2idx.items():
            print('---------------------------', label, idxs)
            picture_reuslt_path = args.output_picture_path + '/' + str(label)
            mkdir_if_no_exist(picture_reuslt_path)
            for idx in idxs:
                picture_path = picture_path_dict[str(idx)]
                shutil.copy(picture_path, picture_reuslt_path)
                # shutil.copy(picture_path, tmp_pth)

def get_dist_nbr(features, args):
    # features = np.fromfile(feature_path, dtype=np.float32)
    features = features.reshape(-1, 2048)
    features = l2norm(features)

    index = knn_faiss(feats=features, k=args.k, knn_method=args.knn_method)
    knns = index.get_knns()
    dists, nbrs = knns2ordered_nbrs(knns)
    return dists, nbrs

def cluster_main(args, extract_features):
    dists, nbrs = get_dist_nbr(features=extract_features, args=args)
    print(dists.shape, nbrs.shape)
    cluster_by_infomap(nbrs, dists, args)


import torch
from torch import Tensor

def log_expm1(
    x: Tensor,
) -> Tensor:
    r"""Logarithm of :math:`e^{x} - 1`.

    If :math:`x` is large, ``torch.exp(x)`` returns ``float("inf")`` whereas:

    .. math::
        \log e^{x} - 1 \approx x.

    Therefore, an approximation for :math:`x > 15` is used, such that the
    output is not infinity for all positive values of :math:`x`.

    Args:
        x: the input tensor.

    Returns:
        the logarithm of :math:`e^{x} - 1`.
    """
    return torch.where(
        x < 15.0,
        torch.log(torch.expm1(torch.minimum(x, torch.tensor(15.0)))),
        x,
    )