import json
import time
import copy

def utc_timestamp():
    return time.strftime("%Y-%m-%dT%H:%M:%S.00Z", time.gmtime())

def response_event(name, header, endpoint):
    event = {
                "header": {
                    "namespace": "Alexa",
                    "name": name,
                    "payloadVersion": "3",
                    "messageId": header["messageId"],
                    "correlationToken": header["correlationToken"]
                },
                "endpoint": {
                    "scope": {
                        "type": "BearerToken",
                        "token": endpoint["scope"]["token"]
                    },
                    "endpointId": endpoint["endpointId"]
                }
            }
    return event

def resolve_response_name(header):
    name = "Response"
    if header["namespace"] == "Alexa" and header["name"] == "ReportState":
        name = "StateReport"
    elif header["namespace"] == "Alexa.Discovery" and header["name"] == "Discover":
        name = "Discover.Response"
    return name

class AlexaEntertainmentCenter:
    def __init__(self, capabilities=[], endpoint_id="Entertainment-Center",
                 manufacturer="Various", friendly_name="Entertainment Center",
                 description="Home Entertainment Center",
                 display_categories=["TV"]):
        self.capabilities = capabilities
        self.endpoint_id = endpoint_id
        self.manufacturer = manufacturer
        self.friendly_name = friendly_name
        self.description = description
        self.display_categories = display_categories

    def discovery(self, header, registered_things):
        endpoint_list = []
        endpoint = {
                "endpointId": self.endpoint_id,
                "manufacturerName": self.manufacturer,
                "friendlyName": self.friendly_name,
                "description": self.description,
                "displayCategories": self.display_categories,
                "capabilities": []
            }
        for capability in self.capabilities:
            endpoint["capabilities"].append(capability.discovery())
        for thing in registered_things:
            endpoint["endpointId"] = thing["thingName"]
            endpoint["friendlyName"] = thing["thingName"]
            endpoint_list.append(copy.deepcopy(endpoint))
        response_header = copy.deepcopy(header)
        response_header["name"] = resolve_response_name(header)
        return_message = {
            "event": {
                "header": response_header,
                "payload": { "endpoints": endpoint_list }
            }
        }
        return return_message

    def discovery_json(self):
        return json.dumps(self.discovery, indent=4, sort_keys=True)

    def response(self, header, endpoint):
        name = resolve_response_name(header)
        response_properties = []
        for capability in self.capabilities:
            response_properties.append(capability.response_properties())
        pass
        response_data = {
            "context": {
                "properties": response_properties
            },
            "event": response_event(name, header, endpoint),
            "payload": {}
        }
        return response_data

class AlexaPowerController:
    def __init__(self, power_state=None):
        self.set_power_state(power_state)

    def set_power_state(self, power_state):
        if str(power_state).upper != 'ON':
            power_state = 'OFF'
        self.power_state = str(power_state).upper()

    def get_power_state(self):
        return self.power_state

    def turn_on(self):
        self.set_power_state('ON')

    def turn_off(self):
        self.set_power_state('OFF')

    def discovery(self):
        capability = {
            "type": "AlexaInterface",
            "interface": "Alexa.PowerController",
            "version": "3",
            "properties": {
                "supported": [
                    {
                        "name": "powerState"
                    }
                ],
                "proactivelyReported": False,
                "retrievable": True
            }
            }
        return capability

    def take_action(self, header, payload):
        if header['name'] == "TurnOn":
            self.turn_on()
        elif header['name'] == "TurnOff":
            self.turn_off()

    def response(self, header, endpoint):
        name = resolve_response_name(header)
        response_data = {
            "context": {
                "properties": self.response_properties()
            },
            "event": response_event(name, header, endpoint),
            "payload": {}
        }
        return response_data

    def response_properties(self):
        if self.power_state is None:
            raise Exception('set power_state before responding')
        properties = [{
                    "namespace": "Alexa.PowerController",
                    "name": "powerState",
                    "value": self.power_state,
                    "timeOfSample": utc_timestamp(),
                    "uncertaintyInMilliseconds": 500
                }]
        return properties

class AlexaSpeaker:
    def __init__(self, muted=None, volume=None):
        self.set_muted(muted)
        self.set_volume(volume)

    def set_muted(self, muted):
        self.muted = muted

    def get_muted(self):
        return self.muted

    def mute_on(self):
        self.set_muted(True)

    def mute_off(self):
        self.set_muted(False)

    def set_volume(self, volume):
        self.volume = volume

    def get_volume(self):
        return self.volume

    def adjust_volume(self, amount):
        if self.volume is None:
            raise Exception('Set the volume before adjusting')
        if amount < 0:
            self.set_volume(self.get_volume() - amount)
        else:
            self.set_volume(self.get_volume() + amount)

    def discovery(self):
        capability = {
            "type": "AlexaInterface",
            "interface": "Alexa.Speaker",
            "version": "3",
            "properties": {
                "supported": [
                    {
                        "name": "muted"
                    },
                    {
                        "name": "volume"
                    }
                ],
                "proactivelyReported": False,
                "retrievable": True
            }
        }
        return capability

    def take_action(self, header, payload):
        if header['name'] == "SetVolume":
            self.set_volume(payload['volume'])
        elif header['name'] == "AdjustVolume":
            self.adjust_volume(payload['volume'])
        elif header['name'] == "SetMute":
            self.set_muted(payload['mute'])

    def response(self, header, endpoint):
        name = resolve_response_name(header)
        response = {
            "context": {
                "properties": self.response_properties()
            },
            "event": response_event(name, header, endpoint)
        }
        return response

    def response_properties(self):
        if self.muted is None:
            raise Exception('Set muted property before responding')
        if self.volume is None:
            raise Exception('set volume property before responding')
        properties = [{
                    "namespace": "Alexa.Speaker",
                    "name": "muted",
                    "value": self.muted,
                    "timeOfSample": utc_timestamp(),
                    "uncertaintyInMilliseconds": 500
                }, {
                    "namespace": "Alexa.Speaker",
                    "name": "volume",
                    "value": self.volume,
                    "timeOfSample": utc_timestamp(),
                    "uncertaintyInMilliseconds": 500
                }]
        return properties


from setuptools import setup, find_packages

with open("requirements.txt", "r") as f:
    requirements = f.read().splitlines()

setup(
    name='ideogram_wrapper',
    version='0.0.2',
    description='Generates images from Ideogram API using textual prompts.',
    author='Flowese',
    packages=find_packages(),
    install_requires=requirements,
)


from app import db

class Job(db.Model):
    __tablename__ = "jobs"

    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(150), unique=True, nullable=False)

    def __init__(self, name):
        self.name = name

    @staticmethod
    def AllJobs():
        records = {}
        try:
            records = Job.query.with_entities(Job.id, Job.name).all()
        except:
            print("Failed to query to the table")

        return records

    @staticmethod
    def get_job(job_id):
        record = {}
        try:
            record = Job.query.filter_by(id=job_id).first()
        except:
            print("Failed to query to the table")

        return record


import numpy as np

# Crear un array 2D manualmente con números de tipo entero de 8 bits
mi_array_2d = np.array([
    [5, 10, 15, 20],
    [25, 30, 35, 40],
    [45, 50, 55, 60],
    [65, 70, 75, 80]
], dtype=np.int8)

# Imprimir el array en pantalla
print(mi_array_2d)

# Imprimir el tipo de array
print("Tipo de array:", mi_array_2d.dtype)


__all__ = ['osh_core']

import time
from datetime import datetime
from os.path import join, exists
from typing import Tuple

from selenium.common.exceptions import *
from selenium.webdriver import Chrome, ChromeOptions, ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

from src.BusinessCentralLayer.middleware.flow_io import AliyunOSS
from src.BusinessCentralLayer.setting import *
from src.BusinessLogicLayer.plugins.faker_any import get_useragent

class _OnlineServiceHallSubmit(object):

    def __init__(self, silence=True, anti=True, debug=False, only_get_screenshot=True) -> None:

        self.url = 'https://ehall.hainanu.edu.cn/qljfwapp/sys/lwHainanuStuTempReport/*default/index.do#/stuTempReport'

        self.debug = debug
        self.only_get_screenshot = only_get_screenshot
        self.silence = silence
        self.anti = anti

        self.osh_model_today = join(SERVER_DIR_SCREENSHOT, str(datetime.now(TIME_ZONE_CN)).split(' ')[0])

        self._check_model()

    # ----------------------------------
    # Public API
    # ----------------------------------
    def rush_cookie_pool(self, user, kernel: str = 'admin') -> Tuple[int, str]:
        api_: Chrome = self._set_startup_option()

        try:
            # 模拟登入
            login_status = self._login(api_, user)
            if login_status == 302:
                logger.warning(f'FAILED -- {user["username"]}-- {OSH_STATUS_CODE[login_status]} -- 网络状况较差或OSH接口繁忙')
                return 302, user['username']
            # 获取SUPER_USER_COOKIE
            time.sleep(1)
            WebDriverWait(api_, 10).until(EC.element_located_to_be_selected)
            cookie: dict or None = api_.get_cookie('MOD_AMP_AUTH')
            # 若cookie有效则进行cookie缓存操作
            if cookie and isinstance(cookie, dict):
                user_cookie: str = f'{cookie.get("name")}={cookie.get("value")};'
                logger.info(f'Get Superuser Cookie -- {user_cookie}')
                # 根据不同的接入模式，使用不同的解决方案存储cookie
                if kernel == 'admin':
                    with open(SERVER_PATH_COOKIES, 'w', encoding='utf-8') as f:
                        f.write(user_cookie)
                elif kernel == 'general':
                    cache_path = os.path.join(SERVER_DIR_CACHE_FOR_TIMER, f'{user["username"]}.txt')
                    with open(cache_path, 'w', encoding='utf-8') as f:
                        f.write(user_cookie)
                return 400, user_cookie
            else:
                return 403, user['username']
        finally:
            self._kill(api_)

    def run(self, user) -> Tuple[int, str] or Tuple[int, BaseException, str]:
        """
        服务器部署后，必须使用Silence Login 方案，而此时对于任务提交的调试有暂时无法攻克的技术难题
        故不要在部署后使用此模块进行表单提交，而应使用osh-runner
        :param user:
        :return:
        """

        # # 若今日已签到则无需登入
        # model_status = self._check_model(user, check_way='oss')
        # if model_status == 900:
        #     logger.info(f'<OshCore> IGNORE {user["username"]} || {OSH_STATUS_CODE[model_status]}')
        #     return model_status, user['username']

        # logger.info(f'<OshCore> Run OnlineServiceHallSubmit || {user["username"]}')

        # 共享任务句柄
        api_: Chrome = self._set_startup_option()

        try:
            # 模拟登入
            login_status = self._login(api_, user)
            if login_status == 302:
                logger.warning(
                    f'<OshCore> FAILED || 网络状况较差或OSH接口繁忙 || {user["username"]} {OSH_STATUS_CODE[login_status]}')
                return login_status, user['username']

            # 跳转到最新打卡页面
            stu_status: int = self._goto_sign_up(api_, user)

            # 在执行_goto_sign_up时已自动保存并上传任务页截图
            # 若启动该分支仅需完成此需求，请立即结束后续操作
            if self.only_get_screenshot:
                self.print_debug_msg("<OshCore> OnlyGetScreenshot || quick driver")
                return stu_status, user["username"]

            # 若需使用本分支执行签到操作，则根据“是否已签到”决定是否执行签到任务
            # 可签到 -> submit
            if stu_status == 903:
                self._submit(api_, user)
                return 300, user["username"]
            else:
                return stu_status, user["username"]

        # NoneType: 用户无任何待处理表单，意味着该用户不适用当前签到方案
        except TypeError as e:
            logger.warning(f'{user["username"]} -- {OSH_STATUS_CODE[304]}')
            return 304, e, user["username"]
        # 未知异常
        except WebDriverException as e:
            logger.exception(f'{user["username"]} -- {e}')
            return 301, e, user["username"]
        # 垃圾回收
        finally:
            self._kill(api_)

    # ----------------------------------
    # Private API
    # ----------------------------------

    def _check_model(self, user=None, check_way='init') -> bool or int:
        if check_way == 'init':
            if not exists(self.osh_model_today):
                os.mkdir(self.osh_model_today)
                logger.info(f'今日首次打卡进程启动，已初始化文档树')
                return True

        elif check_way == 'stale':
            if "{}.png".format(user['username']) in os.listdir(self.osh_model_today):
                return 900

        elif check_way == 'oss':
            if AliyunOSS().snp_exist(user['username']):
                return 900
        else:
            return False

    def _set_startup_option(self) -> Chrome:
        """
        ChromeDriver settings
        @return:
        """

        self.print_debug_msg("初始化驱动")

        options = ChromeOptions()

        # 最高权限运行
        options.add_argument('--no-sandbox')

        # 隐身模式
        options.add_argument('-incognito')

        # 无缓存加载
        # options.add_argument('--disk-cache-')

        # 设置中文
        options.add_argument('lang=zh_CN.UTF-8')

        options.add_experimental_option('excludeSwitches', ['enable-automation'])

        # 更换头部
        options.add_argument(f'user-agent={get_useragent()}')

        # 静默启动
        if self.silence is True:
            options.add_argument('--headless')

        if not self.anti:
            chrome_pref = {"profile.default_content_settings": {"Images": 2, 'javascript': 2},
                           "profile.managed_default_content_settings": {"Images": 2},
                           }
            options.experimental_options['prefs'] = chrome_pref

        return Chrome(options=options, executable_path=CHROMEDRIVER_PATH)

    def _login(self, api: Chrome, user, retry_num=0, max_retry=3) -> bool or int:

        # FIXME: 加入用户密码错误鉴别
        # 重试次数达到阈值
        if retry_num >= max_retry:
            return 302

        api.get(self.url)
        try:
            if not self.anti:
                time.sleep(1)
            WebDriverWait(api, 10).until(EC.presence_of_element_located((By.ID, "username"))).send_keys(
                user['username'])
            api.find_element_by_id('password').send_keys(user['password'])
            api.find_element_by_tag_name('button').click()

            self.print_debug_msg("模拟登录成功")
            return True
        except TimeoutException or NoSuchElementException:
            retry_num += 1
            api.refresh()
            self._login(api, user, retry_num)
        except StaleElementReferenceException:
            time.sleep(1.2)
            self._login(api, user, retry_num)

    @staticmethod
    def _check_status(element) -> int:
        """

        :param element: API定位的最新任务元素
        :return:
        """

        # 解压数据
        stu_missions = {}
        i = 1
        stu_info: list = element.text.split('\n')
        while i < element.text.split('\n').__len__() - 1:
            stu_missions.update({stu_info[i]: stu_info[i + 1]})
            i += 2

        # 今日签到任务tag已成功定位，但未抵达任务开放时间，此时默认上报状态为false
        if datetime.fromisoformat(stu_missions.get('上报开始时间')) > datetime.now(TIME_ZONE_CN).now():
            return 901
        else:
            # 今日/昨日 任务已签到
            if stu_missions.get('是否上报') == '是':
                return 902
            # 今日/昨日 任务未签到
            elif stu_missions.get('是否上报') == '否':
                return 903

        # if stu_missions.get('是否上报') == '是':
        #     return 902
        # elif stu_missions.get('是否上报') == '否':
        #     if datetime.fromisoformat(stu_missions.get('上报开始时间')) > datetime.now(TIME_ZONE_CN).now():
        #         return 901
        #     else:
        #         return 903

    def print_debug_msg(self, msg: str):
        if self.debug:
            print(msg)

    def _goto_sign_up(self, api: Chrome, user=None, retry_num=0, max_retry=3) -> int:
        """

        :param api:
        :param user:
        :param retry_num:
        :param max_retry:
        :return:
        """
        # 超时中断
        if retry_num >= max_retry:
            logger.debug(f'FAILED -- {user["username"]}-- 该测试用例异常，可能原因为：账号密码错误或网络异常')
            return False

        try:
            # 获取最新任务tag
            time.sleep(1)
            latest_mission = WebDriverWait(api, 50).until(EC.presence_of_element_located((
                By.XPATH,
                "//div[@class='mint-layout-container cjarv696w']"
            )))

            # 查询任务状态
            task_status = self._check_status(latest_mission)

            # 可签到
            if task_status == 903:
                if self.only_get_screenshot:
                    logger.error(f"<OshCore> {user['username']} || 截图捕获失败，今日签到任务未执行。")
                latest_mission.click()

            # 已签到
            elif task_status == 902:
                # logger.info(f"{user['username']} -- {OSH_STATUS_CODE[task_status]}")
                self._save_log(api, user)
                # logger.success(f"{user['username']} -- 截图上传成功")

            # 未开放
            elif task_status == 901:
                logger.warning(f'{user["username"]} -- {OSH_STATUS_CODE[task_status]}')
            return task_status

        # 元素更新/任务超时/网络通信异常
        except NoSuchElementException or TimeoutException:
            retry_num += 1
            logger.debug(f'{user["username"]} -- 任务超时，重试次数{retry_num}')
            self._goto_sign_up(api, retry_num=retry_num)

    def _submit(self, api: Chrome, user=None):

        def _scroll_the_window():
            actions = ActionChains(api)
            actions.key_down(Keys.END).perform()
            actions.reset_actions()

        def _click_temperature_selection_bar(retry=0, max_retry=10):
            try:
                time.sleep(1)
                _scroll_the_window()
                api.find_element_by_xpath("//div[@class='__em-selectlist']").click()
            except ElementClickInterceptedException:
                if retry <= max_retry:
                    retry += 1
                    _click_temperature_selection_bar(retry=retry)

        def _click_confirm_button_of_the_temperature():
            time.sleep(1)
            api.find_element_by_xpath("//div[@visible-item-count]//div[contains(@class,'confirm')]").click()

        def _click_submit_button_of_the_temperature():
            time.sleep(0.5)
            api.find_element_by_xpath(
                "//button[@class='mint-button flowEditButton mt-btn-primary mint-button--normal']").click()

        def _click_submit_button_of_the_form():
            try:
                self.print_debug_msg("尝试捕获弹窗")
                time.sleep(0.5)
                err_ = api.find_element_by_xpath("//div[contains(@class,'mint-msgbox-message')]")
                # api.find_element_by_class_name("mint-msgbox-content").text
                if err_ is not None:
                    logger.error(f"<OshCore> {user['username']} || {err_.text}")
            except NoSuchElementException:
                print("123123123")
                pass

            time.sleep(0.5)
            api.find_element_by_xpath("//button[@class='mint-msgbox-btn mint-msgbox-confirm mt-btn-primary']").click()

        def _generate_behavior_response():
            response = self._save_log(api, user, status_='goto')
            if response:
                logger.success(f"<OshCore> {user['username']} || 签到成功")
            else:
                logger.warning(f"<OshCore> {user['username']} || 签到异常")

        # 进入[体温签到页面]
        api.find_element_by_id("app").click()
        self.print_debug_msg("进入[体温签到页面]")

        # 点击[体温选择栏]
        _click_temperature_selection_bar()
        self.print_debug_msg("点击[体温选择栏]")

        # [确认]体温
        _click_confirm_button_of_the_temperature()
        self.print_debug_msg("[确认]体温")

        # 点击[提交数据]按钮
        _click_submit_button_of_the_temperature()
        self.print_debug_msg("点击[提交数据]按钮")

        # 点击[确认提交数据]按钮
        _click_submit_button_of_the_form()
        self.print_debug_msg("点击[确认提交数据]按钮")

        # 判断签到状态 并上传截图
        _generate_behavior_response()
        self.print_debug_msg("判断签到状态 并上传截图")

    def _save_log(self, api: Chrome, user, status_='wait') -> bool:
        """

        :param api:
        :param user:
        :param status_: wait:已签到 ，goto：刚签到，返回上一级后截图
        :return:
        """

        if status_ == 'wait':
            api.save_screenshot(join(self.osh_model_today, f"{user['username']}.png"))
            AliyunOSS().upload_base64(api.get_screenshot_as_base64(), user['username'])
            return True
        elif status_ == 'goto':
            api.back()
            time.sleep(0.5)
            if self._check_status(api.find_element_by_xpath("//div[@class='mint-layout-container cjarv696w']")) == 902:
                self._save_log(api, user)
                return True
            else:
                return False
        else:
            return False

    @staticmethod
    def _kill(api: Chrome) -> None:
        api.delete_all_cookies()
        api.quit()

osh_core = _OnlineServiceHallSubmit


import matplotlib.pyplot as plt

xlst, ylst = [],[]

def even(x):
    if (x % 2) == 1:
        return False
    else:
        return True

def f(n):
    if even(n):
        return n // 2
    else:
        return 3*n+1

def g(n,i):
    xlst.append(i)
    ylst.append(n)
    print(str(n)+" ",end="")
    if n == 1:
        return 1
    else:
        return g(f(n),i+1)

g(27,0)
xmax = max(tuple(xlst))
ymax = max(tuple(ylst))
print("\nNumber of recursive calls: {0}\nMaximum number: {1}".format(xmax,ymax))
plt.plot(xlst,ylst,"r--")
plt.axis([0,xmax,0,ymax])
plt.savefig("./Assignment11/g27(27,0).png")
plt.show()


#!/usr/bin/env python3

def psf_study(quadrant_path, ztfname, filtercode, logger, args):
    from utils import ListTable, match_pixel_space, quadrant_size_px
    import numpy as np
    from saunerie.plottools import binplot
    from scipy.stats import norm
    import matplotlib
    import matplotlib.pyplot as plt
    import pickle
    from matplotlib.ticker import MultipleLocator, FormatStrFormatter, AutoMinorLocator
    matplotlib.use('Agg')

    if not quadrant_path.joinpath("psfstars.list").exists():
        return True

    psf_resid = ListTable.from_filename(quadrant_path.joinpath("psf_resid_tuple.dat"))
    psf_tuple = ListTable.from_filename(quadrant_path.joinpath("psftuple.list"))
    psf_stars = ListTable.from_filename(quadrant_path.joinpath("psfstars.list"))
    stand = ListTable.from_filename(quadrant_path.joinpath("standalone_stars.list"))

    stand.df['mag'] = -2.5*np.log10(stand.df['flux'])
    psf_stars.df['mag'] = -2.5*np.log10(psf_stars.df['flux'])
    psf_residuals_px = (psf_resid.df['fimg'] - psf_resid.df['fpsf']).to_numpy()
    mag = np.linspace(np.min(stand.df['mag']), np.max(stand.df['mag']), 100)

    subx = 4
    suby = 1

    # Fit order 2 polynomials on skewness/mag relation
    def _fit_skewness_subquadrants(axis, sub):
        ranges = np.linspace(0., quadrant_size_px[axis], sub+1)
        skew_polynomials = []
        for i in range(sub):
            range_mask = (stand.df['y'] >= ranges[i]) & (stand.df['y'] < ranges[i+1])
            skew_polynomials.append(np.polynomial.Polynomial.fit(stand.df.loc[range_mask]['mag'], stand.df.loc[range_mask]['gm{}3'.format(axis)], 1))

        return skew_polynomials

    skew_polynomial_x = np.polynomial.Polynomial.fit(stand.df['mag'], stand.df['gmx3'], 1)
    skew_polynomial_y = np.polynomial.Polynomial.fit(stand.df['mag'], stand.df['gmy3'], 1)

    skew_polynomial_subx = _fit_skewness_subquadrants('x', subx)
    skew_polynomial_suby = _fit_skewness_subquadrants('y', suby)

    with open(quadrant_path.joinpath("stamp_skewness.pickle"), 'wb') as f:
        pickle.dump({'poly_x': skew_polynomial_x, 'poly_y': skew_polynomial_y,
                     'poly_x_sub': skew_polynomial_subx, 'poly_y_sub': skew_polynomial_suby}, f)

    plt.figure(figsize=(7., 7.))
    ax = plt.axes()
    ax.tick_params(which='both', direction='in')
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())
    plt.grid(linestyle='--', color='xkcd:sky blue')
    plt.suptitle("Skewness plane for standalone stars stamps in \n {}".format(quadrant_path.name))
    plt.scatter(stand.df['gmx3'], stand.df['gmy3'], s=2., color='black')
    plt.xlabel("$M^g_{xxx}$")
    plt.ylabel("$M^g_{yyy}$")
    plt.axvline(0.)
    plt.axhline(0.)
    plt.axvline(skew_polynomial_x.coef[0], ls='--')
    plt.axhline(skew_polynomial_y.coef[0], ls='--')
    plt.axis('equal')
    plt.savefig(quadrant_path.joinpath("{}_mx3_my3_plane.png".format(quadrant_path.name)), dpi=300.)
    plt.close()

    plt.subplots(ncols=1, nrows=2, sharex=True, figsize=(10., 5.))
    plt.suptitle("Skewness vs magnitude for standalone stars stamps in {}".format(quadrant_path.name))

    ax = plt.subplot(2, 1, 1)
    ax.tick_params(which='both', direction='in')
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())
    plt.grid(linestyle='--', color='xkcd:sky blue')
    plt.scatter(stand.df['mag'], stand.df['gmx3'], s=2.)
    plt.plot(mag, skew_polynomial_x(mag), color='black')
    plt.axhline(0.)
    plt.ylim(-0.5, 0.3)
    plt.xlim(-15., -7)
    plt.ylabel("$M^g_{xxx}$")

    ax = plt.subplot(2, 1, 2)
    ax.tick_params(which='both', direction='in')
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())
    plt.grid(linestyle='--', color='xkcd:sky blue')
    plt.scatter(stand.df['mag'], stand.df['gmy3'], s=2.)
    plt.plot(mag, skew_polynomial_y(mag), color='black')
    plt.axhline(0.)
    plt.xlabel("$m$")
    plt.ylim(-0.3, 0.3)
    plt.xlim(-15., -6)
    plt.ylabel("$M^g_{yyy}$")

    plt.savefig(quadrant_path.joinpath("{}_skewness_magnitude.png".format(quadrant_path.name)), dpi=300.)
    plt.close()
    return

    star_indices = match_pixel_space(psf_stars.df[['x', 'y']].to_records(), psf_resid.df[['xc', 'yc']].rename(mapper={'xc': 'x', 'yc': 'y'}, axis='columns').to_records())
    psf_size = int(np.sqrt(sum(star_indices==0)))
    psf_residuals = np.zeros([len(psf_stars.df), psf_size*psf_size])
    for star_index in range(len(np.bincount(star_indices))):
        star_mask = (star_indices==star_index)
        ij = (psf_resid.df.iloc[star_mask]['j']*psf_size + psf_resid.df.iloc[star_mask]['i'] + int(np.floor(psf_size**2/2))).to_numpy()
        np.put_along_axis(psf_residuals[star_index], ij, psf_residuals_px[star_mask], 0)

    psf_residuals = psf_residuals.reshape(len(psf_stars.df), psf_size, psf_size)

    bins_count = 5
    mag_range = (psf_stars.df['mag'].min(), psf_stars.df['mag'].max())
    bins = np.linspace(*mag_range, bins_count+1)
    psf_residual_means = []
    psf_residual_count = []
    for i in range(bins_count):
        lower_bound = bins[i]
        upper_bound = bins[i+1]

        binned_stars_mask = np.all([psf_stars.df['mag'] < upper_bound, psf_stars.df['mag'] >= lower_bound], axis=0)
        psf_residual_means.append(np.mean(psf_residuals[binned_stars_mask], axis=0))
        psf_residual_count.append(sum(binned_stars_mask))

    plt.subplots(ncols=bins_count, nrows=1, figsize=(5.*bins_count, 5.))
    plt.suptitle("Binned PSF residuals for {}".format(quadrant_path.name))
    for i, psf_residual_mean in enumerate(psf_residual_means):
        plt.subplot(1, bins_count,  1+i)
        plt.imshow(psf_residual_mean)
        plt.axis('off')
        plt.title("${0:.2f} \leq m < {1:.2f}$\n$N={2}$".format(bins[i], bins[i+1], psf_residual_count[i]))

    plt.savefig(quadrant_path.joinpath("{}_psf_residuals.png".format(quadrant_path.name)))
    plt.close()
    # plt.subplots(ncols=2, nrows=1, sharex=True, sharey=True)
    # # plt.subplot(1, 2, 1)
    # # plt.scatter(stand.df['gmx3'], stand.df['gmy3'], c=stand.df['x'], s=2.)
    # # plt.axvline(0.)
    # # plt.axhline(0.)

    # # plt.subplot(1, 2, 2)
    # # plt.scatter(stand.df['gmx3'], stand.df['gmy3'], c=stand.df['y'], s=2.)
    # # plt.axvline(0.)
    # # plt.axhline(0.)
    # # plt.show()

    # plt.subplot(2, 2, 1)
    # plt.scatter(stand.df['mag'], stand.df['gmx3'], c=stand.df['x'], s=1.)
    # plt.axhline(0.)
    # plt.subplot(2, 2, 2)
    # plt.scatter(stand.df['mag'], stand.df['gmy3'], c=stand.df['x'])
    # plt.axhline(0.)
    # plt.subplot(2, 2, 3)
    # plt.scatter(stand.df['mag'], stand.df['gmx3'], c=stand.df['y'])
    # plt.axhline(0.)
    # plt.subplot(2, 2, 4)
    # plt.scatter(stand.df['mag'], stand.df['gmy3'], c=stand.df['y'])
    # plt.axhline(0.)
    # plt.show()

    # psf_resid.df.to_csv("psf_resid_tuple.csv", sep=",")
    # psf_tuple.df.to_csv("psftuple.csv", sep=",")
    # psf_stars.df.to_csv("psfstars.csv", sep=",")

    # psf_resid.df['flux'] = psf_stars.df['flux'].iloc[idx].reset_index(drop=True)
    # psf_resid.df['eflux'] = psf_stars.df['eflux'].iloc[idx].reset_index(drop=True)

    # limits = [np.min(res), np.max(res)]
    # f = np.linspace(*limits, 200)
    # s = np.var(res)
    # m = np.mean(res)
    # print(np.sqrt(s))
    # print(m)

    # plt.hist(res, bins=1000, density=True)
    # plt.plot(f, norm.pdf(f, loc=m, scale=np.sqrt(s)))
    # plt.xlim(limits)
    # plt.show()

    # plt.plot(psf_resid.df['flux'], psf_resid.df['eflux']/psf_resid.df['flux'], '.')
    # plt.show()

    # plt.subplots(nrows=2, ncols=1, figsize=(10., 5.), sharex=True)
    # plt.subplot(2, 1, 1)
    # binned_mag, plot_res, res_dispersion = binplot(psf_resid.df['flux'].to_numpy(), psf_resid.df['fimg'] - psf_resid.df['fpsf'], nbins=10, data=True, rms=True, scale=False)

    # plt.ylabel("res")
    # plt.grid()

    # plt.subplot(2, 1, 2)
    # plt.plot(binned_mag, res_dispersion, color='black')
    # plt.xlabel("Flux [ADU]")
    # plt.ylabel("$\sigma_\\mathrm{res}$")
    # plt.grid()

    # plt.show()

    return True

def psf_study_reduce(band_path, ztfname, filtercode, logger, args):
    import numpy as np
    import pandas as pd
    from deppol_utils import quadrants_from_band_path
    from utils import get_header_from_quadrant_path, ListTable
    import matplotlib
    import matplotlib.pyplot as plt
    from matplotlib.ticker import MultipleLocator, FormatStrFormatter, AutoMinorLocator
    from utils import idx2markerstyle, plot_ztf_focal_plan_values
    import itertools
    import pickle
    from astropy import time
    from matplotlib.cm import ScalarMappable
    from matplotlib.colorbar import Colorbar
    # matplotlib.use('Agg')

    output_path = band_path.joinpath("psf_study")
    output_path.mkdir(exist_ok=True)

    # Compare seeing computed by sextractor vs the one computed by the ZTF pipeline
    quadrant_paths = quadrants_from_band_path(band_path, logger, check_files="psfstars.list", ignore_noprocess=False)

    #chi2_df = pd.read_csv(band_path.joinpath("astrometry/ref2px_plots/chi2_quadrants.csv"), index_col=0)

    # def _extract_seeing(quadrant_path):
    #     hdr = get_header_from_quadrant_path(quadrant_path)
    #     if quadrant_path.name in chi2_df.index.tolist():
    #         chi2 = chi2_df.at[quadrant_path.name, 'chi2']
    #     else:
    #         chi2 = float('nan')

    #     psfstars_list = ListTable.from_filename(quadrant_path.joinpath("psfstars.list"))
    #     return 2.355*hdr['gfseeing'], hdr['seeing'], len(psfstars_list.df), chi2

    # seeings = np.array(list(map(_extract_seeing, quadrant_paths))).T

    def _extract_skewness(quadrant_path):
        hdr = get_header_from_quadrant_path(quadrant_path)
        with open(quadrant_path.joinpath("stamp_skewness.pickle"), 'rb') as f:
            poly = pickle.load(f)
            poly_x = poly['poly_x']
            poly_y = poly['poly_y']
            poly_x_sub = poly['poly_x_sub']
            poly_y_sub = poly['poly_y_sub']

        #out_dict = {'mjd': hdr['obsmjd'], 'quadrant_id': "{}_{}".format(hdr['ccdid'], hdr['qid'])}
        out_dict = {'mjd': hdr['obsmjd'], 'ccdid': int(hdr['ccdid']), 'qid': int(hdr['qid']), 'rcid': int(hdr['rcid']),
                    'sky': hdr['sexsky'], 'skysigma': hdr['sexsigma'], 'gfseeing': hdr['gfseeing'], 'seeing': hdr['seeing']}

        out_dict.update(dict(('x{}'.format(i), coef) for i, coef in enumerate(poly_x.coef)))
        out_dict.update(dict(('y{}'.format(i), coef) for i, coef in enumerate(poly_y.coef)))
        out_dict.update({'x1_sub': [poly.coef[1] for poly in poly_x_sub],
                         'y1_sub': [poly.coef[1] for poly in poly_y_sub]})

        return out_dict

    skewness_df = pd.DataFrame(list(map(_extract_skewness, quadrant_paths)))
    cmap = 'coolwarm'

    unique_mjds = list(set(skewness_df['mjd']))
    xskewness = dict([(mjd, dict([(i+1, dict([(j, None) for j in range(4)])) for i in range(16)])) for mjd in unique_mjds])
    yskewness = dict([(mjd, dict([(i+1, dict([(j, None) for j in range(4)])) for i in range(16)])) for mjd in unique_mjds])

    for mjd in unique_mjds:
        skewness_today_df = skewness_df.loc[skewness_df['mjd']==mjd]
        for row in skewness_today_df.iterrows():
            # Dirty
            row = row[1]
            ccdid = int(row['ccdid'])
            qid = int(row['qid'])

            xskewness[mjd][ccdid][qid-1] = np.tile(np.array(row['x1_sub']), (len(row['x1_sub']), 1))
            yskewness[mjd][ccdid][qid-1] = np.tile(np.array(row['y1_sub']), (len(row['y1_sub']), 1))

    def _plot_focal_plane_skewness(x, vmin, vmax):
        cm = ScalarMappable(cmap=cmap)
        cm.set_clim(vmin=vmin, vmax=vmax)

        if x == 'x1':
            skewness = xskewness
        elif x == 'y1':
            skewness = yskewness

        for mjd in unique_mjds:
            t = time.Time(mjd, format='mjd')
            fig = plt.figure(figsize=(5., 6.), constrained_layout=True)
            f1, f2 = fig.subfigures(ncols=1, nrows=2, height_ratios=[8., 1.])
            plot_ztf_focal_plan_values(f1, skewness[mjd], vmin=vmin, vmax=vmax, cmap=cmap)
            ax = f2.add_subplot()
            Colorbar(ax, cm, orientation='horizontal', label="${}_{}$".format(x[0], x[1]))
            fig.suptitle("Focal plane skewness in ${}$ direction the {}".format(x[0], t.to_value('iso', subfmt='date')), fontsize='large')
            plt.savefig(output_path.joinpath("{}_focal_plane_{}skewness.png".format(t.to_value('iso', subfmt='date'), x[0])), dpi=300.)
            plt.close()

    sharedscale = False
    center = True
    if sharedscale:
        vmin, vmax = np.min([skewness_df['x1'], skewness_df['y1']]), np.max([skewness_df['x1'], skewness_df['y1']])
        xvmin = vmin
        xvmax = vmax
        yvmin = vmin
        yvmax = vmax
    else:
        xvmin, xvmax = np.min(skewness_df['x1']), np.max(skewness_df['x1'])
        yvmin, yvmax = np.min(skewness_df['y1']), np.max(skewness_df['y1'])

    if center:
        if sharedscale:
            m = max(np.abs(vmin), np.abs(vmax))
            xvmin = -m
            xvmax = m
            yvmin = -m
            yvmax = m
        else:
            m = max(np.abs(xvmin), np.abs(xvmax))
            xvmin = -m
            xvmax = m
            m = max(np.abs(yvmin), np.abs(yvmax))
            yvmin = -m
            yvmax = m

    _plot_focal_plane_skewness('x1', xvmin, xvmax)
    _plot_focal_plane_skewness('y1', yvmin, yvmax)

    rcids = set(skewness_df['rcid'])
    plt.subplots(nrows=2, ncols=1, figsize=(10., 5.))
    plt.suptitle("Order 1 polynomial coefficients fit on skewness/mag")
    ax = plt.subplot(2, 1, 1)
    if len(rcids) <= len(idx2markerstyle):
        for i, rcid in enumerate(rcids):
            rcid_mask = (skewness_df['rcid'] == rcid)
            plt.plot(skewness_df.loc[rcid_mask]['mjd'], skewness_df[rcid_mask]['x1'], idx2markerstyle[i], color='black', label=rcid)
        plt.legend(title="Quadrant ID")
    else:
        plt.scatter(skewness_df['mjd'], skewness_df['x1'], c=skewness_df['rcid'].tolist(), s=1.)
        plt.colorbar()
    plt.axhline(0., color='black')
    plt.ylim(vmin, vmax)
    ax.tick_params(which='both', direction='in')
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())
    plt.grid(linestyle='--', color='xkcd:sky blue')
    plt.xlabel("MJD")
    plt.ylabel("$S^x_1$")

    ax = plt.subplot(2, 1, 2)
    if len(rcids) <= len(idx2markerstyle):
        for i, rcid in enumerate(rcids):
            rcid_mask = (skewness_df['rcid'] == rcid)
            plt.plot(skewness_df.loc[rcid_mask]['mjd'], skewness_df[rcid_mask]['y1'], idx2markerstyle[i], color='black', label=rcid)
        plt.legend(title="Quadrant ID")
    else:
        plt.scatter(skewness_df['mjd'], skewness_df['y1'], c=skewness_df['rcid'].tolist(), s=1.)
        plt.colorbar()
    plt.axhline(0., color='black')
    plt.ylim(vmin, vmax)
    ax.tick_params(which='both', direction='in')
    ax.xaxis.set_minor_locator(AutoMinorLocator())
    ax.yaxis.set_minor_locator(AutoMinorLocator())
    plt.grid(linestyle='--', color='xkcd:sky blue')
    plt.xlabel("MJD")
    plt.ylabel("$S^y_1$")

    plt.tight_layout()
    plt.savefig(output_path.joinpath("skewness_mjd.png"), dpi=300.)
    plt.close()

    # ax = plt.subplot(2, 1, 2)
    # if len(qids) <= len(idx2markerstyle):
    #     for i, qid in enumerate(qids):
    #         qid_mask = (skewness_df['quadrant_id'] == qid)
    #         plt.plot(skewness_df.loc[qid_mask]['mjd'], skewness_df[qid_mask]['y1'], idx2markerstyle[i], color='black', label=qid)
    #     plt.legend(title="Quadrant ID")
    # else:
    #     plt.scatter(skewness_df['mjd'], skewness_df['y1'], c=skewness_df['quadrant_id'].tolist(), s=1.)
    #     plt.colorbar()
    # plt.axhline(0., color='black')
    # plt.ylim(*ylim)
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.xlabel("MJD")
    # plt.ylabel("$S^y_1$")

    return

    # ax = plt.axes()
    # plt.plot(seeings[0], seeings[1], '.')
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.xlabel("GF seeing [pixel]")
    # plt.ylabel("ZTF seeing [pixel]")
    # plt.show()
    # plt.close()

    # ax = plt.axes()
    # plt.plot(seeings[2], seeings[3], '.')
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.xlabel("PSF stars count")
    # plt.ylabel("Chi2")
    # plt.show()
    # plt.close()

    # ax = plt.axes()
    # plt.plot(seeings[2], seeings[0]-seeings[1]-np.mean(seeings[0]-seeings[1]), '.')
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.xlabel("PSF stars count")
    # plt.ylabel("GF - ZTF - <GF-ZTF> (seeing) [pixel]")
    # plt.show()
    # plt.close()

    # ax = plt.axes()
    # plt.plot(seeings[3], seeings[0]-seeings[1]-np.mean(seeings[0]-seeings[1]), '.')
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.xlabel("Chi2")
    # plt.ylabel("GF - ZTF - <GF-ZTF> (seeing) [pixel]")
    # plt.show()
    # plt.close()

    # ax = plt.axes()
    # plt.hist(seeings[0]-seeings[1]-np.mean(seeings[0]-seeings[1]), bins='auto', histtype='step', color='black')
    # ax.tick_params(which='both', direction='in')
    # ax.xaxis.set_minor_locator(AutoMinorLocator())
    # ax.yaxis.set_minor_locator(AutoMinorLocator())
    # plt.grid(linestyle='--', color='xkcd:sky blue')
    # plt.show()

def seeing_study(band_path, ztfname, filtercode, logger, args):
    from utils import get_header_from_quadrant_path, plot_ztf_focal_plan_values, ListTable
    from deppol_utils import quadrants_from_band_path
    import pickle
    import numpy as np
    import itertools
    import matplotlib
    import matplotlib.pyplot as plt
    from matplotlib.cm import ScalarMappable
    from matplotlib.colorbar import Colorbar
    quadrant_paths = quadrants_from_band_path(band_path, logger, check_files="psfstars.list", ignore_noprocess=False)

    keys = {'seeing': 'hdr',
            'seseeing': 'hdr',
            'gfseeing': 'gfseeing',
            'momentx1': 'psfstars',
            'momenty1': 'psfstars',
            'momentx2': 'psfstars',
            'momenty2': 'psfstars',
            'momentxy': 'psfstars'}

    if not band_path.joinpath("seeings.pickle").exists() or args.from_scratch:
        vals = {}

        for key in keys:
            vals[key] = {}
            for i in range(1, 17):
                vals[key][i] = {}
                for j in range(4):
                    vals[key][i][j] = []

        for quadrant_path in quadrant_paths:
            hdr = get_header_from_quadrant_path(quadrant_path)
            psfstars_list = ListTable.from_filename(quadrant_path.joinpath("psfstars.list"))

            for key in keys.keys():
                if keys[key] == 'psfstars':
                    vals[key][hdr['ccdid']][hdr['qid']-1].append(psfstars_list.header[key])
                elif key == 'seseeing':
                    vals[key][hdr['ccdid']][hdr['qid']-1].append(2.355*hdr[key])
                elif keys[key] == 'hdr':
                    vals[key][hdr['ccdid']][hdr['qid']-1].append(hdr[key])
                elif key == 'gfseeing':
                    with open(quadrant_path.joinpath("psf.dat"), 'r') as f:
                        lines = f.readlines()
                        gfseeing = float(lines[3].split(" ")[0].strip())
                    vals[key][hdr['ccdid']][hdr['qid']-1].append(2.355*gfseeing)

        with open(band_path.joinpath("seeings.pickle"), 'wb') as f:
            pickle.dump(vals, f)
    else:
        with open(band_path.joinpath("seeings.pickle"), 'rb') as f:
            vals = pickle.load(f)

    for key in keys:
        for key_i in vals[key].keys():
            for key_j in vals[key][key_i].keys():
                vals[key][key_i][key_j] = np.median(list(map(lambda x: float(x), vals[key][key_i][key_j])))

    cmap = 'coolwarm'
    def _plot_focal_plane_seeing(key):
        val_list = [[vals[key][key_i][key_j] for key_j in vals[key][key_i].keys()] for key_i in vals[key].keys()]
        val_list = list(itertools.chain(*val_list))
        vmin = min(val_list)
        vmax = max(val_list)

        cm = ScalarMappable(cmap=cmap)
        cm.set_clim(vmin=vmin, vmax=vmax)

        fig = plt.figure(figsize=(5., 6.), constrained_layout=False)
        #f1, f2, f3 = fig.subfigures(ncols=1, nrows=3, height_ratios=[12., 1., 1.])
        f1, f2 = fig.subfigures(ncols=1, nrows=2, height_ratios=[12., 0.3], wspace=-1., hspace=-1.)
        f1.suptitle("\n{}-{} Focal plane {}".format(ztfname, filtercode, key), fontsize='large')
        plot_ztf_focal_plan_values(f1, vals[key], vmin=vmin, vmax=vmax, cmap=cmap, scalar=True)
        ax2 = f2.add_subplot(clip_on=False)
        f2.subplots_adjust(bottom=-5, top=5.)
        ax2.set_position((0.2, 0, 0.6, 1))
        # ax2 = f2.add_subplot()
        # ax3 = f3.add_subplot(axis='none')
        # Colorbar(ax, cm, orientation='horizontal')
        cb = f1.colorbar(cm, cax=ax2, orientation='horizontal')
        # ax2.autoscale(tight=True)
        ax2.set_clip_on(False)
        ax2.minorticks_on()
        ax2.tick_params(direction='inout')
        plt.savefig(band_path.joinpath("{}-{}_focal_plane_{}.png".format(ztfname, filtercode, key)), dpi=300., bbox_inches='tight', pad_inches=0.1)
        plt.close()

    for key in keys:
        # print("Plotting {}".format(key))
        _plot_focal_plane_seeing(key)

def match_gaia(quadrant_path, ztfname, filtercode, logger, args):
    from itertools import chain
    from utils import read_list, get_wcs_from_quadrant, get_mjd_from_quadrant_path, contained_in_exposure, match_pixel_space, gaiarefmjd, quadrant_width_px, quadrant_height_px
    import pandas as pd
    import numpy as np
    from astropy.coordinates import SkyCoord
    import astropy.units as u

    if not quadrant_path.joinpath("psfstars.list").exists():
        return False

    _, stars_df = read_list(quadrant_path.joinpath("psfstars.list"))
    _, aper_stars_df = read_list(quadrant_path.joinpath("standalone_stars.list"))
    wcs = get_wcs_from_quadrant(quadrant_path)
    obsmjd = get_mjd_from_quadrant_path(quadrant_path)
    center_radec = wcs.pixel_to_world_values(np.array([[quadrant_width_px/2., quadrant_height_px/2.]]))[0]
    center_skycoord = SkyCoord(ra=center_radec[0], dec=center_radec[1], unit='deg')

    # Match aperture photometry and psf photometry catalogs
    i = match_pixel_space(stars_df[['x', 'y']], aper_stars_df[['x', 'y']], radius=1.)
    stars_df = stars_df.iloc[i[i>=0]].reset_index(drop=True)
    aper_stars_df = aper_stars_df.iloc[i>=0].reset_index(drop=True)

    # Add aperture photometry
    aper_fields = [['apfl'+str(i), 'eapfl'+str(i), 'rad'+str(i)] for i in range(10)]
    aper_fields = list(chain(*aper_fields))
    stars_df[aper_fields] = aper_stars_df[aper_fields]

    gaia_stars_df = pd.read_hdf(args.lc_folder.joinpath("{}.hd5".format(ztfname)), key='gaia_cal')
    gaia_stars_df.rename({'pmde': 'pmdec'}, axis='columns', inplace=True)
    gaia_stars_df['gaiaid'] = gaia_stars_df.index

    # First order proper motion correction for indentification
    # We do it in astrometry code
    # gaia_stars_df['ra'] = gaia_stars_df['ra']+(obsmjd-gaiarefmjd)*gaia_stars_df['pmra']/np.cos(gaia_stars_df['dec']/180.*np.pi)/1000./3600./365.25
    # gaia_stars_df['dec'] = gaia_stars_df['dec']+(obsmjd-gaiarefmjd)*gaia_stars_df['pmde']/1000./3600./365.25
    # mas/year -> deg/day
    gaia_stars_df['pmra'] = gaia_stars_df['pmra']/np.cos(np.deg2rad(gaia_stars_df['dec']))/1000./3600./365.25
    gaia_stars_df['pmdec'] = gaia_stars_df['pmdec']/1000./3600./365.25
    gaia_stars_df.dropna(inplace=True)

    ra_corrected = gaia_stars_df['ra']+(obsmjd-gaiarefmjd)*gaia_stars_df['pmra']
    dec_corrected = gaia_stars_df['dec']+(obsmjd-gaiarefmjd)*gaia_stars_df['pmdec']

    # gaia_stars_radec = SkyCoord(gaia_stars_df['ra'], gaia_stars_df['dec'], unit='deg')
    gaia_stars_skycoords = SkyCoord(ra_corrected, dec_corrected, unit='deg')

    sep = center_skycoord.separation(gaia_stars_skycoords)
    idxc = (sep < 0.5*u.deg)
    gaia_stars_skycoords = gaia_stars_skycoords[idxc]
    gaia_stars_df = gaia_stars_df[idxc]

    #gaia_mask = contained_in_exposure(gaia_stars_radec, wcs, return_mask=True)
    #gaia_stars_df = gaia_stars_df.iloc[gaia_mask]
    #x, y = gaia_stars_radec[gaia_mask].to_pixel(wcs)
    x, y = gaia_stars_skycoords.to_pixel(wcs)
    gaia_stars_df['x'] = x
    gaia_stars_df['y'] = y
    gaia_stars_df = gaia_stars_df.dropna()

    try:
        i = match_pixel_space(gaia_stars_df[['x', 'y']].to_records(), stars_df[['x', 'y']].to_records(), radius=2.)
    except Exception as e:
        logger.error("Could not match any Gaia stars!")
        raise ValueError("Could not match any Gaia stars!")

    matched_gaia_stars_df = gaia_stars_df.iloc[i[i>=0]].reset_index(drop=True)
    matched_stars_df = stars_df.iloc[i>=0].reset_index(drop=True)
    logger.info("Matched {} GAIA stars".format(len(matched_gaia_stars_df)))

    with pd.HDFStore(quadrant_path.joinpath("matched_stars.hd5"), 'w') as hdfstore:
        hdfstore.put('matched_gaia_stars', matched_gaia_stars_df)
        hdfstore.put('matched_stars', matched_stars_df)

    return True

def match_gaia_reduce(band_path, ztfname, filtercode, logger, args):
    import pandas as pd
    import numpy as np
    from utils import get_header_from_quadrant_path
    from deppol_utils import quadrants_from_band_path

    quadrant_paths = quadrants_from_band_path(band_path, logger, check_files="match_gaia.success")

    logger.info("Concatenating matched stars measurements from {} quadrants.".format(len(quadrant_paths)))

    matched_stars_list = []
    quadrants_dict = {}

    for quadrant_path in quadrant_paths:
        logger.info(quadrant_path.name)
        matched_gaia_stars_df = pd.read_hdf(quadrant_path.joinpath("matched_stars.hd5"), key='matched_gaia_stars')
        matched_stars_df = pd.read_hdf(quadrant_path.joinpath("matched_stars.hd5"), key='matched_stars')

        matched_gaia_stars_df.rename(columns={'x': 'gaia_x', 'y': 'gaia_y'}, inplace=True)
        matched_stars_df['mag'] = -2.5*np.log10(matched_stars_df['flux'])
        matched_stars_df['emag'] = matched_stars_df['eflux']/matched_stars_df['flux']

        matched_stars_df = pd.concat([matched_stars_df, matched_gaia_stars_df], axis=1)

        quadrant_dict = {}
        header = get_header_from_quadrant_path(quadrant_path)
        quadrant_dict['quadrant'] = quadrant_path.name
        quadrant_dict['airmass'] = header['airmass']
        quadrant_dict['mjd'] = header['obsmjd']
        quadrant_dict['seeing'] = header['seeing']
        quadrant_dict['ha'] = header['hourangd'] #*15
        quadrant_dict['ha_15'] = 15.*header['hourangd']
        quadrant_dict['lst'] = header['oblst']
        quadrant_dict['azimuth'] = header['azimuth']
        quadrant_dict['dome_azimuth'] = header['dome_az']
        quadrant_dict['elevation'] = header['elvation']
        quadrant_dict['z'] = 90. - header['elvation']
        quadrant_dict['telra'] = header['telrad']
        quadrant_dict['teldec'] = header['teldecd']

        quadrant_dict['field'] = header['dbfield']
        quadrant_dict['ccdid'] = header['ccd_id']
        quadrant_dict['qid'] = header['amp_id']
        quadrant_dict['rcid'] = header['dbrcid']

        quadrant_dict['fid'] = header['dbfid']

        quadrant_dict['temperature'] = header['tempture']
        quadrant_dict['head_temperature'] = header['headtemp']
        quadrant_dict['ccdtemp'] = float(header['ccdtmp{}'.format(str(header['ccd_id']).zfill(2))])

        quadrant_dict['wind_speed'] = header['windspd']
        quadrant_dict['wind_dir'] = header['winddir']
        quadrant_dict['dewpoint'] = header['dewpoint']
        quadrant_dict['humidity'] = header['humidity']
        quadrant_dict['wetness'] = header['wetness']
        quadrant_dict['pressure'] = header['pressure']

        quadrant_dict['crpix1'] = header['crpix1']
        quadrant_dict['crpix2'] = header['crpix2']
        quadrant_dict['crval1'] = header['crval1']
        quadrant_dict['crval2'] = header['crval2']
        quadrant_dict['cd_11'] = header['cd1_1']
        quadrant_dict['cd_12'] = header['cd1_2']
        quadrant_dict['cd_21'] = header['cd2_1']
        quadrant_dict['cd_22'] = header['cd2_2']

        quadrant_dict['skylev'] = header['sexsky']
        quadrant_dict['sigma_skylev'] = header['sexsigma']

        for key in quadrant_dict.keys():
            matched_stars_df[key] = quadrant_dict[key]

        matched_stars_list.append(matched_stars_df)
        quadrants_dict[quadrant_path.name] = quadrant_dict

    matched_stars_df = pd.concat(matched_stars_list, axis=0, ignore_index=True)

    # Build gaia star catalog
    gaia_stars_df = pd.read_hdf(args.lc_folder.joinpath("{}.hd5".format(ztfname)), key='gaia_cal')
    gaia_stars_df.rename({'pmde': 'pmdec'}, axis='columns', inplace=True)
    gaia_stars_df['gaiaid'] = gaia_stars_df.index
    gaia_stars_df = gaia_stars_df[~gaia_stars_df.index.duplicated(keep='first')]

    gaia_stars = []
    for gaiaid in set(matched_stars_df['gaiaid']):
        #gaia_stars.append(pd.Series(gaia_stars_df.drop(['rcid', 'field'], axis='columns').loc[gaiaid], name=gaiaid))
        gaia_stars.append(pd.Series(gaia_stars_df.loc[gaiaid], name=gaiaid))

    gaia_stars_df = pd.DataFrame(data=gaia_stars)
    gaia_stars_df.rename({'pmde': 'pmdec'}, axis='columns', inplace=True)
    gaia_stars_df['pmra'] = gaia_stars_df['pmra']/np.cos(np.deg2rad(gaia_stars_df['dec']))/1000./3600./365.25
    gaia_stars_df['pmdec'] = gaia_stars_df['pmdec']/1000./3600./365.25

    # Remove measures with Nan's
    nan_mask = matched_stars_df.isna().any(axis=1)
    matched_stars_df = matched_stars_df[~nan_mask]
    logger.info("Removed {} measurements with NaN's".format(sum(nan_mask)))

    nan_mask = gaia_stars_df.isna().any(axis=1)
    gaia_stars_df = gaia_stars_df[~nan_mask]
    logger.info("Removed {} Gaia stars with NaN's".format(sum(nan_mask)))

    # Compute color
    matched_stars_df['colormag'] = matched_stars_df['bpmag'] - matched_stars_df['rpmag']
    gaia_stars_df['colormag'] = gaia_stars_df['bpmag'] - gaia_stars_df['rpmag']

    # Save to disk
    matched_stars_df.to_parquet(band_path.joinpath("matched_stars.parquet"))
    logger.info("Total matched Gaia stars: {}".format(len(matched_stars_df)))

    quadrants_df = pd.DataFrame.from_dict(quadrants_dict, orient='index')
    quadrants_df.to_parquet(band_path.joinpath("quadrants.parquet"))

    gaia_stars_df.to_parquet(band_path.joinpath("gaia_stars.parquet"))

    return True

# Extract data from standalone stars and plot several distributions
def stats(quadrant_path, ztfname, filtercode, logger, args):
    import warnings
    import pandas as pd
    from utils import read_list
    from astropy.io import fits

    warnings.filterwarnings('ignore',category=pd.io.pytables.PerformanceWarning)

    import pandas as pd

    def _extract_from_list(list_filename, hdfstore):
        list_path = quadrant_path.joinpath(list_filename).with_suffix(".list")

        if not list_path.exists():
            return False

        with open(list_path, mode='r') as f:
            global_params, df = read_list(f)

        hdfstore.put(list_path.stem, df)
        hdfstore.put("{}_globals".format(list_path.stem), pd.DataFrame([global_params]))

        return True

    with pd.HDFStore(quadrant_path.joinpath("lists.hdf5"), mode='w') as hdfstore:
        # From make_catalog
        _extract_from_list("se", hdfstore)

        # From mkcat2
        cont = _extract_from_list("standalone_stars", hdfstore)

        if not cont:
            return True

        _extract_from_list("aperse", hdfstore)

        # From calibrated.fits
        keywords = ['sexsky', 'sexsigma', 'bscale', 'bzero', 'origsatu', 'saturlev', 'backlev', 'back_sub', 'seseeing', 'gfseeing']

        calibrated = {}
        with fits.open(quadrant_path.joinpath("calibrated.fits")) as hdul:
            for keyword in keywords:
                calibrated[keyword] = hdul[0].header[keyword]

            hdfstore.put('calibrated', pd.DataFrame([calibrated]))

        # From makepsf
        cont = _extract_from_list("psfstars", hdfstore)

        if not cont:
            return True

        _extract_from_list("psftuples", hdfstore)

    return True

def stats_reduce(band_path, ztfname, filtercode, logger, args):
    import pandas as pd
    import matplotlib
    import matplotlib.pyplot as plt
    from deppol_utils import quadrants_from_band_path
    from utils import get_header_from_quadrant_path, ListTable
    matplotlib.use('Agg')

    # Seeing histogram
    folders = [folder for folder in band_path.glob("*") if folder.is_dir()]
    quadrant_paths = quadrants_from_band_path(band_path, logger, ignore_noprocess=True)

    logger.info("Plotting fitted seeing histogram")
    seeings = []
    for quadrant_path in quadrant_paths:
        if quadrant_path.joinpath("calibrated.fits").exists():
           hdr = get_header_from_quadrant_path(quadrant_path)
           seeings.append(hdr['seeing'])

    ax = plt.axes()
    plt.suptitle("Seeing distribution (computed by the ZTF pipeline)")
    #plt.hist(seeings, bins=int(len(seeings)/4), color='xkcd:dark grey', histtype='step')
    plt.hist(seeings, bins='auto', color='xkcd:dark grey', histtype='step')
    if args.max_seeing:
        removed_quadrants = len(list(filter(lambda x: x >= args.max_seeing, seeings)))
        plt.axvline(args.max_seeing)
        plt.text(0.2, 0.9, "{:.02f}% quadrants with seeing > {} ({} quadrants)".format(removed_quadrants/len(seeings)*100, args.max_seeing, len(seeings)), transform=ax.transAxes)
    plt.grid()
    plt.xlabel("Seeing FWHM [pixel]")
    plt.ylabel("Count")
    plt.savefig(band_path.joinpath("{}-{}_seeing_dist.png".format(ztfname, filtercode)), dpi=300)
    plt.close()

    quadrant_paths = quadrants_from_band_path(band_path, logger, check_files=['psfstars.list', 'calibrated.fits'], ignore_noprocess=True)
    logger.info("Plotting psfstars count vs seeing")

    seeings = []
    psfstars_counts = []
    for quadrant_path in quadrant_paths:
        hdr = get_header_from_quadrant_path(quadrant_path)
        seeings.append(hdr['seeing'])

        psfstars_list = ListTable.from_filename(quadrant_path.joinpath("psfstars.list"))
        psfstars_counts.append(len(psfstars_list.df))

    ax = plt.axes()
    plt.plot(seeings, psfstars_counts, '.')

    if args.min_psfstars:
        filtered_psfstars_count = len(list(filter(lambda x: x < args.min_psfstars, psfstars_counts)))
        plt.axhline(args.min_psfstars)
        plt.text(0.2, 0.1, "{:.02f}% quadrants ({}) with PSF stars count < {}".format(filtered_psfstars_count/len(quadrant_paths)*100., filtered_psfstars_count, args.min_psfstars), transform=ax.transAxes)

    if args.max_seeing:
        plt.axvline(args.max_seeing)

    plt.grid()
    plt.xlabel("Seeing FWHM [pixel]")
    plt.ylabel("PSF stars count")
    plt.savefig(band_path.joinpath("{}-{}_psfstars_seeing.png".format(ztfname, filtercode)), dpi=300.)
    plt.close()

    with open(band_path.joinpath("{}-{}_failures.txt".format(ztfname, filtercode)), 'w') as f:
        # Failure rates
        def _failure_rate(listname, func):
            success_count = 0
            for quadrant_path in quadrant_paths:
                if quadrant_path.joinpath("{}.list".format(listname)).exists():
                    success_count += 1

            f.writelines(["For {}:\n".format(func),
                          " Success={}/{}\n".format(success_count, len(folders)),
                          " Rate={}\n\n".format(float(success_count)/len(folders))])

        _failure_rate("se", 'make_catalog')
        _failure_rate("standalone_stars", 'mkcat2')
        _failure_rate("psfstars", 'makepsf')

    logger.info("Plotting computing time histograms")
    # Plot results_*.csv histogram
    result_paths = list(band_path.glob("results_*.csv"))
    for result_path in result_paths:
        func = "_".join(str(result_path.stem).split("_")[1:])
        result_df = pd.read_csv(result_path)
        computation_times = (result_df['time_end'] - result_df['time_start']).to_numpy()
        plt.hist(computation_times, bins=int(len(result_df)/4), histtype='step')
        plt.xlabel("Computation time (s)")
        plt.ylabel("Count")
        plt.title("Computation time for {}".format(func))
        plt.grid()
        plt.savefig(band_path.joinpath("{}-{}_{}_compute_time_dist.png".format(ztfname, filtercode, func)), dpi=300)
        plt.close()

def clean(quadrant_path, ztfname, filtercode, logger, args):
    # We want to delete all files in order to get back to the prepare_deppol stage
    files_to_keep = ["elixir.fits", "dead.fits.gz", ".dbstuff"]
    files_to_delete = list(filter(lambda f: f.name not in files_to_keep, list(quadrant_path.glob("*"))))

    for file_to_delete in files_to_delete:
            file_to_delete.unlink()

    return True

def clean_reduce(band_path, ztfname, filtercode, logger, args):
    from shutil import rmtree

    # We want to delete all files in order to get back to the prepare_deppol stage
    files_to_keep = ["prepare.log"]

    # Delete all files
    files_to_delete = list(filter(lambda f: f.is_file() and (f.name not in files_to_keep), list(band_path.glob("*"))))

    [f.unlink() for f in files_to_delete]

    # Delete output folders
    # We delete everything but the quadrant folders
    folders_to_keep = list(band_path.glob("ztf_*"))
    folders_to_delete = [folder for folder in list(band_path.glob("*")) if (folder not in folders_to_keep) or folder.is_file()]

    [rmtree(band_path.joinpath(folder), ignore_errors=True) for folder in folders_to_delete]

def filter_psfstars_count(band_path, ztfname, filtercode, logger, args):
    from deppol_utils import quadrants_from_band_path, noprocess_quadrants
    from utils import ListTable

    quadrant_paths = quadrants_from_band_path(band_path, logger, ignore_noprocess=True)
    noprocess = noprocess_quadrants(band_path)
    flagged_count = 0
    quadrants_to_flag = []

    for quadrant_path in quadrant_paths:
        if quadrant_path.joinpath("psfstars.list").exists():
            quadrant = quadrant_path.name
            psfstars_list = ListTable.from_filename(quadrant_path.joinpath("psfstars.list"))
            psfstars_count = len(psfstars_list.df)

            if psfstars_count <= args.min_psfstars:
                flagged_count += 1

                if quadrant not in noprocess:
                    quadrants_to_flag.append(quadrant)

    if quadrants_to_flag:
        with open(band_path.joinpath("noprocess"), 'a') as f:
            for quadrant_to_flag in quadrants_to_flag:
                f.write("{}\n".format(quadrant_to_flag))

    flagged_count = len(quadrants_to_flag)
    logger.info("{} quadrants flagged as having PSF stars count <= {}.".format(flagged_count, args.min_psfstars))
    logger.info("{} quadrants added to the noprocess list.".format(len(quadrants_to_flag)))

    return True

def filter_astro_chi2(band_path, ztfname, filtercode, logger, args):
    from deppol_utils import noprocess_quadrants
    import pandas as pd
    import numpy as np

    noprocess = noprocess_quadrants(band_path)

    chi2_path = band_path.joinpath("astrometry/ref2px_chi2_quadrants.csv")
    if not chi2_path.exists():
        logger.info("Could not find file {}!".format(chi2_path))
        return False

    chi2_df = pd.read_csv(chi2_path, index_col=0)

    to_filter = (np.any([chi2_df['chi2'] >= args.astro_max_chi2, chi2_df['chi2'].isna()], axis=0))
    # to_filter = (np.any([chi2_df['chi2'].isna()], axis=0))
    logger.info("{} quadrants flagged as having astrometry chi2 >= {} or NaN values.".format(sum(to_filter), args.astro_max_chi2))
    logger.info("List of filtered quadrants:")
    logger.info(chi2_df.loc[to_filter])

    quadrants_to_flag = 0
    with open(band_path.joinpath("noprocess"), 'a') as f:
        for quadrant in chi2_df.loc[to_filter].index.tolist():
            if quadrant not in noprocess:
                f.write("{}\n".format(quadrant))
                quadrants_to_flag += 1

    logger.info("{} quadrants added to the noprocess list.".format(quadrants_to_flag))

    return True

def filter_seeing(band_path, ztfname, filtercode, logger, args):
    from utils import get_header_from_quadrant_path
    from deppol_utils import quadrants_from_band_path, noprocess_quadrants, run_and_log

    quadrant_paths = quadrants_from_band_path(band_path, logger)
    flagged_count = 0
    quadrants_to_flag = []
    noprocess = noprocess_quadrants(band_path)

    for quadrant_path in quadrant_paths:
        quadrant = quadrant_path.name

        try:
            seeing = get_header_from_quadrant_path(quadrant_path)['SEEING']
        except FileNotFoundError as e:
            logger.error(e)
            logger.error("Folder content:")
            run_and_log(["ls", "-lah", quadrant_path], logger)
            continue
        except KeyError as e:
            logger.error(quadrant)
            logger.error(e)
            quadrants_to_flag.append(quadrant)
            continue

        if seeing > args.max_seeing:
            flagged_count += 1

            if quadrant not in noprocess:
                quadrants_to_flag.append(quadrant)

    if quadrants_to_flag:
        with open(band_path.joinpath("noprocess"), 'a') as f:
            for quadrant_to_flag in quadrants_to_flag:
                f.write("{}\n".format(quadrant_to_flag))

    logger.info("{} quadrants flagged as having seeing > {}.".format(flagged_count, args.max_seeing))
    logger.info("{} quadrants added to the noprocess list.".format(len(quadrants_to_flag)))

    return True

def discard_calibrated(quadrant_path, ztfname, filtercode, logger, args):
    pass

discard_calibrated_rm = ["calibrated.fits", "weight.fz"]


import os
import shutil
from gi.repository import Gdk
from gi.repository import GLib
from gi.repository import Gtk
from gi.repository import Pango

from kuwo import Config
from kuwo import Widgets

_ = Config._

MARGIN_LEFT = 15
MARGIN_TOP = 20
ShortcutMode = Config.ShortcutMode

class NoteTab(Gtk.Box):
    def __init__(self):
        super().__init__(orientation=Gtk.Orientation.VERTICAL)
        self.set_border_width(10)

class ColorButton(Gtk.ColorButton):
    def __init__(self, color):
        super().__init__()

class ColorBox(Gtk.Box):
    def __init__(self, label, conf, color_name, use_margin=False):
        super().__init__()
        self.conf = conf
        self.color_name = color_name
        left_label = Gtk.Label(label)
        self.pack_start(left_label, False, True, 0)

        color_button = Gtk.ColorButton()
        color_button.set_use_alpha(True)
        color_rgba = Gdk.RGBA()
        color_rgba.parse(conf[color_name])
        color_button.set_rgba(color_rgba)
        color_button.connect('color-set', self.on_color_set)
        self.pack_end(color_button, False, True, 0)

        if use_margin:
            self.props.margin_left = 20

    def on_color_set(self, color_button):
        color_rgba = color_button.get_rgba()
        if color_rgba.alpha == 1:
            color_rgba.alpha = 0.999
        self.conf[self.color_name] = color_rgba.to_string()

class FontBox(Gtk.Box):
    def __init__(self, label, conf, font_name, use_margin=True):
        super().__init__()
        self.conf = conf
        self.font_name = font_name
        left_label = Gtk.Label(label)
        self.pack_start(left_label, False, True, 0)

        font_button = Gtk.SpinButton()
        adjustment = Gtk.Adjustment(conf[font_name], 4, 72, 1, 10)
        adjustment.connect('value-changed', self.on_font_set)
        font_button.set_adjustment(adjustment)
        font_button.set_value(conf[font_name])
        self.pack_end(font_button, False, True, 0)

        if use_margin:
            self.props.margin_left = 20

    def on_font_set(self, adjustment):
        self.conf[self.font_name] = adjustment.get_value()

class ChooseFolder(Gtk.Box):
    def __init__(self, parent, conf_name, toggle_label):
        super().__init__(orientation=Gtk.Orientation.VERTICAL)
        self.props.margin_left = MARGIN_LEFT
        self.props.margin_top = 5
        self.parent = parent
        self.app = parent.app
        self.conf_name = conf_name
        self.old_dir = self.app.conf[conf_name]

        hbox = Gtk.Box(spacing=5)
        self.pack_start(hbox, False, True, 0)

        self.dir_entry = Gtk.Entry()
        self.dir_entry.set_text(self.old_dir)
        self.dir_entry.props.editable = False
        self.dir_entry.props.can_focus = False
        self.dir_entry.props.width_chars = 20
        hbox.pack_start(self.dir_entry, True, True, 0)

        choose_button = Gtk.Button('...')
        choose_button.connect('clicked', self.on_choose_button_clicked)
        hbox.pack_start(choose_button, False, False, 0)

    def on_choose_button_clicked(self, button):
        def on_dialog_file_activated(dialog):
            new_dir = dialog.get_filename()
            dialog.destroy()
            self.dir_entry.set_text(new_dir)
            if new_dir != self.app.conf[self.conf_name]:
                self.app.conf[self.conf_name] = new_dir
            return

        dialog = Gtk.FileChooserDialog(
                _('Choose a Folder'), self.parent,
                Gtk.FileChooserAction.SELECT_FOLDER,
                (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
                 Gtk.STOCK_OK, Gtk.ResponseType.OK))

        dialog.connect('file-activated', on_dialog_file_activated)
        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            on_dialog_file_activated(dialog)
            return
        dialog.destroy()

class Preferences(Gtk.Dialog):
    def __init__(self, app):
        self.app = app
        super().__init__(
                _('Preferences'), app.window, 0,
                (Gtk.STOCK_CLOSE, Gtk.ResponseType.CLOSE,))
        self.set_modal(True)
        self.set_transient_for(app.window)
        self.set_default_size(600, 320)
        self.set_border_width(5)
        box = self.get_content_area()
        #box.props.margin_left = MARGIN_LEFT

        notebook = Gtk.Notebook()
        box.pack_start(notebook, True, True, 0)

        # generic tab
        generic_box = NoteTab()
        notebook.append_page(generic_box, Gtk.Label(_('Generic')))

        status_button = Gtk.CheckButton(_('Close to system tray'))
        status_button.set_active(app.conf['use-status-icon'])
        status_button.connect('toggled', self.on_status_button_toggled)
        generic_box.pack_start(status_button, False, False, 0)

        notify_button = Gtk.CheckButton(_('Show kwplayer on lock screen'))
        notify_button.set_tooltip_text(
            _('Only works with gdm3/gnome3.8+\n') +
            _('Please disable it on other desktop environments (like KDE)'))
        notify_button.set_active(app.conf['use-notify'])
        notify_button.connect('toggled', self.on_notify_button_toggled)
        generic_box.pack_start(notify_button, False, False, 0)

        # format tab
        format_box = NoteTab()
        notebook.append_page(format_box, Gtk.Label(_('Format')))

        audio_label = Widgets.BoldLabel(_('Prefered Audio Format'))
        format_box.pack_start(audio_label, False, False, 0)
        radio_mp3 = Gtk.RadioButton(_('MP3 (faster)'))
        radio_mp3.props.margin_left = MARGIN_LEFT
        radio_mp3.connect('toggled', self.on_audio_toggled)
        format_box.pack_start(radio_mp3, False, False, 0)
        radio_ape = Gtk.RadioButton(_('APE (better)'))
        radio_ape.join_group(radio_mp3)
        radio_ape.props.margin_left = MARGIN_LEFT
        radio_ape.set_active(app.conf['use-ape'])
        radio_ape.connect('toggled', self.on_audio_toggled)
        format_box.pack_start(radio_ape, False, False, 0)

        video_label = Widgets.BoldLabel(_('Prefered Video Format'))
        video_label.props.margin_top = MARGIN_TOP
        format_box.pack_start(video_label, False, False, 0)
        radio_mp4 = Gtk.RadioButton(_('MP4 (faster)'))
        radio_mp4.props.margin_left = MARGIN_LEFT
        radio_mp4.connect('toggled', self.on_video_toggled)
        format_box.pack_start(radio_mp4, False, False, 0)
        radio_mkv = Gtk.RadioButton(_('MKV (better)'))
        radio_mkv.props.margin_left = MARGIN_LEFT
        radio_mkv.join_group(radio_mp4)
        radio_mkv.set_active(app.conf['use-mkv'])
        radio_mkv.connect('toggled', self.on_video_toggled)
        format_box.pack_start(radio_mkv, False, False, 0)

        # lyrics tab
        lrc_box = NoteTab()
        notebook.append_page(lrc_box, Gtk.Label(_('Lyrics')))

        lrc_normal_text_label = Widgets.BoldLabel(_('Normal Text'))
        lrc_box.pack_start(lrc_normal_text_label, False, True, 0)

        lrc_normal_text_size = FontBox(
                _('text size'), app.conf, 'lrc-text-size', use_margin=True)
        lrc_box.pack_start(lrc_normal_text_size, False, True, 0)

        lrc_normal_text_color = ColorBox(
                _('text color'), app.conf, 'lrc-text-color', use_margin=True)
        lrc_box.pack_start(lrc_normal_text_color, False, True, 0)
        lrc_normal_text_color.props.margin_bottom = 10

        lrc_highlighted_text_label = Widgets.BoldLabel(
                _('Highlighted Text'))
        lrc_box.pack_start(lrc_highlighted_text_label, False, True, 0)

        lrc_highlighted_text_size = FontBox(
                _('text size'), app.conf, 'lrc-highlighted-text-size',
                use_margin=True)
        lrc_box.pack_start(lrc_highlighted_text_size, False, True, 0)

        lrc_highlighted_text_color = ColorBox(
                _('text color'), app.conf, 'lrc-highlighted-text-color',
                use_margin=True)
        lrc_highlighted_text_color.props.margin_bottom = 10
        lrc_box.pack_start(lrc_highlighted_text_color, False, True, 0)

        lrc_word_back_color = ColorBox(
                _('Lyrics Text Background color'), app.conf, 'lrc-back-color')
        lrc_box.pack_start(lrc_word_back_color, False, True, 0)

        # folders tab
        folder_box = NoteTab()
        notebook.append_page(folder_box, Gtk.Label(_('Folders')))

        song_folder_label = Widgets.BoldLabel(_('Place to store sogns'))
        folder_box.pack_start(song_folder_label, False, False, 0)
        song_folder = ChooseFolder(
                self, 'song-dir', _('Moving cached songs to new folder'))
        folder_box.pack_start(song_folder, False, False, 0)

        mv_folder_label = Widgets.BoldLabel(_('Place to store MVs'))
        mv_folder_label.props.margin_top = MARGIN_TOP
        folder_box.pack_start(mv_folder_label, False, False, 0)
        mv_folder = ChooseFolder(
                self, 'mv-dir', _('Moving cached MVs to new folder'))
        folder_box.pack_start(mv_folder, False, False, 0)

        self.notebook = notebook

        # shortcut tab
        self.init_shortcut_tab()

    def init_shortcut_tab(self):
        curr_mode = self.app.conf['shortcut-mode']

        box = NoteTab()
        self.notebook.append_page(box, Gtk.Label(_('Shortcut')))

        self.shortcut_win = Gtk.ScrolledWindow()

        disable_btn = Gtk.RadioButton(_('Disable Keyboard Shortcut'))
        disable_btn.connect(
                'toggled', self.on_shortcut_btn_toggled, ShortcutMode.NONE)
        disable_btn.set_active(curr_mode == ShortcutMode.NONE)
        box.pack_start(disable_btn, False, False, 0)

        default_btn = Gtk.RadioButton(_('Use Default MultiMedia Key'))
        default_btn.connect(
                'toggled', self.on_shortcut_btn_toggled, ShortcutMode.DEFAULT)
        default_btn.join_group(disable_btn)
        default_btn.set_active(curr_mode == ShortcutMode.DEFAULT)
        box.pack_start(default_btn, False, False, 0)

        custom_btn = Gtk.RadioButton(_('Use Custom Keyboard Shortcut'))
        custom_btn.connect(
                'toggled', self.on_shortcut_btn_toggled, ShortcutMode.CUSTOM)
        custom_btn.join_group(default_btn)
        custom_btn.set_active(curr_mode == ShortcutMode.CUSTOM)
        box.pack_start(custom_btn, False, False, 0)

        self.shortcut_win.props.margin_left = 10
        self.shortcut_win.set_sensitive(curr_mode == ShortcutMode.CUSTOM)
        box.pack_start(self.shortcut_win, True, True, 0)

        # name, shortct key, shortcut modifiers
        self.shortcut_liststore = Gtk.ListStore(str, int, int)
        tv = Gtk.TreeView(model=self.shortcut_liststore)
        self.shortcut_win.add(tv)

        name_cell = Gtk.CellRendererText()
        name_col = Gtk.TreeViewColumn('Action', name_cell, text=0)
        tv.append_column(name_col)

        key_cell = Gtk.CellRendererAccel(editable=True)
        key_cell.connect('accel-edited', self.on_shortcut_key_cell_edited)
        key_col = Gtk.TreeViewColumn(
                'Shortcut Key', key_cell, accel_key=1, accel_mods=2)
        tv.append_column(key_col)

        for name in self.app.conf['custom-shortcut']:
            key = self.app.conf['custom-shortcut'][name]
            i18n_name = Config.SHORT_CUT_I18N[name]
            k, m = Gtk.accelerator_parse(key)
            self.shortcut_liststore.append([i18n_name, k, m, ])

    def run(self):
        self.get_content_area().show_all()
        super().run()

    def on_destroy(self):
        print('dialog.on_destroy()')
        Config.dump_conf(self.app.conf)

    # generic tab signal handlers
    def on_status_button_toggled(self, button):
        self.app.conf['use-status-icon'] = button.get_active()

    def on_notify_button_toggled(self, button):
        self.app.conf['use-notify'] = button.get_active()

    # format tab signal handlers
    def on_audio_toggled(self, radiobtn):
        self.app.conf['use-ape'] = radiobtn.get_group()[0].get_active()

    def on_video_toggled(self, radiobtn):
        # radio_group[0] is MKV
        self.app.conf['use-mkv'] = radiobtn.get_group()[0].get_active()

    def on_shortcut_btn_toggled(self, button, mode):
        if button.get_active() is False:
            return
        self.app.conf['shortcut-mode'] = mode
        self.shortcut_win.set_sensitive(mode == ShortcutMode.CUSTOM)

    def on_shortcut_key_cell_edited(self, accel, path, key, mod,
                                    hardware_keycode):
        accel_key = Gtk.accelerator_name(key, mod)
        name = self.shortcut_liststore[path][0]
        self.shortcut_liststore[path][1] = key
        self.shortcut_liststore[path][2] = int(mod)
        self.app.conf['custom-shortcut'][name] = accel_key


import json
from datetime import datetime

class Notebook:
    def __init__(self):
        self.update()

    # Reload algorithms and records
    def update(self):
        with open('records.json') as f:
            self.records = json.load(f)
        f.close()
        self.id_no = len(self.records)

        with open('algos.json') as f:
            self.algos = json.load(f)
        f.close()

    # Load an algorithm
    # para:
    # name, str. Algorithm name
    # return:
    # res, json. Loading the entry
    def load(self,name):
        return 0

    # Create a new algorithm
    # para:
    # algo_name, json. Json file of algorithm
    # time, python time object. Current time
    # description, str. Description of the algorithm
    # return:
    # bool, successfully added or not
    def create(self, algo_name, time = datetime.now().strftime("%d/%m/%Y %H:%M:%S"), description=""):
        # Create algo json
        if algo_name not in self.records.values():
            x = {str(self.id_no):{
                 'name':algo_name,
                 'last_edited_time':time,
                 'description':description,
                 'example':[],
                 'function':[]
                }
            }
            write_json(x,'algos.json')
            write_json({self.id_no:algo_name},'records.json')
            # Increase id by 1
            self.id_no += 1
            self.update()
            return True
        else:
            return False

    # Edit an algorithm
    # para:
    # algo_name, str. Current algorithm name.
    # attribute, str. The attribute to be changed
    # content, str. Replacing the old content of the attribute
    # return:
    # Bool, successfully changed or not
    def edit(self,algo_name,attribute,content):
        self.update()
        return 0

    # Add example record to an entry
    # para:
    # example_jason, json of the example
    # return:
    # bool, successful added or not
    def add_example(self,example_json):
        return 0

    # Add a function to exsist algorithm
    # para:
    # algo_name, str. Name of the algorithm
    # file_path, str. Directory of the file
    # return:
    # bool, wether the function is added or not
    def add_function(self,alg_name,file_path):
        return 0

    # Delete an algorithm
    # para:
    # algo_name, str. Algorithm name
    # return:
    # Bool, Successfully deleted or not
    def delete(self,algo_name):
        return 0

    # Serach an algorithm
    # para:
    # algo_name
    # return:
    # res,json file
    def search(self,algo_name):
        for key,val in self.records.items():
            if val == algo_name:
                return self.algos[key]
        return None

    # Display info
    # para:
    # algo_name,str
    # return:
    # void
    def display(self,algo_name):
        return 0

# function to add to JSON
def write_json(new_data, filename):
    with open(filename,'r+') as file:
        file_data = json.load(file)
        # Join new_data with file_data inside emp_details
        file_data.update(new_data)
        # Sets file's current position at offset.
        file.seek(0)
        # convert back to json.
        json.dump(file_data, file, indent = 4)
        file.close()

note=Notebook()


from datetime import datetime, timedelta
from .envs import settings
import jwt

def create_jwt_token(data: dict) -> str:
    now = datetime.utcnow()

    return jwt.encode(
        {
            **data,
            **{
                "exp": now + timedelta(minutes=15),
                "iat": now,
            },
        },
        settings.jwt_secret,
        algorithm="HS256",
    )

def decode_jwt_token(token: str) -> dict:
    try:
        data = jwt.decode(
            token,
            settings.jwt_secret,
            algorithms=["HS256"],
        )

    except Exception as e:
        print(e)
        return {}

    return data


import sqlite3
import argparse
from core import *
from weblogo import *
import pandas as pd

def get_transcript_sequence(transcript, organism_sqlite_cursor):
    '''
    return the nucleotide sequence for a given transcript as a string
    '''
    sequence = organism_sqlite_cursor.execute(
        f"SELECT sequence FROM transcripts WHERE transcript == '{transcript}'"
    ).fetchall()[0]

    return sequence[0]

def main(args):
    cursor = get_sqlite_cursor(args.s)

    openprot = read_openprot_annotations(args.transcripts)

    probs_df = pd.read_csv(args.p)
    init = []
    for row in probs_df.iterrows():
        init.append([f'{row[1]["transcript"]}_orf', row[1]["prob_uORF_init_rpb"], row[1]["prob_uORF_init_raw"]])
        init.append([f'{row[1]["transcript"]}_cds', row[1]["prob_cds_init_rpb"], row[1]["prob_cds_init_raw"]])

    init_df = pd.DataFrame(init, columns=[
                                'transcript',
                                'rpb_prob',
                                'raw_prob',
    ])
    print(init_df.sort_values('raw_prob'))
    unique_transcripts = list(probs_df['transcript'].unique())

    outfile = open(args.o, 'w')
    for row in openprot.iterrows():
        if "None" in list(row[1]):
            continue

        cds_start = int(row[1]['cds_start'])
        orf_start = int(row[1]['orf_start'])
        sequence = get_transcript_sequence(row[1]['transcript'], cursor)
        outfile.write(f">{row[1]['transcript']}_orf\n")
        outfile.write(f"{sequence[orf_start-10:orf_start+10]}\n")
        print(sequence[orf_start-1:orf_start+2])
        outfile.write(f">{row[1]['transcript']}_cds\n")
        outfile.write(f"{sequence[cds_start-10:cds_start+10]}\n")

    outfile.close
    return True

if __name__ == "__main__":
    parser = argparse.ArgumentParser()

    parser.add_argument("--transcripts", help="path to subset of openprot data ")
    parser.add_argument("-s", help="path to organism sqlite ")
    parser.add_argument("-p", help="path to probabilities csv")
    parser.add_argument("-o", help="path to output fasta file of sequences at start site")

    args = parser.parse_args()
    main(args)


import sys
import json
import numpy as np
import matplotlib.pyplot as plt

# ==========================
# *** Reading JSON files ***

# *** Reading for PMTs ***

def getAoPJson(filename, elecver):
    xst = []
    yaopHbpmt = []
    yaopHberrpmt = []
    yaopCapmt = []
    yaopCaerrpmt = []

    ydiffHbCa = []
    npmts = 3

    for pmt in range(0, npmts):
        tmpfile = open(filename+str(pmt+1)+".json", 'r')
        print("opened:", filename+str(pmt+1)+".json")
        tmpdata = json.load(tmpfile)
        tmpaveAoPHb = []
        tmperrAoPHb = []
        tmpaveAoPCa = []
        tmperrAoPCa = []
        for info in tmpdata[elecver]:
            tmpaveAoPHb.append( info['extAveAoPHb'] )
            tmperrAoPHb.append( info['extRmsAoPHb'] )
            tmpaveAoPCa.append( info['extAveAoPCa'] )
            tmperrAoPCa.append( info['extRmsAoPCa'] )

            if ( pmt==npmts-1 ):
                xst.append( info['stationId'] )
        yaopHbpmt.append( tmpaveAoPHb )
        yaopHberrpmt.append( tmperrAoPHb )
        yaopCapmt.append( tmpaveAoPCa )
        yaopCaerrpmt.append( tmperrAoPCa )

    return xst, yaopHbpmt, yaopHberrpmt, yaopCapmt, yaopCaerrpmt

basenameUub = "../fitaopmlt/uubFitaopPMT"
basenameUb = "../../nouub/fitHist/fitaopmlt/ubFitaopPMT"

outnameHbase = "../../plots/aopHbaseUubUbPMTs.png"
outnameCalib = "../../plots/aoCalibUubUbPMTs.png"

aopUub = getAoPJson(basenameUub, "uub")
aopUb = getAoPJson(basenameUb, "ub")
stLabel = []
npmts = 3

ubouub = 8.33/25.
markers = ['o', 's', '^']

# ==============================
# *** For AoP HBase Plotting ***
diffHb = []
tmpaopUub = np.array( aopUub[1] )
tmpaopUb = np.array( aopUb[1] )

for pmt in range(0, npmts):
    tmp = []
    for st in range(len(aopUub[0])):
        if tmpaopUub[pmt][st] > 4 and tmpaopUb[pmt][st] > 2:
            tmp.append( 1 - ( ubouub*tmpaopUub[pmt][st]/tmpaopUb[pmt][st] ) )
        else:
            tmp.append(1)
    diffHb.append(tmp)

yerr = []
for pmt in range(0, npmts):
    tmp = []
    for st in range( len(aopUub[0]) ):
        if aopUb[1][pmt][st] > 0:
            tmp.append( ( aopUub[2][pmt][st]/aopUb[1][pmt][st] )**2 + ( aopUb[2][pmt][st]*aopUub[1][pmt][st]/aopUb[1][pmt][st]**2 )**2 )
        else:
            tmp.append( 0 )
    yerr.append( np.sqrt( tmp ) )

plt.figure(figsize=(16,9))
for i in range(0, npmts):
    plt.errorbar(aopUub[0], diffHb[i], yerr=yerr[i], fmt=markers[i], markersize=12, label=" PMT"+str(i+1))

plt.legend(fontsize=22, loc=1)
plt.xlabel("Station Id.", fontsize=24)
plt.xticks(rotation=60)
plt.ylabel("(1 - (0,33*AoPuub)/AopUb) HBase [au]", fontsize=24)
plt.xticks(fontsize=22)
plt.yticks(fontsize=22)
plt.ylim(0, 0.6)
#plt.grid(axis='y', color='0.75')
#plt.show()
plt.tight_layout()
plt.savefig(outnameHbase, dpi=100)
print("Average AoP Peak-HBase OK")

tmpsum = []
tmpsum.append( [np.average(diffHb[pmt]) for pmt in range(3)] )
print(np.average(np.array(tmpsum)))

# ==============================
# *** For AoP Calib Plotting ***

diffHb = []
tmpaopUub = np.array( aopUub[3] )
tmpaopUb = np.array( aopUb[3] )

for pmt in range(0, npmts):
    tmp = []
    for st in range(len(aopUub[0])):
        if tmpaopUub[pmt][st] > 4 and tmpaopUb[pmt][st] > 2:
            tmp.append( 1 - ( ubouub*tmpaopUub[pmt][st]/tmpaopUb[pmt][st] ) )
        else:
            tmp.append(1)
    diffHb.append(tmp)

yerr = []
for pmt in range(0, npmts):
    tmp = []
    for st in range( len(aopUub[0]) ):
        if aopUb[1][pmt][st] > 0:
            tmp.append( ( aopUub[4][pmt][st]/aopUb[3][pmt][st] )**2 + ( aopUb[4][pmt][st]*aopUub[3][pmt][st]/aopUb[3][pmt][st]**2 )**2 )
        else:
            tmp.append( 0 )
    yerr.append( np.sqrt( tmp ) )

plt.figure(figsize=(16,9))
for i in range(0, npmts):
    plt.errorbar(aopUub[0], diffHb[i], yerr=yerr[i], fmt=markers[i], markersize=12, label=" PMT"+str(i+1))
    #plt.plot(aopUub[0], diffHb[i], markers[i], markersize=12, label=" PMT"+str(i+1))
plt.legend(fontsize=22, loc=1)
plt.xlabel("Station Id.", fontsize=24)
plt.xticks(rotation=60)
plt.ylabel("(1 - (0,33*AoPuub)/AopUb) Calib [au]", fontsize=24)
plt.xticks(fontsize=22)
plt.yticks(fontsize=22)
plt.ylim(0,0.6)
#plt.grid(axis='y', color='0.75')
#plt.show()
plt.tight_layout()
plt.savefig(outnameCalib, dpi=100)
print("Average AoP Peak-Calib OK")


import tkinter as tk
import re
import numpy as np
from tkinter import scrolledtext
from tkinter import *
from tkinter import ttk
import interview
import survave
import maze

def main_interview():

    class Patient:
        def __init__(self, name, age):
            self.name = name
            self.age = age
        def quiz(self):
            Text_q, Text_dict = interview.convert_question()
            diagnoz = interview.open_window(Text_q, Text_dict, self.name)
            return diagnoz
    class Patient_by_hospital(Patient):
        def game(self):
            survave.run()
            return
    class Patient_by_polyclinic(Patient):
        def game(self):
            maze.founding()
            return

    name = 'LEXA'
    age = 18
    lexa = Patient(name,age)

    mainwindow = tk.Tk()
    mainwindow.title('Вы')
    mainwindow.resizable(width=False, height=False)
    mainwindow.geometry('720x480+400+100')
    photo = tk.PhotoImage(file = 'hospital.png')
    mainwindow.iconphoto(False, photo)
    mainwindow.config(bg = '#DEFDFF')
    label = tk.Label(mainwindow,text = "Ваше имя:",
                     bg = '#DEFDFF',
                     fg = '#99004C',
                     font = ('Arial', 18, 'bold'),
                     pady = 20
                     )
    label .pack()
    entry = tk.Entry(mainwindow)
    entry.pack()
    label1 = tk.Label(mainwindow,text = "Ваш возраст:",
                      bg = '#DEFDFF',
                      fg = '#99004C',
                      font = ('Arial', 18, 'bold'),
                      pady = 20
                      )
    label1.pack()
    entry1 = tk.Entry(mainwindow)
    entry1.pack()
    def out():
        global name, age
        name = entry.get()
        age = entry1.get()

        label2 = tk.Label(mainwindow, text="Закройте вкладку",
                          bg='#DEFDFF',
                          fg='#99004C',
                          font=('Arial', 18, 'bold'),
                          pady=20
                          )
        label2.pack()
        return
    button = tk.Button(mainwindow,text = "Отправить",
                       command = out,
                       bg='#DEFDFF',
                       fg='#99004C',
                       font=('Arial', 18, 'bold'),
                       activebackground='green',
                       pady=20
                       )
    button.pack()
    mainwindow.mainloop()

    lexa = Patient(name,age)
    status = lexa.quiz()
    if status<0.5:
        lexa_stay = Patient_by_hospital(name,age)
    else:
        lexa_stay = Patient_by_polyclinic(name,age)

    lexa_stay.game()


import pytest
import torch
from flash.core.serve.types import Label, Repeated
from flash.core.utilities.imports import _TOPIC_SERVE_AVAILABLE

@pytest.mark.skipif(not _TOPIC_SERVE_AVAILABLE, reason="Not testing serve.")
def test_repeated_deserialize():
    repeated = Repeated(dtype=Label(classes=["classA", "classB"]))
    res = repeated.deserialize(*({"label": "classA"}, {"label": "classA"}, {"label": "classB"}))
    assert res == (torch.tensor(0), torch.tensor(0), torch.tensor(1))

@pytest.mark.skipif(not _TOPIC_SERVE_AVAILABLE, reason="Not testing serve.")
def test_repeated_serialize(session_global_datadir):
    repeated = Repeated(dtype=Label(path=str(session_global_datadir / "imagenet_labels.txt")))
    assert repeated.deserialize(*({"label": "chickadee"}, {"label": "stingray"})) == (
        torch.tensor(19),
        torch.tensor(6),
    )
    assert repeated.serialize((torch.tensor(19), torch.tensor(6))) == ("chickadee", "stingray")
    assert repeated.serialize(torch.tensor([19, 6])) == ("chickadee", "stingray")

@pytest.mark.skipif(not _TOPIC_SERVE_AVAILABLE, reason="Not testing serve.")
def test_repeated_max_len():
    repeated = Repeated(dtype=Label(classes=["classA", "classB"]), max_len=2)

    with pytest.raises(ValueError):
        repeated.deserialize(*({"label": "classA"}, {"label": "classA"}, {"label": "classB"}))
    assert repeated.deserialize(*({"label": "classA"}, {"label": "classB"})) == (
        torch.tensor(0),
        torch.tensor(1),
    )
    with pytest.raises(ValueError):
        repeated.serialize((torch.tensor(0), torch.tensor(0), torch.tensor(1)))
    assert repeated.serialize((torch.tensor(1), torch.tensor(0))) == ("classB", "classA")

    # max_len < 1
    with pytest.raises(ValueError):
        Repeated(dtype=Label(classes=["classA", "classB"]), max_len=0)
    assert Repeated(dtype=Label(classes=["classA", "classB"]), max_len=1) is not None

    # type(max_len) is not int
    with pytest.raises(TypeError):
        Repeated(dtype=Label(classes=["classA", "classB"]), max_len=str)

@pytest.mark.skipif(not _TOPIC_SERVE_AVAILABLE, reason="Not testing serve.")
def test_repeated_non_serve_dtype():
    class NonServeDtype:
        pass

    with pytest.raises(TypeError):
        Repeated(NonServeDtype())

@pytest.mark.skipif(not _TOPIC_SERVE_AVAILABLE, reason="Not testing serve.")
def test_not_allow_nested_repeated():
    with pytest.raises(TypeError):
        Repeated(dtype=Repeated())


from __future__ import print_function

import kubernetes
import psycopg2
import subprocess
import os
import datetime
import time
import itertools
import base64

# Parameters:
BUCKET = os.environ['S3_BUCKET']
NAMESPACE = os.environ['S3_NAMESPACE']
ACCESS_KEY = os.environ['AWS_ACCESS_KEY']
SECRET_KEY = os.environ['AWS_SECRET_KEY']
HOST = os.environ.get('S3_HOST', 's3.amazonaws.com')
HOST_BUCKET = os.environ.get('S3_HOST_BUCKET', '%(bucket)s.s3.amazonaws.com')
s3cmd = 's3cmd --access_key="%s" --secret_key="%s" --host="%s" --host-bucket="%s"' % (ACCESS_KEY, SECRET_KEY, HOST, HOST_BUCKET)
bucket_prefix = 's3://%s/db_backups/%s' % (BUCKET, NAMESPACE)

def get_latest_md5(db_name, table):
    cmd = '%s ls %s/%s/%s/' % (s3cmd, bucket_prefix, db_name, table)
    current_files = subprocess.Popen(['/bin/sh', '-c', cmd], stdout=subprocess.PIPE)
    current_files, _ = current_files.communicate()
    current_files = current_files.split('\n')
    current_files = [line.split() for line in current_files if line.strip() != '']
    current_files.sort(key=lambda line: line[0]+' '+line[1])
    if len(current_files) == 0:
        return None
    else:
        return current_files[-1][-1].split('.')[-3]  # the md5 hash

def handle_table(db_url, table):
    start = time.time()
    db_name = db_url.split('/')[-1]
    # print('\tGot table "%s/%s"' % (db_name, table))
    latest_hash = get_latest_md5(db_name, table)
    # print('\tExisting hash "%s"' % latest_hash)
    cmd = 'pg_dump -t "%s" "%s" | gzip | md5sum' % (table, db_url)
    # print('#',cmd)
    proc = subprocess.Popen(['/bin/sh', '-c', cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    current_hash, _ = proc.communicate()
    current_hash = current_hash.split()[0][:6]
    # print('\tCurrent hash "%s"' % (current_hash, ))
    if latest_hash is None:
        status = 'NEW'
    elif current_hash != latest_hash:
        status = 'MODIFIED'
    else:
        status = 'SAME'
    if status != 'SAME':
        filename = '%s/%s/%s/%s.%s.%s.%s.pg_dump.gz' % \
                   (bucket_prefix, db_name, table, db_name, table, datetime.datetime.now().date().isoformat(), current_hash)
        cmd = '%s put --no-progress --no-encrypt - %s' % (s3cmd, filename)
        cmd = 'pg_dump -t "%s" "%s" | gzip | %s ' % \
              (table, db_url, cmd)
        # print('#',cmd)
        proc = subprocess.Popen(['/bin/sh', '-c', cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate()
        print(out, err)
        # print('-->', proc.returncode)
    print('\tTable %s: %s, %d secs' % (table, status, int(time.time() - start)))

def handle_db(db_url):
    start = time.time()
    print('Encountered DB URL %s' % db_url)
    conn = psycopg2.connect(db_url)
    cursor = conn.cursor()
    cursor.execute("""SELECT table_name FROM information_schema.tables
                      WHERE table_schema = 'public'""")
    count = 0
    for table in cursor.fetchall():
        table = table[0]
        handle_table(db_url, table)
        count += 1
    print('\tDB %s: %d tables, %d secs' % (db_url, count, int(time.time() - start)))

def get_services():
    all_urls = set()
    try:
        kubernetes.config.incluster_config.load_incluster_config()
    except Exception as e:
        print(e)
        print('attempting local kube config')
        kubernetes.config.load_kube_config()
    kubev1 = kubernetes.client.CoreV1Api()
    containers = itertools.chain(*[pod.spec.containers for pod
                                   in kubev1.list_pod_for_all_namespaces(watch=False).items])
    envvars = itertools.chain(*[container.env for container in containers if container.env])
    secrets = {s.metadata.name: s.data for s in kubev1.list_secret_for_all_namespaces(watch=False).items}
    for envvar in envvars:
        if envvar.name == 'DATABASE_URL' and envvar.value_from and envvar.value_from.secret_key_ref:
            secret = secrets.get(envvar.value_from.secret_key_ref.name)
            if secret:
                db_url = secret.get(envvar.value_from.secret_key_ref.key)
                if db_url:
                    db_url = base64.b64decode(db_url).decode()
                    if db_url not in all_urls:
                        handle_db(db_url)
                        all_urls.add(db_url)

if __name__ == "__main__":
    get_services()