import code

import gym
import torch
from tqdm import trange

import numpy as np

import components.prioritized_memory
import components.memory
from components.filesys_manager import ExperimentPath

class BaseTrainer:

    def __init__(self, args):
        # init experiment hyper-parameters
        self.args = args
        self.debug = self.args.debug
        self.allow_printing = True
        self.exp_path = ExperimentPath(f"{self.args.exp_root}/{self.args.name}/{self.args.env}/{args.trainer}/{self.args.seed}")
        self.print("log directory initialized")
        # env init
        self.expl_env = gym.make(self.args.env)
        self.eval_env = gym.make(self.args.env)
        # replay buffer
        if args.prioritized:
            self.replay_buffer = components.prioritized_memory.ReplayMemory(self.args)
        else:
            self.replay_buffer = components.memory.ReplayMemory(self.args)
        # global time steps
        self.t = 0
        # exploration environment
        self.s = torch.from_numpy(self.expl_env.reset()).to(dtype=torch.float32, device=self.args.device)
        self.done = False
        self.episode_num = 0
        self.episode_len = 0
        self.episode_return = 0
        self.episode_discounted_return = 0
        # init model
        self.init_model()
        # log
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "model initialized"])
        self.exp_path['config'].json_write(vars(self.args))
        self.exp_path['model_info'].txt_write(str(self.q_learner))
        self.save('initial')

    def print(self, *args, **kwargs):
        if self.allow_printing:
            print(",", self.exp_path.now(), self.exp_path.str(), ",", *args, **kwargs)

    def init_model(self):
        """ init model """
        self.print("warning: init model not implemented")
        self.q_learner = None
        self.q_target = None
        self.policy = None
        self.q_optimizer = None
        self.policy_optimizer = None

    def sample_batch(self, batch_size):
        return self.replay_buffer.sample(batch_size)

    def expl_action(self, obs) -> torch.Tensor:
        self.print("warning: expl action not implemented")
        a = np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0])
        return torch.from_numpy(a)

    def eval_action(self, obs) -> torch.Tensor:
        self.print("warning: eval action not implemented")
        a = np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0])
        return torch.from_numpy(a)

    def before_learn_on_batch(self):
        self.print("warning: before learn on batch not implemented")

    def learn_on_batch(self, treeidx, s, a, r, s_, d, w):
        """ learn on batch """
        self.print("warning: learn on batch not implemented")

    def after_learn_on_batch(self):
        self.print("warning: after learn on batch not implemented")

    def state_value_pred(self, s):
        """ log the state value of initial state """
        self.print("warning: state value pred not implemented")
        return -1

    def reset_expl_env(self):
        """ reset exploration env """
        self.s = self.expl_env.reset()
        self.s = torch.from_numpy(self.s).to(dtype=torch.float32, device=self.args.device)
        self.done = False
        self.episode_num += 1
        self.episode_len = 0
        self.episode_return = 0
        self.episode_discounted_return = 0

    def advance_expl_env(self, next_obs, reward, done):
        """ advance exploration env for next step """
        self.s = next_obs
        self.done = done
        self.episode_return += reward
        if self.args.reward_clip:
            reward = min(max(reward, -1), 1)
        self.episode_discounted_return += reward * self.args.discount ** self.episode_len
        self.episode_len += 1

    def epsilon(self):
        """ compute current epsilon (linear annealing 1M steps) """
        return 1 - self.t * ((1 - 0.1) / self.args.epsilon_steps) if self.t < self.args.epsilon_steps else 0.1

    def collect_init_steps(self, render=False):
        """ fill the replay buffer before learning """
        for _ in trange(self.args.min_num_steps_before_training):
            if self.done:
                self.reset_expl_env()
            # sample transition
            a = torch.from_numpy(np.random.uniform(low=-1, high=1, size=self.expl_env.action_space.shape[0]))
            s_, r, d, _ = self.expl_env.step(a)
            s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
            r = r.item()
            # store into replay
            if self.args.reward_clip:
                r = min(max(r, -1), 1)
            if 'reward_std' in self.args:
                r += np.random.normal(0, self.args.reward_std)  # clip, add noise
            if self.args.prioritized:
                self.replay_buffer.append(self.s, a, r, d)
            else:
                self.replay_buffer.append(self.s, a, r, s_, d)
            # render
            if render:
                self.expl_env.render()
            # sample next state
            self.advance_expl_env(s_, r, d)
            self.t += 1
        self.reset_expl_env()
        self.print("finished collect init steps")

    def sample_transition(self, render=False):
        """ train one step """
        if self.done:
            self.reset_expl_env()

        # sample transition
        a = self.expl_action(self.s)
        s_, r, d, _ = self.expl_env.step(a)
        s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
        r = r.item()
        # store into replay
        if self.args.reward_clip:
            r = min(max(r, -1), 1)
        if 'reward_std' in self.args:
           r += np.random.normal(0, self.args.reward_std)  # clip, add noise
        if self.args.prioritized:
            self.replay_buffer.append(self.s, a, r, d)
        else:
            self.replay_buffer.append(self.s, a, r, s_, d)
        # render
        if render:
            self.expl_env.render()
        # sample next state
        self.advance_expl_env(s_, r, d)

        # log status
        if self.done:
            self.exp_path['expl_episode_stats[t,return,discounted_return]'].csv_writerow(
                [self.t, self.episode_return, self.episode_discounted_return])
            if self.args.prioritized:
                self.exp_path['debug[t,buffer_size,epsilon]'].csv_writerow(
                    [self.t, self.replay_buffer.data_buffer.index, self.epsilon()]
                )
            else:
                self.exp_path['debug[t,buffer_size]'].csv_writerow(
                    [self.t, len(self.replay_buffer)]
                )

        self.t += 1

    def evaluate(self, render=False, random=False):
        """ evaluate current policy """
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "evaluation started"])
        self.print(f"evaluation started t={self.t}")
        # init eval
        s = self.eval_env.reset()
        s = torch.from_numpy(s).to(dtype=torch.float32, device=self.args.device)
        done = False
        episode_len = 0
        episode_return = 0
        episode_discounted_return = 0
        v_initials = []
        v_initials.append(self.state_value_pred(s))
        returns = []
        discounted_returns = []
        episode_lens = []
        # start eval
        for _ in trange(self.args.num_eval_steps_per_epoch):
            if done:
                # record data
                returns.append(episode_return)
                discounted_returns.append(episode_discounted_return)
                episode_lens.append(episode_len)
                # reset env
                s = self.eval_env.reset()
                s = torch.from_numpy(s).to(dtype=torch.float32, device=self.args.device)
                v_initials.append(self.state_value_pred(s))
                episode_len = 0
                episode_return = 0
                episode_discounted_return = 0
            if len(returns) >= self.args.eval_max_episode:
                # check if enough episodes simulated
                break
            # sample transition
            a = self.eval_action(s)
            if random:
                a = np.random.randint(self.eval_env.action_space())
            s_, r, d, _ = self.eval_env.step(a)
            s_ = torch.from_numpy(s_).to(dtype=torch.float32, device=self.args.device)
            r = r.item()
            # advance env
            s = s_
            done = d
            episode_return += r
            if self.args.reward_clip:
                r = min(max(r, -1), 1)
            episode_discounted_return += r * self.args.discount ** episode_len
            episode_len += 1
            # render
            if render:
                print(f"s={s.shape} a={a} r={r} r_clip={r}")
                self.eval_env.render()
        # finished eval
        self.print(f"returns={returns}\nmean={np.mean(returns) if len(returns) > 0 else 0}")
        if len(returns) == 0:
            self.exp_path['eval_episode_stats[t,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, v_initials[0], episode_return, episode_discounted_return, episode_len])
            self.exp_path['eval_mean_stats[t,num_episode,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, 1, v_initials[0], episode_return, episode_discounted_return, episode_len])

        else:
            for V_init, R, discounted_R, length in zip(v_initials, returns, discounted_returns, episode_lens):
                self.exp_path['eval_episode_stats[t,v_init,return,discounted_return,length]'].csv_writerow(
                    [self.t, V_init, R, discounted_R, length])
            self.exp_path['eval_mean_stats[t,num_episode,v_init,return,discounted_return,length]'].csv_writerow(
                [self.t, len(returns), np.mean(v_initials), np.mean(returns), np.mean(discounted_returns), np.mean(episode_lens)])

        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "evaluation finished"])

    def save(self, name='trainer'):
        """ save this trainer """
        buffer = self.replay_buffer
        expl_env = self.expl_env
        eval_env = self.eval_env
        # skip the things not saving
        self.replay_buffer = None
        self.expl_env = None
        self.eval_env = None
        self.exp_path["checkpoint"][f"{name}_{self.t}.pth"].save_model(self)
        self.exp_path['timestamp[timenow,t,msg]'].csv_writerow([self.exp_path.now(), self.t, "model saved"])
        self.replay_buffer = buffer
        self.expl_env = expl_env
        self.eval_env = eval_env


#!/usr/bin/env python

from __future__ import print_function

import os, sys, json

import numpy as np

from ase.build import bulk, surface
from ase.units import Rydberg, Bohr
from ase.io import read
from ase.visualize import view
from ase.spacegroup import crystal
from ase.calculators.espresso import Espresso

infile = sys.argv[1]
print(infile)
with open(infile) as handle:
    system = json.loads(handle.read())

face = system['face']
layers = system['layers']
vacuum = system['vacuum']
kpts = system['kpts']
ecut = system['ecut']
mode = system['mode']

cwd = os.getcwd()

def cassiterite(show=False):
    a = 4.7382
    c = 3.1871
    sno2 = crystal(['Sn','O'], basis=[(0, 0, 0), (0.3, 0.3, 0.0)],
                   spacegroup='P 4 2/mnm', cellpar=[a,a,c,90,90,90],
                   pbc=True)
    return sno2

def create_surface(atoms,face=(1,1,0),layers=3,vacuum=10.813,kpts=([6,3,1])):
    mySurface = surface( atoms, face, layers)
    mySurface.center(vacuum=vacuum, axis=2)
    kpts = np.asarray(kpts)
    return mySurface

sno2         = cassiterite()
sno2_surface = create_surface(sno2,
                              face=face,
                              layers=layers,
                              vacuum=vacuum,
                              kpts=kpts)

# Put together QE input dict
input_dict = {
    'control': {
        'calculation': 'scf',
        'etot_conv_thr': 1e-6,
        'nstep': 100,
        'outdir': 'sno2_test_face_{0}{1}{2}'.format(face[0], face[1], face[2]),
    },
    'system': {
        'ecutwfc': ecut,
    },
    'electrons': {
        'diagonalization': 'cg',
    },
}

# Put together pseudopotential dict
psp_dict = {'Sn': 'Sn.UPF',
            'O': 'O.UPF',
            }

calc = Espresso(input_data=input_dict,
                kpts=kpts,
                pseudo_dir=cwd + "/../pseudo",
                pseudopotentials=psp_dict,
                )

sno2_surface.set_calculator(calc)

if mode == 'view':
    view(sno2_surface)
elif mode == 'calc':
    calc.calculate(sno2_surface)
    print('SnO2 PE:', sno2_surface.get_potential_energy())


import numpy as np
try:
    import mc
except Exception:
    pass
import cv2
import os
from PIL import Image

import torch
from torch.utils.data import Dataset
import torchvision.transforms as transforms

import utils
from . import reader

class PartialCompDataset(Dataset):

    def __init__(self, config, phase):
        self.dataset = config['dataset']
        if self.dataset == 'COCOA':
            self.data_reader = reader.COCOADataset(config['{}_annot_file'.format(phase)])
        elif self.dataset == 'Mapillary':
            self.data_reader = reader.MapillaryDataset(
                config['{}_root'.format(phase)], config['{}_annot_file'.format(phase)])
        else:
            self.data_reader = reader.KINSLVISDataset(
                self.dataset, config['{}_annot_file'.format(phase)])
        if config['load_rgb']:
            self.img_transform = transforms.Compose([
                transforms.Normalize(config['data_mean'], config['data_std'])
            ])
        self.eraser_setter = utils.EraserSetter(config['eraser_setter'])
        self.sz = config['input_size']
        self.eraser_front_prob = config['eraser_front_prob']
        self.phase = phase

        self.config = config

        self.memcached = config.get('memcached', False)
        self.initialized = False
        self.memcached_client = config.get('memcached_client', None)

    def __len__(self):
        return self.data_reader.get_instance_length()

    def _init_memcached(self):
        if not self.initialized:
            assert self.memcached_client is not None, "Please specify the path of your memcached_client"
            server_list_config_file = "{}/server_list.conf".format(self.memcached_client)
            client_config_file = "{}/client.conf".format(self.memcached_client)
            self.mclient = mc.MemcachedClient.GetInstance(server_list_config_file, client_config_file)
            self.initialized = True

    def _load_image(self, fn):
        if self.memcached:
            try:
                img_value = mc.pyvector()
                self.mclient.Get(fn, img_value)
                img_value_str = mc.ConvertBuffer(img_value)
                img = utils.pil_loader(img_value_str)
            except:
                print('Read image failed ({})'.format(fn))
                raise Exception("Exit")
            else:
                return img
        else:
            return Image.open(fn).convert('RGB')

    def _get_inst(self, idx, load_rgb=False, randshift=False):
        modal, bbox, category, imgfn, _ = self.data_reader.get_instance(idx)
        centerx = bbox[0] + bbox[2] / 2.
        centery = bbox[1] + bbox[3] / 2.
        size = max([np.sqrt(bbox[2] * bbox[3] * self.config['enlarge_box']), bbox[2] * 1.1, bbox[3] * 1.1])
        if size < 5 or np.all(modal == 0):
            return self._get_inst(
                np.random.choice(len(self)), load_rgb=load_rgb, randshift=randshift)

        # shift & scale aug
        if self.phase  == 'train':
            if randshift:
                centerx += np.random.uniform(*self.config['base_aug']['shift']) * size
                centery += np.random.uniform(*self.config['base_aug']['shift']) * size
            size /= np.random.uniform(*self.config['base_aug']['scale'])

        # crop
        new_bbox = [int(centerx - size / 2.), int(centery - size / 2.), int(size), int(size)]
        modal = cv2.resize(utils.crop_padding(modal, new_bbox, pad_value=(0,)),
            (self.sz, self.sz), interpolation=cv2.INTER_NEAREST)

        # flip
        if self.config['base_aug']['flip'] and np.random.rand() > 0.5:
            flip = True
            modal = modal[:, ::-1]
        else:
            flip = False

        if load_rgb:
            rgb = np.array(self._load_image(os.path.join(
                self.config['{}_image_root'.format(self.phase)], imgfn))) # uint8
            rgb = cv2.resize(utils.crop_padding(rgb, new_bbox, pad_value=(0,0,0)),
                (self.sz, self.sz), interpolation=cv2.INTER_CUBIC)
            if flip:
                rgb = rgb[:, ::-1, :]
            rgb = torch.from_numpy(rgb.astype(np.float32).transpose((2, 0, 1)) / 255.)
            rgb = self.img_transform(rgb) # CHW

        if load_rgb:
            return modal, category, rgb
        else:
            return modal, category, None

    def __getitem__(self, idx):
        if self.memcached:
            self._init_memcached()
        randidx = np.random.choice(len(self))
        modal, category, rgb = self._get_inst(
            idx, load_rgb=self.config['load_rgb'], randshift=True) # modal, uint8 {0, 1}
        if not self.config.get('use_category', True):
            category = 1
        eraser, _, _ = self._get_inst(randidx, load_rgb=False, randshift=False)
        eraser = self.eraser_setter(modal, eraser) # uint8 {0, 1}

        # erase
        erased_modal = modal.copy().astype(np.float32)
        if np.random.rand() < self.eraser_front_prob:
            erased_modal[eraser == 1] = 0 # eraser above modal
        else:
            eraser[modal == 1] = 0 # eraser below modal
        erased_modal = erased_modal * category

        # shrink eraser
        max_shrink_pix = self.config.get('max_eraser_shrink', 0)
        if max_shrink_pix > 0:
            shrink_pix = np.random.choice(np.arange(max_shrink_pix + 1))
            if shrink_pix > 0:
                shrink_kernel = shrink_pix * 2 + 1
                eraser = 1 - cv2.dilate(
                    1 - eraser, np.ones((shrink_kernel, shrink_kernel), dtype=np.uint8),
                    iterations=1)
        eraser_tensor = torch.from_numpy(eraser.astype(np.float32)).unsqueeze(0) # 1HW
        # erase rgb
        if rgb is not None:
            rgb = rgb * (1 - eraser_tensor)
        else:
            rgb = torch.zeros((3, self.sz, self.sz), dtype=torch.float32) # 3HW
        erased_modal_tensor = torch.from_numpy(
            erased_modal.astype(np.float32)).unsqueeze(0) # 1HW
        target = torch.from_numpy(modal.astype(np.int)) # HW
        return rgb, erased_modal_tensor, eraser_tensor, target


from pyspark.sql import SparkSession
from pyspark.sql import functions as fs

spark= SparkSession.builder.appName("word_count").getOrCreate()

data= spark.read.text("book.txt")

pro_data= data.select(fs.explode(fs.split(data.value,"\\W+")).alias("words"))
pro_data.filter(pro_data.words !="")

a=pro_data.select("words").groupBy("words").count().show()


import pygame
from constantes import *
from auxiliar import Auxiliar

class Background:
    '''
    Clase para representar un fondo en un juego utilizando Pygame.

    Attributes:
        x (int): La coordenada x de la esquina superior izquierda del fondo.
        y (int): La coordenada y de la esquina superior izquierda del fondo.
        width (int): Ancho del fondo.
        height (int): Alto del fondo.
        path (str): Ruta del archivo de imagen para el fondo.
    '''
    def __init__(self, x, y,width, height,  path):
        '''
        Constructor de la clase. Inicializa las propiedades del fondo.
        '''
        self.image = pygame.image.load(path).convert()
        self.image = pygame.transform.scale(self.image,(width,height))
        self.rect = self.image.get_rect()
        self.rect.x = x
        self.rect.y = y

    def draw(self,screen):
        '''
        Dibuja el fondo en la pantalla especificada.
        '''
        screen.blit(self.image,self.rect)
        if(DEBUG):
            pygame.draw.rect(screen,color=ROJO,rect=self.collition_rect)


"""
enums.py

Contains the different types of objects for the application.
"""
# Import modules and libraries
from enum import Enum

class UserType(Enum):
    """
    Represents the different types of users in the application.
    """
    admin = "admin"
    manager = "manager"
    inspector = "inspector"
    maintenance_worker = "maintenance_worker"
    repair_tech = "repair_tech"
    user = "user" # Default value
    archived = "archived"

class FireClass(Enum):
    """
    Represents the common fire types an extinguisher can put out in the United States.
    """
    A = "A" # Ordinary solid combustibles
    B = "B" # Flammable liquids and gases
    C = "C" # Energized electrical equipment
    D = "D" # Combustible metals
    K = "K" # Oils and fats
    ABC = "ABC" # Multi-purpose dry chemical
    other = "other" # Default value

class TicketType(Enum):
    """
    Represents the type of ticket.
    """
    inspect = "inspect"
    damaged_ext = "damaged_ext"
    repair_ext = "repair_ext"
    damaged_box = "damaged_box"
    repair_box = "repair_box"
    other = "other" # Default value

class TicketState(Enum):
    """
    Represents the state of a ticket.
    """
    open = "open"
    in_progress = "in_progress"
    closed = "closed"
    archived = "archived"


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from icalendar import Calendar, Event
from datetime import *
from dateutil.parser import parse
from pytz import UTC  # timezone

cal_file = '/Users/gyyoon/Desktop/pentaa.ics'

MEETING_STR = '[회의]'
TEATIME_STR = '[Tea-Time]'
WORK_STR = '[업무]'
NONWORK_STR = '[비업무]'
LUNCH_STR = '[점심]'
CLUB_STR = '[동호회]'
WORKCLEAN_STR = '[업무정리]'
TALK_STR = '[면담]'
OUTWORK_STR = '[외근]'
BREAK_STR = '[휴식]'
TOGETHER_CTR = '[회식]'
SEATCLEAN_STR = '[자리정리]'
EVENT_STR = '[행사]'
LATE_SRT = '[지연출근]'

# Count global var
total_cnt = 0
work_cnt = 0
nonwork_cnt = 0
meeting_cnt = 0
lunch_cnt = 0
club_cnt = 0
workclean_cnt = 0
outwork_cnt = 0
unknown_cnt = 0
break_cnt = 0
together_cnt = 0
seatclean_cnt = 0
event_cnt = 0

# Duration global var
work_dur = timedelta(hours=0, minutes=0)
nonwork_dur = timedelta(hours=0, minutes=0)
meeting_dur = timedelta(hours=0, minutes=0)
lunch_dur = timedelta(hours=0, minutes=0)
club_dur = timedelta(hours=0, minutes=0)
workclean_dur = timedelta(hours=0, minutes=0)
outwork_dur = timedelta(hours=0, minutes=0)
break_dur = timedelta(hours=0, minutes=0)
together_dur = timedelta(hours=0, minutes=0)
seatclean_dur = timedelta(hours=0, minutes=0)
event_dur = timedelta(hours=0, minutes=0)
late_dur = timedelta(hours=0, minutes=0)

def process_work_event(comp):
    # Duration global var
    global work_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    work_dur += duration

def process_nonwork_event(comp):
    # Duration global var
    global nonwork_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    nonwork_dur += duration

def process_meeting_event(comp):
    # Duration global var
    global meeting_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    meeting_dur += duration

def process_lunch_event(comp):
    # Duration global var
    global lunch_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    lunch_dur += duration

def process_club_event(comp):
    # Duration global var
    global club_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    club_dur += duration

def process_workclean_event(comp):
    # Duration global var
    global workclean_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    workclean_dur += duration

def process_outwork_event(comp):
    # Duration global var
    global outwork_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    outwork_dur += duration

def process_break_event(comp):
    # Duration global var
    global break_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    break_dur += duration

def process_together_event(comp):
    # Duration global var
    global together_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    together_dur += duration

def process_seatclean_event(comp):
    # Duration global var
    global seatclean_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    seatclean_dur += duration

def process_event_event(comp):
    # Duration global var
    global event_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    event_dur += duration

def process_late_event(comp):
    global late_dur

    start_dt = comp.get('dtstart').dt
    end_dt = comp.get('dtend').dt
    duration = end_dt - start_dt

    late_dur += duration

def print_results():
    print('----------------------------------')

    print('total count : ' + str(total_cnt))
    print('meeting : ' + str(meeting_cnt))
    print('work : ' + str(work_cnt))
    print('nonwork : ' + str(nonwork_cnt))
    print('lunch : ' + str(lunch_cnt))
    print('unknown ' + str(unknown_cnt))

    print('----------------------------------')

    print('workdur : ' + str(work_dur))
    print('nonwork dur : ' + str(nonwork_dur))
    print('meeting dur : ' + str(meeting_dur))
    print('lunch dur : ' + str(lunch_dur))
    print('club dur : ' + str(club_dur))
    print('workclean dur : ' + str(workclean_dur))
    print('outwork dur : ' + str(outwork_dur))
    print('break dur : ' + str(break_dur))
    print('together dur : ' + str(together_dur))
    print('seatclean dur : ' + str(seatclean_dur))
    print('event dur : ' + str(event_dur))
    print('late dur : ' + str(late_dur))

    print('----------------------------------')

class EventProcessor:
    def __init__(self):
        print('Initialize')

    def calc_event_duration(self, comp):
        start_dt = comp.get('dtstart').dt
        end_dt = comp.get('dtend').dt
        duration = end_dt - start_dt

        return duration

def main():
    # Count global var
    global total_cnt
    global work_cnt
    global nonwork_cnt
    global meeting_cnt
    global lunch_cnt
    global club_cnt
    global workclean_cnt
    global outwork_cnt
    global break_cnt
    global together_cnt
    global seatclean_cnt
    global event_cnt
    global unknown_cnt
    global late_dur

    g = open(cal_file, 'rb')
    gcal = Calendar.from_ical(g.read())

    for component in gcal.walk():
        if component.name == "VEVENT":
            start_date = component.get('dtstart').dt
            end_date = component.get('dtend').dt

            # I don't know why, but same formatted strs have differecne types,
            # date or datetime, so I unified them to date type(datetime requires
            # timezone set...)
            if type(start_date) != type(date(2018, 4, 4)):
                start_date = start_date.date()

            if type(end_date) != type(date(2018, 4, 4)):
                end_date = end_date.date()

            # Maybe someday I might get inputs from user that specifies
            # the date range...
            if start_date >= date(2018, 1, 1) and end_date <= date(2018, 12, 31):
                total_cnt += 1

                event_summary = component.get('summary')

                if WORK_STR in event_summary:
                    work_cnt += 1
                    process_work_event(component)
                    print(event_summary)

                elif MEETING_STR in event_summary or TEATIME_STR in \
                        event_summary or TALK_STR in event_summary:
                    meeting_cnt += 1
                    process_meeting_event(component)

                elif NONWORK_STR in event_summary:
                    nonwork_cnt += 1
                    process_nonwork_event(component)

                elif LUNCH_STR in event_summary:
                    lunch_cnt += 1
                    process_lunch_event(component)

                elif CLUB_STR in event_summary:
                    club_cnt += 1
                    process_club_event(component)

                elif WORKCLEAN_STR in event_summary:
                    workclean_cnt += 1
                    process_workclean_event(component)

                elif OUTWORK_STR in event_summary:
                    outwork_cnt += 1
                    process_outwork_event(component)

                elif BREAK_STR in event_summary:
                    break_cnt += 1
                    process_break_event(component)

                elif TOGETHER_CTR in event_summary:
                    together_cnt += 1
                    process_together_event(component)

                elif SEATCLEAN_STR in event_summary:
                    seatclean_cnt += 1
                    process_seatclean_event(component)

                elif EVENT_STR in event_summary:
                    event_cnt += 1
                    process_event_event(component)
                elif LATE_SRT in event_summary:
                    event_cnt += 1
                    process_late_event(component)
                else:
                    unknown_cnt += 1
                    # print(event_summary)
                    # print(component.get('dtstart').dt)
                    # print(component.get('dtend').dt)
                    # print(component.get('dtstamp').dt)
    g.close()

    print_results()

if __name__ == "__main__":
    main()


class Solution:
    def maxSlidingWindow(self, nums: List[int], k: int) -> List[int]:
        output = []
        q = collections.deque()  # index
        l = r = 0
        # O(n) O(n)
        while r < len(nums):
            # pop smaller values from q
            while q and nums[q[-1]] < nums[r]:
                q.pop()
            q.append(r)

            # remove left val from window
            if l > q[0]:
                q.popleft()

            if (r + 1) >= k:
                output.append(nums[q[0]])
                l += 1
            r += 1

        return output


import os
from datetime import datetime, timedelta
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
import googleapiclient.discovery
import googleapiclient.errors
import httplib2.error
import youtube_dl
import logging

class LiveBroadcast():
    def __init__(self, broadcast_id, title, channel_id, channel_name="", m3u8_url=None, protocol="m3u8", mine=False):
        self.id = broadcast_id
        self.title = title
        self.m3u8_url = m3u8_url
        self.url = f"https://www.youtube.com/watch?v={broadcast_id}"
        self.channel_id = channel_id
        self.channel_url = f"https://www.youtube.com/channel/{channel_id}"
        self.channel_name = channel_name
        self.protocol = protocol
        self.mine = mine

class GoogleApis:
    class NetworkException(Exception):
        pass

    class HttpException(Exception):
        pass

    class AuthException(Exception):
        pass

    def __init__(self, api_name, api_version, scopes):
        self.api_name = api_name
        self.api_version = api_version
        self.scopes = scopes
        self.service = None

    def is_authorized(self):
        return self.service is not None

    def get_credentials(self, token_file, client_secrets_file, force_new=False):
        creds = None

        # Get previous credentials from file
        if os.path.exists(token_file):
            if not force_new:
                creds = Credentials.from_authorized_user_file(token_file, self.scopes)
            else:
                creds = None
        # If the credentials don't exist, do oauth
        if not creds or not creds.valid:
            if creds and creds.expired and creds.refresh_token:
                creds.refresh(Request())
            else:
                flow = InstalledAppFlow.from_client_secrets_file(client_secrets_file, self.scopes)
                creds = flow.run_console()
            with open(token_file, "w") as token:
                token.write(creds.to_json())

        return creds

    def auth_key(self, api_key):
        self.service = googleapiclient.discovery.build(self.api_name, self.api_version, developerKey=api_key)

    def auth_oauth(self, token_file, client_secrets_file, force_new=False):
        credentials = self.get_credentials(token_file, client_secrets_file, force_new)
        self.service = googleapiclient.discovery.build(self.api_name, self.api_version, credentials=credentials)

class YoutubeApis(GoogleApis):

    def __init__(self):
        super().__init__("youtube", "v3", ["https://www.googleapis.com/auth/youtube.force-ssl"])

    # Not recommended to use: costs 100 quota units and takes ~5 minutes to detect newly started broadcasts
    def search_livebroadcasts_ytapi(self, channel_id):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        request = self.service.search().list(part="snippet", eventType="live", type="video", channelId=channel_id)
        livestreams = []
        try:
            res = request.execute()
            items = res.get("items", [])
            for item in items:
                single_stream = LiveBroadcast(
                    item.get("id").get("videoId"),
                    item.get("snippet").get("title"),
                    channel_id
                )

                livestreams.append(single_stream)
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))

        return livestreams

    def search_livebroadcasts(self, channel_id):
        channel_url = f"https://www.youtube.com/channel/{channel_id}"
        options = {
            "playlistend": 1, # only the first item
            "quiet": True
        }
        livestreams = []
        with youtube_dl.YoutubeDL(options) as yt_dl:
            try:
                res = yt_dl.extract_info(channel_url, download=False)
                res_item = res["entries"][0]["entries"][0]

                if res_item["protocol"] == "m3u8":
                    single_stream = LiveBroadcast(
                        res_item["id"],
                        res_item["title"],
                        channel_id,
                        channel_name=res_item["channel"],
                        m3u8_url=res_item["url"]
                    )
                    livestreams.append(single_stream)
            except youtube_dl.utils.DownloadError as e:
                raise GoogleApis.NetworkException(f"youtube-dl failed to search live broadcasts: {str(e)}")
            except (IndexError, KeyError):
                pass # no livestreams found
        return livestreams

    def parse_livestream_res(self, res):
        ingestion_info = res.get("cdn").get("ingestionInfo")
        res_data = {
            "id": res.get("id", ""), # ex 'AniW-ozy_koWoLjDw3F2Rg1618885401806773'
            "rtmp_url": ingestion_info.get("ingestionAddress", ""),
            "rtmp_key": ingestion_info.get("streamName", "")
        }
        return res_data

    def list_videos(self, video_id):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        request = self.service.videos().list(
            part="contentDetails,id,snippet,status",
            id=video_id
        )
        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))

        return res.get("items")[0]

    # Creates the RTMP ingestion point that can be reused for every stream
    def insert_livestream(self, title, fps="variable", resolution="variable"):
        # fps can be "30fps", "60fps"
        # resolution "1080p", "720p", "480p", etc
        # both can be set to "variable" for automatic detection
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        request = self.service.liveStreams().insert(
            # part="snippet,cdn,id,status",
            part = "id,cdn",
            body={
                "cdn": {
                    "ingestionType": "rtmp",
                    "resolution": resolution,
                    "frameRate": fps
                },
                "snippet": {
                    "title": title
                }
            }
        )

        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))

        return self.parse_livestream_res(res)

    def create_variable_livestream(self, title):
        livestreams = self.list_livestream()
        variable_stream = None
        for livestream in livestreams:
            if livestream.get("cdn").get("resolution") == "variable":
                variable_stream = livestream
                break

        # Seems like YT will always create a default variable stream if deleted
        variable_stream_data = None
        if variable_stream is None:
            logging.info("Variable livestream not found, creating new one")
            variable_stream_data = self.insert_livestream(title)
        else:
            variable_stream_data = self.parse_livestream_res(variable_stream)

        return variable_stream_data

    def list_livestream(self):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        request = self.service.liveStreams().list(
            part="id,cdn,snippet,status",
            mine=True
        )

        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))

        return res.get("items", [])

    # Creates the actual stream video instance that viewers see
    def insert_broadcast(self, title, description=None, archive=True, privacy="public"):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        # Privacy may be: "public", "private", "unlisted"
        broadcast_date = datetime.utcnow()
        #broadcast_date += timedelta(minutes=1)

        request = self.service.liveBroadcasts().insert(
            part="id,snippet,contentDetails,status",
            body={
                "contentDetails": {
                    "enableDvr": archive,
                    "enableAutoStart": True,
                    "enableAutoStop": False
                },
                "snippet": {
                    "scheduledStartTime": broadcast_date.isoformat(),
                    "title": title,
                    "description": description
                },
                "status": {
                    "privacyStatus": privacy
                }
            }
        )
        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))

        res_data = {
            "id": res.get("id", "") # ex '1b9GoutrU7k'
        }

        return res_data

    def list_broadcast(self):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        # acceptable status values: complete, live, testing
        request =  self.service.liveBroadcasts().list(
            part="id,snippet,contentDetails,status",
            mine=True
        )
        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))
        return res.get("items", [])

    def transition_broadcast(self, broadcast_id, status):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        # acceptable status values: complete, live, testing
        request = self.service.liveBroadcasts().transition(
            broadcastStatus=status,
            id=broadcast_id,
            part="id"
        )
        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))
        return res

    def bind_broadcast(self, broadcast_id, stream_id):
        if not self.is_authorized():
            raise GoogleApis.AuthException("Requires OAuth")
        request = self.service.liveBroadcasts().bind(
            id=broadcast_id,
            part="id,snippet,contentDetails,status",
            streamId=stream_id
        )
        res = None
        try:
            res = request.execute()
        except googleapiclient.errors.HttpError as e:
            raise GoogleApis.HttpException(str(e))
        except httplib2.error.ServerNotFoundError as e:
            raise GoogleApis.NetworkException(str(e))
        return res

    def create_rtmp_broadcast(self, title, description, privacy):
        # First, check if a stream exists
        stream_data = self.create_variable_livestream("Variable stream")
        broadcast_data = self.insert_broadcast(title, description, privacy=privacy)
        data = {
            "video_id": broadcast_data["id"],
            "rtmp_url": stream_data["rtmp_url"],
            "rtmp_key": stream_data["rtmp_key"]
        }
        self.bind_broadcast(data["video_id"], stream_data["id"])
        return data

    # TODO support other quality levels?
    # TODO distinguish between net and param exceptions
    def get_stream_m3u8_url(self, video_url):
        options = {
            "noplaylist": True,
        }
        playlist_url = None
        with youtube_dl.YoutubeDL(options) as yt_dl:
            try:
                res = yt_dl.extract_info(video_url, download=False)
                playlist_url = res["url"]
            except youtube_dl.utils.DownloadError as e:
                raise GoogleApis.NetworkException(f"youtube-dl failed to download m3u8: {str(e)}")
        return playlist_url


from collections import defaultdict
import os

import pandas as pd

path = '/home/djf/djf/POI/CDRF/data/Foursquare_NYC.txt'

dic = defaultdict(int)

f = open(path, 'r')
lines = f.readlines()

for line in lines:
    user, t, lat, lon, POI = line.strip().split('\t')
    dic[int(POI)] += 1

counts = [item[1] for item in dic.items()]

counts = sorted(counts)

with open('counts.txt', 'w') as f:
    for item in counts[:-10]:
        f.write('{}\n'.format(item))


from django.contrib import admin

from .models import InstrumentItem
# Register your models here.

class InstrumentItemAdmin(admin.ModelAdmin):
    search_fields = ['definition']
    list_filter = ['instrument']
    list_display = ['definition','instrument','discrimination','difficulty','guessing','upper_asymptote']
admin.site.register(InstrumentItem, InstrumentItemAdmin)


#!/usr/bin/env python3
import argparse
from pprint import pprint
import json

import hlib

def check_temp(cl, device_id):

    device_name = None

    # first, try and find the device as a device
    url = f"/clip/v2/resource/device/{device_id}"

    resp = cl.get(url)
    if resp.status_code == 200:
        data = resp.json()
        device = data['data'][0]

        # see if the device has a temperature device
        temp_id = None
        services = device.get('services', [])
        for service in services:
            if service['rtype'] == 'temperature':
                temp_id = service['rid']
                break

        if temp_id is None:
            print("Device has no temperature sensor")
            for service in services:
                print(f"- {service['rtype']}")
            return None, None

        device_name = device['metadata']['name']
        device_id = temp_id

    # now get the temperature
    url = f"/clip/v2/resource/temperature/{device_id}"

    resp = cl.get(url)
    if resp.status_code != 200:
        print(f"Request failed with {resp.status_code} {resp.reason}")
        return None

    data = resp.json()
    device = data['data'][0]

    temp = device['temperature']['temperature']
    return float(temp), device_name

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('device_id', help='id of the device to query', type=str, nargs='+')
    args = parser.parse_args()

    bridge = hlib.find_bridge()
    if bridge is None:
        print("Error: failed to locate a bridge")
        return

    cfg = hlib.load_config()
    cl = hlib.new_client(bridge, cfg['user_name'])

    print("Temperatures:")
    for idx, device_id in enumerate(args.device_id):
        temp, device_name = check_temp(cl, device_id)
        if temp is None:
            continue

        if device_name is None:
            print(f"{idx:02d} {temp}")
        else:
            print(f"{idx:02d} {temp} at {device_name}")

    # check for pushover configuratin
    # token = cfg.get('pushover_token', None)
    # clients = cfg.get('pushover_clients', None)
    # if token and clients:
    #     messages = []
    #     if len(red):
    #         messages.append("RED: incorrectly configured lights")
    #         for light_id, light_name in red.items():
    #             messages.append(f"  - {light_id} {light_name}")
    #
    #     if len(green):
    #         messages.append("GREEN: correctly configured lights")
    #         for light_id, light_name in green.items():
    #             messages.append(f"  - {light_id} {light_name}")
    #
    #     if len(red):
    #         title = "RED: incorrectly configured lights"
    #     else:
    #         title = "GREEN: correctly configured lights"
    #
    #     message = "\n".join(messages)
    #     hlib.send_message(token, clients, message, title)

if __name__ == "__main__":
    main()


# coding: utf-8

import xlrd

def getProgramList(filepath):
	program_list = dict()
	program_list[u'网络剧'] = []
	program_list[u'网络电影'] = []
	program_list[u'网络综艺'] = []

	data = xlrd.open_workbook(filepath)

	table = data.sheet_by_name(u'网络剧')
	nrows = table.nrows
	ncols = table.ncols
	for i in range(2, nrows):
		one_program = dict()
		one_program[u'节目名称'] = table.cell(i, 1).value
		one_program[u'播出网站'] = table.cell(i, 3).value
		program_list[u'网络剧'].append(one_program)

	table = data.sheet_by_name(u'网络电影')
	nrows = table.nrows
	ncols = table.ncols
	for i in range(2, nrows):
		one_program = dict()
		one_program[u'节目名称'] = table.cell(i, 1).value
		one_program[u'播出网站'] = table.cell(i, 3).value
		program_list[u'网络电影'].append(one_program)

	table = data.sheet_by_name(u'网络综艺')
	nrows = table.nrows
	ncols = table.ncols
	for i in range(2, nrows):
		one_program = dict()
		one_program[u'节目名称'] = table.cell(i, 1).value
		one_program[u'播出网站'] = table.cell(i, 3).value
		program_list[u'网络综艺'].append(one_program)

	return program_list

def get_next_keyword(filepath, idx):
	data = xlrd.open_workbook(filepath)
	table = data.sheet_by_name('Sheet1')
	nrows = table.nrows
	ncols = table.ncols

	return table.cell(idx, 0).value

if __name__ == "__main__":
	program_list = getProgramList(u"征片情况汇总0213.xlsx")
	print(program_list)


import cv2
import numpy as np
import csv
import math

img = cv2.imread(".//new_dataset//250.jpg")
img = cv2.GaussianBlur(img , (5 , 5) , 0)

n = 64
div = 256//n #n is the number of bins, here n = 64
rgb = cv2.split(img)
q = []
for ch in rgb:
    vf = np.vectorize(lambda x, div: int(x//div)*div)
    quantized = vf(ch, div)
    q.append(quantized.astype(np.uint8))
img = cv2.merge(q)

row , col , channels = img.shape

connectivity = 8
tau = 0
if tau == 0:
    tau = row*col*0.1

rgb = cv2.split(img)
q = []
for ch in rgb:
    vf = np.vectorize(lambda x, div: int(x//div)*div)
    quantized = vf(ch, div)
    q.append(quantized.astype(np.uint8))
img = cv2.merge(q)
bgr = cv2.split(img)
total = []
for ch in bgr:
    for k in range(0 , 256 , div):
        temp = ch.copy()
        temp = (temp == k).astype(np.uint8)
        output = cv2.connectedComponentsWithStats(temp , connectivity , cv2.CV_32S)
        num_labels = output[0]
        labels = output[0]
        stats = output[2]
        centroids = output[3]
        alpha = 0
        beta = 0
        req = stats[1:]
        for r in req:
            if(r[4] >= tau):
                alpha += r[4]
            else:
                beta += r[4]
        total.append(alpha)
        total.append(beta)

dist = []
name = []
with open('ccv_feat.csv' , 'r') as csvFile:
    reader = csv.reader(csvFile)
    i = 0
    for row in reader:
        distance = math.sqrt(sum([(float(a) - float(b)) ** 2 for a , b in zip(row , total)]))
        dist.append(distance)
        nam = str(i)+".jpg"
        name.append(nam)
        i += 1

di = {}
for i in range(len(name)):
    di[name[i]] = dist[i]
sorted_di = sorted(di.items() , key = lambda kv : kv[1])

pic = []
t = 0
first_key = list(sorted_di)[:10]
for key , val in first_key:
    key = ".//new_dataset//"+key
    res = cv2.imread(key)
    na = str(t)
    cv2.imshow(na , res)
    t += 1


__docformat__ = "reStructuredText"
__author__ = "davidh"

import logging
import os
import re
import sys
from collections import namedtuple
from datetime import datetime, timedelta
from glob import glob

LOG = logging.getLogger(__name__)

FILENAME_RE = r"HS_H08_(?P<date>\d{8})_(?P<time>\d{4})_(?P<band>B\d{2})_FLDK_(?P<res>R\d+)\.(?P<ext>.+)"
fn_re = re.compile(FILENAME_RE)

DT_FORMAT = "%Y%m%d_%H%M"
CASE_NAME_FORMAT = "{start}_{end}_{delta:02d}"

DataCase = namedtuple("DataCase", ["topic_title", "start", "end", "delta", "bands"])

### Guam Cases ###
guam_cases: dict = {}
# Kathy's Cases
guam_cases["Introduction"] = []
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 17, 21, 0, 0), datetime(2015, 7, 18, 20, 0, 0), timedelta(minutes=60), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 18, 1, 0, 0), datetime(2015, 7, 18, 3, 20, 0), timedelta(minutes=10), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2015, 7, 18, 14, 0, 0), datetime(2015, 7, 18, 16, 0, 0), timedelta(minutes=10), "all"
    )
)
guam_cases["Introduction"].append(
    DataCase("Introduction", datetime(2016, 3, 9, 0, 0, 0), datetime(2016, 3, 9, 4, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Introduction"].append(
    DataCase(
        "Introduction", datetime(2016, 3, 9, 1, 30, 0), datetime(2016, 3, 9, 4, 0, 0), timedelta(minutes=10), "all"
    )
)

# Scott's Cases
guam_cases["Water Vapor"] = []
guam_cases["Water Vapor"].append(
    DataCase(
        "Water Vapor", datetime(2015, 10, 7, 0, 0, 0), datetime(2015, 10, 8, 0, 0, 0), timedelta(minutes=30), "all"
    )
)
guam_cases["Water Vapor"].append(
    DataCase(
        "Water Vapor", datetime(2016, 2, 19, 19, 0, 0), datetime(2016, 2, 20, 5, 0, 0), timedelta(minutes=60), "all"
    )
)

# Tim's Cases
guam_cases["Weighting Functions"] = []
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 2, 30, 0),
        datetime(2015, 9, 20, 2, 30, 0),
        timedelta(minutes=0),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 0, 0, 0),
        datetime(2015, 9, 20, 6, 0, 0),
        timedelta(minutes=60),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 1, 30, 0),
        datetime(2015, 9, 20, 2, 30, 0),
        timedelta(minutes=10),
        "all",
    )
)
guam_cases["Weighting Functions"].append(
    DataCase(
        "Weighting Functions",
        datetime(2015, 9, 20, 1, 0, 0),
        datetime(2015, 9, 20, 3, 0, 0),
        timedelta(minutes=10),
        "all",
    )
)

# Jordan's Cases
guam_cases["Extra"] = []
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 17, 12, 0, 0), datetime(2015, 8, 18, 12, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 17, 22, 0, 0), datetime(2015, 8, 18, 1, 0, 0), timedelta(minutes=10), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 24, 15, 0, 0), datetime(2015, 8, 15, 21, 0, 0), timedelta(minutes=60), "all")
)
guam_cases["Extra"].append(
    DataCase("Extra", datetime(2015, 8, 25, 2, 0, 0), datetime(2015, 8, 25, 5, 0, 0), timedelta(minutes=10), "all")
)

def main():
    import argparse

    parser = argparse.ArgumentParser(description="Regenerate or generate mirrored AHI data structure")
    parser.add_argument(
        "base_ahi_dir",
        default="/odyssey/isis/tmp/davidh/sift_data/ahi",
        help="Base AHI directory for the geotiff data files " "(next child directory is the full dated directory)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        dest="verbosity",
        action="count",
        default=int(os.environ.get("VERBOSITY", 2)),
        help="each occurrence increases verbosity 1 level through " "ERROR-WARNING-Info-DEBUG (default Info)",
    )
    parser.add_argument("--overwrite", action="store_true", help="Overwrite existing hardlinks")
    args = parser.parse_args()

    levels = [logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG]
    level = levels[min(3, args.verbosity)]
    logging.basicConfig(level=level)

    if not os.path.isdir(args.base_ahi_dir):
        raise NotADirectoryError("Directory does not exist: %s" % (args.base_ahi_dir,))

    os.chdir(args.base_ahi_dir)

    for section_name, cases in guam_cases.items():
        for case in cases:
            start_str = case.start.strftime(DT_FORMAT)
            end_str = case.end.strftime(DT_FORMAT)
            # Note this only uses the minutes!
            case_name = CASE_NAME_FORMAT.format(
                start=start_str, end=end_str, delta=int(case.delta.total_seconds() / 60.0)
            )
            case_dir = os.path.join(args.base_ahi_dir, section_name, case_name)
            if not os.path.isdir(case_dir):
                LOG.debug("Creating case directory: %s", case_dir)
                os.makedirs(case_dir)
            else:
                LOG.error("Case directory already exists: %s", case_dir)
                continue

            t = case.start
            while t <= case.end:
                glob_pattern = t.strftime("%Y_%m_%d_%j/%H%M/*_%Y%m%d_%H%M_B??_*.merc.tif")
                t = t + case.delta

                matches = glob(glob_pattern)
                if len(matches) == 0:
                    LOG.error("Zero files found matching pattern: %s", glob_pattern)
                    continue
                for input_pathname in matches:
                    fn = os.path.basename(input_pathname)
                    link_path = os.path.join(case_dir, fn)
                    if os.path.exists(link_path) and not args.overwrite:
                        LOG.debug("Link '%s' already exists, skipping...", link_path)
                        continue
                    LOG.debug("Creating hardlink '%s' -> '%s'", link_path, input_pathname)
                    os.link(input_pathname, link_path)
                if int(case.delta.total_seconds()) == 0:
                    LOG.debug("Only one file needed to meet delta of 0")
                    break
            LOG.info("done mirroring files")

if __name__ == "__main__":
    sys.exit(main())


import os

import pandas as pd
from sklearn.model_selection import KFold, train_test_split

root_path = os.path.dirname(__file__)
asset_path = os.path.join(root_path, '../assets')

# load the full titanic example data
data = pd.read_csv(os.path.join(root_path, '../data/train.csv'))

# train / test split
train_data, test_data = train_test_split(data, test_size=0.2)

# number of data samples for the train and test sets
N_TRAIN_DATA_SAMPLES = 10
N_TEST_DATA_SAMPLES = 2

train_test_configs = [
    {
        'data': train_data,
        'n_samples': N_TRAIN_DATA_SAMPLES,
        'data_samples_root': os.path.join(asset_path, 'train_data_samples'),
        'data_samples_content': [],
    },
    {
        'data': test_data,
        'n_samples': N_TEST_DATA_SAMPLES,
        'data_samples_root': os.path.join(asset_path, 'test_data_samples'),
        'data_samples_content': [],
    },
]

# generate data samples
for conf in train_test_configs:
    kf = KFold(n_splits=conf['n_samples'], shuffle=True)
    splits = kf.split(conf['data'])
    for _, index in splits:
        conf['data_samples_content'].append(conf['data'].iloc[index])

# save data samples
for conf in train_test_configs:
    for i, data_sample in enumerate(conf['data_samples_content']):
        filename = os.path.join(conf['data_samples_root'], f'data_sample_{i}/data_sample_{i}.csv')
        os.makedirs(os.path.dirname(filename))
        with open(filename, 'w') as f:
            data_sample.to_csv(f)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
try:
  import petsclinter as pl
except ModuleNotFoundError as mnfe:
  try:
    petsc_dir = os.environ['PETSC_DIR']
  except KeyError as ke:
    raise RuntimeError('Must set PETSC_DIR environment variable') from ke
  sys.path.insert(0, os.path.join(petsc_dir, 'lib', 'petsc', 'bin', 'maint', 'petsclinter'))
  import petsclinter as pl

def __prepare_ns_args(ns_args, parser):
  slepc_mansecs     = ['eps','lme','mfn','nep','pep','svd','sys']
  slepc_aux_mansecs = ['bv','ds','fn','rg','st']

  if ns_args.slepc_dir is None:
    raise RuntimeError('Could not determine SLEPC_DIR from environment, please set via options')

  extra_compiler_flags = [
    '-I' + os.path.join(ns_args.slepc_dir, 'include'),
    '-I' + os.path.join(ns_args.slepc_dir, ns_args.petsc_arch, 'include')
  ]
  with open(os.path.join(ns_args.slepc_dir, ns_args.petsc_arch, 'lib', 'slepc', 'conf', 'slepcvariables'), 'r') as sv:
    line = sv.readline()
    while line:
      if 'INCLUDE' in line:
        for inc in line.split('=', 1)[1].split():
          extra_compiler_flags.append(inc)
      line = sv.readline()

  extra_header_includes = []
  mansecimpls           = [m + 'impl.h' for m in slepc_mansecs + slepc_aux_mansecs] + [
    'slepcimpl.h', 'vecimplslepc.h'
  ]
  for header_file in os.listdir(os.path.join(ns_args.slepc_dir, 'include', 'slepc', 'private')):
    if header_file in mansecimpls:
      extra_header_includes.append(f'#include <slepc/private/{header_file}>')

  if ns_args.src_path == parser.get_default('src_path'):
    ns_args.src_path = os.path.join(ns_args.slepc_dir, 'src')
  if ns_args.patch_dir == parser.get_default('patch_dir'):
    ns_args.patch_dir = os.path.join(ns_args.slepc_dir, 'slepcLintPatches')

  # prepend these
  ns_args.extra_compiler_flags = extra_compiler_flags + ns_args.extra_compiler_flags
  # replace these
  if not ns_args.extra_header_includes:
    ns_args.extra_header_includes = extra_header_includes

  return ns_args

def command_line_main():
  import argparse
  import petsclinter.main

  slepc_classid_map = {
    '_p_BV *'  : 'BV_CLASSID',
    '_p_DS *'  : 'DS_CLASSID',
    '_p_FN *'  : 'FN_CLASSID',
    '_p_RG *'  : 'RG_CLASSID',
    '_p_ST *'  : 'ST_CLASSID',
    '_p_EPS *' : 'EPS_CLASSID',
    '_p_PEP *' : 'PEP_CLASSID',
    '_p_NEP *' : 'NEP_CLASSID',
    '_p_SVD *' : 'SVD_CLASSID',
    '_p_MFN *' : 'MFN_CLASSID',
    '_p_LME *' : 'LME_CLASSID',
  }

  for struct_name, classid_name in slepc_classid_map.items():
    pl.checks.register_classid(struct_name, classid_name)

  parser      = argparse.ArgumentParser(prog='slepclinter', add_help=False)
  group_slepc = parser.add_argument_group(title='SLEPc location settings')
  group_slepc.add_argument('--SLEPC_DIR', required=False, default=os.environ.get('SLEPC_DIR', None), help='if this option is unused defaults to environment variable $SLEPC_DIR', dest='slepc_dir')

  args, parser = pl.main.parse_command_line_args(parent_parsers=[parser])
  args         = __prepare_ns_args(args, parser)

  return pl.main.namespace_main(args)

if __name__ == '__main__':
  sys.exit(command_line_main())


# %%
from pprint import pprint

from datasets import load_dataset

test_file = "../data/ekonspacing/test_small.txt"
val_file = "../data/ekonspacing/val_small.txt"
train_file = "../data/ekonspacing/train_small.txt"

# %%
dataset = load_dataset(
    "ekonspacing.py",
    name="small",
    data_files={"train": str(train_file), "validation": str(val_file), "test": str(test_file)},
    download_mode="force_redownload",
)
print(dataset)
# %%
pprint(dataset["train"][0])
pprint(dataset["test"][0])
# %%


import logging

from .html import extract_html_text
from .pdf import extract_pdf_text

logger = logging.getLogger(__name__)

def extract_text(file_path: str) -> str:
    """Extract text from any kind of file as long as it's html or pdf"""
    try:
        if file_path.endswith('.html'):
            return extract_html_text(file_path)

        if file_path.endswith('.pdf'):
            return extract_pdf_text(file_path)

        raise ValueError(f'Unknown file type {file_path}')
    except Exception as an_exception:  # pylint: disable=W0718
        logger.warning(f'Failed to extract {file_path}: {an_exception}')
        return '' # don't return None or else you can't pipeline this


import cv2
import math

# Source: https://richardpricejones.medium.com/drawing-a-rectangle-with-a-angle-using-opencv-c9284eae3380
# Made slight adjustments to color
def draw_angled_rec(x0, y0, width, height, angle, img, color):

    _angle = angle * math.pi / 180.0
    b = math.cos(_angle) * 0.5
    a = math.sin(_angle) * 0.5
    pt0 = (int(x0 - a * height - b * width),
           int(y0 + b * height - a * width))
    pt1 = (int(x0 + a * height - b * width),
           int(y0 - b * height - a * width))
    pt2 = (int(2 * x0 - pt0[0]), int(2 * y0 - pt0[1]))
    pt3 = (int(2 * x0 - pt1[0]), int(2 * y0 - pt1[1]))

    if color == 'green':
        cv2.line(img, pt0, pt1, (0, 255, 0), 5)
        cv2.line(img, pt1, pt2, (0, 255, 0), 5)
        cv2.line(img, pt2, pt3, (0, 255, 0), 5)
        cv2.line(img, pt3, pt0, (0, 255, 0), 5)
    elif color == 'red':
        cv2.line(img, pt0, pt1, (0, 0, 255), 5)
        cv2.line(img, pt1, pt2, (0, 0, 255), 5)
        cv2.line(img, pt2, pt3, (0, 0, 255), 5)
        cv2.line(img, pt3, pt0, (0, 0, 255), 5)
    else:
        cv2.line(img, pt0, pt1, (255, 0, 255), 5)
        cv2.line(img, pt1, pt2, (255, 0, 255), 5)
        cv2.line(img, pt2, pt3, (255, 0, 255), 5)
        cv2.line(img, pt3, pt0, (255, 0, 255), 5)

img = cv2.imread('minAreaRect_Test.png')
img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(img_gray, 150, 255, cv2.THRESH_BINARY)
# detect the contours on the binary image using cv2.CHAIN_APPROX_NONE
contours, hierarchy = cv2.findContours(image=thresh, mode=cv2.RETR_TREE, method=cv2.CHAIN_APPROX_NONE)
slender_rat = 3
min_width = 10
max_width = 120
min_len = 120
first = 1
image_copy = img.copy()
cv2.imwrite('minAreaRect_Test_Result.png',image_copy)
# cv2.drawContours(image=image_copy, contours=contours, contourIdx=-1, color=(0, 255, 0), thickness=2, lineType=cv2.LINE_AA)
# Used to add text on pattern
counter = 0
for cnt in contours:
    if counter == 0:        # First contour encompasses entire image
        counter += 1
        continue

    heir = hierarchy[0][counter][3]                         # [next, previous, first child, parent].
    if heir == 0:
        rect = cv2.minAreaRect(cnt)
        x = int(rect[0][0])
        y = int(rect[0][1])
        w = int(rect[1][0])
        h = int(rect[1][1])
        theta = int(rect[2])
        draw_angled_rec(x, y, w, h, theta, image_copy, 'green')
        image_tmp = cv2.putText(img=image_copy, text=str(theta)+'[deg]', org=(x, y), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        image_tmp = cv2.putText(img=image_copy, text='w='+str(w), org=(x, y+100), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        image_tmp = cv2.putText(img=image_copy, text='h='+str(h), org=(x, y+200), fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=3, color=(0,0,0), thickness=5)
        cv2.imwrite('minAreaRect_Test_Result.png',image_copy)
    counter += 1


from sys import maxsize as maxint

class Solution:
    def minSubArrayLen(self, s, nums):
        cs = 0
        start = 0
        min_len = maxint
        for end in range(len(nums)):
            cs += nums[end]
            while cs >= s and start <= end:
                min_len = min(min_len, end - start + 1)
                if start == end:
                    break
                cs -= nums[start]
                start += 1
        return min_len if min_len != maxint else 0


#imports
import numpy as np
import matplotlib.pyplot as plt
import scipy.constants as const
from scipy.special import iv as I0
from scipy.special import kv as K0

#Define Global Variables
L_geo = 55.6e-9
Z0 = 50.0
F0_base = 0.95e9        #At lowest Temp
squares= 27223
c_couple = 1.5e-14
TC = 1.5
Delta_0 = (3.5*const.Boltzmann*TC)/2
sigma_n = 6.0e7          # Normal stae conductvity if superconducting film
Thick = 20e-9            # Thickness of superconducting fil
w = 2 * np.pi * F0_base
me = const.m_e
miu_0 = 4*np.pi*10**-7
pi = np.pi

#Main code
def main():
    #Define temperature range with step 0.01K
    step = 0.1
    temp = np.arange(0.2, 0.3   , step)

    #Find sigma1 and sigma 2 and Lint
    sigma1, sigma2 = find_sigma1_sigma2(sigma_n ,Thick, TC, Delta_0, w, temp)
    Lint = find_Lint_square(Thick, w, sigma2) * squares

    #Find lk
    Lk = find_lk(Thick, w, sigma2)

    #Find Res
    sigma12Ratio = sigma1/sigma2
    Res = Lk*w*sigma12Ratio *squares

    #IDC for Lowest Temp (0.2K)
    Ltot_lowest = Lint[0] + L_geo
    IDC = find_IDC(w, Ltot_lowest, c_couple)

    #Find S21
    Sweep_points = 20000
    BW = 5e6
    I_raw = np.zeros((Sweep_points, len(temp)), dtype="float")
    Q_raw = np.copy(I_raw)
    Phase = np.copy(Q_raw)
    S21_Volt = np.copy(I_raw)
    for i in range(0, len(Lint)):
        Sweep, S21_Volt[:,i], Phase[:,i], I_raw[:,i], Q_raw[:,i],_,_,_,_,_ = Capacitive_Res_Sim(F0_base, c_couple, Z0, L_geo, Lint[i], Res[i], BW, Sweep_points, IDC)
        plt.plot(Sweep/1e9, S21_Volt[:,i], label=str("{:.2f}".format(temp[i])))

    #Graph labels and title
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True, title="Temperature / K")
    plt.xlabel('Frequency / GHz', fontsize=13)

    plt.ylabel('Output Amplitude', fontsize=13)
    #plt.title("S21 Amplitude For Varying Temperatures")
    plt.xlim(0.9490, 0.9505)
    plt.locator_params(nbins=6)
    plt.savefig("S21 Plot with Resistance")
    plt.rcParams['figure.dpi'] = 300
    plt.figure()

    #Q vs I plots
    for i in range(0, len(Lint)):
        plt.plot(I_raw[:,i], Q_raw[:,i], linewidth=1,label=str("{:.2f}".format(temp[i])))

    #Minimum S21 at lowest temp
    S21_Base = min(S21_Volt[:,0])
    I_Base = np.zeros(len(temp), dtype="float")
    Q_Base = np.copy(I_Base)

    #Obtain F0_base and I and Q values for Lowest Temp
    for i in range(0, len(S21_Volt[:,0])):
        if S21_Base == S21_Volt[i,0]:
            F0_Base = Sweep[i]

    #Plot I and Q values at F0_Base
    for i in range(0, len(temp)):
        for j in range(0, len(Sweep)):
            if F0_Base == Sweep[j]:
                I_Base[i] = I_raw[j,i]
                Q_Base[i] = Q_raw[j,i]
                plt.plot(I_Base[i], Q_Base[i], markersize=4, marker="x", color='black')

    #labels
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True, title="Temperature / K")
    plt.xlabel('I / V', fontsize=13)
    plt.ylabel('Q / V', fontsize=13)
    #plt.title("Q vs I Plot for Varying Temperature")
    plt.savefig("Q vs I plot for varying temp")
    plt.figure()

    #Finding F0 for the different Temperatures
    F0 = np.zeros(len(temp))
    for i in range(0, len(temp)):
        S21_min = min(S21_Volt[:,i])
        for j in range(0, len(Sweep)):
            if S21_min == S21_Volt[j,i]:
                F0[i] = Sweep[j]

    #Plotting F0 vs Temp
    plt.plot(temp, F0/1e9, color='k', linewidth="1", label="Minimum Of S21")
    plt.xlabel('Temperature / K', fontsize=13)
    plt.ylabel('F0 / GHz', fontsize=13)
    plt.rcParams['figure.dpi'] = 300
    #plt.title("F0 vs Temperature")

    #Finding dI/dF and dQ/dF for lowest temperature
    #Using numerical derivatives
    step = abs((Sweep[0]-Sweep[-1])/Sweep_points)
    for i in range(0, len(Sweep)):
        if Sweep[i] == F0_Base:
            didf = (I_raw[i+1,0] - I_raw[i-1,0])/(2*step)
            dqdf = (Q_raw[i+1,0] - Q_raw[i-1,0])/(2*step)

    #Use Magic Formula
    di = np.zeros(len(temp))
    dq = np.copy(di)
    di = abs(I_Base - I_Base[0])
    dq = abs(Q_Base - Q_Base[0])
    dF0 = Magic_Formula(di, dq, didf, dqdf)

    #Find F0 for different temp
    F0_Magic = F0_Base - abs(dF0)
    plt.plot(temp, F0_Magic/1e9, label="dF0 Formula")
    plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), fancybox=True)
    plt.ticklabel_format(useOffset=False)
    plt.rcParams['figure.dpi'] = 1000
    plt.xlim(0.20, 0.22)
    plt.ylim(0.949980, 0.95)
    plt.savefig("Magic Formula plot")

#KID Simulating Function
def Capacitive_Res_Sim(F0, C_couple, Z0, L_geo, L_int, Res, Sweep_BW, Sweep_points, Capacitance):
    """ Help file here"""
    j=complex(0,1)
    Cc=C_couple
    F_min=F0-(Sweep_BW/2.0)
    F_max=F0+(Sweep_BW/2.0)
    Sweep=np.linspace(F_min, F_max, Sweep_points)
    W=Sweep*2.0*pi
    W0=2.0*pi*F0
    L=L_geo+L_int
    C=Capacitance
    Zres= 1.0/((1./((j*W*L)+Res))+(j*W*C))   # Impedance of resonator section
    Zc=1.0/(j*W*Cc)  #impedance of coupler
    ZT=Zres+Zc
    YT=1.0/ZT
    S21 = 2.0/(2.0+(YT*Z0))
    I_raw=S21.real
    Q_raw=S21.imag
    shift=((1.0-min(I_raw))/2.0)+min(I_raw)
    I_cent=I_raw-shift
    Q_cent=Q_raw
    Phase=Atan(abs(Q_cent/I_cent))
    QU=(W0*L)/Res
    QL=(C*2)/(W0*(Cc**2)*Z0)
    S21_Volt=abs(S21)
    I_offset=shift
    return  (Sweep, S21_Volt, Phase, I_raw, Q_raw, I_cent, Q_cent, QU, QL, I_offset)

#Function to find sigma1 and sigma2
def find_sigma1_sigma2(sigma_n ,Thick, TC, Delta_0, w, T):
    #An interpolation formula for delta_T
    delta_T = Delta_0*np.tanh(1.74*np.sqrt((TC/T)-1))

    #Define constants to simplify eqn
    multiplying_constant = delta_T/(const.hbar * w)
    e_const_1 = - Delta_0/(const.Boltzmann*T)
    e_const_2 = (const.hbar*w)/(2*const.Boltzmann*T)

    #Parts of the sigma1 Ratio
    A = 2*multiplying_constant
    B = np.exp(e_const_1)
    C = K0(0, e_const_2)
    D = 2*(np.sinh(e_const_2))

    #Find Sigma 1 and Sigma 2
    sigma1Ratio = A * B * C * D
    sigma2Ratio = np.pi*multiplying_constant*(1 - (2*np.exp(e_const_1)*np.exp(-e_const_2)*I0(0,e_const_2)))
    sigma2 = sigma2Ratio * sigma_n
    sigma1 = sigma1Ratio * sigma_n
    return sigma1, sigma2

def find_lk(Thick, w, sigma2):
    #Depth
    lower_fraction = miu_0*sigma2*w
    Lambda_T_MB = (1/lower_fraction)**0.5
    fraction = Thick/(2*Lambda_T_MB)

    #Terms for lk
    A = (miu_0*Lambda_T_MB)/4
    B = coth(fraction)
    C = fraction*(csch(fraction))**2

    #R vs T
    lk = A*(B+C)
    return lk

def find_Lint_square(Thick, w, sigma2):
    #Depth
    lower_fraction = miu_0*sigma2*w
    Lambda_T_MB = (1/lower_fraction)**0.5

    #Internal Inductance
    fraction = Thick/(2*Lambda_T_MB)
    L_int = (miu_0*Lambda_T_MB/2)*coth(fraction)
    return L_int

#Define coth and csch
def coth(x):
    return np.cosh(x)/np.sinh(x)

def csch(x):
    return 1/np.sinh(x)

def Atan(x):
    return np.arctan(x)

#Find IDC function
def find_IDC(w0, Ltot, Cc):
    IDC = 1/((w0**2)*Ltot) - Cc
    return IDC

def Magic_Formula(di, dq, didf, dqdf):
    return (di*didf + dq*dqdf)/(didf**2 + dqdf**2)

main()


from django.db import models
from wagtail.admin.edit_handlers import MultiFieldPanel, RichTextFieldPanel, StreamFieldPanel
from wagtail.core.fields import RichTextField, StreamField
from wagtail.snippets.models import register_snippet

from ..modules import text_processing
from .. import configurations
from ..blogs.blocks import SectionBlock

@register_snippet
class BlogPost(models.Model):
    post_title = RichTextField(
        features=[], blank=False, null=True,
    )
    post_summary = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=False, null=True,
    )
    post_introduction = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=True, null=True,
    )
    post_conclusion = RichTextField(
        features=configurations.RICHTEXT_FEATURES, blank=True, null=True,
    )
    sections = StreamField(
        [
            ('section', SectionBlock()),
        ], blank=False
    )

    panels = [
        MultiFieldPanel(
            [
                RichTextFieldPanel('post_title'),
                RichTextFieldPanel('post_summary'),
                RichTextFieldPanel('post_introduction'),
                StreamFieldPanel('sections'),
                RichTextFieldPanel('post_conclusion'),
            ], heading='Post Content'
        ),
    ]

    @property
    def sections_with_title(self):
        sections = []
        for section in self.sections:
            if section.value['title']:
                sections.append(section)
        return sections

    def __str__(self):
        return text_processing.html_to_str(
            self.post_title
        )


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# @Author: José Sánchez-Gallego (gallegoj@uw.edu)
# @Date: 2023-01-19
# @Filename: test_callback.py
# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)

import unittest.mock

import click
from click.testing import CliRunner

from unclick.core import build_command_string

@click.command()
@click.argument("ARG1", type=str)
@click.argument("ARG2", type=int, required=False)
@click.option("--flag1", "-f", is_flag=True, help="A flag.")
def my_command1(*args, **kwrgs):
    """Test command."""

    return

def _command_invoke(command: click.Command, string: str):
    """Checks that command is called and returns a mock of the callback."""

    with unittest.mock.patch.object(command, "callback") as mock_callback:
        runner = CliRunner()

        result = runner.invoke(command, string[len(command.name or "") :])
        assert result.exit_code == 0

        mock_callback.assert_called()

    return mock_callback

def test_callback():
    command_string = build_command_string(my_command1, "hi", 2, flag1=True)

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi", arg2=2, flag1=True)

def test_callback2():
    command_string = build_command_string(my_command1, "hi")

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi", arg2=None, flag1=False)

def test_callback_string_with_spaces():
    command_string = build_command_string(my_command1, "hi how are you")

    mock_callback = _command_invoke(my_command1, command_string)
    mock_callback.assert_called_once_with(arg1="hi how are you", arg2=None, flag1=False)


# -*- coding: utf-8 -*-
__docformat__ = "restructuredtext en"

"""
the PLUGIN extend
File:           app_plugin_ext.py
Copyright:      Blink AG
Author:         Steffen Kube <steffen@blink-dx.com>
"""
from blinkdms.code.lib.oTASK import oTASK
from blinkdms.code.lib.main_imports import *
from blinkdms.code.lib.obj import table_obj_chk

class gPlugin_ext:

    def __init__(self, _html, infoarr, mod_is_adm_space):
        self._html = _html
        self.infoarr = infoarr
        self._mod_is_adm_space = mod_is_adm_space

    def html_left(self, db_obj):
        '''
        left HTML frame update history ...
        '''
        # TBD: later ...
        #objhist_lib = objhist()
        #objects = objhist_lib.get_objects_sort(db_obj)
        #self._html.add_meta('objhist', objects)
        # if len(objects):
        #    self._html.add_meta( 'objhist_i', session['sessvars'].get('objhist_i',{})  )
        pass

    def task_info(self, db_obj):
        '''
        left HTML frame update history ...
        '''

        if not session.get('loggedin', 0):
            return

        task_lib = oTASK.UserTask(session['sesssec']['user_id'])
        num = task_lib.getNumOpenTasks(db_obj)
        self._html.add_meta('task.num', num)

    def post_actions(self, db_obj):
        # self.html_left(db_obj)
        self.task_info(db_obj)

    def set_object(self, tablename, objid):
        self.tablename = tablename
        self.objid = objid

    def sec_check_obj(self, db_obj):
        '''
        do TABLE and OBJECT access checks ..
        '''
        if not self._mod_is_adm_space:  # if not in the ADMIN-Area
            return

        tablename = self.tablename
        # tablib   = table_cls(tablename)

        # if GlobMethods.is_admin():
        #    pass
        # if tablib.is_bo() :
        #    self.infoarr['table.icon.show'] = 1
        # return

        # throws errors
        act = {
            'tab': ['read'],
            'obj': ['read']
        }
        if len(self.infoarr.get('objtab.acc_check', {})):
            act = self.infoarr['objtab.acc_check']

        table_obj_chk.do_tab_obj_access_chk(db_obj, tablename, self.objid, act)


"""
Query builder examples.

NOTES:

# Infix notation (natural to humans)
    NOT ((FROM='11' OR TO="22" OR TEXT="33") AND CC="44" AND BCC="55")
# Prefix notation (Polish notation, IMAP version)
    NOT (((OR OR FROM "11" TO "22" TEXT "33") CC "44" BCC "55"))
# Python query builder
    NOT(AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55'))

# python to prefix notation steps:
1. OR(1=11, 2=22, 3=33) ->
    "(OR OR FROM "11" TO "22" TEXT "33")"
2. AND("(OR OR FROM "11" TO "22" TEXT "33")", cc='44', bcc='55') ->
    "AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55')"
3. NOT("AND(OR(from_='11', to='22', text='33'), cc='44', bcc='55')") ->
    "NOT (((OR OR FROM "1" TO "22" TEXT "33") CC "44" BCC "55"))"
"""

import datetime as dt
from imap_tools import AND, OR, NOT, A, H, U

# date in the date list (date=date1 OR date=date3 OR date=date2)
q1 = OR(date=[dt.date(2019, 10, 1), dt.date(2019, 10, 10), dt.date(2019, 10, 15)])
# '(OR OR ON 1-Oct-2019 ON 10-Oct-2019 ON 15-Oct-2019)'

# date not in the date list (NOT(date=date1 OR date=date3 OR date=date2))
q2 = NOT(OR(date=[dt.date(2019, 10, 1), dt.date(2019, 10, 10), dt.date(2019, 10, 15)]))
# 'NOT ((OR OR ON 1-Oct-2019 ON 10-Oct-2019 ON 15-Oct-2019))'

# subject contains "hello" AND date greater than or equal dt.date(2019, 10, 10)
q3 = A(subject='hello', date_gte=dt.date(2019, 10, 10))
# '(SUBJECT "hello" SINCE 10-Oct-2019)'

# from contains one of the address parts
q4 = OR(from_=["@spam.ru", "@tricky-spam.ru"])
# '(OR FROM "@spam.ru" FROM "@tricky-spam.ru")'

# marked as seen and not flagged
q5 = AND(seen=True, flagged=False)
# '(SEEN UNFLAGGED)'

# (text contains tag15 AND subject contains tag15) OR (text contains tag10 AND subject contains tag10)
q6 = OR(AND(text='tag15', subject='tag15'), AND(text='tag10', subject='tag10'))
# '(OR (TEXT "tag15" SUBJECT "tag15") (TEXT "tag10" SUBJECT "tag10"))'

# (text contains tag15 OR subject contains tag15) OR (text contains tag10 OR subject contains tag10)
q7 = OR(OR(text='tag15', subject='tag15'), OR(text='tag10', subject='tag10'))
# '(OR (OR TEXT "tag15" SUBJECT "tag15") (OR TEXT "tag10" SUBJECT "tag10"))'

# header IsSpam contains '++' AND header CheckAntivirus contains '-'
q8 = A(header=[H('IsSpam', '++'), H('CheckAntivirus', '-')])
# '(HEADER "IsSpam" "++" HEADER "CheckAntivirus" "-")'

# UID range
q9 = A(uid=U('1034', '*'))
# '(UID 1034:*)'

# complex from README
q10 = A(OR(from_='from@ya.ru', text='"the text"'), NOT(OR(A(answered=False), A(new=True))), to='to@ya.ru')
# '((OR FROM "from@ya.ru" TEXT "\\"the text\\"") NOT ((OR (UNANSWERED) (NEW))) TO "to@ya.ru")'


import numpy as np
import json
import CMS_lumi
import os
import copy
import ROOT

def main():

    #work_dir = "."
    #plots_dir = "."

    ROOT.gROOT.SetBatch()
    ROOT.gStyle.SetOptStat(0000)
    ROOT.gStyle.SetPalette(ROOT.kVisibleSpectrum)

    file_obs = ROOT.TFile("higgsCombineTest.GoodnessOfFit.mH100.root","READ")
    file_toys = ROOT.TFile("higgsCombineTest.GoodnessOfFit.mH100.Toys.root","READ")

    limit_obs = np.zeros((1), dtype="float64")
    limit_toys = np.zeros((1), dtype="float64")
    m_h = np.zeros((1), dtype="float64")

    ########### Data #################
    tree_obs = file_obs.Get("limit")
    tree_obs.SetBranchAddress("limit",limit_obs)
    tree_obs.SetBranchAddress("mh",m_h)
    tree_obs.GetEntry(0)

    ########### Toys #################
    tree_toys = file_toys.Get("limit")
    tree_toys.SetBranchAddress("limit",limit_toys)

    #Histo =ROOT.TH1D("Histo","",40,(0.85)*tree_toys.GetMinimum("limit"),(0.95)*tree_toys.GetMaximum("limit"))
    Histo =ROOT.TH1D("Histo","",40,40,180)

    for iEvent in range(tree_toys.GetEntries()):
        tree_toys.GetEntry(iEvent)
        Histo.Fill(limit_toys,1.)
    Histo.Scale(1/Histo.Integral())

    bin_obs = Histo.GetXaxis().FindBin(limit_obs)
    p_value = Histo.Integral(bin_obs,Histo.GetNbinsX())
    print("p-value: " + str(p_value))

    canvas = ROOT.TCanvas("canvas","canvas",900,900)
    canvas.cd()
    canvas.SetTicks()
    canvas.SetLeftMargin(0.15)
    canvas.SetRightMargin(0.05)
    canvas.SetBottomMargin(0.13)

    Histo.GetYaxis().SetRangeUser(0.,1.6*Histo.GetMaximum())
    Histo.SetLineColor(ROOT.kSpring-8)
    Histo.SetLineWidth(1)
    Histo.SetLineStyle(1)
    Histo.SetFillColorAlpha(ROOT.kSpring-8,0.20)
    Histo.SetFillStyle(1001)
    Histo.GetYaxis().SetTitleOffset(1.5)
    Histo.GetYaxis().SetTitle("Normalized to unity")
    Histo.GetYaxis().SetTitleSize(0.038)
    Histo.GetYaxis().SetLabelSize(0.032)
    Histo.GetYaxis().SetTickLength(0.03)
    Histo.GetXaxis().SetLabelFont(42)
    Histo.GetXaxis().SetLabelSize(0.032)
    Histo.GetXaxis().SetTitleSize(0.038)
    Histo.GetXaxis().SetTitleOffset(1.3)
    Histo.GetXaxis().SetTitle("-2 ln #lambda (saturated)")
    Histo.GetXaxis().SetTickLength(0.03)

    Histo_PValue = Histo.Clone("Histo_PValue")
    Histo_PValue.SetLineColor(ROOT.kRed)
    Histo_PValue.SetLineWidth(0)
    Histo_PValue.SetFillColorAlpha(ROOT.kRed,0.40)

    arrow = ROOT.TArrow(Histo.GetBinLowEdge(bin_obs),0.002,Histo.GetBinLowEdge(bin_obs),0.4*Histo.GetBinContent(bin_obs),0.02,"<|")
    arrow.SetAngle(50)
    arrow.SetLineWidth(3)
    arrow.SetLineColor(ROOT.kRed)
    arrow.SetFillColorAlpha(ROOT.kRed,0.45)

    for iBin in range(Histo_PValue.GetXaxis().FindBin(limit_obs)):
        Histo_PValue.SetBinContent(iBin,0.)

    Histo.Draw("h")
    Histo_PValue.Draw("sameh")
    arrow.Draw()

    leg = ROOT.TLegend(0.55,0.55,0.9,0.75)
    leg.SetTextSize(0.035)
    leg.SetBorderSize(0)
    leg.AddEntry(arrow,"Observed","l")
    leg.AddEntry(Histo,"Expected (Toys)","lf")
    leg.Draw()

    Latex = ROOT.TLatex()
    Latex.SetNDC()
    Latex.SetTextColor(ROOT.kGray+2)
    Latex.SetTextSize(0.04)
    Latex.SetTextFont(12)
    Latex.DrawLatex(1-canvas.GetRightMargin()-0.75,1-canvas.GetTopMargin()-0.1,"Sig+Bkgd")
    Latex.DrawLatex(1-canvas.GetRightMargin()-0.75,1-canvas.GetTopMargin()-0.15,"p-value: " + str(round(p_value,3)))
    Latex.DrawLatex(canvas.GetLeftMargin()+0.01,1+0.02-canvas.GetTopMargin(),'#color[4]{SL}')

    CMS_lumi.CMS_lumi(canvas, 15, 3)

    canvas.Update()
    canvas.Print("GoF.pdf","Portrait pdf")

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
import os
import sys
import webbrowser

from invoke import task

docs_dir = 'docs'
build_dir = os.path.join(docs_dir, '_build')

@task
def readme(ctx, browse=False):
    ctx.run("rst2html.py README.rst > README.html")
    if browse:
        webbrowser.open_new_tab('README.html')

def build_docs(ctx, browse):
    ctx.run("sphinx-build %s %s" % (docs_dir, build_dir), echo=True)
    if browse:
        browse_docs(ctx)

@task
def clean_docs(ctx):
    ctx.run('rm -rf %s' % build_dir)

@task
def browse_docs(ctx):
    path = os.path.join(build_dir, 'index.html')
    webbrowser.open_new_tab(path)

@task
def docs(ctx, clean=False, browse=False, watch=False):
    """Build the docs."""
    if clean:
        clean_docs(ctx)
    if watch:
        watch_docs(ctx, browse=browse)
    else:
        build_docs(ctx, browse=browse)

@task
def watch_docs(ctx, browse=False, port=1234):
    """Run build the docs when a file changes."""
    try:
        import sphinx_autobuild  # noqa
    except ImportError:
        print('ERROR: watch task requires the sphinx_autobuild package.')
        print('Install it with:')
        print('    pip install sphinx-autobuild')
        sys.exit(1)
    ctx.run('sphinx-autobuild {0} --port={port} {1} {2}'.format(
        '--open-browser' if browse else '', docs_dir, build_dir, port=port),
            echo=True, pty=True)


'''
Complete the following 3 searching problems using techniques
from class and from Ch15 of the textbook website
'''

#1.  (7pts) Write code which finds and prints the longest
# word in the provided dictionary.  If there are more
# than one longest word, print them all.
import re
def split_line(line):
    #this function takes in a line of text and returns
    #a list of words in the line
    return re.findall('[A-Za-z]+(?:\'[A-Za-z]+)?', line)

file = open("dictionary.txt")

longest = [" "]

for line in file:
    words = split_line(line)
    for word in words:
        if len(word.upper()) > len(longest[0]):
            longest = [word]
        elif len(word) == len(longest[0]):
            longest.append(word)
print(longest)

file.close()
#2.  (10pts)  Write code which finds
# The total word count AND average word length
# in "AliceInWonderLand.txt"
file = open("AliceInWonderLand.txt")

word_count = 0
average = 0

for line in file:
    words = split_line(line)
    for word in words:
        word_count += 1
        average += len(word)
average /= word_count

print("There are", word_count, "words in Alice in Wonderland")
print("The average word length is:", average)

file.close()
# CHOOSE ONE OF THE FOLLOWING TWO PROBLEMS

#3 (13pts)  How many times does "Cheshire" occur in "AliceInWonderLand.txt"?
# How many times does "Cat" occur?
cheshire = 0
cat = 0

file = open("AliceInWonderLand.txt")

for line in file:
    words = split_line(line)
    for word in words:
        if word.upper() == "CHESHIRE":
            cheshire += 1
        elif word.upper() == "CAT":
            cat += 1

print("Cheshire is said", cheshire, "times.")
print("Cat is said", cat, "times.")

file.close()
# How many times does "Cheshire" immediately followed by "Cat" occur?
combo = 0
previous = ""
current = ""

file = open("AliceInWonderLand.txt")

for line in file:
    words = split_line(line)
    for word in words:
        current = word
        if previous.upper() == "CHESHIRE" and current.upper() == "CAT":
            combo += 1
        previous = current

print("Cheshire cat is said", combo, "Times")
file.close()

#### OR #####

#3  (13pts)Find the most frequently occurring
# seven letter word in "AliceInWonderLand.txt"

# Challenge problem (for fun).  What words appear in the text of "Alice in Wonderland" that DO NOT occur in "Alice Through the Looking Glass".  Make a list.  You can substitute this for any of the above problems.


from mlagents_envs.environment import UnityEnvironment
from mlagents_envs.environment import ActionTuple
from mlagents_envs.side_channel.engine_configuration_channel import EngineConfigurationChannel

import numpy as np
import mlagents.trainers

from collections import namedtuple

obs = namedtuple(
        'obs',
        ['vector', 'front', 'right', 'back', 'left', 'raycast'])

class Drone(object):

    def __init__(self, time_scale=1.0, filename='mac.app', port=11000):

        self.engine_configuration_channel = EngineConfigurationChannel()
        print(f"VERSION : {mlagents.trainers.__version__}")
        self.env = UnityEnvironment(
                file_name=filename,
                worker_id=port,
                side_channels=[self.engine_configuration_channel])
        self.env.reset()

        self.behavior_name = list(self.env.behavior_specs.keys())[0]
        self.spec = self.env.behavior_specs[self.behavior_name]
        self.engine_configuration_channel.set_configuration_parameters(time_scale=time_scale)
        self.dec, self.term = self.env.get_steps(self.behavior_name)

    def reset(self):

        self.env.reset()
        self.dec, self.term = self.env.get_steps(self.behavior_name)
        self.tracked_agent = -1

        self.state = [self.dec.obs[i][0] for i in range(6)]
        self.state = obs(
                vector=self.state[0], front=self.state[1],
                right=self.state[2], back=self.state[3],
                left=self.state[4], raycast=self.state[5])

        return self.state

    def step(self, action):

        if self.tracked_agent == -1 and len(self.dec) >= 1:
            self.tracked_agent = self.dec.agent_id[0]

        action = np.clip(action, -1, 1)
        action_tuple = ActionTuple()
        action_tuple.add_continuous(np.array([action]))

        self.env.set_actions(self.behavior_name, action_tuple)
        self.env.step()

        self.dec, self.term = self.env.get_steps(self.behavior_name)

        reward = 0
        done = False
        if self.tracked_agent in self.dec:
            reward += self.dec[self.tracked_agent].reward
        if self.tracked_agent in self.term:
            reward += self.term[self.tracked_agent].reward
            done = True

        if done:
            return self.state, reward, done

        self.state = [self.dec.obs[i][0] for i in range(6)]
        self.state = obs(
                vector=self.state[0], front=self.state[1],
                right=self.state[2], back=self.state[3],
                left=self.state[4], raycast=self.state[5])

        return self.state, reward, done

if __name__ == '__main__':

    import matplotlib.pyplot as plt

    env = Drone(
            time_scale=0.1,
            filename='/Users/chageumgang/Desktop/baselines/mac.app')

    episode = 0
    while True:

        state = env.reset()
        done = False
        score = 0
        episode += 1

        while not done:

            action = np.random.rand(3)
            next_state, reward, done = env.step(action)
            score += reward

            print(next_state.vector.shape)
            print(next_state.raycast.shape)
            print(next_state.front.shape)

            '''
            fig = plt.figure()
            ax1 = fig.add_subplot(2, 2, 1)
            ax1.imshow(state.front)
            ax1 = fig.add_subplot(2, 2, 2)
            ax1.imshow(state.back)
            ax1 = fig.add_subplot(2, 2, 3)
            ax1.imshow(state.right)
            ax1 = fig.add_subplot(2, 2, 4)
            ax1.imshow(state.left)
            plt.show(block=False)
            plt.pause(0.1)
            '''

            state = next_state

        print(episode, score)


import pandas as pd
from bs4 import BeautifulSoup as bs

#Criando objeto BS
def get_file(file_name):
    content = []
    with open(file_name, 'r') as file:
        content = file.readlines()
        content = ''.join(content)
        soup = bs(content,'xml')
    return soup

#Buscando parents
def get_parents(soup):
    parents = []
    for p in soup.parents:
        if p.has_attr("NAME") and p.find('TRANSFORMATION'):
            parents.append(p["NAME"])
    return parents

#Buscando valores
def get_values(soup, tag1):
    mapping = soup.find_all(tag1)
    names = []
    descriptions = []
    parents = []
    for map in mapping:
        names.append(map.get('NAME'))
        descriptions.append(map.get('DESCRIPTION'))
        parents.append(get_parents(map)) #soup>map
    list_values = [parents, names, descriptions]
    return list_values

#Criando DF
def create_df(list_values):
    dict = {
        'PARENTS': list_values[0],
        'NAME': list_values[1],
        'DESCRIPTION': list_values[2]
    }
    df = pd.DataFrame(dict)
    return df

#Criando excel
def create_file(filename, df):
    return df.to_excel(excel_writer=filename, sheet_name=filename, header=True)

#Separando colunas
def transform_column_parents(file):
    df_origin = pd.read_excel(file)
    list_column = df_origin['PARENTS'].values.tolist()
    list_column = [eval(item) for item in list_column]
    try:
        df_column = pd.DataFrame(list_column, columns=['MAPPING', 'FOLDER', 'REPOSITORY'])
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    except:
        df_column = pd.DataFrame(list_column, columns=['FOLDER', 'REPOSITORY'])
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    else:
        print('working')
        df_clean = df_origin.drop(['PARENTS'], axis=1)
        df_clean = df_origin.drop(['Unnamed: 0'], axis=1)
        df_complete = pd.concat([df_column, df_clean], axis=1)
    return df_complete

if __name__ == '__main__':
    #Strings
    name = '../mapping.XML'
    tag_mapping = 'MAPPING'
    tag_transformation = 'TRANSFORMATION'
    tag_target = 'TARGET'
    tag_source = 'SOURCE'
    filename_mapping = 'mapping.xlsx'
    filename_transformation = 'transformation.xlsx'
    filename_target = 'target.xlsx'
    filename_source = 'source.xlsx'

    #Criando objeto soup
    soup = get_file(name)

    #buscando valores de mapping, criando df e criando xlsx
    list_mapping = get_values(soup, tag_mapping)
    df_mapping = create_df(list_mapping)
    create_file(filename_mapping, df_mapping)

    #buscando valores de transformation, criando df e criando xlsx
    list_transformation = get_values(soup, tag_transformation)
    df_transformation = create_df(list_transformation)
    create_file(filename_transformation, df_transformation)

    #buscando valores de target, criando df e criando xlsx
    list_target = get_values(soup, tag_target)
    df_target = create_df(list_target)
    create_file(filename_target, df_target)

    #buscando valores de source, criando df e criando xlsx
    list_source = get_values(soup, tag_source)
    df_source = create_df(list_source)
    create_file(filename_source, df_source)

    #Bloco para dividir em tabelas os parents
    file1 = '../source.xlsx'
    file2 = '../mapping.xlsx'
    file3 = '../transformation.xlsx'
    file4 = '../target.xlsx'

    #Sources
    df1 = transform_column_parents(file1)
    create_file(df1, 'source.xlsx')

    #Mappings
    df2 = transform_column_parents(file2)
    create_file(df2, 'mapping.xlsx')

    #Transformations
    df3 = transform_column_parents(file3)
    create_file(df3, 'transformation.xlsx')

    #Targets
    df4 = transform_column_parents(file4)
    create_file(df4, 'target.xlsx')


class Codec:

    def serialize(self, root):
        """Encodes a tree to a single string.
        :type root: TreeNode
        :rtype: str
        """
        ret = []
        if root == None : return ""

        q = deque()
        q.append(root)
        while q :
            k = len(q)
            for i in range (k):
                node = q.popleft()
                if node:
                    ret.append(str(node.val))
                    q.append(node.left)
                    q.append(node.right)
                else:
                    ret.append("N")

        return ",".join(ret)

    def deserialize(self, data):
        """Decodes your encoded data to tree.

        :type data: str
        :rtype: TreeNode
        """

        nodes = data.split(",")
        if not nodes[0]:
            return None
        q = deque()
        root = TreeNode(nodes[0])
        q.append(root)
        i = 1
        while i < len(nodes[1:]):
            if not q :
                return root
            curnode = q.popleft()
            if nodes[i] != "N":
                curnode.left = TreeNode(int(nodes[i]))
                q.append(curnode.left)
            i+=1
            if nodes[i] != "N":
                curnode.right = TreeNode(int(nodes[i]))
                q.append(curnode.right)
            i+=1

        return root


'''
Algorithm: just count how many characters(frequency more than one regard it as one)
'''
#!/bin/python3

import sys
from collections import Counter
def stringConstruction(s):
    # Complete this function
    return len(Counter(s).values())

if __name__ == "__main__":
    q = int(input().strip())
    for a0 in range(q):
        s = input().strip()
        result = stringConstruction(s)
        print(result)


#!/home/shailja/.virtualenv/my_env/bin/python3

import requests
import bs4
import sys

content =  sys.argv[1]

def display_actual_text(text,para_no):
    text = text[para_no]
    [s.extract() for s in text(['style', 'script', '[document]', 'head', 'title'])]
    visible_text = text.getText()
    print(visible_text)

wiki_url = 'https://en.wikipedia.org/wiki/{}'

request = requests.get(wiki_url.format(content))
soup = bs4.BeautifulSoup(request.text,'lxml')
actual_text=soup.select('p')

# this is to handle if wikipedia don't have any article with this name
try:
    display_actual_text(actual_text,1)
except:
    print(f"Wikipedia does not have an article with this exact name.")
    exit(1)

    # wiki_url = 'https://en.wikipedia.org/wiki/Special:Search?go=Go&search={}&ns0=1'

    # request = requests.get(wiki_url.format(content))
    # soup = bs4.BeautifulSoup(request.text,'lxml')
    # #relevant_text=soup.select("div", {"id": "bodyContent"})                          #soup.select('#mw-body')
    # #relevant_text=relevant_text.select("div.searchresults")
    # content = soup.select(".mw-search-result")[0].select('td')[1]
    # title=content.a['title']
    # relevant_text=content.select(".searchresult")[0]
    # [s.extract() for s in relevant_text(['style', 'script', '[document]', 'head', 'title'])]
    # visible_text = relevant_text.getText()
    # print(title)
    # print(visible_text)

# for more data

show_next = False
para_no = 2

# this is to handle if search is found and reaches to the end of content in wikipidea
try:
    while not show_next:
        show_next = input("!")
        print()
        display_actual_text(actual_text,para_no)
        para_no+=1

except:
    print("End")