import os
import pygame
from pygame.locals import *
import random
import array2d

def load_sound(name):
    class NoneSound:
        def play(self): pass
    if not pygame.mixer:
        return NoneSound()
    fullname = os.path.join('sound', name)
    sound = pygame.mixer.Sound(fullname)
    return sound

def random_color():
    return (random.randint(0,255), random.randint(0,255), random.randint(0,255))

def xor(a,b):
    if bool(a) != bool(b):
        return True
    else:
        return False

class Difficulty:
    def __init__(self, name, width, height, mines, cellsize):
        self.name = name
        self.width = width
        self.height = height
        self.mines = mines
        self.cell_size = cellsize

class Mines:
    def __init__(self):
        self.difficulty = 1
        self.difficulties = []
        self.difficulties.append(Difficulty("Easy"  ,  9,  9, 10, 35))
        self.difficulties.append(Difficulty("Medium", 16, 16, 40, 25))
        self.difficulties.append(Difficulty("Expert", 30, 16, 99, 20))
        self.difficulties.append(Difficulty("More Expert", 40, 25, 200, 18))
        self.difficulties.append(Difficulty("Narrow", 30, 5, 28, 20))
        self.difficulties.append(Difficulty("More Narrow", 40, 4, 35, 18))
        self.difficulties.append(Difficulty("Ridiculous", 50, 30, 320, 15))

        self.SW = 800
        self.SH = 600

        self.w = self.difficulties[self.difficulty].width
        self.h = self.difficulties[self.difficulty].height
        self.cell_size = self.difficulties[self.difficulty].cell_size
        self.mines = self.difficulties[self.difficulty].mines
        self.flagged = 0

        self.grid = list((array2d.Array2d(self.w,self.h,0),array2d.Array2d(self.w,self.h,0),array2d.Array2d(self.w,self.h,0)))

        self.xoff = (self.SW/2)-(self.cell_size*self.w/2)
        self.yoff = (self.SH/2)-(self.cell_size*self.h/2)+10

        self.gameover = False
        self.win = False

        self.small_font = pygame.font.Font(pygame.font.get_default_font(),12)
        self.medium_font = pygame.font.Font(pygame.font.get_default_font(),18)
        self.large_font = pygame.font.Font(pygame.font.get_default_font(),24)

        self.colors_bg = list(((50,55,65), (80,85,95), (70,75,85)))
        self.colors_text = (155,155,160)
        self.colors_text_win = (120,230,120)
        self.colors_text_lose = (230,120,120)
        self.colors_flag = list(((100,50,50),(200,100,100)))
        self.colors_flag2 = list(((50,50,100),(100,100,200)))
        self.colors_unrevealed = (165, 165, 165)
        self.colors_highlight = (180, 185, 190)
        self.colors_notouch = (148, 175, 190)
        self.colors_notouch_win = (175, 195, 178)
        self.colors_revealed = (185, 185, 185)
        self.colors_mine = (215, 185, 185)
        self.colors_back = (40,40,45)
        self.colors_back_win = (75,85,78)
        self.colors_back_lose = (85,78,75)
        self.colors_numbers = (0,0,0)

        self.background = pygame.Surface((self.SW,self.SH))
        self.background.fill((222,222,0))
        self.background.fill(self.colors_bg[0])

        self.sound_boom = load_sound('boomboomuhoh.wav')
        self.sound_boop = load_sound('boop.wav')
        self.sound_beeh = load_sound('beeh.wav')
        self.sound_wsh = load_sound('wsh.wav')
        self.sound_yes = load_sound('yes.wav')

        for x in range(int(self.SW/16)):
            for y in range(int(self.SH/12)):
                if xor(x%2,y%2):
                    pygame.draw.rect(self.background, self.colors_bg[1], (int(x*(self.SW/16))+1, int(y*(self.SH/12))+1, int(self.SW/16)-2, int(self.SH/12)-2) )
                else:
                    pygame.draw.rect(self.background, self.colors_bg[2], (int(x*(self.SW/16))+1, int(y*(self.SH/12))+1, int(self.SW/16)-2, int(self.SH/12)-2) )

        self.screen = pygame.display.set_mode((self.SW, self.SH))
        pygame.display.set_caption('Mines - ' + self.difficulties[self.difficulty].name)

    def surrounding(self, pos):
        count = 0

        if pos[0]:
            if pos[1]:
                if self.grid[0].v[pos[0]-1][pos[1]-1]:
                    count += 1
            if self.grid[0].v[pos[0]-1][pos[1]]:
                count += 1
            if pos[1] < self.h-1:
                if self.grid[0].v[pos[0]-1][pos[1]+1]:
                    count += 1

        if pos[0] < self.w-1:
            if pos[1] < self.h-1:
                if self.grid[0].v[pos[0]+1][pos[1]+1]:
                    count += 1
            if self.grid[0].v[pos[0]+1][pos[1]]:
                count += 1
            if pos[1]:
                if self.grid[0].v[pos[0]+1][pos[1]-1]:
                    count += 1

        if pos[1]:
            if self.grid[0].v[pos[0]][pos[1]-1]:
                count += 1
        if pos[1] < self.h-1:
            if self.grid[0].v[pos[0]][pos[1]+1]:
                count += 1

        return count

    def check_win(self):
        self.win = True
        for x in range(self.w):
            for y in range(self.h):
                if not self.grid[0].v[x][y] == self.grid[2].v[x][y]:
                    self.win = False

        if self.win:
            self.gameover = True
            for x in range(self.w):
                for y in range(self.h):
                    if not self.grid[0].v[x][y]:
                        self.grid[1].v[x][y] = 1

    def reveal(self,x,y):
        if not self.gameover and not self.grid[2].v[x][y]:
            self.grid[1].v[x][y] = 1
        else:
            return 0

        if self.grid[0].v[x][y]:
            self.sound_boom.play()
            self.gameover = True

        if not self.surrounding((x,y)) and not self.gameover:
            if x and y and not self.grid[1].v[x-1][y-1]:
                self.reveal(x-1,y-1)
            if x and not self.grid[1].v[x-1][y]:
                self.reveal(x-1,y)
            if x and y < self.h-1 and not self.grid[1].v[x-1][y+1]:
                self.reveal(x-1,y+1)

            if x < self.w-1 and y and not self.grid[1].v[x+1][y-1]:
                self.reveal(x+1,y-1)
            if x < self.w-1 and not self.grid[1].v[x+1][y]:
                self.reveal(x+1,y)
            if x < self.w-1 and y < self.h-1 and not self.grid[1].v[x+1][y+1]:
                self.reveal(x+1,y+1)

            if y and not self.grid[1].v[x][y-1]:
                self.reveal(x,y-1)
            if y < self.h-1 and not self.grid[1].v[x][y+1]:
                self.reveal(x,y+1)

    def flag(self,x,y):
        if not self.gameover and not self.win and not self.grid[1].v[x][y]:
            if self.grid[2].v[x][y] == 1:
                self.grid[2].v[x][y] = 2
                self.flagged -= 1
            elif self.grid[2].v[x][y] == 2:
                self.grid[2].v[x][y] = 0
            else:
                self.sound_yes.play()
                self.grid[2].v[x][y] = 1
                self.flagged += 1
            self.check_win()

    def set_mines(self):
        for x in range(self.w):
            for y in range(self.h):
                self.grid[0].v[x][y] = 0
                self.grid[1].v[x][y] = 0
                self.grid[2].v[x][y] = 0

        done = False
        minesset = 0

        while not done:
            x = random.randint(0,(self.w-1))
            y = random.randint(0,(self.h-1))
            if self.grid[0].v[x][y] == 0:
                self.grid[0].v[x][y] = 1
                minesset += 1
            if minesset == self.mines:
                done = True

    def update_bg(self):
        self.background.fill((222,222,0))
        self.background.fill(self.colors_bg[0])
        for x in range(int(self.SW/16)):
            for y in range(int(self.SH/12)):
                if xor(x%2,y%2):
                    pygame.draw.rect(self.background, self.colors_bg[1], (int(x*(self.SW/16))+1, int(y*(self.SH/12))+1, int(self.SW/16)-2, int(self.SH/12)-2) )
                else:
                    pygame.draw.rect(self.background, self.colors_bg[2], (int(x*(self.SW/16))+1, int(y*(self.SH/12))+1, int(self.SW/16)-2, int(self.SH/12)-2) )

    def reset(self):
        self.win = False
        self.gameover = False
        self.flagged = 0

        self.w = self.difficulties[self.difficulty].width
        self.h = self.difficulties[self.difficulty].height
        self.cell_size = self.difficulties[self.difficulty].cell_size
        self.mines = self.difficulties[self.difficulty].mines

        self.xoff = (self.SW/2)-(self.cell_size*self.w/2)
        self.yoff = (self.SH/2)-(self.cell_size*self.h/2)+10

        self.grid = list((array2d.Array2d(self.w,self.h,0),array2d.Array2d(self.w,self.h,0),array2d.Array2d(self.w,self.h,0)))
        self.set_mines()
        pygame.display.set_caption('Mines - ' + self.difficulties[self.difficulty].name)

    def randomize_colors(self):
        self.colors_bg = list((random_color(), random_color(), random_color()))
        self.colors_text = random_color()
        self.colors_text_win = random_color()
        self.colors_text_lose = random_color()
        self.colors_flag = list((random_color(),random_color()))
        self.colors_flag2 = list((random_color(),random_color()))
        self.colors_unrevealed = random_color()
        self.colors_highlight = random_color()
        self.colors_notouch = random_color()
        self.colors_notouch_win = random_color()
        self.colors_revealed = random_color()
        self.colors_mine = random_color()
        self.colors_back = random_color()
        self.colors_back_win = random_color()
        self.colors_back_lose = random_color()
        self.colors_numbers = random_color()
        self.update_bg()

    def load_colors(self, fn = "default.color"):
        if os.path.exists(fn):
            fin = open(fn,"r")
            lines = fin.readlines()
            for line in lines:
                exec("self.colors_"+line)
            self.update_bg()
        else:
            print(fn + " does not exist.\n")

    def print_colors(self):
        print("bg = " + str(self.colors_bg))
        print("text = " + str(self.colors_text))
        print("text_win = " + str(self.colors_text_win))
        print("text_lose = " + str(self.colors_text_lose))
        print("flag = " + str(self.colors_flag))
        print("flag2 = " + str(self.colors_flag2))
        print("unrevealed = " + str(self.colors_unrevealed))
        print("highlight = " + str(self.colors_highlight))
        print("notouch = " + str(self.colors_notouch))
        print("notouch_win = " + str(self.colors_notouch_win))
        print("revealed = " + str(self.colors_revealed))
        print("mine = " + str(self.colors_mine))
        print("back = " + str(self.colors_back))
        print("back_win = " + str(self.colors_back_win))
        print("back_lose = " + str(self.colors_back_lose))
        print("numbers = " + str(self.colors_numbers))
        print("")

    def draw(self):
        self.screen.blit(self.background,(0,0))

        if self.win:
            pygame.draw.rect(self.screen, self.colors_back_win,(self.xoff-2, self.yoff-2, self.cell_size*self.w+4, self.cell_size*self.h+4))
        elif self.gameover:
            pygame.draw.rect(self.screen, self.colors_back_lose,(self.xoff-2, self.yoff-2, self.cell_size*self.w+4, self.cell_size*self.h+4))
        else:
            pygame.draw.rect(self.screen, self.colors_back,(self.xoff-2, self.yoff-2, self.cell_size*self.w+4, self.cell_size*self.h+4))

        for x in range(self.w):
            for y in range(self.h):
                if self.grid[1].v[x][y]:
                    if self.grid[0].v[x][y]:
                        pygame.draw.rect(self.screen, self.colors_mine, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                    else:
                        if self.surrounding((x,y)):
                            pygame.draw.rect(self.screen, self.colors_revealed, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                            self.screen.blit(self.small_font.render(str(self.surrounding((x,y))),True, self.colors_numbers),
                                             (self.xoff+x*self.cell_size+(self.cell_size/2)-(self.small_font.size(str(self.surrounding((x,y))))[0]/2)-1,
                                              self.yoff+y*self.cell_size+(self.cell_size/2)-(self.small_font.size(str(self.surrounding((x,y))))[1]/2)-1))
                        else:
                            if self.win:
                                pygame.draw.rect(self.screen, self.colors_notouch_win, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                            else:
                                pygame.draw.rect(self.screen, self.colors_notouch, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                else:
                    p = pygame.mouse.get_pos()
                    if (int((p[0] - self.xoff)/self.cell_size) == x) and (int((p[1] - self.yoff)/self.cell_size) == y):
                        pygame.draw.rect(self.screen, self.colors_highlight, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                    else:
                        pygame.draw.rect(self.screen,self.colors_unrevealed, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                    if self.grid[0].v[x][y] and self.gameover and not self.win:
                        pygame.draw.rect(self.screen, self.colors_mine, (self.xoff+x*self.cell_size, self.yoff+y*self.cell_size, self.cell_size-1, self.cell_size-1))
                    if self.grid[2].v[x][y] == 1:
                        pygame.draw.rect(self.screen, self.colors_flag[0], (self.xoff+x*self.cell_size+4, self.yoff+y*self.cell_size+4, self.cell_size-1-8, self.cell_size-1-8))
                        pygame.draw.rect(self.screen, self.colors_flag[1], (self.xoff+x*self.cell_size+5, self.yoff+y*self.cell_size+5, self.cell_size-1-10, self.cell_size-1-10))
                    if self.grid[2].v[x][y] == 2:
                        pygame.draw.rect(self.screen, self.colors_flag2[0], (self.xoff+x*self.cell_size+4, self.yoff+y*self.cell_size+4, self.cell_size-1-8, self.cell_size-1-8))
                        pygame.draw.rect(self.screen, self.colors_flag2[1], (self.xoff+x*self.cell_size+5, self.yoff+y*self.cell_size+5, self.cell_size-1-10, self.cell_size-1-10))

        if self.gameover:
            if not self.win:
                self.screen.blit(self.medium_font.render("GAME OVER", True, self.colors_text_lose), ((self.SW/2)-(self.medium_font.size("GAME OVER")[0]/2),10))
            else:
                self.screen.blit(self.medium_font.render("YOU WIN!", True, self.colors_text_win), ((self.SW/2)-(self.medium_font.size("YOU WIN!")[0]/2),10))
        self.screen.blit(self.medium_font.render("Change Difficulty With +/-",True, self.colors_text), (10,self.SH-25))

        pygame.draw.line(self.screen, self.colors_text, (10, 8+self.medium_font.size("Minesweeper")[1]), (10+self.medium_font.size("Minesweeper")[0], 8+self.medium_font.size("Minesweeper")[1]))
        self.screen.blit(self.medium_font.render("Minesweeper", True, self.colors_text), (10,10))
        self.screen.blit(self.medium_font.render("Difficulty: "+self.difficulties[self.difficulty].name, True, self.colors_text), (10,35))
        self.screen.blit(self.medium_font.render("Mines Left:"+str(self.mines-self.flagged), True, self.colors_text), (self.xoff, self.yoff - 25))

        if pygame.mouse.get_pos()[0] >= self.SW-25 and pygame.mouse.get_pos()[1] <= 25:
            pygame.draw.rect(self.screen, self.colors_highlight, (self.SW-25,0,25,25))
        else:
            pygame.draw.rect(self.screen, self.colors_unrevealed, (self.SW-25,0,25,25))
        self.screen.blit(self.medium_font.render("R", True, (50,50,50)), (self.SW-18, 4))

        pygame.display.flip()

    def play(self):
        self.set_mines()

        running = True
        while running:
            for event in pygame.event.get():
                if event.type == QUIT:
                    running = False
                elif event.type == MOUSEBUTTONDOWN:
                    if event.pos[0] >= self.SW-20 and event.pos[1] <= 20:
                        self.reset()
                    if event.pos[0] >= self.xoff and event.pos[0] < self.xoff+(self.cell_size*self.w):
                        if event.pos[1] >= self.yoff and event.pos[1] < self.yoff+(self.cell_size*self.h):
                            if event.button == 1:
                                self.reveal(int((event.pos[0]-self.xoff)/self.cell_size), int((event.pos[1]-self.yoff)/self.cell_size))
                                if not self.gameover:
                                    self.sound_wsh.play()
                            elif event.button == 3:
                                self.flag(int((event.pos[0]-self.xoff)/self.cell_size), int((event.pos[1]-self.yoff)/self.cell_size))
                elif event.type == KEYDOWN:
                    if event.key == K_KP1:
                        self.load_colors("color/1.color")
                    if event.key == K_KP2:
                        self.load_colors("color/2.color")
                    if event.key == K_KP3:
                        self.load_colors("color/3.color")
                    if event.key == K_KP4:
                        self.load_colors("color/4.color")
                    if event.key == K_KP5:
                        self.load_colors("color/5.color")
                    if event.key == K_KP6:
                        self.load_colors("color/6.color")
                    if event.key == K_KP7:
                        self.load_colors("color/7.color")
                    if event.key == K_KP8:
                        self.load_colors("color/8.color")
                    if event.key == K_KP9:
                        self.load_colors("color/9.color")

                    if event.key == K_r:
                        pygame.mixer.stop()
                        self.reset()
                    if event.key == K_z:
                        self.randomize_colors()
                    if event.key == K_x:
                        self.load_colors()
                    if event.key == K_c:
                        self.print_colors()
                    if event.key == K_KP_PLUS:
                        if self.difficulty < len(self.difficulties) - 1:
                            self.difficulty += 1
                            self.reset()
                        else:
                            self.difficulty = 0
                            self.reset()
                    if event.key == K_KP_MINUS:
                        if self.difficulty > 0:
                            self.difficulty -= 1
                            self.reset()
                        else:
                            self.difficulty = len(self.difficulties)-1
                            self.reset()
            self.draw()

def main():
    pygame.init()
    game = Mines()
    game.play()
    pygame.quit()

if __name__ == '__main__':
    main()


'''
Problem Solving Baekjoon 5525
Author: Injun Son
Date: August 27, 2020
'''
import sys

N = int(input())
M = int(input())
S = input()

answer = 0
pattern = 0
i= 1
while i< M-1:
    if S[i-1]=='I' and S[i]=='O' and S[i+1]=='I':
        pattern +=1
        if pattern ==N:
            answer +=1
            pattern -=1
        i+=1
    else:
        pattern = 0
    i+=1

print(answer)


from csv import reader
from flask import Flask, request, Response
from flask_jwt_extended import JWTManager
from json import dumps
from redis import Redis
from shared.config import ShopConfiguration
from shared.errors import MissingField, ValidationError
from shared.shop_models import db
from shared.utils import check_is_warehouse
from tempfile import NamedTemporaryFile

app = Flask(__name__)
app.config.from_object(ShopConfiguration)
jwt = JWTManager(app)

@app.route('/update', methods=['POST'])
@check_is_warehouse()
def update():
    if not request.files or not 'file' in request.files:
        raise MissingField('file')
    values: list[str] = []
    with NamedTemporaryFile(suffix='.csv') as tf:
        request.files['file'].save(tf.name)
        with open(tf.name) as f:
            csv_file = reader(f)
            for index, row in enumerate(csv_file):
                if len(row) != 4:
                    raise ValidationError(f'Incorrect number of values on line {index}.')
                categories = row[0].split('|')
                name = row[1]
                quantity = 0
                price = 0.0
                try:
                    quantity = int(row[2])
                    price = float(row[3])
                except ValueError:
                    # Error reporting happens below
                    pass
                if quantity <= 0:
                    raise ValidationError(f'Incorrect quantity on line {index}.')
                if price <= 0:
                    raise ValidationError(f'Incorrect price on line {index}.')
                values.append(dumps({
                    'name': name,
                    'quantity': quantity,
                    'price': price,
                    'categories': categories
                }))
    with Redis(app.config['REDIS_HOST']) as client:
        client.rpush('products', *values)
    return Response(status=200)

if __name__ == '__main__':
    db.init_app(app)
    app.run('0.0.0.0', 5002, debug=True)


import numpy as np

from whylogs.core.metrics.model_metrics import ModelMetrics
from whylogs.proto import ModelProfileMessage, ModelType

SUPPORTED_TYPES = ("binary", "multiclass")

class ModelProfile:
    """
    Model Class for sketch metrics for model outputs

    Attributes
    ----------
    metrics : ModelMetrics
        the model metrics object
    model_type : ModelType
        Type of mode, CLASSIFICATION, REGRESSION, UNKNOWN, etc.
    output_fields : list
        list of fields that map to model outputs
    """

    def __init__(self, output_fields=None, metrics: ModelMetrics = None):
        super().__init__()

        if output_fields is None:
            output_fields = []
        self.output_fields = output_fields
        if metrics is None:
            metrics = ModelMetrics()
        self.metrics = metrics

    def add_output_field(self, field: str):
        if field not in self.output_fields:
            self.output_fields.append(field)

    def compute_metrics(
        self,
        targets,
        predictions,
        scores=None,
        model_type: ModelType = None,
        target_field=None,
        prediction_field=None,
        score_field=None,
    ):
        """
        Compute and track metrics for confusion_matrix

        Parameters
        ----------
        targets : List
            targets (or actuals) for validation, if these are floats it is assumed the model is a regression type model
        predictions : List
            predictions (or inferred values)
        scores : List, optional
            associated scores for each prediction (for binary and multiclass problems)
        target_field : str, optional
        prediction_field : str, optional
        score_field : str, optional (for binary and multiclass problems)

        Raises
        ------
        NotImplementedError

        """

        metric_type = self.metrics.init_or_get_model_type(scores)

        if metric_type == ModelType.REGRESSION:
            self.metrics.compute_regression_metrics(
                predictions=predictions,
                targets=targets,
                target_field=target_field,
                prediction_field=prediction_field,
            )

        elif metric_type == ModelType.CLASSIFICATION:

            # if score are not present set them to 1.
            if scores is None:
                scores = np.ones(len(targets))

            scores = np.array(scores)

            # compute confusion_matrix
            self.metrics.compute_confusion_matrix(
                predictions=predictions,
                targets=targets,
                scores=scores,
                target_field=target_field,
                prediction_field=prediction_field,
                score_field=score_field,
            )
        else:
            raise NotImplementedError(f"Model type {metric_type} not supported yet")

    def to_protobuf(self):
        return ModelProfileMessage(output_fields=self.output_fields, metrics=self.metrics.to_protobuf())

    @classmethod
    def from_protobuf(cls, message: ModelProfileMessage):
        # convert google.protobuf.pyext._message.RepeatedScalarContainer to a list
        output_fields = [f for f in message.output_fields]
        return ModelProfile(
            output_fields=output_fields,
            metrics=ModelMetrics.from_protobuf(message.metrics),
        )

    def merge(self, model_profile):
        if model_profile is None:
            return self

        output_fields = list(set(self.output_fields + model_profile.output_fields))
        metrics = self.metrics.merge(model_profile.metrics)

        return ModelProfile(output_fields=output_fields, metrics=metrics)


# ---------------------------------------------
# Copyright (c) OpenMMLab. All rights reserved.
# ---------------------------------------------

from termios import BS0
import numpy as np
import torch
import torch.nn as nn
from mmcv.cnn import xavier_init
from mmcv.cnn.bricks.transformer import build_transformer_layer_sequence
from mmcv.runner.base_module import BaseModule

from mmdet.models.utils.builder import TRANSFORMER
from torch.nn.init import normal_
from projects.mmdet3d_plugin.models.utils.visual import save_tensor
from mmcv.runner.base_module import BaseModule
from torchvision.transforms.functional import rotate

from ...bevformer.modules.temporal_self_attention import TemporalSelfAttention
from ...bevformer.modules.spatial_cross_attention import MSDeformableAttention3D
from ...bevformer.modules.decoder import CustomMSDeformableAttention
from projects.mmdet3d_plugin.models.utils.bricks import run_time

from mmcv.runner import force_fp32, auto_fp16
import pdb

@TRANSFORMER.register_module()
class OccupancyTransformer_normal(BaseModule):
    """Implements the Detr3D transformer.
    Args:
        as_two_stage (bool): Generate query from encoder features.
            Default: False.
        num_feature_levels (int): Number of feature maps from FPN:
            Default: 4.
        two_stage_num_proposals (int): Number of proposals when set
            `as_two_stage` as True. Default: 300.
    """

    def __init__(self,
                 num_feature_levels=4,
                 num_cams=6,
                 two_stage_num_proposals=300,
                 encoder=None,
                 decoder=None,
                 embed_dims=256,
                 use_cams_embeds=True,
                 rotate_center=[100, 100],
                 use_decoder=True,
                 **kwargs):
        super(OccupancyTransformer_normal, self).__init__(**kwargs)

        self.encoder = build_transformer_layer_sequence(encoder)
        if use_decoder:
            self.decoder = build_transformer_layer_sequence(decoder)
        self.embed_dims = embed_dims
        self.num_feature_levels = num_feature_levels
        self.num_cams = num_cams
        self.fp16_enabled = False

        self.use_cams_embeds = use_cams_embeds

        self.two_stage_num_proposals = two_stage_num_proposals
        self.init_layers()
        self.rotate_center = rotate_center

    def init_layers(self):
        """Initialize layers of the Detr3DTransformer."""
        self.level_embeds = nn.Parameter(torch.Tensor(
            self.num_feature_levels, self.embed_dims))
        self.cams_embeds = nn.Parameter(
            torch.Tensor(self.num_cams, self.embed_dims))

    def init_weights(self):
        """Initialize the transformer weights."""
        for p in self.parameters():
            if p.dim() > 1:
                nn.init.xavier_uniform_(p)
        for m in self.modules():
            if isinstance(m, MSDeformableAttention3D) or isinstance(m, TemporalSelfAttention) \
                    or isinstance(m, CustomMSDeformableAttention):
                try:
                    m.init_weight()
                except AttributeError:
                    m.init_weights()
        normal_(self.level_embeds)
        normal_(self.cams_embeds)

    @auto_fp16(apply_to=('mlvl_feats', 'bev_queries', 'bev_pos'))
    def get_bev_features(
            self,
            mlvl_feats,
            bev_queries,
            bev_h,
            bev_w,
            grid_length=[0.512, 0.512],
            bev_pos=None,
            use_3d_conv=False,
            bev_z=None,
            direct_proj=False,
            fpn_index=None,
            **kwargs):
        """
        obtain bev features.
        """

        bs = mlvl_feats[0].size(0)
        bev_queries = bev_queries.unsqueeze(1).repeat(1, bs, 1)
        bev_pos = bev_pos.flatten(2).permute(2, 0, 1)

        feat_flatten = []
        spatial_shapes = []
        for lvl, feat in enumerate(mlvl_feats):
            bs, num_cam, c, h, w = feat.shape
            spatial_shape = (h, w)
            feat = feat.flatten(3).permute(1, 0, 3, 2) #num_cam, bs, hw, c
            if not direct_proj:
                if self.use_cams_embeds:
                    feat = feat + self.cams_embeds[:, None, None, :].to(feat.dtype)
                feat = feat + self.level_embeds[None, None, lvl:lvl + 1, :].to(feat.dtype)
            spatial_shapes.append(spatial_shape)
            feat_flatten.append(feat)

        feat_flatten = torch.cat(feat_flatten, 2)
        spatial_shapes = torch.as_tensor(spatial_shapes, dtype=torch.long, device=bev_pos.device)
        level_start_index = torch.cat((spatial_shapes.new_zeros((1,)), spatial_shapes.prod(1).cumsum(0)[:-1]))
        feat_flatten = feat_flatten.permute(0, 2, 1, 3)  # (num_cam, H*W, bs, embed_dims)

        bev_embed = self.encoder(
                bev_queries,
                feat_flatten,
                feat_flatten,
                bev_h=bev_h,
                bev_w=bev_w,
                bev_pos=bev_pos,
                spatial_shapes=spatial_shapes,
                level_start_index=level_start_index,
                use_3d_conv=use_3d_conv,
                bev_z=bev_z,
                direct_proj=direct_proj,
                **kwargs
            )

        return bev_embed


import os

import math
import numpy as np
import torch.optim as optim
import torch
from torch_geometric.loader import DataLoader
from lifelines.utils import concordance_index as get_cindex
from torch.utils.data import Subset
from sklearn.model_selection import KFold

# from .MgraphDTA.dataset import *
from .MgraphDTA.preprocessing import GNNDataset
from .MgraphDTA.model import *
from .MgraphDTA.utils import *
from tqdm import tqdm
from lifelines.utils import concordance_index as ci
from sklearn.metrics import mean_squared_error
# from MgraphDTA.preprocessing import *

from .structure_dataset import DTADataset
import pandas as pd
import torch_geometric.data as data
from .model_NHGNN import NHGNN_DTA

class MergedDataset(data.Dataset):
    def __init__(self, dataset1, dataset2, transform=None, pre_transform=None):
        super(MergedDataset, self).__init__(None, transform, pre_transform)
        assert len(dataset1) == len(dataset2), "Datasets must be of equal length!"
        self.dataset1 = dataset1
        self.dataset2 = dataset2

    def len(self):
        return len(self.dataset1)

    def get(self, idx):
        data1 = self.dataset1[idx]
        data2 = self.dataset2[idx]

        # Simply return a tuple of the two data points
        return data1, data2

class MGraphDTA(nn.Module):
    def __init__(self, data_input_path, data_path, drug_vocab, target_vocab, auto_dataset: bool = True, filter_num=32, out_dim=1,
                 model_name="Mgraph_1", model_dir="saved_models", dataset_name="kiba", seed=42, test_ratio=0.15, tar_len=2600):
        super().__init__()
        # data
        self.data_input_path = data_input_path
        self.dataset_name = dataset_name
        if auto_dataset:
            self.dataset = self.get_dataset(data_input_path, dataset_name, data_path, target_vocab, tar_len)
            self.train_set, self.test_set = self.split_dataset(self.dataset, seed, test_ratio)

        # model
        self.model_name = model_name
        self.model_dir = model_dir
        # self.protein_encoder = TargetRepresentation(block_num, vocab_protein_size, embedding_size)
        self.protein_struc_model = NHGNN_DTA(pretrain_path=f"saved_models/TFusion_{dataset_name}.pt",
                                             data_path=f"data_input/{dataset_name}/raw/data.csv",
                                             saved_path=f"saved_models/NHGNN_{dataset_name}_test1.pkl",
                                             drug_vocab=drug_vocab, target_vocab=target_vocab, seed=seed, test_ratio=test_ratio)
        self.ligand_encoder = GraphDenseNet(num_input_features=22, out_dim=filter_num * 3, block_config=[8, 8, 8],
                                            bn_sizes=[2, 2, 2])

        self.classifier = nn.Sequential(
            nn.Linear(352, 1024),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(1024, 1024),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(1024, 256),
        )
        self.classifier2 = nn.Sequential(
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(256, 256)
        )

        # self.classifier3 = nn.Sequential(
        #     nn.Linear(256, 256),
        #     nn.ReLU(),
        #     nn.Dropout(0.1),
        #     nn.Linear(256, 1)
        # )
        # self.dd1 = nn.Linear(256, 256)
        # self.dd2 = nn.Linear(256, 1)

    def get_dataset(self, data_input_path, dataset_name, data_path, target_vocab, tar_len):
        def target_to_graph(target_key, target_sequence, contact_dir):
            target_edge_index = []
            target_size = len(target_sequence)
            contact_file = os.path.join(contact_dir, target_key + '.npy')
            contact_map = np.load(contact_file)
            contact_map += np.matrix(np.eye(contact_map.shape[0]))
            index_row, index_col = np.where(contact_map > 0.8)
            for i, j in zip(index_row, index_col):
                target_edge_index.append([i, j])
            target_edge_index = np.array(target_edge_index)
            return target_size, target_edge_index

        print(f"processing proterin structure dataset of {dataset_name}...")

        df = pd.read_csv(data_path)
        smiles = set(df['compound_iso_smiles'])
        target = set(df['target_key'])

        target_seq = {}
        for i in range(len(df)):
            target_seq[df.loc[i, 'target_key']] = df.loc[i, 'target_sequence']
        target_graph = {}
        pconsc4_path = os.path.join(os.path.dirname(os.path.dirname(data_path)), "pconsc4")
        for k in target_seq:
            seq = target_seq[k]
            _, graph = target_to_graph(k, seq, pconsc4_path)
            target_graph[seq] = graph

        target_emb = {}
        target_len = {}
        for k in target_seq:
            seq = target_seq[k]
            content = []
            flag = 0
            for i in range(len(seq)):
                if flag >= len(seq):
                    break
                if (flag + 1 < len(seq)):
                    if target_vocab.stoi.__contains__(seq[flag:flag + 2]):
                        content.append(target_vocab.stoi.get(seq[flag:flag + 2]))
                        flag = flag + 2
                        continue
                content.append(target_vocab.stoi.get(seq[flag], target_vocab.unk_index))
                flag = flag + 1

            if len(content) > tar_len - 2:
                content = content[:tar_len - 2]

            X = [target_vocab.sos_index] + content + [target_vocab.eos_index]
            target_len[k] = len(content)
            if tar_len > len(X):
                padding = [target_vocab.pad_index] * (tar_len - len(X))
                X.extend(padding)
            target_emb[seq] = torch.tensor(X)

        fpath = os.path.join(data_input_path, dataset_name)
        root_path = os.path.dirname(os.path.dirname(data_path))
        data_set1 = GNNDataset(fpath)
        data_set2 = DTADataset(root=root_path, path=data_path, target_graph=target_graph, target_len=target_len)
        data_set = MergedDataset(data_set1, data_set2)
        return data_set

    def split_dataset(self, dataset, seed, test_ratio):
        test_size = round(len(dataset) * test_ratio)
        train_dataset, test_dataset = torch.utils.data.random_split(
            dataset=dataset,
            lengths=[len(dataset) - test_size, test_size],
            generator=torch.Generator().manual_seed(seed)
        )
        return train_dataset, test_dataset

    def forward(self, data, data2, output_vector=False, freeze_front=False):
        protein_x = self.protein_struc_model.forward_pro(data2)
        ligand_x = self.ligand_encoder(data)

        xx = torch.cat([protein_x, ligand_x], dim=-1)
        x = self.classifier(xx)
        out = self.classifier2(x)

        if output_vector:
            return out

        # if freeze_front:
        #     out = self.dd1(out) * out
        #     out = self.dd2(out)
        #     return out

        out = torch.norm(out, dim=-1)

        # if output_vector:
        #     return torch.cat((xx, out), dim=-1), torch.cat((x, out), dim=-1)
        return out

    def val(self, test_set, batch_size=128, use_cuda=True):
        dataloader_test = DataLoader(test_set, batch_size=batch_size, shuffle=False)
        # dataloader_test = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False, collate_fn=self.collate)
        model = self
        model.eval()
        total_pred = torch.Tensor()
        total_label = torch.Tensor()
        print("validating...")
        with torch.no_grad():
            for data, data2 in tqdm(dataloader_test, ncols=80):
                if use_cuda:
                    data = data.cuda()
                out = model(data, data2)
                total_pred = torch.cat((total_pred, out.cpu()), 0)
                total_label = torch.cat((total_label, data.y.cpu()), 0)
        all_ci = ci(total_label.detach().numpy().flatten(), total_pred.detach().numpy().flatten())
        all_mse = mean_squared_error(total_label.detach().numpy().flatten(), total_pred.detach().numpy().flatten())
        model.train()
        return all_mse, all_ci

    def fit(self, train_set, test_set, save_model=True, device=torch.device('cuda'), early_stop_epoch=400, lr=5e-4,
            freeze_front=False):
        print(f"len(train_set)={len(train_set)}")
        print(f"len(test_set)={len(test_set)}")

        train_loader = DataLoader(train_set, batch_size=256, shuffle=True)

        epochs = 3000
        steps_per_epoch = 50
        num_iter = math.ceil((epochs * steps_per_epoch) / len(train_loader))
        break_flag = False

        model = self.to(device)

        optimizer = optim.Adam(model.parameters(), lr=lr)
        criterion = nn.MSELoss()

        global_step = 0
        global_epoch = 0

        running_loss = AverageMeter()
        running_cindex = AverageMeter()
        running_best_mse = BestMeter("min")

        model.train()

        if freeze_front:
            for param in model.parameters():
                param.requires_grad = False
            for param in model.dd1.parameters():
                param.requires_grad = True
            for param in model.dd2.parameters():
                param.requires_grad = True
            optimizer = optim.Adam(filter(lambda p : p.requires_grad, model.parameters()), lr=lr)

        for i in range(num_iter):
            if break_flag:
                break

            for data, data2 in train_loader:
                model.train()

                global_step += 1
                data = data.to(device)
                pred = model(data, data2, freeze_front=freeze_front)

                loss = criterion(pred.view(-1), data.y.view(-1))
                cindex = get_cindex(data.y.detach().cpu().numpy().reshape(-1),
                                    pred.detach().cpu().numpy().reshape(-1))

                optimizer.zero_grad()
                loss.backward()
                optimizer.step()

                running_loss.update(loss.item(), data.y.size(0))
                running_cindex.update(cindex, data.y.size(0))

                if global_step % steps_per_epoch == 0:

                    global_epoch += 1

                    epoch_loss = running_loss.get_average()
                    epoch_cindex = running_cindex.get_average()
                    running_loss.reset()
                    running_cindex.reset()

                    mse, ci = self.val(test_set)

                    msg = "epoch-%d, loss-%.4f, cindex-%.4f, mse-%.4f ci-%.4f" % (
                    global_epoch, epoch_loss, epoch_cindex, mse, ci)
                    print(msg)

                    if mse < running_best_mse.get_best():
                        running_best_mse.update(mse)
                        if save_model:
                            save_model_dict(model, self.model_dir, self.model_name)
                    else:
                        count = running_best_mse.counter()
                        if count > early_stop_epoch:
                            break_flag = True
                            break


import scrapy

class JobsSpider(scrapy.Spider):
    name = "jobs"
    allowed_domains = ["craigslist.org"]
    start_urls = ['https://vancouver.craigslist.org/search/sof']

    def parse(self, response):
        titles = response.xpath('//a[@class="result-title hdrlnk"]/text()').extract()
        print(titles)


# import os
# print(os.getcwd())

# Assume words.txt is under data folder
# f = open('data/words.txt')
# line = f.readline()
# print(line)

# f = open('data/words.txt')

# number_of_words = 0

# for line in f:
#     word = line.strip()
#     number_of_words += 1

# print(number_of_words)

def find_long_words():
    """
    prints only the words with more than 20 characters
    """
    f = open('data/words.txt')  # Assume words.txt is under data folder

    for line in f:
        word = line.strip()
        if len(word) > 20:
            print(word, len(word))

# find_long_words()

def has_no_e(word):
    """
    returns True if the given word doesn't have the letter "e" in it
    """
    # for letter in word:
    #     if letter == "e" or letter == "E":
    #         return False
    # return True

    return 'e' not in word.lower()

# print(has_no_e('Babson'))
# print(has_no_e('College'))
# print(has_no_e('EA sports'))

def find_words_no_e():
    """
    returns the percentage of the words that don't have the letter "e"
    """
    f = open("data/words.txt")
    num_words_no_e = 0
    num_of_words = 0
    num_words_with_e = 0
    for line in f:
        num_of_words += 1
        word = line.strip()
        if has_no_e(word):
            # print(word)
            num_words_no_e += 1
        else:
            num_words_with_e += 1
    return num_words_no_e / num_of_words

# perc_no_e = find_words_no_e()
# print(f'The percentage of the words with no "e" is {perc_no_e*100:.2f}%.')

def avoids(word, forbidden):
    """
    returns True if the given word does not use any of the forbidden letters
    """
    for letter in word:
        if letter in forbidden:
            return False
    return True

# print(avoids('Babson', 'abcde'))
# print(avoids('College', 'e'))
# print(avoids('Boston', 'xyz'))

def find_words_no_vowels():
    """
    returns the percentage of the words that don't have vowel letters
    """
    f = open("data/words.txt")
    num_words_no_vowel = 0
    num_of_words = 0
    for line in f:
        num_of_words += 1
        word = line.strip()
        if avoids(word, 'aeiou'):
            # print(word)
            num_words_no_vowel += 1
    return num_words_no_vowel / num_of_words

# perc_no_vowel = find_words_no_vowels()
# print(f'The percentage of the words without vowel letters is {perc_no_vowel*100:.2f}%.')

def uses_only(word, available):
    """
    takes a word and a string of letters, and that returns True if the word
    contains only letters in the string available.
    """
    for letter in word:
        if letter not in available:
            return False
    return True

# print(uses_only('Babson', 'aBbsonxyz'))
# print(uses_only('college', 'aBbsonxyz'))

def find_words_only_use_planet():
    """"""
    f = open('data/words.txt')
    num_words_only_use_planet = 0
    for line in f:
        word = line.strip()
        if uses_only(word, 'planet'):
            num_words_only_use_planet += 1
            print(word)
    return num_words_only_use_planet

# print(
#     'Number of words that use only letters from "planets" is',
#     find_words_only_use_planet(),
# )

def uses_all(word, required):
    """
    takes a word and a string of required letters, and that returns True if
    the word uses all the required letters at least once.
    """
    # for letter in required:
    #     if letter not in word:
    #         return False
    # return True
    return uses_only(required, word)

# please write test cases

# print(uses_all('Babson', 'abs'))
# print(uses_all('college', 'abs'))
# print(uses_all('Babson', 'aeoiu'))
# print(uses_all('Babesonious', 'aeoiu'))

def find_words_using_all_vowels():
    """
    return the number of the words that use all the vowel letters
    """
    f = open('data/words.txt')
    num_words_use_all_vowels = 0
    for line in f:
        word = line.strip()
        if uses_all(word, 'aeiou'):
            num_words_use_all_vowels += 1
            print(word)
    return num_words_use_all_vowels

# print('The number of words that use all the vowels:', find_words_using_all_vowels())

def is_abecedarian(word):
    """
    returns True if the letters in a word appear in alphabetical order
    (double letters are ok).
    """
    left = word[0]
    for letter in word:
        if left > letter:
            return False
        left = letter
    return True

# print(is_abecedarian('abs'))
# print(is_abecedarian('college'))

def find_abecedarian_words():
    """
    returns the number of abecedarian words
    """
    fin = open('data/words.txt')
    counter = 0
    for line in fin:
        word = line.strip()
        if is_abecedarian(word):
            counter += 1
    return counter

# print(find_abecedarian_words())

def is_abecedarian_using_recursion(word):
    """
    returns True if the letters in a word appear in alphabetical order
    (double letters are ok).
    """
    if len(word) <= 1:
        return True
    if word[0] > word[1]:
        return False
    # print('now checking, ', word[1:])
    return is_abecedarian_using_recursion(word[1:])

# print(is_abecedarian_using_recursion('abcdef'))

def is_abecedarian_using_while(word):
    """
    returns True if the letters in a word appear in alphabetical order
    (double letters are ok).
    """
    i = 0
    while i < len(word) - 1:
        if word[i + 1] < word[i]:
            return False
        i = i + 1
    return True

# print(is_abecedarian_using_while('abcdef'))

def main():
    """"""
    find_long_words()

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# This program is part of OParl and may be used to build the specification

import os
import shlex
import shutil
import subprocess
import sys
from argparse import ArgumentParser
from glob import glob
from os import path

from scripts.json_schema2markdown import schema_to_markdown

SPECIFICATION_BUILD_ACTIONS = [
    'all',
    'clean',
    'test',
    'live',
    'html',
    'pdf',
    'odt',
    'docx',
    'txt',
    'epub',
    'archives',
    'zip',
    'gz',
    'bz'
]

SPECIFICATION_BUILD_TOOLS = [
    'pandoc',
    'dot',
    'xelatex',
    'gs',
    'convert',
    'python3',
    'tar',
    'zip'
]

SPECIFICATION_BUILD_FLAGS = {
    'gs': '-dQUIET -dSAFER -dBATCH -dNOPAUSE -sDisplayHandle=0 -sDEVICE=png16m -r600 -dTextAlphaBits=4',
    'pandoc': '--from markdown --standalone --table-of-contents --toc-depth=2 --number-sections'
}

def configure_argument_parser():
    parser = ArgumentParser(
        prog='./build.py',
        epilog='''
            build.py is part of the OParl Specification and thus distributed
            under the terms of the Creative Commons SA 4.0 License.
        '''
    )

    parser.add_argument(
        '--language',
        '-l',
        help='Specification language',
        default='de',
        action='store',
    )

    parser.add_argument(
        '--version',
        '-V',
        help='This will be displayed as version in the specification. Defaults to `git desribe`',
        action='store',
    )

    parser.add_argument(
        '--print-basename',
        help='This will output the base name used for build output',
        action='store_true',
        dest='print_basename'
    )

    parser.add_argument(
        '--latex-template',
        help='Change the latex template used for PDF generation',
        action='store',
        default='resources/template.tex'
    )

    parser.add_argument(
        '--html-style',
        help='Change the CSS file used for styling the HTML output',
        action='store',
        default='resources/html5.css'
    )

    parser.add_argument(
        '--list-actions',
        help='List available build actions',
        dest='list_actions',
        action='store_true'
    )

    parser.add_argument(
        'action',
        help='Build action to take, available actions are: {}'.format(', '.join(SPECIFICATION_BUILD_ACTIONS)),
        action='store',
        nargs='*'
    )

    return parser

def check_build_action(action):
    if len(action) == 0:
        return 'all'

    if action[0] in SPECIFICATION_BUILD_ACTIONS:
        return action[0]

    raise Exception(
        'Unknown build action: {}, choose one of: {}'.format(action, ', '.join(SPECIFICATION_BUILD_ACTIONS)))

def get_git_describe_version():
    return subprocess.check_output('git describe', shell=True, universal_newlines=True).strip()

def check_available_tools():
    tools = {}
    for tool in SPECIFICATION_BUILD_TOOLS:
        executable = shutil.which(tool)
        if executable:
            tools[tool] = executable
        else:
            raise Exception('{} not found, aborting.'.format(tool))

    return tools

def get_filename_base(language, version):
    return 'OParl-{}-{}'.format(version, language)

def prepare_builddir(filename_base):
    os.makedirs('build/src/images')
    os.makedirs('build/{}'.format(filename_base))

def prepare_schema(language):
    language_file = 'schema/strings.yml'
    if language != 'de':
        language_file = 'locales/{}/schema/strings.yml'.format(language)

    output_file = 'build/src/3-99-schema.md'

    schema_to_markdown('schema', 'examples', output_file, language, language_file)

def prepare_markdown(language):
    glob_pattern = 'src/*.md'
    if language != 'de':
        glob_pattern = 'locales/{}/src/*.md'.format(language)

    files = glob(glob_pattern)
    for f in files:
        shutil.copy2(f, 'build/src/')

    os.mkdir('build/extra')
    with open('build/extra/detailed-version.md', 'w') as fp:
        fp.write("Version {}\n".format(get_git_describe_version()))

def prepare_images(tools):
    glob_pattern = 'src/images/*.*'

    files = glob(glob_pattern)
    for f in files:
        convert_command = ''
        filename, extension = os.path.splitext(f)
        fout = path.join('build', 'src', 'images', os.path.basename(filename) + '.png')

        shutil.copy2(f, fout)

        if extension == '.pdf':
            convert_command = '{} {} -sOutputFile={} -f {}'.format(
                tools['gs'],
                SPECIFICATION_BUILD_FLAGS['gs'],
                fout,
                f
            )

        if extension == '.dot':
            convert_command = '{} -Tpng {} -o {}'.format(
                tools['dot'],
                f,
                fout
            )

        if extension == '.svg':
            convert_command = '{} {} {}'.format(
                tools['convert'],
                f,
                fout
            )

        cmd = shlex.split(convert_command)
        try:
            subprocess.run(cmd, check=True)
        except subprocess.CalledProcessError:
            raise Exception(
                'Errored on image prep for {}, please check the command:\n{}'.format(
                    f, convert_command
                )
            )

def create_symlinks(filename_base):
    """ Allow having stable links when working on the spec """
    os.makedirs('build/latest')
    for specfile in os.listdir('build/{}'.format(filename_base)):
        existing = os.path.abspath('build/{}/{}'.format(filename_base, specfile))
        new = 'build/latest/oparl{}'.format(os.path.splitext(specfile)[-1])
        os.symlink(existing, new)

def get_pandoc_version(pandoc_bin):
    version_tuple = subprocess.getoutput('{} --version'.format(pandoc_bin)).split('\n')[0].split(' ')[1].split('.')
    return [int(v) for v in version_tuple]

def run_pandoc(pandoc_bin, filename_base, output_format, extra_args='', extra_files=''):
    output_file = 'build/{}/{}.{}'.format(filename_base, filename_base, output_format)
    if path.exists(output_file):
        return

    source_files = glob('build/src/*.md')
    source_files.sort(key=lambda file: path.basename(file))
    # This gets the version into the titlepage of the pdf
    detailed_version = '-M detailed-version={}'.format(get_git_describe_version())

    # NOTE: Once we can safely assume pandoc 2.0, we can use resource-path
    #       for neater include management in the markdown files
    # pandoc_command = '{} {} {} --resource-path=.:build/src -o {} {} {}'.format(
    pandoc_command = '{} {} {} -o {} {} {} {}'.format(
        pandoc_bin,
        SPECIFICATION_BUILD_FLAGS['pandoc'],
        extra_args,
        output_file,
        extra_files,
        detailed_version,
        ' '.join(source_files)
    )

    cmd = shlex.split(pandoc_command)
    try:
        subprocess.run(cmd, check=True, stdout=sys.stdout, stderr=sys.stderr)
    except subprocess.CalledProcessError:
        raise Exception(
            'Errored on pandoc: {}'.format(pandoc_command)
        )

class Action:
    @staticmethod
    def clean():
        shutil.rmtree('build', ignore_errors=True)

    @staticmethod
    def test():
        # TODO: validate.py appears to be broken
        pass

    @staticmethod
    def live(tools, _, filename_base):
        args = '--to html5 --section-divs --no-highlight --template=resources/live.html'
        run_pandoc(tools['pandoc'], filename_base, 'html', extra_args=args)

    @staticmethod
    def html(tools, options, filename_base):
        args = '--to html5 --css {} --section-divs --self-contained'.format(options.html_style)
        run_pandoc(tools['pandoc'], filename_base, 'html', extra_args=args,
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-bild.md')

    @staticmethod
    def pdf(tools, options, filename_base):
        args = '--pdf-engine=xelatex'
        if get_pandoc_version(tools['pandoc'])[0] < 2:
            args = '--latex-engine=xelatex'

        args += ' --template {}'.format(options.latex_template)

        run_pandoc(tools['pandoc'], filename_base, 'pdf', extra_args=args)

    @staticmethod
    def odt(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'odt',
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-text.md')

    @staticmethod
    def docx(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'docx',
                   extra_files='build/extra/detailed-version.md resources/lizenz-als-text.md')

    @staticmethod
    def txt(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'txt')

    @staticmethod
    def epub(tools, _, filename_base):
        run_pandoc(tools['pandoc'], filename_base, 'epub', extra_files='build/extra/detailed-version.md')

    @staticmethod
    def all(tools, options, filename_base):
        Action.html(tools, options, filename_base)
        Action.pdf(tools, options, filename_base)
        Action.odt(tools, options, filename_base)
        Action.docx(tools, options, filename_base)
        Action.txt(tools, options, filename_base)
        Action.epub(tools, options, filename_base)

    @staticmethod
    def zip(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.zip'.format(filename_base)
        subprocess.run(['zip', '-qr', archive_name, filename_base], cwd='build')

    @staticmethod
    def gz(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.tar.gz'.format(filename_base)
        subprocess.run(['tar', '-czf', archive_name, filename_base], cwd='build')

    @staticmethod
    def bz(tools, options, filename_base):
        Action.all(tools, options, filename_base)
        archive_name = '{}.tar.bz2'.format(filename_base)
        subprocess.run(['tar', '-cjf', archive_name, filename_base], cwd='build')

    @staticmethod
    def archives(tools, options, filename_base):
        Action.zip(tools, options, filename_base)
        Action.gz(tools, options, filename_base)
        Action.bz(tools, options, filename_base)

def main():
    options = configure_argument_parser().parse_args()
    action = check_build_action(options.action)

    if options.version is None:
        options.version = get_git_describe_version()

    filename_base = get_filename_base(options.language, options.version)

    if options.print_basename:
        print(filename_base)
        exit(0)

    if options.list_actions:
        for action in SPECIFICATION_BUILD_ACTIONS:
            print('- ' + action)
        exit()

    tools = check_available_tools()

    # always clean
    Action.clean()

    if action == 'clean':
        exit(0)

    prepare_builddir(filename_base)
    prepare_schema(options.language)
    prepare_markdown(options.language)
    prepare_images(tools)

    # Avoid much boilerplate
    getattr(Action, action)(tools, options, filename_base)

    create_symlinks(filename_base)

if __name__ == '__main__':
    main()


#Program 1 – The A-Team
#Description: Design and write a program that reads the text from a provided text file into a list, displays the
# text on-screen, makes some alterations to the text and outputs the changed text to the screen, then saves the
# altered text as a new file.

#Student #: W0420180
#Student Name: Austin Lake

import random

def main():
    # YOUR CODE STARTS HERE, each line must be indented (one tab)

    # Get both file paths to Original and Altered text files
    file = "C:/Users/Austi/OneDrive/Documents/School/IT Programming/Programming/Assignment 4/a4-p1-theateam-Austin-Lake/ateam_Original.txt"
    new_file = "C:/Users/Austi/OneDrive/Documents/School/IT Programming/Programming/Assignment 4/a4-p1-theateam-Austin-Lake/ateam_Altered.txt"

    # Open Original file make alterations to each line, store each line into a list, and return that list
    lines = OpenFile(file)

    # Open Altered file and write each line of the altered list into it
    WriteToFile(new_file, lines)

    # YOUR CODE ENDS HERE

def OpenFile(file) -> list:

    with open(file, 'r') as a_file:

        print(a_file.read())
        lines = []
        a_file.seek(0)
        random_line = random.randrange(1, len(a_file.readlines()))
        a_file.seek(0)

        for index, line in enumerate(a_file.readlines(), start=1):
            if len(line) > 20 and index != random_line:
                print(f'{index}. {line.lower()}')
                lines.append(f'{index}. {line.lower()}')
            elif len(line) <= 20 and index != random_line:
                print(f'{index}. {line.upper()}')
                lines.append(f'{index}. {line.upper()}')

        return lines

def WriteToFile(file, a_list):

    with open(file, 'w') as a_file:

        for i in range(len(a_list)):
            a_file.write(a_list[i])

main()


"""
Clean reenrollment asssessment table from raw
"""
from optparse import OptionParser
from tulsa.config import create_engine_from_config_file
from tulsa.etl.sql_str_normer import normalize_colname, normalize_tablename
from collections import Counter
import sys
import json

def load_intorbust(engine):
    '''
    load the intorbust sql function into the connection.
    intorbust makes all the values of a column an int or null if not possible.

    :param engine: the engine to load intorbust into
    :type engine: sqlalchemy engine
    :param sqlfile: the filename/location of the intorbust code
    :type sqlfile: str
    '''

    sql_str = r'''create or replace function isdigits(text) returns boolean
       as 'select $1 ~ ''^(-)?[0-9]+$'' as result'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(text) returns int
    as 'select
       	  case isdigits($1) when TRUE then cast($1 as int)
	  else null
	  end'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(numeric) returns numeric
       as 'select $1'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;

    create or replace function intorbust(float) returns float
       as 'select $1'
       language sql
       immutable
       RETURNS NULL ON NULL INPUT;
    '''
    engine.execute(sql_str)

def get_tables_with_prefix(engine, prefix, schemaname):
    '''
    return a list of the table names that begin with prefix in a specific
    schema

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param prefix: the prefix to use in selecting tables
    :type prefix: str
    :param schemaname: schemaname to search with
    :type schemaname: str

    :returns: list[str] -- the table names beginning with prefix of schema
    '''
    get_tables_query = """select tablename
                              from pg_catalog.pg_tables
                              where schemaname = '{schemaname}'
                              and tablename ~ '{prefix}.?';
                              """.format(prefix=prefix, schemaname=schemaname)
    query_tables = engine.execute(get_tables_query).fetchall()
    query_tables_strs = [q['tablename'] for q in query_tables]
    return query_tables_strs

def column_names(engine, table, schemaname):
    '''
    return the columns names of a table and a schema

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param table: the table name in question
    :type table: str
    :param schemaname: the schema name from which the table exists
    :type schemaname: str
    :returns: list[str] -- the column names of the table
    '''
    return engine.execute('''SELECT * FROM {schemaname}."{table}"
                          '''.format(table=table,
                                     schemaname=schemaname)).keys()

def drop_table(engine, table, schemaname):
    '''
    drop table from schema, cascading if necessary

    :param engine: the engine to use
    :type engine: sqlalchemy engine
    :param table: the table name to drop
    :type table: str
    :param schemaname: the schema name from which the table exists
    :type schemaname: str
    '''
    dropsql = '''DROP TABLE IF EXISTS {}.{} CASCADE;
              '''.format(schemaname, table)
    engine.execute(dropsql)

def clean_data_create_select_column_from_old(
        old_colnames_list, columns_in_all_tables, corrections):
    '''
    Creates select column subclauses with AS statments to clean names,
    returns "as" string of select colnames as new.

    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are common
                                                after normalization
    :type columns_in_all tables: set
    :returns str: a string of "as" statements
     '''
    new_colnames_list = [normalize_colname(name, corrections=corrections)
                         for name in old_colnames_list]
    old_new_colnames = list(zip(old_colnames_list, new_colnames_list))
    intersection_old_new_colnames = [(old, new) for
                                     old, new in old_new_colnames
                                     if new in columns_in_all_tables]
    as_statements = []
    for old, new in intersection_old_new_colnames:
            if new in ['studentid']:
                as_statement = 'intorbust("{old}") AS {new}'.format(old=old,
                                                                    new=new)
                as_statements.append(as_statement)
            else:
                as_statement = '"{old}" AS {new}'.format(old=old, new=new)
                as_statements.append(as_statement)

    as_string = ', '.join(as_statements)
    return as_string

def clean_data_create_select_statement(tablename, old_colnames_list,
                                       columns_in_all_tables, corrections):
    '''
    Creates select statements given AS statments

    :param tablename: the table name from which were selecting
    :type tablename: str
    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are
                                        common after normalization
    :type columns_in_all tables: set
    :returns str: select statement
    '''
    as_statements = clean_data_create_select_column_from_old(
        old_colnames_list, columns_in_all_tables, corrections)
    select_statement = """SELECT {as_statements}
                        FROM raw_data."{tablename}"
                        """.format(as_statements=as_statements,
                                   tablename=tablename)
    return select_statement

def clean_data_create_copy_statement(raw_tablename, old_colnames_list,
                                     columns_in_all_tables, corrections):
    '''
    makes a CREATE TABLE statement given a raw_tablename and does casting to
    numeric where possible

    :param raw_tablename: the table name from which were selecting
    :type str:
    :param old_colnames_list: the colnames from raw that need to be cleaned
    :type old_colnames_list: list[str]
    :param columns_in_all_tables: the column names that are common
                                                after normalization
    :type columns_in_all tables: set
    :returns str: sql create table statement
    '''
    select_statement = clean_data_create_select_statement(
        raw_tablename, old_colnames_list, columns_in_all_tables, corrections)
    clean_table_name = normalize_tablename(raw_tablename)
    copy_statement = """CREATE TABLE clean_data.{tablename} AS
                        {select_statement};
                    """.format(tablename=clean_table_name,
                               select_statement=select_statement)

    return copy_statement

def create_select_statement(tablename, colnames_list, time_from_title):
    """
    create select statement from column name
    additionally making dates form titles if specificed
    format should be _{YY}_{season} where YY is a year
    and season is one of 'fall', 'winter', or 'spring'

    :param tablename: the table name to select from
    :type tablename: str
    :param colnames_list: the colnames to select
    :type colnames: list[str]
    :param time_from_title: whether to try and get date information from title
    :type time_from_tile: bool
    :returns str: sql select statements
    """
    extra_time_cols = ''
    if time_from_title:
        tablename_parts = tablename.split('_')
        cal_year = tablename_parts[1]
        season = tablename_parts[-1]
        if season == 'fall':
            measured_year = cal_year + '_' + str(int(cal_year) + 1)
        elif season in ['winter', 'spring']:
            measured_year = str(int(cal_year) - 1) + '_' + cal_year
        extra_time_cols = """, '{measured_year}'::varchar as measured_year,
                            '{season}'::varchar as season,
                            '{cal_year}'::int as cal_year
                          """.format(measured_year=measured_year,
                                     season=season,
                                     cal_year=cal_year)
    all_columns = ', '.join(colnames_list)
    select_statement = """SELECT {all_columns} {extra_time_cols}
                        FROM clean_data.{tablename}
                        """.format(tablename=tablename,
                                   extra_time_cols=extra_time_cols,
                                   all_columns=all_columns)
    return select_statement

def create_view_statement(columns_dict, view_name, time_from_title):
    """
    create view statement from all the tables and columns in columns dict
    additionally making dates form titles if specificed

    :param columns_dict: a mapping between table names and their column names
    :type columns_dict: dict[str: list]
    :param view_name: what to name the view
    :type view_name: str
    :param time_from_title: whether to try and get date information from title
    :type time_from_tile: bool
    :returns str: sql create view statement
    """

    select_statements_list = [create_select_statement(tablename,
                                                      old_colnames_list,
                                                      time_from_title)
                              for tablename, old_colnames_list
                              in columns_dict.items()]
    select_statements = ''' UNION
                        '''.join(select_statements_list)
    view_statement = """CREATE OR REPLACE VIEW clean_data.{view_name} AS
                        {select_statements};
                    """.format(select_statements=select_statements,
                               view_name=view_name)
    return view_statement

def main(engine, options):
    load_intorbust(engine)

    # find the raw tables to operate over
    raw_tables = []
    if options.raw_table:
        raw_tables.append(options.raw_table)
    elif options.raw_table_prefix:
        prefix_table_names = get_tables_with_prefix(engine,
                                                    options.raw_table_prefix,
                                                    'raw_data')
        raw_tables.extend(prefix_table_names)
    else:
        print('no tables specified... use --prefix or --table')
        sys.exit(1)
    print('going to operate on these raw tables: {}'.format(raw_tables))

    # make a dictionary of raw_tables and their associated column names
    raw_dict = {raw_table: column_names(engine, raw_table, 'raw_data')
                for raw_table in raw_tables}

    if options.corrections:
        corrections = json.load(open(options.corrections, 'r'))
    else:
        corrections = None

    # find the common columns in all tables -- after column name normalization.
    column_count = Counter()
    for raw_table, raw_columns in raw_dict.items():
        normed_colnames = [normalize_colname(col, corrections=corrections)
                           for col in raw_columns]
        for normed_colname in normed_colnames:
            column_count[normed_colname] += 1

    columns_in_all_tables = [col for col, cnt in column_count.items()
                             if cnt == len(raw_tables)]

    # make the destination tables and drop them if necessary
    clean_tables = [normalize_tablename(raw_table) for raw_table in raw_tables]
    for clean_table in clean_tables:
        drop_table(engine, clean_table, 'clean_data')

    # perform the raw -> clean copy
    for raw_table, raw_columns in raw_dict.items():
        copy_statement = clean_data_create_copy_statement(
            raw_table,
            raw_columns,
            columns_in_all_tables,
            corrections=corrections)

        print(copy_statement)
        engine.execute(copy_statement)
    print("look for clean_data.{}".format(clean_tables))

    # now to make the view
    # make a dictionary of clean_tables and their associated column names
    clean_dict = {clean_table: column_names(engine, clean_table, 'clean_data')
                  for clean_table in clean_tables}

    view_statement = create_view_statement(clean_dict,
                                           options.view_name,
                                           options.time_from_title)
    engine.execute(view_statement)

if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option('-t', '--table', dest='raw_table',
                      help='specify 1 raw table name')
    parser.add_option('-p', '--table_prefix', dest='raw_table_prefix',
                      help='specificy a prefix of all raw table names')
    parser.add_option('-c', '--config', dest='config_file',
                      help='where to get database credential file')
    parser.add_option('-v', '--view_name', dest='view_name',
                      help='what the final view name should be')
    parser.add_option('-e', '--date_cols_from_title',
                      action="store_true",
                      dest='time_from_title',
                      help='''turn this flag on if you want date/season columns
                              extracted from the table names''')
    parser.add_option('-o', '--corrections',
                      dest='corrections',
                      help='''point to a json dict of column names you want to
                              change''')

    (options, args) = parser.parse_args()

    engine = create_engine_from_config_file(options.config_file)

    if not options.config_file:
        print('no database credentials json file specified use --config /path')
        sys.exit(1)
    if not options.view_name:
        print('no destination view name specified use --view_name name')
        sys.exit(1)
    else:
        main(engine, options)


from pipeline_dsl.resources.resource import AbstractResource, ConcourseResource
from pipeline_dsl.concourse import concourse_context
from typing import Optional
import os
import re

class GithubReleaseResource:
    def __init__(self, name: str) -> None:
        self.name = name
        self.path = os.path.abspath(self.name)

    def __str__(self) -> str:
        return self.name

    def tag(self, default: str = None) -> Optional[str]:
        if concourse_context():
            with open(os.path.join(self.path, "tag")) as f:
                return f.read().strip()
        return default

class GithubRelease(AbstractResource[GithubReleaseResource]):
    def __init__(
        self,
        owner: str,
        repo: str,
        access_token: str = None,
        pre_release: bool = False,
        release: bool = True,
        github_api_url: str = None,
        github_v4_api_url: str = None,
        github_uploads_url: str = None,
        tag_filter: str = None,
        order_by: str = None,
    ):
        self.owner = owner
        self.repo = repo
        self.access_token = access_token
        self.pre_release = pre_release
        self.release = release
        self.github_api_url = github_api_url
        self.github_v4_api_url = github_v4_api_url
        if github_api_url and not github_v4_api_url:
            self.github_v4_api_url = re.sub(r"/v[0-9]/*", "/graphql", github_api_url)
        self.github_uploads_url = github_uploads_url
        self.tag_filter = tag_filter
        self.order_by = order_by

    def resource_type(self) -> Optional[dict]:
        return {
            "name": "github-release",
            "type": "docker-image",
            "source": {
                "repository": "concourse/github-release-resource",
                "tag": "1.6.3",
            },
        }

    def concourse(self, name: str) -> ConcourseResource:
        result = ConcourseResource(
            name=name,
            type="github-release",
            icon="github",
            source={
                "owner": self.owner,
                "repository": self.repo,
                "access_token": self.access_token,
                "pre_release": self.pre_release,
                "release": self.release,
                "github_api_url": self.github_api_url,
                "github_v4_api_url": self.github_v4_api_url,
                "github_uploads_url": self.github_uploads_url,
                "tag_filter": self.tag_filter,
                "order_by": self.order_by,
            },
        )
        result.source = dict(
            filter(lambda x: x[1] is not None, result.source.items()),
        )
        return result

    def get(self, name: str) -> GithubReleaseResource:
        return GithubReleaseResource(name)


import numpy as np
import torch
from torch.nn import functional as F

from mmdet.core.bbox import BaseBBoxCoder
from mmdet.core.bbox.builder import BBOX_CODERS

INF = 1e8
EPS = 1e-12
PI = np.pi

@BBOX_CODERS.register_module()
class UniYOLOCoder(BaseBBoxCoder):
    """Bbox Coder for MonoFlex.

    Args:
        depth_mode (str): The mode for depth calculation.
            Available options are "linear", "inv_sigmoid", and "exp".
        base_depth (tuple[float]): References for decoding box depth.
        depth_range (list): Depth range of predicted depth.
        combine_depth (bool): Whether to use combined depth (direct depth
            and depth from keypoints) or use direct depth only.
        uncertainty_range (list): Uncertainty range of predicted depth.
        base_dims (tuple[tuple[float]]): Dimensions mean and std of decode bbox
            dimensions [l, h, w] for each category.
        dims_mode (str): The mode for dimension calculation.
            Available options are "linear" and "exp".
        multibin (bool): Whether to use multibin representation.
        num_dir_bins (int): Number of Number of bins to encode
            direction angle.
        bin_centers (list[float]): Local yaw centers while using multibin
            representations.
        bin_margin (float): Margin of multibin representations.
        code_size (int): The dimension of boxes to be encoded.
        eps (float, optional): A value added to the denominator for numerical
            stability. Default 1e-3.
    """

    def __init__(self,
                 depth_mode='linear',
                 num_alpha_bins=12,
                 eps=1e-3):
        super(UniYOLOCoder, self).__init__()
        # output related
        self.num_alpha_bins = num_alpha_bins
        self.depth_mode = depth_mode
        self.eps = eps

    def decode_bboxes2d(self, bbox_preds, priors):
        xys = (bbox_preds[..., :2] * priors[:, 2:]) + priors[:, :2]
        whs = bbox_preds[..., 2:].exp() * priors[:, 2:]

        tl_x = (xys[..., 0] - whs[..., 0] / 2)
        tl_y = (xys[..., 1] - whs[..., 1] / 2)
        br_x = (xys[..., 0] + whs[..., 0] / 2)
        br_y = (xys[..., 1] + whs[..., 1] / 2)

        decoded_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], -1)
        return decoded_bboxes

    def decode_bboxes3d(self, bbox_3d_preds, alpha_preds, priors, cam2imgs):
        direct_depth = bbox_3d_preds[..., 2]
        direct_depth_log_var = bbox_3d_preds[..., 3]
        dims = bbox_3d_preds[..., 4:].view(-1, 3)
        depth = self.decode_depth(direct_depth)
        sigma = torch.exp(-direct_depth_log_var)

        center2d = (bbox_3d_preds[..., :2] * priors[:, 2:]) + priors[:, :2]
        # 1. decode alpha
        alpha = self.decode_alpha_multibin(alpha_preds[..., :self.num_alpha_bins],
                                            alpha_preds[..., self.num_alpha_bins:])  # (b, k, 1)

        # 2. recover rotY
        rot_y = self.recover_rotation(center2d, alpha, cam2imgs)  # (b, k, 1)

        # 2.5 recover box3d_center from center2d and depth
        center3d = torch.cat([center2d, depth.view(1, -1, 1)], dim=-1).squeeze(0)
        center3d = self.pts2Dto3D(center3d, np.array(cam2imgs)).unsqueeze(0)

        # 3. compose 3D box
        batch_bboxes_3d = torch.cat([center3d, bbox_3d_preds[..., 4:], rot_y], dim=-1)

        return batch_bboxes_3d, sigma.view(1, -1)

    def decode_depth(self, direct_depth):

        if self.depth_mode == 'exp':
            direct_depth = direct_depth.exp()
        elif self.depth_mode == 'scale':
            direct_depth = direct_depth * 64 / priors[:, 3:4]
        elif self.depth_mode == 'inv_sigmoid':
            direct_depth = 1 / torch.sigmoid(direct_depth) - 1

        return direct_depth

    def pts2Dto3D(self, points, view):
        """
        Args:
            points (torch.Tensor): points in 2D images, [N, 3], \
                3 corresponds with x, y in the image and depth.
            view (np.ndarray): camera instrinsic, [3, 3]

        Returns:
            torch.Tensor: points in 3D space. [N, 3], \
                3 corresponds with x, y, z in 3D space.
        """
        assert view.shape[0] <= 4
        assert view.shape[1] <= 4
        assert points.shape[1] == 3

        points2D = points[:, :2]
        depths = points[:, 2].view(-1, 1)
        unnorm_points2D = torch.cat([points2D * depths, depths], dim=1)

        viewpad = torch.eye(4, dtype=points2D.dtype, device=points2D.device)
        viewpad[:view.shape[0], :view.shape[1]] = points2D.new_tensor(view)
        inv_viewpad = torch.inverse(viewpad).transpose(0, 1)

        # Do operation in homogenous coordinates.
        nbr_points = unnorm_points2D.shape[0]
        homo_points2D = torch.cat(
            [unnorm_points2D,
             points2D.new_ones((nbr_points, 1))], dim=1)
        points3D = torch.mm(homo_points2D, inv_viewpad)[:, :3]

        return points3D

    def inv_affine(self, points, trans_mats):
        # number of points
        N = points.shape[0]
        # batch_size
        N_batch = trans_mats.shape[0]
        batch_id = torch.arange(N_batch).unsqueeze(1)
        obj_id = batch_id.repeat(1, N // N_batch).flatten()
        trans_mats_inv = trans_mats.inverse()[obj_id]
        centers2d_extend = torch.cat((points, points.new_ones(N, 1)),
                                     dim=1)
        # expand project points as [N, 3, 1]
        centers2d_extend = centers2d_extend.unsqueeze(-1)
        # transform project points back on original image
        centers2d_img = torch.matmul(trans_mats_inv, centers2d_extend).squeeze(2)

        return centers2d_img[:, :2]

    def angle2class(self, angle):
        ''' Convert continuous angle to discrete class and residual. '''
        angle = angle % (2 * PI)
        assert (angle >= 0 and angle <= 2 * PI)
        angle_per_class = 2 * PI / float(self.num_alpha_bins)
        shifted_angle = (angle + angle_per_class / 2) % (2 * PI)
        class_id = int(shifted_angle / angle_per_class)
        residual_angle = shifted_angle - (class_id * angle_per_class + angle_per_class / 2)
        return class_id, residual_angle

    def class2angle(self, cls, residual):
        ''' Inverse function to angle2class. '''
        angle_per_class = 2 * PI / float(self.num_alpha_bins)
        angle_center = cls * angle_per_class
        angle = angle_center + residual
        return angle

    def decode_alpha_multibin(self, alpha_cls, alpha_offset):
        alpha_score, cls = alpha_cls.max(dim=-1)
        cls = cls.unsqueeze(2)
        alpha_offset = alpha_offset.gather(2, cls)
        alpha = self.class2angle(cls, alpha_offset)

        alpha[alpha > PI] = alpha[alpha > PI] - 2 * PI
        alpha[alpha < -PI] = alpha[alpha < -PI] + 2 * PI
        return alpha

    def recover_rotation(self, kpts, alpha, calib):
        device = kpts.device
        calib = torch.tensor(calib).type(torch.FloatTensor).to(device).unsqueeze(0)

        si = torch.zeros_like(kpts[:, :, 0:1]) + calib[:, 0:1, 0:1]
        rot_y = alpha + torch.atan2(kpts[:, :, 0:1] - calib[:, 0:1, 2:3], si)

        while (rot_y > PI).any():
            rot_y[rot_y > PI] = rot_y[rot_y > PI] - 2 * PI
        while (rot_y < -PI).any():
            rot_y[rot_y < -PI] = rot_y[rot_y < -PI] + 2 * PI

        return rot_y

    def encode(self, gt_bboxes_3d):
        pass

    def decode(self, bbox, base_centers2d, labels, downsample_ratio, cam2imgs):
        pass


# -*- coding: utf-8 -*-
"""
Created on Wed Oct  2 19:20:55 2019

@author: User
"""

from playsound import playsound
# INSTALL: pip install playsound
import time

'''
d = int(input())
i = int(input())

if direct == 'right':
    d = 0
elif direct == 'left':
    d = 1
else:
    print('direction not found')
'''
def soundplay(d,deg):
    if d == 0:
        for i in range(deg):
            playsound('beep-01a.wav')
    elif d == 1:
        for i in range(deg):
            playsound('beep-02.wav')
    else:
        playsound('button-30.wav')

inputs = [(0,1), (0,2), (0,3),(1,1),(0,1),(1,2),(2,0)]

for i in inputs:
    soundplay(i[0],i[1])
    time.sleep(2)


def order_weight(string):
    order = {}
    for ind, weight in enumerate(string.split()):
        w = sum(map(int, weight))
        try:
            order[w].append(weight)
        except:
            order[w] = [weight]

    weights = []
    for k in sorted(order):
        weights += sorted(order[k])
    return " ".join(w for w in weights)


import pandas as pd
import numpy as np
import re
import warnings
warnings.filterwarnings('ignore')
df = pd.read_csv('Airbnb_Texas_Rentals.csv')

# drop some columns
df = df.drop(['Unnamed: 0','url'], axis=1)

# Missing values?
df.isna().sum()

# drop all NA's
df = df.dropna().reset_index(drop=True)

# rename some columns
df = df.rename(columns={'average_rate_per_night': 'avg_cost',
                        'bedrooms_count': 'bedrooms',
                        'date_of_listing' : 'date'})

# split '$' dollar sign from rate_per_night and convert to an integer
df['avg_cost'] = df.avg_cost.apply(lambda x: x.split('$')[1]).astype(int)

# log scale price values
df['avg_cost_log'] = np.log1p(df['avg_cost']).astype('float32')

# convert 'bedrooms_count' from object to numeric.

df = df.replace({'bedrooms' : {'Studio' : 0}})   # Studio has 0 separate bedrooms.
df.bedrooms = df.bedrooms.astype(int)

# FUNCTIONS for text cleaning

import spacy
import en_core_web_sm as en
nlp = en.load()
from spacy.tokenizer import Tokenizer

tokenizer = Tokenizer(nlp.vocab)

from bs4 import BeautifulSoup

# HTML
def remove_htmlTags(text):
    untagged = BeautifulSoup(text, 'lxml')
    return(untagged.get_text())

def chars(text):
    text = re.sub(r"((?:\\\\?n+))", ' ', text)
    return(re.sub(r"(?:\s+)", ' ', text))

# change 'br', 'bed(s)', 'bedrooms', 'bdr' to 'bedroom'
def bedroom(text):
    text = re.sub("/",' ', text)
    pattern = r"((?:([0-9]+)\s?br)|(?:([0-9]+)\s?bed(?!s)(rooms?)?)|(?:([0-9]+)\s?bdr?))"
    if re.findall(pattern, text):
        for m in re.compile(pattern).finditer(text):
            group = [grp for grp in list(m.groups()) if grp is not None]
            return(re.sub(group[0], group[1]+' bedroom', text))
    else:
        return(text)

# change 'ba', 'bath', 'bathrooms' to 'bathroom'
def bathroom(text):
    text = re.sub("/",' ', text)
    pattern = r"((?:([0-9]+)\s?(?:ba(th)?(rooms?)?)))"
    if re.findall(pattern, text):
        for m in re.compile(pattern).finditer(text):
            group = [grp for grp in list(m.groups()) if grp is not None]
            return(re.sub(group[0], group[1]+' bathroom', text))
    else:
        return(text)

# sf, sqft etc to Square Feet
def sqft(text):
    pattern = r"((?:\s?sqft)|(?:\s?sf)|(?:\s?(?:sq\.?\s?ft\.?)))"
    if re.findall(pattern, text.lower()):
        for m in re.compile(pattern).finditer(text):
            return(re.sub(m.group(), ' square feet ', text))
    else:
        return(text)

# Lemmatize words
def lemmas(text):
    text = [t.lemma_.lower().strip() if t.lemma_ != '-PRON-' else t.lower_ for t in text]
    return(text)

################################
# Combining all functions to one

def text_cleaner(text, low = True, html = True, charLines = True, urls = True,
                 machines = True, bed = True, bath = True, sf = True,
                 punctuation = True, white_space = True, lemmatization = True):

    # lowercase
    if low:
        text = text.lower()
        text = re.sub('w/', 'with ', text)

    if html:
        text = remove_htmlTags(text)

    if charLines:
        text = chars(text)

    # remove airbnb.com urls
    if urls:
        text = re.sub(r"(?:https://(www.)?airbnb.com/rooms/[0-9]+)", ' ', text)

    # W/D, A/C devices
    if machines:
        text = re.sub('w/d', 'wash dryer', text)
        text = re.sub('a/c', 'air condition', text)

    if bed:
        text = bedroom(text)

    if bath:
        text = bathroom(text)
        text = re.sub('bth', ' bathroom', text)

    if sf:
        text = sqft(text)

    # punctuation, white spaces and dot removal
    if punctuation and white_space:
        text = re.sub(r"[^a-zA-Z0-9\.]", ' ', text)
        text = re.sub(' +', ' ', text)
        text = re.sub(r"(?<!\d)\.(?!\d)", '', text)

    # spaCy Tokenizer API
    text = tokenizer(text)

    if lemmatization:
        text = lemmas(text)
        text = [t for t in text if len(t)>1 or t.isdigit()]

    return(text)

# Apply functions on 'Title' and 'Description'

df['title'] = list(map(text_cleaner, df['title']))

df['description'] = list(map(text_cleaner, df['description']))

# Check for empty data points after cleaning text documnets.

print('empty rows in description: ', len([empty for empty in df.description if empty == []]))
print('empty rows in title: ', len([empty for empty in df.title if empty == []]))

# fill empty rows.
for desc in df.description:
    if desc == []:
        # replace with a bigram
        desc.append('no_description')

for desc in df.title:
    if desc == []:
        # replace with a bigram
        desc.append('no_title')

# Length of Tokens

df['tok_len'] = list(map(len, df['description']))
df['tok_len'] = np.log1p(df['tok_len']).astype('float32')

### ###
# This feature extracts the distance between property and metro area of big TX cities.
# As closer as to the city center is a property, the price might be larger.
# On the other hand if a property has 10 bedrooms, is possible to be at the suburbs or even more far.

# DISTANCE
from math import cos, sin, asin, sqrt, radians

rad_coors = [] # coordiantres in rads

# big cities of Texas, that are in the dataset
for cor in [[32.436161, -99.77401],
            [35.132131, -101.908448],
            [30.309522, -97.73171],
            [30.084549, -94.173759],
            [27.619349, -97.222016],
            [32.786636, -96.874669],
            [32.747097, -97.286434],
            [29.294488, -94.783691],
            [29.829352, -95.081549],
            [29.503068, -98.447688]]:
    rad_coors.append([radians(l) for l in cor]) #angles need to be in radians

# compute distance using ‘haversine’ formula.
def distance(lat, lon):
    R = 6371 #earth’s mean radius
    lat, lon = map(radians, (lat, lon))
    dists = []
    for cor in rad_coors:
        a = sin((cor[0]-lat) * 0.5)**2 + cos(cor[0])*cos(lat)*sin((cor[1]-lon)*0.5)**2
        dists.append(2 * R * asin(sqrt(a)))
        d = np.min(dists)
    return(d)

# map on new column
df['distance'] = list(map(distance, df['latitude'], df['longitude']))
# convert to log
df['Log_dist'] = np.log1p(df['distance']).astype('float32')

####### Vocabulary #######

from collections import Counter

tokens_count = Counter()

cols = ['title', 'description']
for col in cols:
    for word in df[col]:
        tokens_count.update(word)

print('unique total tokens = {}'.format(len(tokens_count)))

min_count = 10 # threshold for words with count less than 10
tokens = {token for token,count in tokens_count.items() if count >= min_count}

PAD, UNK = 'UNK', 'PAD'
tokens = [UNK,PAD] + sorted(tokens)
print('vocabulary size: {}'.format(len(tokens)))

tokens_id = {idx: tok for tok, idx in enumerate(tokens)}

### GloVe
# Pre-trained GloVe 100d word vectors.

embeddings_index = {}

# get vectors
f = open('glove100d/glove-twitter-100.txt', 'r+', encoding='utf-8')
for line in f:
    values = line.split()
    word = values[0]
    coefs = np.asarray(values[1:], dtype='float32')
    embeddings_index[word] = coefs
f.close()

print('Found {} word vectors.'.format(len(embeddings_index)))

# dimensions of word vector space
EMBED_SIZE = 100

embedding_matrix = np.zeros((len(tokens_id), EMBED_SIZE))
for word,i in tokens_id.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        embedding_matrix[i] = embedding_vector

print('size of matrix: {}'.format(len(embedding_matrix)))

# Create some -Bigrams- with trimming.

from gensim.models.phrases import Phraser, Phrases

# Generate Bigrams and reject those with count less than 50 and score under 100.
bigrams = Phraser(Phrases(df.description, min_count=50, threshold=100))

description = [bigrams[text] for text in list(df.description)]

title = [bigrams[text] for text in list(df.title)]

# Vectorize cities of Texas. Keep Top 100 of them.

# extract top 100 cities and towns
cities = set([Counter(df.city).most_common(100)[i][0] for i in range(99)])

df['city'] = df['city'].apply(lambda city: city if city in cities else 'Other')

text_feats = ['title', 'description']
categorical_feats = ['city']
contin_feats = ['Log_dist', 'tok_len']

from sklearn.feature_extraction import DictVectorizer

categorical_vectorizer = DictVectorizer(dtype=np.float32, sparse=False)

categorical_vectorizer.fit(df[categorical_feats].apply(dict, axis = 1))

# example of first 10 encodings.
from itertools import islice
list(islice(categorical_vectorizer.vocabulary_.items(), 5))

### MODEL

import keras
from keras.layers import (Input, Embedding, Conv1D, GlobalMaxPooling1D, Concatenate, Dense, SpatialDropout1D, Dropout)
from keras.optimizers import Adam
from keras.preprocessing.sequence import pad_sequences

# Maximum length of sequeces.

D_MAXLEN = 50 # description
T_MAXLEN = 6  # title

# PAD description
desc_seq = [[tokens_id.get(t,0) for t in desc] for desc in description]
desc_seq = pad_sequences(desc_seq, D_MAXLEN, padding='post', truncating='post')

# PAD title
title_seq = [[tokens_id.get(t,0) for t in ttl] for ttl in title]
title_seq = pad_sequences(title_seq, T_MAXLEN, padding='post', truncating='post')

import keras.backend as K
K.clear_session()

def multi_model(n_tokens = len(tokens), EMBED_SIZE = 100,
                embed_matrix = embedding_matrix,
                n_categ_feats = len(categorical_vectorizer.vocabulary_),
                hidden_size = 128, Sp_dr = None):

    # Embedding layer for textual data
    embed_layer = Embedding(n_tokens,
                            EMBED_SIZE,
                            weights = [embedding_matrix],
                            trainable = False)

    title = Input(shape = [None], name = 'title')
    title_embed = embed_layer(title)
    #title_embed = SpatialDropout1D(Sp_dr)(title_embed)
    title_conv = Conv1D(filters = hidden_size, kernel_size = 2, activation = 'relu')(title_embed)
    title_out = GlobalMaxPooling1D()(title_conv)

    descr = Input(shape = [None], name = 'description')
    descr_embed = embed_layer(descr)
    #descr_embed = SpatialDropout1D(Sp_dr)(descr_embed)
    descr_conv = Conv1D(filters = hidden_size, kernel_size = 3, activation = 'relu')(descr_embed)
    descr_out = GlobalMaxPooling1D()(descr_conv)

    # Fully connected layers
    city = Input(shape=[n_categ_feats])
    city_out = Dense(hidden_size)(city)

    bedroom = Input(shape = (1,))
    bedroom_out = Dense(hidden_size)(bedroom)

    #distance = Input(shape = (1,))
    #distance_out = Dense(hidden_size)(distance)

    t_length = Input(shape = (1,))
    t_length_out = Dense(hidden_size)(t_length)

    concat = Concatenate()([title_out, descr_out, city_out, bedroom_out, t_length_out]) #, distance_out])#, t_length_out])

    output = Dropout(0.1)(concat)

    output = Dense(128, activation='relu')(output)

    output = Dropout(0.1)(output)

    output_layer = Dense(1)(output)

    model = keras.models.Model(inputs=[title, descr, city, bedroom, t_length], # distance], #, t_length],
                               outputs=[output_layer])

    model.compile(loss = ['mean_squared_error'],
                  optimizer = Adam(lr=0.001),
                  metrics = ['mean_absolute_error'])

    return(model)

##
## 10 Fold Cross Validation
from sklearn.model_selection import KFold

kf = KFold(n_splits=10)
kf.get_n_splits(df.index)

X3 = categorical_vectorizer.transform(df[categorical_feats].apply(dict, axis=1))

results = []
scores = []

for train_index, test_index in kf.split(df.index):
    X_tr1, X_tst1 = title_seq[train_index], title_seq[test_index]
    X_tr2, X_tst2 = desc_seq[train_index], desc_seq[test_index]
    X_tr3, X_tst3 = X3[train_index], X3[test_index]
    X_tr4, X_tst4 = df.bedrooms.values[train_index], df.bedrooms.values[test_index]
    X_tr5, X_tst5 = df.tok_len.values[train_index], df.tok_len.values[test_index]

    y_tr, y_tst = df.avg_cost_log.values[train_index], df.avg_cost_log.values[test_index]

    model = None
    model = multi_model()

    result = model.fit([X_tr1, X_tr2, X_tr3, X_tr4, X_tr5], y_tr,
                        epochs = 20, verbose = 0, batch_size = 32,
                        validation_split = 0.2,
                        validation_data = ([X_tst1, X_tst2, X_tst3, X_tst4, X_tst5], y_tst))

    results.append(result)
    loss, score = model.evaluate([X_tst1, X_tst2, X_tst3, X_tst4, X_tst5], y_tst, batch_size=32)
    scores.append(score)

print('score: {} +- {}'.format((1-np.mean(scores))*100, np.std(scores)))

### PLOTS

import matplotlib.pyplot as plt

for i in range(len(results)):
    plt.figure(1, figsize=(10,10))
    plt.plot(results[i].history['val_mean_absolute_error'])
    plt.xlabel('epochs', fontsize =13)
    plt.ylabel('Mean Absolute Error', fontsize =13)
    plt.title('val Mean Absolute Error over 20 epochs and 10 folds', fontsize = 15)
plt.show()

# minimum score of 'scores' list
min_sc = np.argmin(scores)

# plot of val MSE of 7th fold.
plt.figure(1, figsize=(10,10))
plt.plot(results[min_sc].history['mean_absolute_error'])
plt.plot(results[min_sc].history['val_mean_absolute_error'])
plt.xlabel('epochs', fontsize =13)
plt.ylabel('Mean Absolute Error', fontsize =13)
plt.legend(['MAE','val MAE'])
plt.title('Mean Absolute Error over 20 epochs', fontsize = 15)
plt.show()

# plot of val los of 7th fold
plt.figure(1, figsize=(10,10))
plt.plot(results[min_sc].history['loss'])
plt.plot(results[min_sc].history['val_loss'])
plt.xlabel('epochs', fontsize =13)
plt.ylabel('MSE loss', fontsize =13)
plt.legend(['MSE loss','MSE val loss'])
plt.title('Mean Squared Error loss', fontsize = 15)
plt.show()

# y_train / y_test variances

import matplotlib.pyplot as plt
import seaborn as sns

i=1
for train_index, test_index in kf.split(df.index):
    y_tr, y_tst = df.avg_cost_log.values[train_index], df.avg_cost_log.values[test_index]
    sns.distplot(df.avg_cost_log.values[train_index], hist = False)
    sns.distplot(df.avg_cost_log.values[test_index], hist = False)
    plt.title('test set {}, ({}-{})'.format(str(i), str(test_index[0]),str(test_index[-1])), fontsize = 14)
    i+=1
    plt.show()

##END