# TODO :
#  ElasticSearch : 검색기능 구현(DB)
#  DRF Swagger(ysag) : API 문서화 작업용
#  Celery(+Redis, + Naver SENS) : 문자인증을 위한 Naver SENS API 비동기 작동
#  POSTMAN 설치 후 사용(DRF API Check)
import json
import os
import random
import datetime
import calendar
from django.contrib.auth import get_user_model
from django.core.exceptions import ObjectDoesNotExist
from rest_framework import generics, status
from rest_framework.authtoken.models import Token
from rest_framework.authtoken.serializers import AuthTokenSerializer
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from rest_framework.throttling import AnonRateThrottle
from rest_framework.views import APIView
import requests

from tables.models import TableLog
from .serializers import UserSerializer, PhoneNumberVerificationSerializer, CheckUniqueIDSerializer, \
    SocialAuthTokenSerializer

User = get_user_model()

class SignupView(generics.CreateAPIView):
    '''
    회원가입 API.
    아래 4개 필수 항목을 입력해 전달하면, 타입 유효성 검사 후 가입 처리
        'username',
        'password',
        'name',
        'phone_number'
    나머지는 기입하지 않더라도 디폴트값 혹은 Null값 입력
    '''
    queryset = User.objects.all()
    serializer_class = UserSerializer

    def perform_create(self, serializer):
        instance = serializer.save()
        instance.set_password(instance.password)
        instance.save()

        date_range = [
            datetime.date.today().replace(day=1) + datetime.timedelta(i)
            for i in range(0, calendar.monthrange(datetime.date.today().year, datetime.date.today().month)[1])
        ]
        for date in date_range:
            for time in ['Breakfast', 'Lunch', 'Dinner', 'Snack']:
                TableLog.objects.get_or_create(user=User.objects.get(pk=instance.pk), date=date, time=time)

class CheckUniqueIDView(APIView):
    '''
    유저 ID 중복검사를 위한 View
    validation 과정에서 입력한 ID가 이미 존재하는지 체크
    "unique_id" : True / False 를 리턴한다.
    '''
    def post(self, request):
        serializer = CheckUniqueIDSerializer(data=request.data)
        if serializer.is_valid():
            return Response({
                "unique_id": True,
                "message": "사용 가능한 아이디입니다."
            }, status=status.HTTP_200_OK)
        return Response({
            "unique_id": False,
            "message": "이미 존재하는 아이디입니다."
        }, status=status.HTTP_200_OK)

root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
secrets = json.load(open(os.path.join(os.path.join(root_dir, '.secrets'), 'base.json')))

class PhoneNumberVerificationView(APIView):
    '''
    휴대폰 번호 인증을 위한 NAVER SENS API 연동
    휴대폰 번호를 전달하면 정규식 검증(10-11자리 숫자로 이루어진 문자열 여부 확인, - 제외)
    유효한 형식임이 확인되면 NAVER SENS를 통해 입력된 번호로 랜덤 인증번호(1000~9999 사이) 발송
    발송에 성공한 경우 {"verifiation" : <인증번호> , "message" : <인증 성공>} 전달
    실패했을 경우 {"verification" : False, "message" : <인증 실패> 전달
    '''
    # throttle classes : 익명 유저의 verification 신청 횟수 제한
    throttle_classes = (AnonRateThrottle,)

    def post(self, request):
        serializer = PhoneNumberVerificationSerializer(data=request.data)
        if serializer.is_valid():
            service_id = secrets['SENS_SERVICE_ID']
            random_num = str(random.randrange(1000, 9999))
            send_url = f'https://api-sens.ncloud.com/v1/sms/services/{service_id}/messages'
            headers = {
                "Content-Type": "application/json; charset=utf-8",
                "X-NCP-auth-key": secrets['X-NCP-AUTH-KEY'],
                "X-NCP-service-secret": secrets['X-NCP-SERVICE-SECRET']
            }
            body = {
                "type": "SMS",
                "from": secrets['FROM_PHONE_NUMBER'],
                "to": [
                    serializer.data['phone_number']
                ],
                "content": "인증번호는 " + random_num + "입니다."
            }
            res = requests.post(send_url, headers=headers, data=json.dumps(body))
            if not res.json()['status'] == '200':
                return Response({"verification": False, "verificationNumber": "", "message": "인증번호 발송에 실패했습니다."},
                                status=status.HTTP_400_BAD_REQUEST)
            return Response({"verification": True, "verificationNumber": random_num, "message": "인증번호가 발송되었습니다."},
                            status=status.HTTP_202_ACCEPTED)

class AuthTokenView(APIView):
    '''
    Login View.
    Post 요청으로 username, password를 받아
    serializer에서 사용자인증(authenticate)에 성공하면
    해당 사용자와 연결된 토큰 정보를 리턴하거나 없다면 새로 생성한다.
    '''
    def post(self, request):
        serializer = AuthTokenSerializer(data=request.data)
        if serializer.is_valid():
            user = serializer.validated_data['user']
            token, created = Token.objects.get_or_create(user=user)
            return Response({"token": token.key}, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class SocialAuthTokenView(APIView):
    '''
    Social Login View.
    Post 요청으로 iOS-SNS API 통신으로 전달받은 user_id를 확인하여
    이미 있던 계정이면 그에 해당하는 토큰을, 없다면 새롭게 토큰을 생성한다.
    '''
    def post(self, request):
        serializer = SocialAuthTokenSerializer(data=request.data)
        if serializer.is_valid():
            token, created = Token.objects.get_or_create(user=serializer.user)[0]
            return Response({"token": token.key}, status=status.HTTP_200_OK)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class LogoutView(APIView):
    '''
    로그아웃 View. 유저에 할당되었던 토큰을 삭제해준다.
    delete method로 request를 요청해야함
    유저가 토큰을 가지고 있을 경우에만 접근 가능(IsAuthenticated)
    '''
    permission_classes = (IsAuthenticated,)

    def delete(self, request):
        try:
            request.user.auth_token.delete()
        except (AttributeError, ObjectDoesNotExist):
            return Response({"logout": True, "message": "이미 로그아웃 처리되었습니다."},
                            status=status.HTTP_204_NO_CONTENT)
        return Response({"logout": True}, status=status.HTTP_200_OK)

class UserProfileView(generics.RetrieveUpdateAPIView):
    permission_classes = (IsAuthenticated,)
    queryset = User.objects.all()
    serializer_class = UserSerializer

    # members/profile/로 받기 때문에, pk가 추가 인자로 들어오지 않는다.
    # 따라서 lookup_urlkwarg / lookup_field > 기본값 "pk"가 주어지지 않은 경우
    # request.user를 선택하여 리턴하도록 한다.
    def get_object(self):
        lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
        if lookup_url_kwarg not in self.kwargs:
            return self.request.user

    # method for creating password hashing relation
    def perform_update(self, serializer):
        super(UserProfileView, self).perform_update(serializer)
        instance = serializer.save()
        instance.set_password(instance.password)
        instance.save()


import os
from PIL import Image
from scipy.ndimage.filters import gaussian_filter
import cv2
import shutil       # To copy the file
import sys
import numpy as np
import torch
from torchvision import transforms
from torch.utils.data import Dataset
import torch.nn.functional as F
import torchvision
from tqdm import tqdm
import urllib.request
import py7zr

import backbone as bb

aitex_folder = './datasets/AITEX/'
aitex_train_dir = aitex_folder + 'trainset/'
aitex_test_dir = aitex_folder + 'testset/'
aitex_mask_dir = aitex_folder + 'Mask_images/'
aitex_config_file = aitex_folder + 'config'

Defect_path = aitex_folder + 'Defect_images/'
NODefect_path = aitex_folder + 'NODefect_images/'
CUT_PATCHES = 1

AITEX_CLASS_NAMES = ['00', '01', '02', '03', '04', '05', '06']

PATCH_SIZE = 256            # patch size
STRIDE = PATCH_SIZE         # stride of patch
ANOMALY_THRESHOLD = 0       # threshold to consider a patch as anomalous

class AitexDataSet(Dataset):
    def __init__(self, class_name='03', resize=256, cropsize=224, is_train=True):
        self.is_train = is_train
        self.class_name = class_name
        self.resize = resize
        self.cropsize = cropsize
        self.transform = transforms.Compose([transforms.Resize(resize, Image.ANTIALIAS),
                                                transforms.CenterCrop(cropsize),
                                                transforms.ToTensor(),
                                                transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                                  std=[0.229, 0.224, 0.225])])
        self.transform_mask = transforms.Compose([transforms.Resize(resize, Image.NEAREST),
                                                    transforms.CenterCrop(cropsize),
                                                    transforms.ToTensor()])
        self.main_dir = aitex_train_dir+self.class_name+'/' if self.is_train else aitex_test_dir+self.class_name+'/'
        self.all_imgs = sorted(os.listdir(self.main_dir))
        self.mask_dir = aitex_mask_dir
        if not self.is_train:
            self.all_mask = sorted(os.listdir(self.mask_dir))

    def __len__(self):
        return len(self.all_imgs)

    def __getitem__(self, idx):
        img_loc = os.path.join(self.main_dir, self.all_imgs[idx])
        image = Image.open(img_loc).convert('RGB')
        tensor_image = self.transform(image)                                    # x in mvtec class
        mask_name = self.all_imgs[idx].replace('.png', '_mask.png')
        if os.path.isfile(self.mask_dir + '/' + mask_name):
            mask_loc = os.path.join(self.mask_dir, mask_name)
            mask = Image.open(mask_loc).convert('L')
            tensor_mask = self.transform_mask(mask)                             # mask in mvtec class
        else:
            tensor_mask = torch.zeros([1, self.cropsize, self.cropsize])
        if int(torch.sum(tensor_mask)) > ANOMALY_THRESHOLD:                     # y in mvtec class
            defective = 1
        else:
            defective = 0
        return tensor_image, defective, tensor_mask

    def getName(self, idx, mask=False):
        if mask:
            return self.all_imgs[idx].replace('.png', '_mask.png')
        else:
            return self.all_imgs[idx]

def resizeAitex(dataset, original_width=4096, original_height=256):
    img_ = dataset.squeeze(0).numpy()
    img_ = cv2.normalize(img_, None, alpha = 0, beta = 255, norm_type = cv2.NORM_MINMAX, dtype = cv2.CV_32F)
    img_ = img_.astype(np.uint8)

    # Blur the image for better edge detection
    img_blur = cv2.GaussianBlur(img_,(3,3), sigmaX=0, sigmaY=0)

    # Canny Edge Detection
    edges = cv2.Canny(image=img_blur, threshold1=100, threshold2=200)

    vector = np.zeros(original_width)
    for i in range(original_width):
        for j in range(original_height):
            vector[i] += edges[j][i]
    derivative = np.gradient(vector)
    max = np.argmax(derivative)
    cut = (int(max/PATCH_SIZE) + CUT_PATCHES) * PATCH_SIZE

    crop_img = transforms.functional.crop(dataset, top=0, left=cut, height=original_height, width=(original_width-cut))
    new_widths = crop_img.shape[2]
    new_heights = crop_img.shape[1]

    return crop_img, new_widths, new_heights

# --------------- Functions for patches ---------------

def DivideInPatches(img, size, stride):
    patches = img.unfold(1, size, stride).unfold(2, size, stride)
    patches = patches.contiguous().view(patches.size(0), -1, size, size).permute(1,0,2,3)
    return patches

def countAitexAnomalies():
    masks = sorted(os.listdir(aitex_mask_dir))
    number_of_defects = 0
    defective = []
    transform_mask = transforms.ToTensor()
    for mask_name in masks:
        mask_loc = os.path.join(aitex_mask_dir, mask_name)
        mask = Image.open(mask_loc).convert('L')
        tensor_mask = transform_mask(mask)
        if int(torch.sum(tensor_mask)) > ANOMALY_THRESHOLD:
            number_of_defects += 1
            defective.append(True)
        else:
            defective.append(False)
    return number_of_defects, defective

# --------------- Functions to create Aitex Dataset ---------------

def Reformat_Image(ImageFilePath, new_width, new_height, color, offset):

    image = Image.open(ImageFilePath, 'r')
    image_size = image.size
    width = image_size[0]
    height = image_size[1]

    if color == 'white':
        color = (255, 255, 255, 255)
    elif color == 'black':
        color = (0, 0, 0, 255)

    if offset == 'center':
        offset = (int(round(((new_width - width) / 2), 0)), int(round(((new_height - height) / 2), 0)))
    elif offset == 'right':
        offset = (0, 0)
    elif offset == 'left':
        offset = ((new_width - width), (new_height - height))

    background = Image.new('RGBA', (new_width, new_height), color)

    background.paste(image, offset)
    background.save(ImageFilePath)

def DeleteFolder(path):
    shutil.rmtree(path)

def MergeMasks(name):
    mask1 = Image.open(name+'_mask1.png').convert('L')
    mask2 = Image.open(name+'_mask2.png').convert('L')
    mask1 = np.array(mask1)
    mask2 = np.array(mask2)
    mask = np.add(mask1, mask2)
    mask = Image.fromarray(mask)
    mask.save(name+'_mask.png',"png")
    os.remove(name+'_mask1.png')
    os.remove(name+'_mask2.png')

def BinarizeMasks(Mask_path):
    thresh = 128
    maxval = 255

    all_imgs = sorted(os.listdir(Mask_path))
    for i in all_imgs:
        im_gray = np.array(Image.open(Mask_path+i).convert('L'))
        im_bin = (im_gray > thresh) * maxval
        Image.fromarray(np.uint8(im_bin)).save(Mask_path+i)

def RenameFolder(oldname, newname):
    os.rename(oldname, newname)

def FlipImage(filename):
    image = Image.open(filename)
    image = np.fliplr(image)
    Image.fromarray(np.uint8(image)).save(filename)

def CreateAitexDataset(resize, log_file):
    try:
        bb.myPrint("Preparing the AITEX dataset...", log_file)
        NODefect_subdirectories = {
            '2311694-2040n7u': '00',
            '2608691-202020u': '01',
            '2306894-210033u': '02',
            '2311694-1930c7u': '03',
            '2311517-195063u': '04',
            '2306881-210020u': '05',
            '2311980-185026u': '06'
            }

        os.makedirs(aitex_train_dir, exist_ok=True)
        os.makedirs(aitex_test_dir, exist_ok=True)

        for i in range(len(NODefect_subdirectories)):
            RenameFolder(NODefect_path+list(NODefect_subdirectories.keys())[i], NODefect_path+list(NODefect_subdirectories.values())[i])
            os.makedirs(aitex_train_dir+list(NODefect_subdirectories.values())[i], exist_ok=True)
            os.makedirs(aitex_test_dir+list(NODefect_subdirectories.values())[i], exist_ok=True)

        MergeMasks(aitex_mask_dir+'0044_019_04')   # Merge and delete 0044_019_04.png masks
        MergeMasks(aitex_mask_dir+'0097_030_03')   # Merge and delete 0097_030_03.png masks
        BinarizeMasks(aitex_mask_dir)
        Reformat_Image(Defect_path + '0094_027_05.png', 4096, 256, 'white', 'right')
        Reformat_Image(aitex_mask_dir + '0094_027_05_mask.png', 4096, 256, 'black', 'right')
        os.remove(Defect_path + '0100_025_08.png')
        FlipImage(Defect_path + '0094_027_05.png')
        FlipImage(aitex_mask_dir + '0094_027_05_mask.png')

        defect_images = os.listdir(Defect_path)
        nodefect_images = []

        for i in range(len(NODefect_subdirectories)):
            for j in os.listdir(NODefect_path + list(NODefect_subdirectories.values())[i]):
                nodefect_images.append(list(NODefect_subdirectories.keys())[i] + '/' + j)

        for i in range(len(NODefect_subdirectories)):
            new_folder = Defect_path+list(NODefect_subdirectories.values())[i] + '/'
            os.makedirs(new_folder, exist_ok=True)
            for img in defect_images:
                if list(NODefect_subdirectories.values())[i]+'.png' in img:
                    shutil.move(Defect_path + img, new_folder + img)

        Mask_path_temp = aitex_folder + '/Mask_images_temp/'
        RenameFolder(aitex_mask_dir, Mask_path_temp)
        os.makedirs(aitex_mask_dir, exist_ok=True)

        for i in range(len(NODefect_subdirectories)):
            last_image = os.listdir(NODefect_path+list(NODefect_subdirectories.values())[i] + '/')[-1]
            new_folder = Defect_path+list(NODefect_subdirectories.values())[i] + '/' + last_image
            old_folder = NODefect_path+list(NODefect_subdirectories.values())[i] + '/' + last_image
            shutil.move(old_folder, new_folder)

        transform = transforms.Compose([
            transforms.ToTensor()
        ])

        for i in range(len(NODefect_subdirectories)):
            train_folder_temp = NODefect_path+list(NODefect_subdirectories.values())[i] + '/'
            all_train_imgs = sorted(os.listdir(train_folder_temp))
            for img in all_train_imgs:
                img_loc = os.path.join(train_folder_temp, img)
                image = Image.open(img_loc).convert('L')
                tensor_image = transform(image)
                if resize:
                    tensor_image, _, _ = resizeAitex(tensor_image)
                train_patches = DivideInPatches(tensor_image, PATCH_SIZE, STRIDE)
                for idx, patch in enumerate(train_patches):
                    name = img.replace('.png', '_'+str(idx)+'.png')
                    name = os.path.join(aitex_train_dir+list(NODefect_subdirectories.values())[i] + '/', name)
                    torchvision.utils.save_image(patch, name)
            test_folder_temp = Defect_path+list(NODefect_subdirectories.values())[i] + '/'
            all_test_imgs = sorted(os.listdir(test_folder_temp))
            for img in all_test_imgs:
                img_loc = os.path.join(test_folder_temp, img)
                image = Image.open(img_loc).convert('L')
                tensor_image = transform(image)
                if resize:
                    tensor_image, new_widths, _ = resizeAitex(tensor_image)
                test_patches = DivideInPatches(tensor_image, PATCH_SIZE, STRIDE)
                for idx, patch in enumerate(test_patches):
                    name = img.replace('.png', '_'+str(idx)+'.png')
                    name = os.path.join(aitex_test_dir+list(NODefect_subdirectories.values())[i] + '/', name)
                    torchvision.utils.save_image(patch, name)
                mask_name = img.replace('.png', '_mask.png')
                if os.path.isfile(Mask_path_temp + mask_name):
                    mask_loc = os.path.join(Mask_path_temp, mask_name)
                    mask = Image.open(mask_loc).convert('L')
                    tensor_mask = transform(mask)
                else:
                    tensor_mask = torch.zeros([1, 256, 4096])
                if resize:
                    tensor_mask = transforms.functional.crop(tensor_mask, top=0, left=(4096-new_widths), height=256, width=new_widths)
                test_masks = DivideInPatches(tensor_mask, PATCH_SIZE, STRIDE)
                for idx, patch in enumerate(test_masks):
                    name = mask_name.replace('_mask.png', '_'+str(idx)+'_mask.png')
                    name = os.path.join(aitex_mask_dir, name)
                    torchvision.utils.save_image(patch, name)

        DeleteFolder(Defect_path)
        DeleteFolder(NODefect_path)
        DeleteFolder(Mask_path_temp)
        f = open(aitex_config_file, "a")
        f.write(str(resize))
        f.close()
    except Exception as e:
        bb.myPrint(e, log_file)
        bb.myPrint("Error in CreateAitexDataset function!", log_file)
        DeleteFolder(aitex_folder)
        sys.exit(-1)

def prepareAitex(resize, log_file):
    if os.path.isdir(aitex_folder):
        if (os.path.isdir(aitex_train_dir) and os.path.isdir(aitex_test_dir) and os.path.isdir(aitex_mask_dir)):
            f = open(aitex_config_file, "r")
            resize_ = f.readline()
            f.close()
            resize_ = True if resize_ == "True" else False
            if resize == resize_:
                return
            else:
                DeleteFolder(aitex_folder)
                download(log_file)
                CreateAitexDataset(resize, log_file)
        else:
            DeleteFolder(aitex_folder)
            download(log_file)
            CreateAitexDataset(resize, log_file)
    else:
        download(log_file)
        CreateAitexDataset(resize, log_file)

# --------------- Functions to download Aitex Dataset ---------------

URL = 'https://www.aitex.es/wp-content/uploads/2019/07/'

ARCHIVES = [
    'Defect_images.7z',
    'NODefect_images.7z',
    'Mask_images.7z'
]

def download(log_file):
    bb.myPrint("Download AITEX dataset...", log_file)
    os.makedirs(aitex_folder, exist_ok=True)
    try:
        for idx in range(len(ARCHIVES)):
            if not os.path.isfile(aitex_folder+ARCHIVES[idx]):
                download_url(URL+ARCHIVES[idx], aitex_folder+ARCHIVES[idx])
            with py7zr.SevenZipFile(aitex_folder+ARCHIVES[idx], mode='r') as z:
                z.extractall(path=aitex_folder)
            os.remove(aitex_folder+ARCHIVES[idx])
        return
    except Exception as e:
        bb.myPrint(str(e), log_file)
        bb.myPrint("Can't download AITEX dataset. Retry later.", log_file)
        sys.exit(-1)

class DownloadProgressBar(tqdm):
    def update_to(self, b=1, bsize=1, tsize=None):
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)

def download_url(url, output_path):
    with DownloadProgressBar(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t:
        urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)


from lscore.midi.midifile import MIDIFile
from lscore.midi.midiinstruments import *
from lscore.lsystem.lsystem import *

from  musicalinterpretation import MusicalInterpretation
import scales

""" Schenkerian Rendering - see
Growing Music: musical interpretations of L-Systems by
Peter Worth and Susan Stepney

"""
class SchenkerianRendering(MusicalInterpretation):
	def __init__(self,scale,
					  tempo = 120,
					  instrument_name = DISTORTION_GUITAR,
					  multiplier = 4,
					  output_name = 'output.mid'):
		MusicalInterpretation.__init__(self,scale,tempo,output_name,instrument_name,multiplier)

	def create_score(self,string):
		timeOffset = 0
		currentNote = 0
		#Monta pilha com a string
		#O topo da pilha é o primeiro elemento da string
		strStack = list(string)
		noteStack = list()
		timeStack = list()
		code = str()

		midi = MIDIFile(self.output_name)
		midi.text(string)
		mpqn = 60000000/self.tempo
		midi.setTempo(mpqn)
		midi.timeSignature(4,2,24,8)
		midi.patchChange(0,self.instrument_name)

		noteLength = 0
		while True:
			try:
				tok = strStack.pop(0)
				if tok is 'F':
					noteLength += 1
				if tok is '+':
					currentNote +=1
				elif tok is '-':
					currentNote -=1
				elif tok is '[':
					noteStack.append(currentNote)
					timeStack.append(noteLength)
					noteLength = 0
				elif tok is ']':
					if noteLength is not 0:
						note = self.scale.getMIDINote(currentNote)
						midi.noteOn(0,note,127)
						ticks = (midi.timeDivision / self.multiplier)*noteLength
						midi.noteOff(0,note,127,ticks)

					currentNote = noteStack.pop()
					noteLength = timeStack.pop()
			except IndexError:
				note = self.scale.getMIDINote(currentNote)
				midi.noteOn(0,note,127)
				ticks = (midi.timeDivision / self.multiplier)*noteLength
				midi.noteOff(0,note,127,ticks)
				break
		midi.eof()
		midi.write()

if __name__ == "__main__":
	major = scales.Scale(scales.MAJOR)
	s = SchenkerianRendering(major)
	l = LSystem("X")
	l.addProduction("X","F[+X][-X]FX")
	l.addProduction("F","FF")
	s.create_score(l.generateString(3))


from __future__ import absolute_import

from concurrent.futures import ThreadPoolExecutor

import grpc

from apache_beam.portability.api import beam_runner_api_pb2_grpc
from apache_beam.portability.api.beam_runner_api_pb2_grpc import TestStreamServiceServicer

class TestStreamServiceController(TestStreamServiceServicer):
  def __init__(self, events, endpoint=None):
    self._server = grpc.server(ThreadPoolExecutor(max_workers=10))

    if endpoint:
      self.endpoint = endpoint
      self._server.add_insecure_port(self.endpoint)
    else:
      port = self._server.add_insecure_port('[::]:0')
      self.endpoint = '[::]:{}'.format(port)

    beam_runner_api_pb2_grpc.add_TestStreamServiceServicer_to_server(
        self, self._server)
    self._events = events

  def start(self):
    self._server.start()

  def stop(self):
    self._server.stop(0)
    self._server.wait_for_termination()

  def Events(self, request, context):
    """Streams back all of the events from the streaming cache."""

    for e in self._events:
      yield e


import unittest

from mock import Mock

from apache_beam.metrics.cells import DistributionData
from apache_beam.runners.google_cloud_dataflow.dataflow_runner import DataflowRunner
from apache_beam.runners.google_cloud_dataflow.internal import apiclient
from apache_beam.runners.google_cloud_dataflow.internal.clients import dataflow
from apache_beam.utils.pipeline_options import PipelineOptions

class UtilTest(unittest.TestCase):

  @unittest.skip("Enable once BEAM-1080 is fixed.")
  def test_create_application_client(self):
    pipeline_options = PipelineOptions()
    apiclient.DataflowApplicationClient(
        pipeline_options,
        DataflowRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)

  def test_default_job_name(self):
    job_name = apiclient.Job.default_job_name(None)
    regexp = 'beamapp-.*-[0-9]{10}-[0-9]{6}'
    self.assertRegexpMatches(job_name, regexp)

  def test_split_int(self):
    number = 12345
    split_number = apiclient.to_split_int(number)
    self.assertEqual((split_number.lowBits, split_number.highBits),
                     (number, 0))
    shift_number = number << 32
    split_number = apiclient.to_split_int(shift_number)
    self.assertEqual((split_number.lowBits, split_number.highBits),
                     (0, number))

  def test_translate_distribution(self):
    metric_update = dataflow.CounterUpdate()
    distribution_update = DistributionData(16, 2, 1, 15)
    apiclient.translate_distribution(distribution_update, metric_update)
    self.assertEqual(metric_update.distribution.min.lowBits,
                     distribution_update.min)
    self.assertEqual(metric_update.distribution.max.lowBits,
                     distribution_update.max)
    self.assertEqual(metric_update.distribution.sum.lowBits,
                     distribution_update.sum)
    self.assertEqual(metric_update.distribution.count.lowBits,
                     distribution_update.count)

  def test_translate_means(self):
    metric_update = dataflow.CounterUpdate()
    accumulator = Mock()
    accumulator.sum = 16
    accumulator.count = 2
    apiclient.MetricUpdateTranslators.translate_scalar_mean_int(accumulator,
                                                                metric_update)
    self.assertEqual(metric_update.integerMean.sum.lowBits, accumulator.sum)
    self.assertEqual(metric_update.integerMean.count.lowBits, accumulator.count)

    accumulator.sum = 16.0
    accumulator.count = 2
    apiclient.MetricUpdateTranslators.translate_scalar_mean_float(accumulator,
                                                                  metric_update)
    self.assertEqual(metric_update.floatingPointMean.sum, accumulator.sum)
    self.assertEqual(
        metric_update.floatingPointMean.count.lowBits, accumulator.count)

if __name__ == '__main__':
  unittest.main()


# -*- coding: utf-8 -*-
"""
Created on Sat Aug  3 12:48:55 2019

@author: sudesh.amarnath
"""

import boto3
import os
import glob
import findspark
findspark.init('/home/ubuntu/spark-2.1.1-bin-hadoop2.7')
import pyspark
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('test').getOrCreate()
from pyspark.sql.functions import concat_ws,concat,lit,Column,regexp_replace

def upload_file(file_name, bucket, object_name=None):
    if object_name is None:
        object_name = file_name
    s3_client = boto3.client('s3')
    response = s3_client.upload_file(file_name, bucket, object_name)
s3 = boto3.resource('s3')
my_bucket = s3.Bucket('sudeshrandom')

for file in my_bucket.objects.all():
    if ".json" in file.key:
        file_name=file.key
        fname=file_name.replace('new/','')
        with open(fname,'w') as f:
            wfile_path='/home/ubuntu/processed/'+fname.replace('.json','')
            obj = my_bucket.Object(file.key)
            f.write(obj.get()['Body'].read().decode('utf-8'))
            df= spark.read.option("multiLine", "true").json(fname)
            df.select(concat_ws("",df['info.seed']).alias("id"),\
                        concat(concat_ws("",df['results.name.first']),lit(' '),concat_ws("",df['results.name.last'])).alias('Full_Name'),\
                        concat_ws("",df['results.gender']).alias("Gender"),\
                        concat_ws("",df['results.dob.date']).astype('date').alias("DoB"),\
                        concat_ws("",df['results.email']).alias("Email"),\
                        concat_ws("",df['results.phone']).alias("home_phone"),\
                        concat_ws("",df['results.cell']).alias("cell_phone"),\
                        concat_ws("",df['results.location.street']).alias("Street"),\
                        concat_ws("",df['results.location.city']).alias("City"),\
                        concat_ws("",df['results.location.state']).alias("State"),\
                        concat_ws("",df['results.nat']).alias("Country"),\
                        concat_ws("",df['results.location.postcode']).astype('int').alias("Postcode"),\
                        concat_ws("",df['results.location.coordinates.latitude']).alias("Latitude"),\
                        concat_ws("",df['results.location.coordinates.longitude']).alias("Longitude")
                        ).coalesce(1).write.option("mapreduce.fileoutputcommitter.marksuccessfuljobs","false").option("header","true").csv(wfile_path)
            os.remove(fname)

            for s3_file in glob.glob(wfile_path+'/*.csv'):
                cname='processed/'+fname.replace('.json','.csv')
                upload_file(s3_file,'sudeshrandom',cname)


"""Re-export of some bazel rules with repository-wide defaults."""

load("@npm//@angular/bazel:index.bzl", _ng_module = "ng_module", _ng_package = "ng_package")
load("@build_bazel_rules_nodejs//:index.bzl", _pkg_npm = "pkg_npm")
load("@npm//@bazel/jasmine:index.bzl", _jasmine_node_test = "jasmine_node_test")
load("@npm//@bazel/esbuild:index.bzl", "esbuild")
load(
    "@npm//@bazel/concatjs:index.bzl",
    _ts_library = "ts_library",
)

DEFAULT_TSCONFIG_BUILD = "//modules:bazel-tsconfig-build.json"
DEFAULT_TSCONFIG_TEST = "//modules:bazel-tsconfig-test"

def _getDefaultTsConfig(testonly):
    if testonly:
        return DEFAULT_TSCONFIG_TEST
    else:
        return DEFAULT_TSCONFIG_BUILD

def ts_library(
        tsconfig = None,
        testonly = False,
        deps = [],
        devmode_module = None,
        **kwargs):
    deps = deps + ["@npm//tslib", "@npm//@types/node"]
    if testonly:
        deps.append("@npm//@types/jasmine")

    if not tsconfig:
        tsconfig = _getDefaultTsConfig(testonly)

    if not devmode_module:
        devmode_module = "commonjs"

    _ts_library(
        tsconfig = tsconfig,
        testonly = testonly,
        devmode_module = devmode_module,
        devmode_target = "es2022",
        prodmode_target = "es2022",
        deps = deps,
        **kwargs
    )

EXPRESS_VERSION = "^4.15.2"
EXPRESS_TYPES_VERSION = "^4.17.0"

NGUNIVERSAL_SCOPED_PACKAGES = ["@nguniversal/%s" % p for p in [
    "builders",
    "common",
    "express-engine",
]]

PKG_GROUP_REPLACEMENTS = {
    "\"NG_UPDATE_PACKAGE_GROUP\"": """[
      %s
    ]""" % ",\n      ".join(["\"%s\"" % s for s in NGUNIVERSAL_SCOPED_PACKAGES]),
    "EXPRESS_VERSION": EXPRESS_VERSION,
    "EXPRESS_TYPES_VERSION": EXPRESS_TYPES_VERSION,
}

def ng_module(name, package_name, module_name = None, tsconfig = None, testonly = False, deps = [], **kwargs):
    deps = deps + ["@npm//tslib", "@npm//@types/node"]

    if not tsconfig:
        tsconfig = _getDefaultTsConfig(testonly)

    if not module_name:
        module_name = package_name

    _ng_module(
        name = name,
        module_name = package_name,
        package_name = package_name,
        flat_module_out_file = name,
        tsconfig = tsconfig,
        testonly = testonly,
        deps = deps,
        **kwargs
    )

def jasmine_node_test(deps = [], **kwargs):
    local_deps = [
        "@npm//source-map-support",
    ] + deps

    _jasmine_node_test(
        deps = local_deps,
        configuration_env_vars = ["compile"],
        **kwargs
    )

def ng_test_library(name, entry_point = None, deps = [], tsconfig = None, **kwargs):
    local_deps = [
        # We declare "@angular/core" as default dependencies because
        # all Angular component unit tests use the `TestBed` and `Component` exports.
        "@npm//@angular/core",
    ] + deps

    if not tsconfig:
        tsconfig = _getDefaultTsConfig(1)

    ts_library_name = name + "_ts_library"
    ts_library(
        name = ts_library_name,
        testonly = 1,
        tsconfig = tsconfig,
        deps = local_deps,
        **kwargs
    )

    esbuild(
        name,
        testonly = 1,
        args = {
            "keepNames": True,
            # ensure that esbuild prefers .mjs to .js if both are available
            # since ts_library produces both
            "resolveExtensions": [
                ".mjs",
                ".js",
            ],
        },
        output = name + "_spec.js",
        entry_point = entry_point,
        format = "iife",
        # We cannot use `ES2017` or higher as that would result in `async/await` not being downleveled.
        # ZoneJS needs to be able to intercept these as otherwise change detection would not work properly.
        target = "es2016",
        platform = "node",
        deps = [":" + ts_library_name],
    )

def ng_package(deps = [], **kwargs):
    common_substitutions = dict(kwargs.pop("substitutions", {}), **PKG_GROUP_REPLACEMENTS)
    substitutions = dict(common_substitutions, **{
        "0.0.0-PLACEHOLDER": "0.0.0",
    })
    stamped_substitutions = dict(common_substitutions, **{
        "0.0.0-PLACEHOLDER": "{STABLE_PROJECT_VERSION}",
    })

    _ng_package(
        deps = deps,
        externals = [
            "domino",
            "xhr2",
            "jsdom",
            "critters",
            "express-engine",
            "express",
        ],
        substitutions = select({
            "//:stamp": stamped_substitutions,
            "//conditions:default": substitutions,
        }),
        **kwargs
    )

def pkg_npm(name, **kwargs):
    common_substitutions = dict(kwargs.pop("substitutions", {}), **PKG_GROUP_REPLACEMENTS)
    substitutions = dict(common_substitutions, **{
        "0.0.0-PLACEHOLDER": "0.0.0",
    })
    stamped_substitutions = dict(common_substitutions, **{
        "0.0.0-PLACEHOLDER": "{STABLE_PROJECT_VERSION}",
    })

    _pkg_npm(
        name = name,
        substitutions = select({
            "//:stamp": stamped_substitutions,
            "//conditions:default": substitutions,
        }),
        **kwargs
    )


#!/usr/bin/env python3

import pandas as pd

def best_record_company():
    df = pd.read_csv('src/UK-top40-1964-1-2.tsv', sep='\t')
    pubs = df.groupby('Publisher')
    best = pubs['WoC'].sum().max()
    return pubs.filter(lambda df: df['WoC'].sum().max() == best)

def main():
    print(best_record_company())

if __name__ == "__main__":
    main()


import numpy as np
import itertools
import math
from math import comb
import random
import numpy as np
import itertools

plural_dictionary = {
    "fruit": "fruits",
    "apple": "apples",
    "orange": "oranges",
    "banana": "bananas",
    "strawberry": "strawberries",
    "grape": "grapes",

    "vegetable": "vegetables",
    "carrot": "carrots",
    "broccoli": "broccoli",
    "tomato": "tomatoes",
    "potato": "potatoes",
    "cabbage": "cabbages",

    "animal": "animals",
    "dog": "dogs",
    "cat": "cats",
    "elephant": "elephants",
    "giraffe": "giraffes",
    "dolphin": "dolphins",

}

object_dictionary = {
    "fruit": {
        "items": [
            "apple",
            "orange",
            "banana",
            "strawberry",
            "grape",
        ],
        "range": [1, 5]
    },

    "vegetable": {
        "items": [
            "carrot",
            "broccoli",
            "tomato",
            "potato",
            "cabbage"
        ],
        "range": [1, 3]
    },

    "animal":{
        "items": [
            "dog",
            "cat",
            "elephant",
            "giraffe",
            "dolphin"
        ],
        "range": [1, 3]
    }
}

def get_category(sampled_items):
    categories = set()
    for item in sampled_items:
        for category, details in object_dictionary.items():
            if item in details["items"]:
                categories.add(category)
                break
    return categories

def factorial(n):
    if n == 0:
        return 1
    else:
        return n * factorial(n-1)


""" Tests for scramble generation background tasks. """

from unittest.mock import Mock, patch, call

import pytest
from huey.exceptions import TaskException

from cubersio.tasks import huey
from cubersio.tasks.scramble_generation import check_scramble_pool, ScramblePoolTopOffInfo, top_off_scramble_pool
from cubersio.util.events.resources import EVENT_3x3, EVENT_10x10, EVENT_COLL, EVENT_FTO, EVENT_REX

# Put Huey in immediate mode so the tasks execute synchronously
huey.immediate = True

def _setup_mock(**kwargs):
    """ Utility function for setting up a mocked Event or EventDefinition. Need to use Mock::configure because Events
    and EventDefinitions have a `name` attribute which we need to override, and `name` is usually a special reserved
    attribute for Mock. """

    mock_event = Mock()
    mock_event.configure_mock(**kwargs)

    return mock_event

@patch('cubersio.tasks.scramble_generation.get_all_events')
@patch('cubersio.tasks.scramble_generation.top_off_scramble_pool')
def test_check_scramble_pool(mock_top_off_scramble_pool, mock_get_all_events):
    """ Test that the scrambler pool checker task makes the appropriate calls to top_off_scramble_pool based on the
    number of remaining scrambles for each event. """

    # 3x3 and FTO need scrambles, they are below the 2x weekly scrambles threshold.
    # 10x10 has enough scrambles, and COLL doesn't have its scrambles pre-generated.
    mock_get_all_events.return_value = [
        _setup_mock(name=EVENT_3x3.name, id=1, scramble_pool=list(range(5)), totalSolves=5),
        _setup_mock(name=EVENT_10x10.name, id=2, scramble_pool=list(range(5)), totalSolves=1),
        _setup_mock(name=EVENT_COLL.name, id=3, scramble_pool=list(range(5)), totalSolves=5),
        _setup_mock(name=EVENT_FTO.name, id=4, scramble_pool=list(), totalSolves=5),
    ]

    check_scramble_pool()

    mock_get_all_events.assert_called_once()
    assert mock_top_off_scramble_pool.call_count == 2
    mock_top_off_scramble_pool.assert_has_calls([
        call(ScramblePoolTopOffInfo(1, EVENT_3x3.name, 5)),
        call(ScramblePoolTopOffInfo(4, EVENT_FTO.name, 10))
    ])

@pytest.mark.parametrize('top_off_info', [
    ScramblePoolTopOffInfo(event_id=10, event_name=EVENT_REX.name, num_scrambles=5),
    ScramblePoolTopOffInfo(event_id=42, event_name=EVENT_FTO.name, num_scrambles=15),
])
@patch('cubersio.tasks.scramble_generation.add_scramble_to_scramble_pool')
@patch('cubersio.tasks.scramble_generation.get_event_definition_for_name')
def test_top_off_scramble_pool_multi_scramble_puzzles(mock_get_event_definition_for_name,
                                                      mock_add_scramble_to_scramble_pool,
                                                      top_off_info: ScramblePoolTopOffInfo):
    """ Test that top_off_scramble_pool calls the event resource scrambler correctly for those events where scrambles
    are generated in bulk because it's faster. """

    scrambles = list(range(top_off_info.num_scrambles))
    mock_event_def = _setup_mock(name=top_off_info.event_name)
    mock_event_def.get_multiple_scrambles.return_value = scrambles

    mock_get_event_definition_for_name.return_value = mock_event_def

    top_off_scramble_pool(top_off_info)

    mock_get_event_definition_for_name.assert_called_once_with(top_off_info.event_name)
    mock_event_def.get_multiple_scrambles.assert_called_once_with(top_off_info.num_scrambles)

    assert mock_add_scramble_to_scramble_pool.call_count == top_off_info.num_scrambles

    expected_calls = [call(scramble, top_off_info.event_id) for scramble in scrambles]
    mock_add_scramble_to_scramble_pool.assert_has_calls(expected_calls)

@patch('cubersio.tasks.scramble_generation.add_scramble_to_scramble_pool')
@patch('cubersio.tasks.scramble_generation.get_event_definition_for_name')
def test_top_off_scramble_pool_single_scramble_puzzles(mock_get_event_definition_for_name,
                                                       mock_add_scramble_to_scramble_pool):
    """ Test that top_off_scramble_pool calls the event resource scrambler correctly for those events where scrambles
    are generated one at a time. """

    top_off_info = ScramblePoolTopOffInfo(event_id=11, event_name=EVENT_3x3.name, num_scrambles=5)

    scrambles = list(range(top_off_info.num_scrambles))
    mock_event_def = _setup_mock(name=top_off_info.event_name)
    mock_event_def.get_scramble.side_effect = scrambles

    mock_get_event_definition_for_name.return_value = mock_event_def

    top_off_scramble_pool(top_off_info)

    mock_get_event_definition_for_name.assert_called_once_with(top_off_info.event_name)

    assert mock_event_def.get_scramble.call_count == top_off_info.num_scrambles
    assert mock_add_scramble_to_scramble_pool.call_count == top_off_info.num_scrambles

    expected_calls = [call(scramble, top_off_info.event_id) for scramble in scrambles]
    mock_add_scramble_to_scramble_pool.assert_has_calls(expected_calls)

@patch('cubersio.tasks.scramble_generation.get_event_definition_for_name')
def test_top_off_scramble_pool_raises_for_nonexistent_event(mock_get_event_definition_for_name):
    """ Test that top_off_scramble_pool raises RuntimeError for a bogus event. """

    mock_get_event_definition_for_name.return_value = None
    with pytest.raises(TaskException) as te:
        top_off_scramble_pool(ScramblePoolTopOffInfo(event_id=1, event_name="blah", num_scrambles=5)).get()
    assert f"Can't find an EventResource for event blah" in te.value.metadata['error']


""" Test File """

from flask import Flask
from redis import Redis

app = Flask(__name__)
redis_client = Redis(
    host='redis_db',
    port=6379
)

@app.route('/')
def hello():
    """
    Main app route, simply returns a Hello
    """
    count_key = redis_client.get('count')
    count = int(count_key) if count_key else 0
    redis_client.set('count', count+1)

    return f'Hello World {count}'

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=int('5000'), debug=True)


import tidypolars as tp
from tidypolars import col
import polars as pl
from tidypolars.utils import _repeat

def test_arrange1():
    """Can arrange ascending"""
    df = tp.Tibble(x = ['a', 'a', 'b'], y = [2, 1, 3])
    actual = df.arrange('y')
    expected = tp.Tibble(x = ['a', 'a', 'b'], y = [1, 2, 3])
    assert actual.frame_equal(expected), "arrange ascending failed"
    assert type(actual) == tp.Tibble, "arrange didn't return a Tibble"

def test_arrange2():
    """Can arrange descending"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [2, 1, 3]})
    actual = df.arrange(tp.desc('x'), 'y')
    expected = tp.Tibble({'x': ['b', 'a', 'a'], 'y': [3, 1, 2]})
    assert actual.frame_equal(expected), "arrange descending failed"

def test_arrange_across():
    """Can arrange across"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [1, 2, 3], 'z': [1, 2, 3]})
    actual = df.arrange(
        tp.across(['x']),
        tp.across(['y', 'z'], tp.desc)
    )
    expected = tp.Tibble(x = ['a', 'a', 'b'], y = [2, 1, 3], z = [2, 1, 3])
    assert actual.frame_equal(expected), "arrange across failed"

def test_bind_cols_single():
    """Can bind_cols"""
    df1 = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [1, 2, 3]})
    df2 = tp.Tibble({'z': [4, 4, 4]})
    actual = df1.bind_cols(df2)
    expected = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [1, 2, 3], 'z':[4, 4, 4]})
    assert actual.frame_equal(expected), "bind_cols failed"
    assert type(actual) == tp.Tibble, "bind_cols didn't return a Tibble"

def test_bind_cols_multiple():
    """Can bind_cols multiple"""
    df1 = tp.Tibble(x = range(3))
    df2 = tp.Tibble(y = range(3))
    df3 = tp.Tibble(z = range(3))
    actual = df1.bind_cols(df2, df3)
    expected = tp.Tibble(x = range(3), y = range(3), z = range(3))
    assert actual.frame_equal(expected), "multiple bind_cols failed"

def test_bind_rows_single():
    """Can bind rows"""
    df1 = tp.Tibble({'x': ['a', 'a'], 'y': [2, 1]})
    df2 = tp.Tibble({'x': ['b'], 'y': [3]})
    actual = df1.bind_rows(df2)
    expected = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [2, 1, 3]})
    assert actual.frame_equal(expected), "bind_rows failed"
    assert type(actual) == tp.Tibble, "bind_rows didn't return a Tibble"

def test_bind_rows_auto_align():
    """Can bind rows"""
    df1 = tp.Tibble(x = ['a', 'a'], y = [2, 1])
    df2 = tp.Tibble(y = [3], x = ['b'])
    actual = df1.bind_rows(df2)
    expected = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [2, 1, 3]})
    assert actual.frame_equal(expected), "bind_rows auto-align failed"

def test_bind_rows_multiple():
    """Can bind rows (multiple)"""
    df1 = tp.Tibble({'x': ['a', 'a'], 'y': [2, 1]})
    df2 = tp.Tibble({'x': ['b'], 'y': [3]})
    df3 = tp.Tibble({'x': ['b'], 'y': [3]})
    actual = df1.bind_rows(df2, df3)
    expected = tp.Tibble({'x': ['a', 'a', 'b', 'b'], 'y': [2, 1, 3, 3]})
    assert actual.frame_equal(expected), "bind_rows multiple failed"

def test_clone():
    df = tp.Tibble(x = range(3), y = range(3))
    actual = df.clone()
    assert type(actual) == tp.Tibble, "clone didn't return a Tibble"

def test_count_no_args():
    """Can count rows (no args)"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [1, 1, 1]})
    actual = df.count()
    expected = tp.Tibble({'n': [3]})
    assert actual.frame_equal(expected), "count with no args failed"

def test_count_one_arg():
    """Can count rows (one arg)"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [1, 1, 1]})
    actual = df.count('x', sort = True)
    expected = tp.Tibble({'x': ['a', 'b'], 'n': [2, 1]})
    assert actual.frame_equal(expected), "count with one arg failed"

def test_distinct_empty():
    """Can distinct columns"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': ['a', 'a', 'b']})
    actual = df.distinct()
    expected = tp.Tibble({'x': ['a', 'b'], 'y': ['a', 'b']})
    assert actual.frame_equal(expected), "empty distinct failed"
    assert type(actual) == tp.Tibble, "distinct didn't return a Tibble"

def test_distinct_select():
    """Can distinct columns"""
    df = tp.Tibble({'x': ['a', 'a', 'b'], 'y': [2, 1, 3]})
    actual = df.distinct('x')
    expected = tp.Tibble({'x': ['a', 'b']})
    assert actual.frame_equal(expected), "distinct with select failed"

def test_drop():
    """Can drop columns"""
    df = tp.Tibble(x = range(3), y = range(3))
    actual = df.drop('x')
    expected = tp.Tibble(y = range(3))
    assert actual.frame_equal(expected), "drop failed"
    assert type(actual) == tp.Tibble, "drop didn't return a Tibble"

def test_drop_null_empty():
    """Can drop nulls from all cols"""
    df = tp.Tibble(x = [1, None, 3], y = [None, 2, 3], z = range(1, 4))
    actual = df.drop_null()
    expected = tp.Tibble(x = [3], y = [3], z = [3])
    assert actual.frame_equal(expected), "empty drop_null failed"
    assert type(actual) == tp.Tibble, "drop_null didn't return a Tibble"

def test_drop_null_select():
    """Can drop nulls with selection"""
    df = tp.Tibble(x = [1, None, 3], y = [None, 2, 3], z = range(1, 4))
    actual = df.drop_null('x')
    expected = tp.Tibble(x = [1, 3], y = [None, 3], z = [1, 3])
    assert actual.frame_equal(expected, null_equal = True), "drop_null with selection failed"

def test_fill():
    """Can fill"""
    df = tp.Tibble({'chr': ['a', None], 'int': [1, None]})
    actual = df.fill('chr', 'int')
    expected = tp.Tibble({'chr': ['a', 'a'], 'int': [1, 1]})
    assert actual.frame_equal(expected), "fill failed"
    assert type(actual) == tp.Tibble, "fill didn't return a Tibble"

def test_filter():
    """Can filter multiple conditions"""
    df = tp.Tibble({'x': range(10), 'y': range(10)})
    actual = df.filter(col('x') <= 3, col('y') < 2)
    expected = tp.Tibble({'x': range(2), 'y': range(2)})
    assert actual.frame_equal(expected), "filter failed"
    assert type(actual) == tp.Tibble, "filter didn't return a Tibble"

def test_filter_grouped():
    df = tp.Tibble(x = range(3), y = ['a', 'a', 'b'])
    actual = df.filter(col('x') <= col('x').mean(), by = 'y').arrange('y')
    expected = tp.Tibble(x = [0, 2], y = ['a', 'b'])
    assert actual.frame_equal(expected), "grouped filter failed"
    assert type(actual) == tp.Tibble, "grouped filter didn't return a Tibble"

def test_full_join():
    """Can perform a full join"""
    df1 = tp.Tibble(x = ['a', 'a', 'b'], y = range(3))
    df2 = tp.Tibble(x = ['a'], z = range(1))
    actual = df1.full_join(df2)
    expected = tp.Tibble(x = ['a', 'a', 'b'], y = [0, 1, 2], z = [0, 0, None])
    assert actual.frame_equal(expected, null_equal = True), "full_join failed"
    assert type(actual) == tp.Tibble, "full_join didn't return a Tibble"

def test_inner_join():
    """Can perform a inner join"""
    df1 = tp.Tibble(x = ['a', 'a', 'b'], y = range(3))
    df2 = tp.Tibble(x = ['a'], z = range(1))
    actual = df1.inner_join(df2)
    expected = tp.Tibble(x = ['a', 'a'], y = [0, 1], z = [0, 0])
    assert actual.frame_equal(expected), "inner_join failed"
    assert type(actual) == tp.Tibble, "inner_join didn't return a Tibble"

def test_left_join():
    """Can perform a left join"""
    df1 = tp.Tibble(x = ['a', 'a', 'b'], y = range(3))
    df2 = tp.Tibble(x = ['a', 'b'], z = range(2))
    actual = df1.left_join(df2)
    expected = tp.Tibble(x = ['a', 'a', 'b'], y = range(3), z = [0, 0 ,1])
    assert actual.frame_equal(expected), "left_join failed"
    assert type(actual) == tp.Tibble, "left_join didn't return a Tibble"

def test_mutate():
    """Can edit existing columns and can add columns"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    actual = df.mutate(double_x = col('x') * 2,
                       y = col('y') + 10,
                       y_plus_3 = col('y') + 3)
    expected = tp.Tibble(
        x = _repeat(1, 3),
        y = _repeat(12, 3),
        double_x = _repeat(2, 3),
        y_plus_3 = _repeat(15, 3)
    )
    assert actual.frame_equal(expected), "mutate failed"
    assert type(actual) == tp.Tibble, "mutate didn't return a Tibble"

def test_mutate_across():
    """Can mutate multiple columns simultaneously"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    actual = df.mutate(tp.across(tp.Int64, lambda x: x * 2),
                       x_plus_y = col('x') + col('y'))
    expected = tp.Tibble(
        {'x': _repeat(2, 3),
         'y': _repeat(4, 3),
         'x_plus_y': _repeat(6, 3)}
    )
    assert actual.frame_equal(expected), "mutate across failed"

def test_mutate_constant():
    """Can add a constant value without tp.lit"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    actual = df.mutate(z = "z")
    expected = tp.Tibble(
        x = _repeat(1, 3),
        y = _repeat(2, 3),
        z = _repeat('z', 3)
    )
    assert actual.frame_equal(expected), "mutate failed"

def test_names():
    """Can get column names"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    assert df.names == ['x', 'y'], "names failed"

def test_ncol():
    """Can number of columns"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    assert df.ncol == 2, "ncol failed"

def test_nrow():
    """Can number of rows"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    assert df.nrow == 3, "nrow failed"

def test_pivot_longer1():
    "Can pivot all (unspecified) cols to long"
    df = tp.Tibble({'x': [1, 2], 'y': [3, 4]})
    actual = df.pivot_longer()
    expected = tp.Tibble({'name': ['x', 'x', 'y', 'y'], 'value': range(1, 5)})
    assert actual.frame_equal(expected), "unspecified pivot_longer failed"
    assert type(actual) == tp.Tibble, "pivot_longer didn't return a Tibble"

def test_pivot_longer2():
    """Can pivot all (specified) cols to long"""
    df = tp.Tibble({'x': [1, 2], 'y': [3, 4]})
    actual = df.pivot_longer(['x', 'y'])
    expected = tp.Tibble({'name': ['x', 'x', 'y', 'y'], 'value': range(1, 5)})
    assert actual.frame_equal(expected), "specified pivot_longer failed"

def test_pivot_wider1():
    """Can pivot all cols to wide"""
    df = tp.Tibble({'label': ['x', 'y', 'z'], 'val': range(1, 4)})
    actual = df.pivot_wider(names_from = 'label', values_from = 'val').select('x', 'y', 'z')
    expected = tp.Tibble({'x': [1], 'y': [2], 'z': [3]})
    assert actual.frame_equal(expected), "pivot_wider all cols failed"
    assert type(actual) == tp.Tibble, "pivot_wider didn't return a Tibble"

def test_pivot_wider2():
    """Can pivot cols to wide with id col"""
    df = tp.Tibble({'id': _repeat(1, 3), 'label': ['x', 'y', 'z'], 'val': range(1, 4)})
    actual = df.pivot_wider(names_from = 'label', values_from = 'val').select('id', 'x', 'y', 'z')
    expected = tp.Tibble({'id': [1], 'x': [1], 'y': [2], 'z': [3]})
    assert actual.frame_equal(expected), "pivot_wider with id failed"

def test_pivot_wider3():
    """Can pivot cols to wide with values filled"""
    df = tp.Tibble({'id': _repeat(1, 3), 'label': ['x', 'y', 'z'], 'val': range(1, 4)})
    actual = (
        df.pivot_wider(names_from = 'label', values_from = 'id', values_fill = 0)
        .select('val', 'x', 'y', 'z').arrange('val')
    )
    expected = tp.Tibble({'val': [1, 2, 3], 'x': [1, 0, 0], 'y': [0, 1, 0], 'z': [0, 0, 1]})
    assert actual.frame_equal(expected), "pivot_wider with values filled failed"

def test_pivot_wider4():
    """Can pivot cols to wide with values filled - doesn't affect id col"""
    df = tp.Tibble(id = [None, 2], var = ["x", "y"], val = [1, 2])
    actual = (
        df.pivot_wider(names_from = "var", values_from = "val", values_fill = 0)
        .select('id', 'x', 'y')
        .arrange('y')
    )
    expected = tp.Tibble({'id': [None, 2], 'x': [1, 0], 'y': [0, 2]})
    assert actual.frame_equal(expected), "pivot_wider with values filled failed"

def test_print():
    """Printing doesn't alter class of df"""
    df = tp.Tibble(x = range(3), y = range(3))
    repr(df)
    print(df)
    assert isinstance(df, tp.Tibble), "Printing failed"

def test_pull():
    """Can use pull"""
    df = tp.Tibble({'x': _repeat(1, 3), 'y': _repeat(2, 3)})
    actual = df.pull('x')
    expected = df.to_polars().get_column('x')
    assert actual.series_equal(expected), "pull failed"

def test_relocate_before():
    """Can relocate before columns"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.relocate('y', 'z', before = 'x')
    expected = df.select('y', 'z', 'x')
    assert actual.frame_equal(expected), "relocate before failed"
    assert type(actual) == tp.Tibble, "relocate didn't return a Tibble"

def test_relocate_after():
    """Can relocate after columns"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.relocate('z', 'y', after = 'x')
    expected = df.select('x', 'z', 'y')
    assert actual.frame_equal(expected), "relocate after failed"

def test_relocate_empty():
    """Can relocate to the beginning"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.relocate('z', 'y')
    expected = df.select('z', 'y', 'x')
    assert actual.frame_equal(expected), "relocate to the beginning failed"

def test_rename_dplyr_kwargs():
    """Can rename - dplyr interface (kwargs)"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.rename(new_x = 'x', new_y = 'y')
    expected = tp.Tibble({'new_x': range(3), 'new_y': range(3), 'z': range(3)})
    assert actual.frame_equal(expected), "dplyr rename failed"
    assert type(actual) == tp.Tibble, "rename didn't return a Tibble"

def test_rename_dplyr_strings():
    """Can rename - dplyr interface (strings)"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.rename('new_x', 'x', 'new_y', 'y')
    expected = tp.Tibble({'new_x': range(3), 'new_y': range(3), 'z': range(3)})
    assert actual.frame_equal(expected), "dplyr rename failed"

def test_rename_pandas():
    """Can rename - pandas interface"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.rename({'x': 'new_x', 'y': 'new_y'})
    expected = tp.Tibble({'new_x': range(3), 'new_y': range(3), 'z': range(3)})
    assert actual.frame_equal(expected), "pandas rename failed"

def test_replace_null():
    """Can replace nulls"""
    df = tp.Tibble(x = [0, None], y = [None, None])
    actual = df.replace_null(dict(x = 1, y = 2))
    expected = tp.Tibble(x = [0, 1], y = [2, 2])
    assert actual.frame_equal(expected), "replace_null method failed"
    assert type(actual) == tp.Tibble, "replace_null didn't return a Tibble"

def test_set_names():
    """Can set_names"""
    df = tp.Tibble(x = range(3), y = range(3))
    actual = df.set_names(['a', 'b'])
    expected = tp.Tibble(a = range(3), b = range(3))
    assert actual.frame_equal(expected), "set_names failed"
    assert type(actual) == tp.Tibble, "set_names didn't return a Tibble"

def test_select():
    """Can select columns"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.select('x', 'z')
    expected = df[['x', 'z']]
    assert actual.frame_equal(expected), "select failed"
    assert type(actual) == tp.Tibble, "select didn't return a Tibble"

def test_separate():
    """Can separate"""
    df = tp.Tibble(x = ['a_a', 'b_b', 'c_c'])
    actual = df.separate('x', into = ['left', 'right']).arrange('left')
    expected = tp.Tibble(left = ['a', 'b', 'c'], right = ['a', 'b', 'c'])
    assert actual.frame_equal(expected), "separate failed"
    assert type(actual) == tp.Tibble, "separate didn't return a Tibble"

def test_slice():
    """Can slice"""
    df = tp.Tibble({'x': range(3), 'y': ['a', 'a', 'b']})
    actual = df.slice(0, 2)
    expected = tp.Tibble({'x': [0, 2], 'y': ['a', 'b']})
    assert actual.frame_equal(expected), "slice failed"
    assert type(actual) == tp.Tibble, "slice didn't return a Tibble"

def test_slice_head():
    """Can slice_head"""
    df = tp.Tibble({'x': range(3), 'y': ['a', 'a', 'b']})
    actual = df.slice_head(2)
    expected = tp.Tibble({'x': [0, 1], 'y': ['a', 'a']})
    assert actual.frame_equal(expected), "slice_head failed"
    assert type(actual) == tp.Tibble, "slice_head didn't return a Tibble"

def test_slice_tail():
    """Can slice_tail by group"""
    df = tp.Tibble({'x': range(3), 'y': ['a', 'a', 'b']})
    actual = df.slice_tail(2)
    expected = tp.Tibble({'x': [1, 2], 'y': ['a', 'b']})
    assert actual.frame_equal(expected), "slice_tail failed"
    assert type(actual) == tp.Tibble, "slice_tail didn't return a Tibble"

def test_summarise():
    """Can use summarise alias"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.summarise(avg_x = col('x').mean())
    expected = tp.Tibble({'avg_x': [1]})
    assert actual.frame_equal(expected), "summarise failed"

def test_summarize():
    """Can use summarize"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    actual = df.summarize(avg_x = col('x').mean())
    expected = tp.Tibble({'avg_x': [1]})
    assert actual.frame_equal(expected), "ungrouped summarize failed"
    assert type(actual) == tp.Tibble, "summarize didn't return a Tibble"

def test_summarize_grouped():
    """Can use summarize by group"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': ['a', 'a', 'b']})
    actual = df.summarize(avg_x = col('x').mean(), by = 'z').arrange('z')
    expected = tp.Tibble(z = ['a', 'b'], avg_x = [.5, 2])
    assert actual.frame_equal(expected), "grouped summarize failed"

def test_summarize_across():
    """Can use summarize_across"""
    df = tp.Tibble(x = range(3), y = range(3), z = range(3))
    actual = df.summarize(tp.across(['x', 'y'], tp.max, names_prefix = "max_"),
                          avg_x = col('x').mean())
    expected = tp.Tibble({'max_x': [2], 'max_y': [2], 'avg_x': [1]})
    assert actual.frame_equal(expected), "ungrouped summarize across failed"

def test_to_dict():
    """Can convert to a dictionary"""
    df = tp.Tibble({'x': range(3), 'y': range(3)})
    assert type(df.to_dict()) == dict

def test_to_polars():
    """Can convert to a polars DataFrame"""
    df = tp.Tibble({'x': range(3), 'y': range(3), 'z': range(3)})
    assert isinstance(df.to_polars(), pl.DataFrame), "to_polars failed"

def test_unite():
    """Can unite columns"""
    df = tp.Tibble(a = ["a", "a", "a"], b = ["b", "b", "b"], c = range(3))
    actual = df.unite("new_col", ["a", "b"])
    expected = tp.Tibble(new_col = ["a_b"] * 3, c = range(3))
    assert actual.frame_equal(expected), "unite failed"
    assert type(actual) == tp.Tibble, "unite didn't return a Tibble"

def test_funs_in_a_row():
    """Tests if shallow copy is working properly"""
    df = tp.Tibble(x = range(3), y = range(3), z = range(3))
    df.distinct()
    df.drop('x')
    df.drop_null()
    df.filter(col('x') < 7)
    df.head()
    df.mutate(col('x') * 2)
    df.relocate('y', before = 'x')
    df.rename({'x': 'new_x'})
    df.select('x', 'y')
    df.slice(1)
    df.slice_head()
    df.slice_tail()
    df.tail()
    df.arrange('x', 'y')
    assert True, "Functions in a row failed"


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def mergeTwoLists(self, l1, l2):
        """
        :type l1: ListNode
        :type l2: ListNode
        :rtype: ListNode
        """
        dummy = cur = ListNode(0)

        #l1, l2의 수를 비교해 작은 수 부터 연결
        while l1 and l2:
            if l1.val >= l2.val:
                cur.next = ListNode(l2.val)
                l2 = l2.next
            else:
                cur.next = ListNode(l1.val)
                l1 = l1.next
            cur = cur.next

        #남은 수 연결
        while l1:
            cur.next = ListNode(l1.val)
            l1 = l1.next
            cur = cur.next
        while l2:
            cur.next = ListNode(l2.val)
            l2 = l2.next
            cur = cur.next

        return dummy.next


import csv
import numpy as np
from random import sample

class Data(object):
    """
    Abstraction for the training data for classification
    """
    def __init__(self, classes, input_dimension):
        """
        Initialize the Data object.

        Arguments:
            `classes`        :  a array of all possible classes in the data
            `input_dimension`: a integer indicating the number of input dimensions
                               of training a classification NN
        """
        self.num_classes  = len(classes)
        self.class_dict = classes
        self.input_dimension = input_dimension

    def read_n_process(self, filename):
        """
        Read a .csv file and sets two parameters

        1. `self.samples` : a (n x k) array where n is the
                          number of data points and k is the
                          dimension of input data points

        2. `self.labels` : a (n x 1) array of one-hot encoding vector
                          for the label of each data point

        3. `self.d`     : a (n x m) array where n is the
                          number of data points and m is the
                          number of categories in a row of
                          the csv file.

        Each row is categorized in the following way:

        [sepal-length, sepal-width, petal-length, petal-width, class]

        where `class` is a one-hot encoding vector of the three categories:
        1. Iris Setosa
        2. Iris Versicolour
        3. Iris Virginica

        Ex) [1, 0, 0] is the encoding for Iris Setosa

        NOTE: all elements in self.d are type string
              but elements in input are converted to float when possible

        Arguments:
            `filename` : string
        """
        data = []
        with open(filename) as csvfile:
            reader = csv.reader(csvfile)
            for line in reader:
                row = line
                data.append(row)

        self.d = np.array(data)
        self.num_samples = self.d.shape[0]
        self.samples_per_class = self.num_samples // self.num_classes
        # Pool of all available lookup indices for every class
        self.index_pool = set(range(self.samples_per_class))
        # Input is the first n column of the data
        self.samples = np.array(self.d[:, :self.input_dimension], dtype=np.float)
        # The last column is the label
        self.labels = np.array([self.one_hot_encode(s) \
                               for s in self.d[:, self.input_dimension:]])

    def one_hot_encode(self, s):
        """
        Given the string of a class, return the one-hot-encoding
        of that class
        """
        v = [0] * self.num_classes
        for index, class_name in enumerate(self.class_dict):
            if s == class_name:
                v[index] = 1
        return v

    def encoding_to_class(self, v):
        """
        Return the class associated with the one-hot encoding vector
        """
        index = np.argmax(v)
        return self.class_dict[index]

    def split_samples(self, training_size, validation_size):
        """
        Randomly split the samples into training and testing sample sets.
        Not overlap between training, validating and testing set.

        Assume sample pools to have uniform distribution of sample across
        all classes.

        Arguments:
           `training_size` : number of training samples per class
           `validation_size` : number of samples used for validation
                              (NOTE: the remaining samples are for testing)
        """
        assert training_size + validation_size <= self.samples_per_class, \
                "Training size + validiont size cannot exceed total number of samples"
        training_indices     = sample(self.index_pool, training_size)
        nontraining_indices  = self.index_pool.difference(set(training_indices))
        validation_indices   = list((sample(nontraining_indices, validation_size)))
        testing_indices      = list(nontraining_indices.difference(validation_indices))
        self.training_samples, self.training_labels = \
                self.select_samples_n_labels_from_classes(training_indices, self.samples, self.labels)
        self.validation_samples, self.validation_labels = \
                self.select_samples_n_labels_from_classes(validation_indices, self.samples, self.labels)
        self.testing_samples, self.testing_labels   = \
                self.select_samples_n_labels_from_classes(testing_indices, self.samples, self.labels)
        # A test set with str labels. For printing purpose
        self.labeled_testing_set = self.select_raw_data_with_labels(testing_indices)

    def resample(self, samples_per_class, sample_pool, label_pool):
        """
        Randomly select a batch of sample from the pool of data
        with equal distribution across all classes.

        Assuming data has equal distribution of samples across all classes
        and that every element from the same class is neighboring each other in the pool

        Example:
            [ClassA1, ClassA2, ..., ClassB1, ClassB2, ..., ClassC1, ClassC2, ...]

        Arguments:
            `sample_per_class` : (NOTE should be a multiple of the number of classes)
            `sample_pool` : a (n x k) array of samples to randomly select from
            `label_pool`  : a (n x k) array of sample labels
        """
        index_pool = range(sample_pool.shape[0] // self.num_classes)
        batch_indices = sample(index_pool, samples_per_class)
        return self.select_samples_n_labels_from_classes(batch_indices, sample_pool, label_pool)

    def select_samples_n_labels_from_classes(self, indices, sample_pool, label_pool=None):
        """
        Select the element from each classes in the sample_pool at each index

        Assume equal distribution of sample across all classes in the sample_pool
        and that every element from the same class is neighboring each other in the pool

        Example:
            [ClassA1, ClassA2, ..., ClassB1, ClassB2, ..., ClassC1, ClassC2, ...]

        Arguments:
            `indices` : an array of indices of the sample pool
            `sample_pool` : an array of samples
            `label_pool`  : an array of sample labels (must be same size as `sample_pool`)

        """
        samples_per_class = sample_pool.shape[0] // self.num_classes
        assert len(indices) <= samples_per_class, \
            "Number of selected samples per class cannot exceed total number of samples per class"

        sample_batch = np.array([])
        label_batch  = np.array([])
        for i in range(len(indices)):
            for j in range(self.num_classes):
                sample_index = indices[i] + j * samples_per_class
                sample_batch = np.vstack((sample_batch, sample_pool[sample_index])) \
                                if sample_batch.size else sample_pool[sample_index]
                label_batch = np.vstack((label_batch, label_pool[sample_index])) \
                                if label_batch.size else label_pool[sample_index]
        return (sample_batch, label_batch)

    def select_raw_data_with_labels(self, indices):
        """
        Select the element from the raw input set (self.d)
        at each index
        """
        samples_per_class = self.d.shape[0] // self.num_classes
        data_batch  = np.array([])
        for i in range(len(indices)):
            for j in range(self.num_classes):
                data_index = indices[i] + j * samples_per_class
                data_batch = np.vstack((data_batch, self.d[data_index])) \
                                if data_batch.size else self.d[data_index]

        return data_batch


#!/usr/bin/env python

from __future__ import print_function
import sys
import math
import re
import string
from operator import itemgetter
from ConfigBundle import *
from urlparse import *
import simplejson

##  @package Document
#   Provides operations on pieces of text/documents

#import global variable defined in ConfigBundle module
global applicationConfig

##  This class provides an object that unifies all extra data needed for document parsing
#
#   This object contains any punctuation marks that will be stripped from articles, along
#   with any common words that should be prevented from appearing as keywords (due to their frequency)
class DocumentParsingSettings:

    ##  Initializes a new object and loads the data from the supplied files
    #   @param      punctuationMarksFilename        The file to load the punctuation marks from
    #   @param      ignoredWordsFilename            The file to load the words to ignore from
    #   @param      minLength                       Minimum acceptable length for a word
    def __init__(self, punctuationMarksFilename, ignoredWordsFilename, minLength):

        ##  @var minLength
        #   Minimum length for a word to take it into account
        self.minLength = minLength

        ##  @var punctuationMarks
        #   Punctuation marks to strip from content
        self.punctuationMarks = []

        ##  @var ignoredWords
        #   Words to explicitly not take into account
        self.ignoredWords = []

        #if additional debug output is requested, print the filenames
        if applicationConfig.debugOutput is True:
            print("Will load default parsing settings from {0}, {1}".format(punctuationMarksFilename, ignoredWordsFilename))

        #read the data from the files supplied
        try:
            defaultWordsTxt = Document.FromFile(ignoredWordsFilename)
            self.ignoredWords = defaultWordsTxt.text.split()

            defaultPunctuationMarks = Document.FromFile(punctuationMarksFilename)
            self.punctuationMarks = defaultPunctuationMarks.text.split()
        #things that can go wrong here are filesystem-related (very rarely device-related)
        except IOError as exc:
            print("Error reading from file: {0}".format(exc.strerror))
            raise

        #if additional debug output is requested, print the punctuation marks and the wods to ignore
        if applicationConfig.debugOutput is True:
            print("The following words will not be counted:", end=" ")
            for word in self.ignoredWords:
                print(word, end=" ")
            print("\nThe following punctuation marks will be stripped:", end=" ")
            for punctuationMark in self.punctuationMarks:
                print(punctuationMark, end=" ")
            print("\n")

##  Common text(file) related operations
#
#   This implements text file loading along with TF metrics.
class Document:

    ##  Initialize a new Document
    #   @param      text        The text
    def __init__(self, text):

        ##  @var text
        #   The loaded text
        self.text = text

        ##  @var words
        #   The text splitted in words
        self.words = []

        ##  @var tf
        #   Frequency of appearance for each word
        self.tf = []

    ##  Method for loading text from a file
    #   @param      filename        The name of the file to load text from
    @staticmethod
    def FromFile(filename):
        text = ""
        try:
            file = open(filename, "rU")
            for line in file:
                text += line
        except IOError as exc:
            print("Error reading from file: ", exc.strerror)
            raise
        finally:
            file.close()

        #after we have successfully loaded text from a file, create a new Document object
        newDoc = Document(text)
        return newDoc

    ##  Calculate term-frequency table
    #   @param      parsingSettings     Parse the document with these settings
    #   @param      returnPart          Boolean variable indicating wether to return a part of the calculated table or the complete table
    #   @param      termsWanted         If a partial table is requested, size of the table
    def CalculateTF(self, parsingSettings, returnPart, termsWanted):
        rawWords = self.text.lower().split() #spit the text into words
        frequencies = {}
        for word in rawWords:
            for punctuationMark in parsingSettings.punctuationMarks: #strip characters
                if punctuationMark in word:
                    word = word.replace(punctuationMark, "")
            if word not in parsingSettings.ignoredWords and len(word) >= parsingSettings.minLength: #check if in (common) words to ignore
                self.words.append(word)
                if word in frequencies:
                    frequencies[word] += 1 #increment occurences
                else:
                    frequencies[word] = 1 #add to dictionary

        #convert the word array & the frequency dict in an array of arrays
        for word, count in frequencies.items():
            tmp = [word, count]
            self.tf.append(tmp)

        sortedTF = sorted(self.tf, key=lambda tup: tup[1], reverse=True)
        if returnPart is True: #if a partial table is requested
            return sortedTF[:termsWanted]
        else:
            return sortedTF

#nothing to run directly from here
if __name__ == "__main__":
    print("Nothing to run here! Execute the main application file instead!")


from fastapi import APIRouter
import traceback
from .nl_to_sql_utils import get_similar_severities, get_most_relevant_severity, get_sql_query
from .nl_to_sql_prompts import tables_info
from pydantic import BaseModel

router = APIRouter()

class NLtoSQL(BaseModel):
    """Request body for streaming."""
    query: str

@router.post("/nl_to_sql")
async def nl_to_sql(body: NLtoSQL):
    """
    Args:
        query (str): user query for which we want to find attack techniques
    Returns:
        json object with following fields
            query(str),
            most_relevant_severity(str),
            sql_query(str)
    """

    try:
        query = body.query
        similar_severities = get_similar_severities(query)
        inputs = {
            "user_query":query,
            "severities":similar_severities
        }
        most_relevant_severity = await get_most_relevant_severity(inputs)
        inputs = {
            "tables_info":tables_info,
            "severity_value":most_relevant_severity
        }
        sql_query = await get_sql_query(inputs)
        return {
            "query": query,
            "most_relevant_severity": most_relevant_severity,
            "sql_query": sql_query
        }
    except Exception as e:
        traceback.print_exc()


import pytest
from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
from dbt.tests.adapter.utils.data_types.test_type_bigint import (
    models__actual_sql as bigint_model,
)
from dbt.tests.adapter.utils.data_types.test_type_bigint import (
    models__expected_sql as bigint_expected,
)
from dbt.tests.adapter.utils.data_types.test_type_boolean import (
    BaseTypeBoolean,
)
from dbt.tests.adapter.utils.data_types.test_type_boolean import (
    models__actual_sql as bool_model,
)
from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
from dbt.tests.adapter.utils.data_types.test_type_float import (
    models__actual_sql as float_model,
)
from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt
from dbt.tests.adapter.utils.data_types.test_type_int import (
    models__actual_sql as int_model,
)
from dbt.tests.adapter.utils.data_types.test_type_numeric import (
    BaseTypeNumeric,
)
from dbt.tests.adapter.utils.data_types.test_type_numeric import (
    models__actual_sql as num_model,
)
from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString
from dbt.tests.adapter.utils.data_types.test_type_string import (
    models__actual_sql as string_model,
)
from dbt.tests.adapter.utils.data_types.test_type_timestamp import (
    BaseTypeTimestamp,
)
from dbt.tests.adapter.utils.data_types.test_type_timestamp import (
    models__actual_sql as ts_model,
)
from firebolt import __version__ as sdk_version

schema_actual_table_yml = """
version: 2
models:
  - name: actual
    config:
      materialized: table
"""

schema_expected_table_yml = """
version: 2
models:
  - name: expected
    config:
      materialized: table
"""

class TestTypeBigInt(BaseTypeBigInt):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'expected.yml': schema_expected_table_yml,
            'expected.sql': bigint_expected,
            'actual.yml': schema_actual_table_yml,
            'actual.sql': self.interpolate_macro_namespace(bigint_model, 'type_bigint'),
        }

class TestTypeFloat(BaseTypeFloat):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(float_model, 'type_float'),
            'actual.yml': schema_actual_table_yml,
        }

class TestTypeInt(BaseTypeInt):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(int_model, 'type_int'),
            'actual.yml': schema_actual_table_yml,
        }

@pytest.mark.skipif(
    sdk_version <= '0.15.0', reason='Decimal type implemented in firebolt-sdk>0.15.0'
)
class TestTypeNumeric(BaseTypeNumeric):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(num_model, 'type_numeric'),
            'actual.yml': schema_actual_table_yml,
        }

class TestTypeString(BaseTypeString):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(string_model, 'type_string'),
            'actual.yml': schema_actual_table_yml,
        }

class TestTypeTimestamp(BaseTypeTimestamp):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(ts_model, 'type_timestamp'),
            'actual.yml': schema_actual_table_yml,
        }

@pytest.mark.skip('True boolean is feature-flagged')
class TestTypeBoolean(BaseTypeBoolean):
    @pytest.fixture(scope='class')
    def models(self):
        return {
            'actual.sql': self.interpolate_macro_namespace(bool_model, 'type_boolean'),
            'actual.yml': schema_actual_table_yml,
        }


# Given a string of words, you need to find the highest scoring word.

# Each letter of a word scores points according to its position in the alphabet: a = 1, b = 2, c = 3 etc.

# For example, the score of abad is 8 (1 + 2 + 1 + 4).

# You need to return the highest scoring word as a string.

# If two words score the same, return the word that appears earliest in the original string.

# All letters will be lowercase and all inputs will be valid.

def high(x):
    # CAN BE SUBSTITUTED BY USING ORD()-96
    # alpha = "abcdefghijklmnopqrstuvwxyz"
    # count = 1
    # d = {} #Empty dictionary to add values into
    # for i in alpha:
    #     d[i] = count
    #     count+=1
    score = 0
    highest_score = 0
    winning_word = ''
    for word in x.split():
        for letter in word:
            score += (ord(letter)-96)
        if score > highest_score:
            highest_score = score
            winning_word = word
        score = 0
    return winning_word

print(high('man i need a taxi up to ubud'))
print(high('what time are we climbing up the volcano'))
print(high('take me to semynak'))
print(high('aa b'))
print(high('b aa'))
print(high('bb d'))
print(high('d bb'))
print(high("aaa b"))

# def high(x):
#     return max(x.split(), key=lambda k: sum(ord(c) - 96 for c in k))

# def high(x):
#     words=x.split(' ')
#     list = []
#     for i in words:
#         scores = [sum([ord(char) - 96 for char in i])]
#         list.append(scores)
#     return words[list.index(max(list))]

# def high(x):
#     highest_score = 0
#     for word in x.split(' '):
#         score = sum(ord(c)-96 for c in word)
#         if score > highest_score:
#             highest_score = score
#             highest_word = word

#     return highest_word


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import re
import os
import copy
import yaml
from enum import Enum

from utils.util import mace_check
from utils.util import MaceLogger
from py_proto import mace_pb2

CPP_KEYWORDS = [
    'alignas', 'alignof', 'and', 'and_eq', 'asm', 'atomic_cancel',
    'atomic_commit', 'atomic_noexcept', 'auto', 'bitand', 'bitor',
    'bool', 'break', 'case', 'catch', 'char', 'char16_t', 'char32_t',
    'class', 'compl', 'concept', 'const', 'constexpr', 'const_cast',
    'continue', 'co_await', 'co_return', 'co_yield', 'decltype', 'default',
    'delete', 'do', 'double', 'dynamic_cast', 'else', 'enum', 'explicit',
    'export', 'extern', 'false', 'float', 'for', 'friend', 'goto', 'if',
    'import', 'inline', 'int', 'long', 'module', 'mutable', 'namespace',
    'new', 'noexcept', 'not', 'not_eq', 'nullptr', 'operator', 'or', 'or_eq',
    'private', 'protected', 'public', 'register', 'reinterpret_cast',
    'requires', 'return', 'short', 'signed', 'sizeof', 'static',
    'static_assert', 'static_cast', 'struct', 'switch', 'synchronized',
    'template', 'this', 'thread_local', 'throw', 'true', 'try', 'typedef',
    'typeid', 'typename', 'union', 'unsigned', 'using', 'virtual', 'void',
    'volatile', 'wchar_t', 'while', 'xor', 'xor_eq', 'override', 'final',
    'transaction_safe', 'transaction_safe_dynamic', 'if', 'elif', 'else',
    'endif', 'defined', 'ifdef', 'ifndef', 'define', 'undef', 'include',
    'line', 'error', 'pragma',
]

def sanitize_load(s):
    # do not let yaml parse ON/OFF to boolean
    for w in ["ON", "OFF", "on", "off"]:
        s = re.sub(r":\s+" + w + "$", r": '" + w + "'", s)

    # sub ${} to env value
    s = re.sub(r"\${(\w+)}", lambda x: os.environ[x.group(1)], s)
    return yaml.load(s)

def parse(path):
    with open(path) as f:
        config = sanitize_load(f.read())

    return config

def parse_device_info(path):
    conf = parse(path)
    return conf["devices"]

class ModelKeys(object):
    platform = "platform"
    runtime = "runtime"
    models = 'models'
    graph_optimize_options = "graph_optimize_options"
    input_tensors = "input_tensors"
    input_shapes = "input_shapes"
    input_data_types = "input_data_types"
    input_data_formats = "input_data_formats"
    input_ranges = "input_ranges"
    output_tensors = "output_tensors"
    output_shapes = "output_shapes"
    output_data_types = "output_data_types"
    output_data_formats = "output_data_formats"
    check_tensors = "check_tensors"
    check_shapes = "check_shapes"
    model_file_path = "model_file_path"
    model_sha256_checksum = "model_sha256_checksum"
    weight_file_path = "weight_file_path"
    weight_sha256_checksum = "weight_sha256_checksum"
    quantize_range_file = "quantize_range_file"
    quantize = "quantize"
    quantize_schema = "quantize_schema"
    quantize_large_weights = "quantize_large_weights"
    quantize_stat = "quantize_stat"
    change_concat_ranges = "change_concat_ranges"
    winograd = "winograd"
    cl_mem_type = "cl_mem_type"
    data_type = "data_type"
    subgraphs = "subgraphs"
    validation_inputs_data = "validation_inputs_data"

class DataFormat(Enum):
    NONE = 0
    NHWC = 1
    NCHW = 2
    HWIO = 100
    OIHW = 101
    HWOI = 102
    OHWI = 103
    AUTO = 1000

def parse_data_format(str):
    str = str.upper()
    mace_check(str in [e.name for e in DataFormat],
               "unknown data format %s" % str)
    return DataFormat[str]

class DeviceType(Enum):
    CPU = 0
    GPU = 2
    HEXAGON = 3
    HTA = 4
    APU = 5
    CPU_GPU = 100

DEVICE_MAP = {
    "cpu": DeviceType.CPU,
    "gpu": DeviceType.GPU,
    "hexagon": DeviceType.HEXAGON,
    "dsp": DeviceType.HEXAGON,
    "hta": DeviceType.HTA,
    "apu": DeviceType.APU,
    "cpu+gpu": DeviceType.CPU_GPU
}

def parse_device_type(str):
    mace_check(str in DEVICE_MAP, "unknown device %s" % str)
    return DEVICE_MAP[str]

class Platform(Enum):
    TENSORFLOW = 0
    CAFFE = 1
    ONNX = 2
    MEGENGINE = 3
    KERAS = 4
    PYTORCH = 5

def parse_platform(str):
    str = str.upper()
    mace_check(str in [e.name for e in Platform],
               "unknown platform %s" % str)
    return Platform[str]

DATA_TYPE_MAP = {
    'float32': mace_pb2.DT_FLOAT,
    'int32': mace_pb2.DT_INT32,
}

def parse_data_type(str):
    if str == "float32":
        return mace_pb2.DT_FLOAT
    if str == "float16":
        return mace_pb2.DT_FLOAT16
    elif str == "int32":
        return mace_pb2.DT_INT32
    else:
        mace_check(False, "data type %s not supported" % str)

def parse_internal_data_type(str):
    if str == 'fp32_fp32':
        return mace_pb2.DT_FLOAT
    elif str == 'bf16_fp32':
        return mace_pb2.DT_BFLOAT16
    elif str == 'fp16_fp16':
        return mace_pb2.DT_FLOAT16
    else:
        return mace_pb2.DT_HALF

def to_list(x):
    if isinstance(x, list):
        return x
    else:
        return [x]

def parse_int_array(xs):
    if len(xs) is 0:
        return [1]
    return [int(x) for x in xs.split(",")]

def parse_float_array(xs):
    return [float(x) for x in xs.split(",")]

def normalize_input_data_types(conf, input_count):
    default_input_dt = conf[ModelKeys.data_type]
    if default_input_dt == mace_pb2.DT_HALF:
        default_input_dt = mace_pb2.DT_FLOAT  # Compatible with old version
    conf_input_dts = to_list(conf.get(ModelKeys.input_data_types, []))
    if len(conf_input_dts) == 0:
        input_data_types = [default_input_dt]
    else:
        input_data_types = [parse_data_type(dt) for dt in conf_input_dts]

    if len(input_data_types) == 1 and input_count > 1:
        input_data_types = [input_data_types[0]] * input_count
    mace_check(len(input_data_types) == input_count,
               "the number of input_data_types should be "
               "the same as input tensors")
    conf[ModelKeys.input_data_types] = input_data_types

def normalize_output_data_types(conf, output_count):
    default_output_dt = conf[ModelKeys.data_type]
    if default_output_dt == mace_pb2.DT_HALF:
        default_output_dt = mace_pb2.DT_FLOAT  # Compatible with old version
    conf_output_dts = to_list(conf.get(ModelKeys.output_data_types, []))
    if len(conf_output_dts) == 0:
        output_data_types = [default_output_dt]
    else:
        output_data_types = [parse_data_type(dt) for dt in conf_output_dts]

    if len(output_data_types) == 1 and output_count > 1:
        output_data_types = [output_data_types[0]] * output_count
    mace_check(len(output_data_types) == output_count,
               "the number of output_data_types should be "
               "the same as output tensors")
    conf[ModelKeys.output_data_types] = output_data_types

def normalize_model_config(conf):
    conf = copy.deepcopy(conf)
    if ModelKeys.subgraphs in conf:
        subgraph = conf[ModelKeys.subgraphs][0]
        del conf[ModelKeys.subgraphs]
        conf.update(subgraph)

    conf[ModelKeys.platform] = parse_platform(conf[ModelKeys.platform])
    conf[ModelKeys.runtime] = parse_device_type(conf[ModelKeys.runtime])

    if ModelKeys.quantize in conf and conf[ModelKeys.quantize] == 1:
        conf[ModelKeys.data_type] = mace_pb2.DT_FLOAT
    else:
        if ModelKeys.data_type in conf:
            conf[ModelKeys.data_type] = parse_internal_data_type(
                conf[ModelKeys.data_type])
        else:
            conf[ModelKeys.data_type] = mace_pb2.DT_HALF

    # parse input
    conf[ModelKeys.input_tensors] = to_list(conf[ModelKeys.input_tensors])
    conf[ModelKeys.input_tensors] = [str(i) for i in
                                     conf[ModelKeys.input_tensors]]
    input_count = len(conf[ModelKeys.input_tensors])
    conf[ModelKeys.input_shapes] = [parse_int_array(shape) for shape in
                                    to_list(conf[ModelKeys.input_shapes])]
    mace_check(
        len(conf[ModelKeys.input_shapes]) == input_count,
        "input node count and shape count do not match")

    normalize_input_data_types(conf, input_count)

    input_data_formats = [parse_data_format(df) for df in
                          to_list(conf.get(ModelKeys.input_data_formats,
                                           ["NHWC"]))]
    if len(input_data_formats) == 1 and input_count > 1:
        input_data_formats = [input_data_formats[0]] * input_count
    mace_check(len(input_data_formats) == input_count,
               "the number of input_data_formats should be "
               "the same as input tensors")
    conf[ModelKeys.input_data_formats] = input_data_formats

    input_ranges = [parse_float_array(r) for r in
                    to_list(conf.get(ModelKeys.input_ranges,
                                     ["-1.0,1.0"]))]
    if len(input_ranges) == 1 and input_count > 1:
        input_ranges = [input_ranges[0]] * input_count
    mace_check(len(input_ranges) == input_count,
               "the number of input_ranges should be "
               "the same as input tensors")
    conf[ModelKeys.input_ranges] = input_ranges

    # parse output
    conf[ModelKeys.output_tensors] = to_list(conf[ModelKeys.output_tensors])
    conf[ModelKeys.output_tensors] = [str(i) for i in
                                      conf[ModelKeys.output_tensors]]
    output_count = len(conf[ModelKeys.output_tensors])
    conf[ModelKeys.output_shapes] = [parse_int_array(shape) for shape in
                                     to_list(conf[ModelKeys.output_shapes])]
    mace_check(len(conf[ModelKeys.output_tensors]) == output_count,
               "output node count and shape count do not match")

    normalize_output_data_types(conf, output_count)

    output_data_formats = [parse_data_format(df) for df in
                           to_list(conf.get(ModelKeys.output_data_formats,
                                            ["NHWC"]))]
    if len(output_data_formats) == 1 and output_count > 1:
        output_data_formats = [output_data_formats[0]] * output_count
    mace_check(len(output_data_formats) == output_count,
               "the number of output_data_formats should be "
               "the same as output tensors")
    conf[ModelKeys.output_data_formats] = output_data_formats

    if ModelKeys.check_tensors in conf:
        conf[ModelKeys.check_tensors] = to_list(conf[ModelKeys.check_tensors])
        conf[ModelKeys.check_shapes] = [parse_int_array(shape) for shape in
                                        to_list(conf[ModelKeys.check_shapes])]
        mace_check(len(conf[ModelKeys.check_tensors]) == len(
            conf[ModelKeys.check_shapes]),
                   "check tensors count and shape count do not match.")

    MaceLogger.summary(conf)

    return conf