# -*- coding: utf-8 -*-

import json
import os.path
from typing import Tuple

from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.http import HttpRequest, HttpResponseRedirect, JsonResponse
from django.shortcuts import redirect

from catmaid.control import cropping
from catmaid.control.authentication import requires_user_role
from catmaid.control.common import urljoin
from catmaid.models import UserRole, RegionOfInterest, Project, Relation, \
        Stack, ClassInstance, RegionOfInterestClassInstance

from celery.task import task
from celery.utils.log import get_task_logger

User = get_user_model()

# Prefix for stored ROIs
file_prefix = "roi_"
# File extension of the stored ROIs
file_extension = "png"
# The path were cropped files get stored in
roi_path = os.path.join(settings.MEDIA_ROOT,
    settings.MEDIA_ROI_SUBDIRECTORY)
# A common logger for the celery tasks
logger = get_task_logger(__name__)
# Locks will expire after two minutes
LOCK_EXPIRE = 60 * 2

@requires_user_role([UserRole.Browse])
def get_roi_info(request:HttpRequest, project_id=None, roi_id=None) -> JsonResponse:
    """ Returns a JSON string filled with information about
    the region of interest with ID <roi_id>.
    """
    roi = RegionOfInterest.objects.get(id=roi_id)

    info = {
        'id': roi.id,
        'zoom_level': roi.zoom_level,
        'location': [roi.location_x, roi.location_y, roi.location_z],
        'width': roi.width,
        'height': roi.height,
        'rotation_cw': roi.rotation_cw,
        'stack_id': roi.stack.id,
        'project_id': roi.project.id}

    return JsonResponse(info)

def _add_roi(project_id, stack_id, user_id, x_min, x_max, y_min, y_max, z,
             zoom_level, rotation_cw) -> RegionOfInterest:
    """ Add a new ROI database object and return it.
    """

    # Calculate ROI center and extent
    cx = (x_max + x_min) * 0.5
    cy = (y_max + y_min) * 0.5
    cz = z
    width = abs(x_max - x_min)
    height = abs(y_max - y_min)

    # Create a new ROI class instance
    roi = RegionOfInterest()
    roi.user_id = user_id
    roi.editor_id = user_id
    roi.project_id = project_id
    roi.stack_id = stack_id
    roi.zoom_level = zoom_level
    roi.location_x = cx
    roi.location_y = cy
    roi.location_z = cz
    roi.width = width
    roi.height = height
    roi.rotation_cw = rotation_cw
    roi.save()

    # Create cropped image, if wanted
    if settings.ROI_AUTO_CREATE_IMAGE:
        file_name, file_path = create_roi_path(roi.id)
        user = User.objects.get(pk=user_id)
        create_roi_image(user, project_id, roi.id, file_path)

    return roi

@requires_user_role(UserRole.Annotate)
def add_roi(request:HttpRequest, project_id=None) -> JsonResponse:
    # Try to get all needed POST parameters
    x_min = float(request.POST['x_min'])
    x_max = float(request.POST['x_max'])
    y_min = float(request.POST['y_min'])
    y_max = float(request.POST['y_max'])
    z = float(request.POST['z'])
    zoom_level = int(request.POST['zoom_level'])
    rotation_cw = int(request.POST['rotation_cw'])
    stack_id = int(request.POST['stack'])

    roi = _add_roi(project_id, stack_id, request.user.id, x_min, x_max,
                   y_min, y_max, z, zoom_level, rotation_cw)

    # Build result data set
    status = {'status': "Created new ROI with ID %s." % roi.id}

    return JsonResponse(status)

@requires_user_role(UserRole.Annotate)
def link_roi_to_class_instance(request:HttpRequest, project_id=None, relation_id=None,
        stack_id=None, ci_id=None) -> JsonResponse:
    """ With the help of this method one can link a region of interest
    (ROI) to a class instance. The information about the ROI is passed
    as POST variables.
    """
    # Try to get all needed POST parameters
    x_min = float(request.POST['x_min'])
    x_max = float(request.POST['x_max'])
    y_min = float(request.POST['y_min'])
    y_max = float(request.POST['y_max'])
    z = float(request.POST['z'])
    zoom_level = int(request.POST['zoom_level'])
    rotation_cw = int(request.POST['rotation_cw'])

    # Get related objects
    project = Project.objects.get(id=project_id)
    stack = Stack.objects.get(id=stack_id)
    ci = ClassInstance.objects.get(id=ci_id)
    rel = Relation.objects.get(id=relation_id)

    roi = _add_roi(project.id, stack.id, request.user.id, x_min, x_max, y_min,
                   y_max, z, zoom_level, rotation_cw)

    # Link ROI and class instance
    roi_ci = RegionOfInterestClassInstance()
    roi_ci.user = request.user
    roi_ci.project = project
    roi_ci.relation = rel
    roi_ci.region_of_interest = roi
    roi_ci.class_instance = ci
    roi_ci.save()

    # Build result data set
    status = {'status': "Created new ROI with ID %s." % roi.id}

    return JsonResponse(status)

@requires_user_role([UserRole.Annotate, UserRole.Browse])
def remove_roi_link(request:HttpRequest, project_id=None, roi_id=None) -> JsonResponse:
    """ Removes the ROI link with the ID <roi_id>. If there are no more
    links to the actual ROI after the removal, the ROI gets removed as well.
    """
    # Remove ROI link
    roi_link = RegionOfInterestClassInstance.objects.get(id=roi_id)
    roi_link.delete()
    # Remove ROI if there are no more links to it
    remaining_links = RegionOfInterestClassInstance.objects.filter(
        region_of_interest=roi_link.region_of_interest)
    if remaining_links.count() == 0:
        # Delete the ROI class instance
        roi_link.region_of_interest.delete()
        # Make sure, there is no cropped image left
        file_name, file_path = create_roi_path(roi_id)
        file_info = ""
        if os.path.exists(file_path) and os.path.isfile(file_path):
            try:
                os.remove(file_path)
                file_info = " The same goes for its cropped image."
            except OSError as e:
                file_info = " However, its cropped image couldn't be removed."
        # Create status data
        status = {'status': "Removed ROI link with ID %s. The ROI " \
            "itself has been deleted as well.%s" % (roi_id, file_info)}
    else:
        status = {'status': "Removed ROI link with ID %s. The ROI " \
            "itself has not been deleted, because there are still " \
            "links to it." % roi_id}

    return JsonResponse(status)

def create_lock_name(roi_id) -> str:
    """ Creates a name for the image creation lock.
    """
    return "%s-lock-%s" % ('catmaid.create_roi_image', roi_id)

def create_roi_image(user, project_id, roi_id, file_path) -> bool:
    """ Tries to acquire a lock for a creating the cropped image
    of a certain ROI. If able to do this, launches the celery task
    which removes the lock when done.
    """
    lock_id = create_lock_name(roi_id)
    # cache.add fails if the key is already exists
    acquire_lock = lambda: cache.add(lock_id, "true", LOCK_EXPIRE)
    if not acquire_lock():
        logger.debug("ROI %s is already taken care of by another worker" % roi_id)
        return False
    else:
        create_roi_image_task.delay(user, project_id, roi_id, file_path)
        return True

@task(name='catmaid.create_roi_image')
def create_roi_image_task(user, project_id, roi_id, file_path) -> str:
    lock_id = create_lock_name(roi_id)
    # memcache delete is very slow, but we have to use it to take
    # advantage of using add() for atomic locking
    release_lock = lambda: cache.delete(lock_id)
    logger.debug("Creating cropped image for ROI with ID %s" % roi_id)
    try:
        # Get ROI
        roi = RegionOfInterest.objects.get(id=roi_id)
        # Prepare parameters
        hwidth = roi.width * 0.5
        x_min = roi.location_x - hwidth
        x_max = roi.location_x + hwidth
        hheight = roi.height * 0.5
        y_min = roi.location_y - hheight
        y_max = roi.location_y + hheight
        z_min = z_max = roi.location_z
        single_channel = False
        # Create a cropping job
        job = cropping.CropJob(user, project_id, [roi.stack.id],
            x_min, x_max, y_min, y_max, z_min, z_max, roi.rotation_cw,
            roi.zoom_level, single_channel)
        # Create the PIL images
        cropped_stacks = cropping.extract_substack( job )
        if len(cropped_stacks) == 0:
            raise Exception("Couldn't create ROI image")
        # There is only one image here
        img = cropped_stacks[0]
        img.save(str(file_path))
    finally:
        release_lock()

    return "Created image of ROI %s" % roi_id

def create_roi_path(roi_id) -> Tuple[str, str]:
    """ Creates a tuple (file name, file path) for the given ROI ID.
    """
    file_name = file_prefix + str(roi_id) + "." + file_extension
    file_path = os.path.join(roi_path, file_name)

    return (file_name, file_path)

@requires_user_role([UserRole.Browse])
def get_roi_image(request, project_id=None, roi_id=None) -> HttpResponseRedirect:
    """ Returns the URL to the cropped image, described by the ROI.  These
    images are cached, and won't get removed automatically. If the image is
    already present its URL is used and returned. For performance reasons it
    might be a good idea, to add this test to the web-server config.
    """
    file_name, file_path = create_roi_path(roi_id)
    if not os.path.exists(file_path):
        # Start async processing
        create_roi_image(request.user, project_id, roi_id, file_path)
        # Use waiting image
        url = urljoin(settings.STATIC_URL,
            "images/wait_bgwhite.gif")
    else:
        # Create real image di
        url_base = urljoin(settings.MEDIA_URL,
            settings.MEDIA_ROI_SUBDIRECTORY)
        url = urljoin(url_base, file_name)

    return redirect(url)


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
"""
Time slice drought stats
Created on Thu Jan 14 14:04:09 2021

@author: lizz
"""

import numpy as np
import matplotlib.pyplot as plt
import gSPEI as gSPEI

## Labels: (P)arametric or (NP)nonparametric;
## Standardization (1) lumped or (2) split by starting month
fpath_NP2 = './data/SPEI_Files/nonparametric-var_stom_c/'

## Settings in filenames
integration_times = np.arange(3, 28, 4) # all SPEI integration times used
modelnames = ['CanESM2', 'CCSM4', 'CNRM-CM5', 'CSIRO-Mk3-6-0', 'GISS-E2-R', 'INMCM4', 'MIROC-ESM', 'NorESM1-M'] # all models used in comparison
scenarios = ['Rcp4p5', 'Rcp8p5'] # climate scenarios

## Basins in the order they are written
basin_names = ['INDUS','TARIM','BRAHMAPUTRA','ARAL SEA','COPPER','GANGES','YUKON','ALSEK','SUSITNA','BALKHASH','STIKINE','SANTA CRUZ',
'FRASER','BAKER','YANGTZE','SALWEEN','COLUMBIA','ISSYK-KUL','AMAZON','COLORADO','TAKU','MACKENZIE','NASS','THJORSA','JOEKULSA A F.',
'KUSKOKWIM','RHONE','SKEENA','OB','OELFUSA','MEKONG','DANUBE','NELSON RIVER','PO','KAMCHATKA','RHINE','GLOMA','HUANG HE','INDIGIRKA',
'LULE','RAPEL','SANTA','SKAGIT','KUBAN','TITICACA','NUSHAGAK','BIOBIO','IRRAWADDY','NEGRO','MAJES','CLUTHA','DAULE-VINCES',
'KALIXAELVEN','MAGDALENA','DRAMSELV','COLVILLE']

BasinArea=[1139075,1051731,518011,1233148,64959,1024462,829632,28422,49470,423657,51147,30599,
239678,30760,1745094,258475,668561,191032,5880854,390631,17967,1752001,21211,7527,7311,
118114,97485,42944,2701040,5678,787256,793704,1099380,73066,54103,190522,42862,988062,341227,
25127,15689,11882,7961,58935,107215,29513,24108,411516,130062,18612,17118,41993,
17157,261204,17364,57544] # area of each basin in km2

basin_glacier_area = [26893.8, 24645.4, 16606.7, 15176.7, 12998., 11216., 9535.4, 5614.8, 4304.,
                      3945.4, 3467.6, 3027.8, 2495.1, 2372.3, 2317.4, 2295.9, 1878.4, 1677.3,
                      1634.1, 1601.2, 1583.6, 1519.2, 1337.3, 1251.8, 1098.6, 1032.8, 904.2, 742.3,
                      739.5, 683.4, 485.7, 408.4, 374.7, 347.3, 312.7, 285.0, 269.4, 267.9, 248.4,
                      247.2, 238.1, 198.9, 159.5, 146., 134.5, 86.4, 76.2, 71.2, 64.1, 57.3, 46.5,
                      40.6, 37.9, 33.3, 32.1, 31.9]

yrs = np.linspace(1900, 2101, num=2412)
SPEI_by_model_C = {m: {} for m in modelnames} # create dictionary indexed by model name
for m in modelnames:
    norunoff_f_m = fpath_NP2+'NRunoff_{}_{}_{}_Conduct.txt'.format(integration_times[3], m, scenarios[0])
    wrunoff_f_m = fpath_NP2+'WRunoff_{}_{}_{}_Conduct.txt'.format(integration_times[3], m, scenarios[0])
    SPEI_by_model_C[m]['NRunoff'] = np.loadtxt(norunoff_f_m)
    SPEI_by_model_C[m]['WRunoff'] = np.loadtxt(wrunoff_f_m)
    SPEI_by_model_C[m]['diff'] = SPEI_by_model_C[m]['WRunoff'] - SPEI_by_model_C[m]['NRunoff']

## Re-structure dictionary and create pandas DataFrames aggregated by basin
SPEI_by_basin = gSPEI.sort_models_to_basins(SPEI_by_model_C)

## Analyse multi-model ensemble mean & quantiles for drought statistics
r_w = gSPEI.basin_ensemble_mean(SPEI_by_basin, 'TARIM', 'WRunoff')
r_n = gSPEI.basin_ensemble_mean(SPEI_by_basin, 'TARIM', 'NRunoff')

basin_stats_hist = {b: gSPEI.basin_summary_stats(SPEI_by_basin, basin_name=b, modelnames=modelnames, period=(1980,2010)) for b in basin_names}
basin_stats_midC = {b: gSPEI.basin_summary_stats(SPEI_by_basin, basin_name=b, modelnames=modelnames, period=(2030,2060)) for b in basin_names}
basin_stats_endC = {b: gSPEI.basin_summary_stats(SPEI_by_basin, basin_name=b, modelnames=modelnames, period=(2070,2100)) for b in basin_names}

## All stats for historical period
fig, (ax1,ax2,ax3) = plt.subplots(1,3, sharex=True, figsize=(12,4))
for b, a, ag in zip(basin_names, BasinArea, basin_glacier_area):
    pg = ag/a # percent glaciated
    ax1.scatter(pg, basin_stats_hist[b][0][1]-basin_stats_hist[b][0][0], color='k')
    ax2.scatter(pg, basin_stats_hist[b][1][1]-basin_stats_hist[b][1][0], color='k')
    ax3.scatter(pg, -1*(basin_stats_hist[b][2][1]-basin_stats_hist[b][2][0]), color='k')
ax1.set(xlabel='Glacier area fraction', ylabel='Diff. number of droughts 1980-2010', xscale='log')
ax2.set(xlabel='Glacier area fraction', ylabel='Diff. drought duration 1980-2010', xscale='log')
ax3.set(xlabel='Glacier area fraction', ylabel='Diff. drought deficit 1980-2010', xscale='log',
        xlim=(1E-4, 0.22))
plt.tight_layout()
plt.show()

## Drought number over time
fig, (ax1,ax2,ax3) = plt.subplots(1,3, sharey=True, sharex=True, figsize=(12,4))
for b, a, ag in zip(basin_names, BasinArea, basin_glacier_area):
    pg = ag/a # percent glaciated
    ax1.scatter(pg, basin_stats_hist[b][0][1]-basin_stats_hist[b][0][0], color='k')
    ax2.scatter(pg, basin_stats_midC[b][0][1]-basin_stats_midC[b][0][0], color='k')
    ax3.scatter(pg, basin_stats_endC[b][0][1]-basin_stats_endC[b][0][0], color='k')
ax1.set(xlabel='Glacier area fraction', ylabel='Diff. number of droughts 1980-2010', xscale='log')
ax2.set(xlabel='Glacier area fraction', ylabel='Diff. number of droughts 2030-2060', xscale='log')
ax3.set(xlabel='Glacier area fraction', ylabel='Diff. number of droughts 2070-2100', xscale='log',
        xlim=(1E-4, 0.22))
plt.tight_layout()
plt.show()

## Drought duration over time
fig, (ax1,ax2,ax3) = plt.subplots(1,3, sharey=True, sharex=True, figsize=(12,4))
for b, a, ag in zip(basin_names, BasinArea, basin_glacier_area):
    pg = ag/a # percent glaciated
    ax1.scatter(pg, basin_stats_hist[b][1][1]-basin_stats_hist[b][1][0], color='k')
    ax2.scatter(pg, basin_stats_midC[b][1][1]-basin_stats_midC[b][1][0], color='k')
    ax3.scatter(pg, basin_stats_endC[b][1][1]-basin_stats_endC[b][1][0], color='k')
ax1.set(xlabel='Glacier area fraction', ylabel='Diff. drought duration 1980-2010', xscale='log')
ax2.set(xlabel='Glacier area fraction', ylabel='Diff. drought duration 2030-2060', xscale='log')
ax3.set(xlabel='Glacier area fraction', ylabel='Diff. drought duration 2070-2100', xscale='log',
        xlim=(1E-4, 0.22))
plt.tight_layout()
plt.show()

## Drought deficit over time
fig, (ax1,ax2,ax3) = plt.subplots(1,3, sharey=True, sharex=True, figsize=(12,4))
for b, a, ag in zip(basin_names, BasinArea, basin_glacier_area):
    pg = ag/a # percent glaciated
    ax1.scatter(pg, -1*(basin_stats_hist[b][2][1]-basin_stats_hist[b][2][0]), color='k')
    ax2.scatter(pg, -1*(basin_stats_midC[b][2][1]-basin_stats_midC[b][2][0]), color='k')
    ax3.scatter(pg, -1*(basin_stats_endC[b][2][1]-basin_stats_endC[b][2][0]), color='k')
ax1.set(xlabel='Glacier area fraction', ylabel='Diff. drought deficit 1980-2010', xscale='log')
ax2.set(xlabel='Glacier area fraction', ylabel='Diff. drought deficit 2030-2060', xscale='log')
ax3.set(xlabel='Glacier area fraction', ylabel='Diff. drought deficit 2070-2100', xscale='log',
        xlim=(1E-4, 0.22))
plt.tight_layout()
plt.show()


import os
import numpy as np
import codecs
import json
from glob import glob
import cv2
import shutil
from sklearn.model_selection import train_test_split

# 1.存放的json标签路径
labelme_path = "/Volumes/My_Passport/LBT/DATASET/anti-vibration_hammer/timdatasets/json/"

# 原始labelme标注数据路径
saved_path = "VOC2012/"
# 保存路径
isUseTest = False  # 是否创建test集

# 2.创建要求文件夹
if not os.path.exists(saved_path + "Annotations"):
    os.makedirs(saved_path + "Annotations")
if not os.path.exists(saved_path + "JPEGImages/"):
    os.makedirs(saved_path + "JPEGImages/")
if not os.path.exists(saved_path + "ImageSets/Main/"):
    os.makedirs(saved_path + "ImageSets/Main/")

# 3.获取待处理文件
files = glob(labelme_path + "*.json")
files = [i.replace("\\", "/").split("/")[-1].split(".json")[0] for i in files]
print(files)

# 4.读取标注信息并写入 xml
for json_file_ in files:
    json_filename = labelme_path + json_file_ + ".json"
    json_file = json.load(open(json_filename, "r", encoding="utf-8"))
    height, width, channels = cv2.imread('/Volumes/My_Passport/LBT/DATASET/anti-vibration_hammer/timdatasets/images/' + json_file_ + ".jpg").shape
    with codecs.open(saved_path + "Annotations/" + json_file_ + ".xml", "w", "utf-8") as xml:

        xml.write('<annotation>\n')
        xml.write('\t<folder>' + 'WH_data' + '</folder>\n')
        xml.write('\t<filename>' + json_file_ + ".jpg" + '</filename>\n')
        xml.write('\t<source>\n')
        xml.write('\t\t<database>WH Data</database>\n')
        xml.write('\t\t<annotation>WH</annotation>\n')
        xml.write('\t\t<image>flickr</image>\n')
        xml.write('\t\t<flickrid>NULL</flickrid>\n')
        xml.write('\t</source>\n')
        xml.write('\t<owner>\n')
        xml.write('\t\t<flickrid>NULL</flickrid>\n')
        xml.write('\t\t<name>WH</name>\n')
        xml.write('\t</owner>\n')
        xml.write('\t<size>\n')
        xml.write('\t\t<width>' + str(width) + '</width>\n')
        xml.write('\t\t<height>' + str(height) + '</height>\n')
        xml.write('\t\t<depth>' + str(channels) + '</depth>\n')
        xml.write('\t</size>\n')
        xml.write('\t\t<segmented>0</segmented>\n')
        for multi in json_file["shapes"]:
            points = np.array(multi["points"])
            labelName = multi["label"]
            xmin = min(points[:, 0])
            xmax = max(points[:, 0])
            ymin = min(points[:, 1])
            ymax = max(points[:, 1])
            label = multi["label"]
            if xmax <= xmin:
                pass
            elif ymax <= ymin:
                pass
            else:
                xml.write('\t<object>\n')
                xml.write('\t\t<name>' + labelName + '</name>\n')
                xml.write('\t\t<pose>Unspecified</pose>\n')
                xml.write('\t\t<truncated>1</truncated>\n')
                xml.write('\t\t<difficult>0</difficult>\n')
                xml.write('\t\t<bndbox>\n')
                xml.write('\t\t\t<xmin>' + str(int(xmin)) + '</xmin>\n')
                xml.write('\t\t\t<ymin>' + str(int(ymin)) + '</ymin>\n')
                xml.write('\t\t\t<xmax>' + str(int(xmax)) + '</xmax>\n')
                xml.write('\t\t\t<ymax>' + str(int(ymax)) + '</ymax>\n')
                xml.write('\t\t</bndbox>\n')
                xml.write('\t</object>\n')
                print(json_filename, xmin, ymin, xmax, ymax, label)
        xml.write('</annotation>')

# 5.复制图片到 VOC2007/JPEGImages/下
image_files = glob("/Volumes/My_Passport/LBT/DATASET/anti-vibration_hammer/timdatasets/images/" + "*.jpg")
print("copy image files to VOC2012/JPEGImages/")
for image in image_files:
    shutil.copy(image, saved_path + "JPEGImages/")

# 6.拆分训练集、测试集、验证集
txtsavepath = saved_path + "ImageSets/Main/"
ftrainval = open(txtsavepath + '/trainval.txt', 'w')
ftest = open(txtsavepath + '/test.txt', 'w')
ftrain = open(txtsavepath + '/train.txt', 'w')
fval = open(txtsavepath + '/val.txt', 'w')
total_files =  glob("./VOC2012/Annotations/*.xml")
total_files = [i.split("/")[-1].split(".xml")[0] for i in total_files]
trainval_files = []
test_files = []
if isUseTest:
    trainval_files, test_files = train_test_split(total_files, test_size=0.2)
else:
    trainval_files = total_files
for file in trainval_files:
    ftrainval.write(file + "\n")

# split
train_files, val_files = train_test_split(trainval_files, test_size=0.2)

# train
for file in train_files:
    ftrain.write(file + "\n")

# val
for file in val_files:
    fval.write(file + "\n")
for file in test_files:
    print(file)
    ftest.write(file + "\n")
ftrainval.close()
ftrain.close()
fval.close()
ftest.close()
"""
import os
import numpy as np
import codecs
import json
import glob
import cv2
import shutil
from sklearn.model_selection import train_test_split

# 1.标签路径
labelme_path = "/Volumes/My_Passport/LBT/DATASET/anti-vibration_hammer/timdatasets/json/"  # 原始labelme标注数据路径
saved_path = "VOC2012/"  # 保存路径

# 2.创建要求文件夹
dst_annotation_dir = os.path.join(saved_path, 'Annotations')
if not os.path.exists(dst_annotation_dir):
    os.makedirs(dst_annotation_dir)
dst_image_dir = os.path.join(saved_path, "JPEGImages")
if not os.path.exists(dst_image_dir):
    os.makedirs(dst_image_dir)
dst_main_dir = os.path.join(saved_path, "ImageSets", "Main")
if not os.path.exists(dst_main_dir):
    os.makedirs(dst_main_dir)

# 3.获取待处理文件
org_json_files = sorted(glob.glob(os.path.join(labelme_path, '*.json')))
org_json_file_names = [i.split("\\")[-1].split(".json")[0] for i in org_json_files]
org_img_files = sorted(glob.glob(os.path.join(labelme_path, '*.jpg')))
org_img_file_names = [i.split("\\")[-1].split(".jpg")[0] for i in org_img_files]

# 4.labelme file to voc dataset
for i, json_file_ in enumerate(org_json_files):
    json_file = json.load(open(json_file_, "r", encoding="utf-8"))
    image_path = os.path.join(labelme_path, org_json_file_names[i]+'.jpg')
    img = cv2.imread(image_path , -1)
    height, width, channels = img.shape
    dst_image_path = os.path.join(dst_image_dir, "{:06d}.jpg".format(i))
    cv2.imwrite(dst_image_path, img)
    dst_annotation_path = os.path.join(dst_annotation_dir, '{:06d}.xml'.format(i))
    with codecs.open(dst_annotation_path, "w", "utf-8") as xml:
        xml.write('<annotation>\n')
        xml.write('\t<folder>' + 'Pin_detection' + '</folder>\n')
        xml.write('\t<filename>' + "{:06d}.jpg".format(i) + '</filename>\n')
        # xml.write('\t<source>\n')
        # xml.write('\t\t<database>The UAV autolanding</database>\n')
        # xml.write('\t\t<annotation>UAV AutoLanding</annotation>\n')
        # xml.write('\t\t<image>flickr</image>\n')
        # xml.write('\t\t<flickrid>NULL</flickrid>\n')
        # xml.write('\t</source>\n')
        # xml.write('\t<owner>\n')
        # xml.write('\t\t<flickrid>NULL</flickrid>\n')
        # xml.write('\t\t<name>ChaojieZhu</name>\n')
        # xml.write('\t</owner>\n')
        xml.write('\t<size>\n')
        xml.write('\t\t<width>' + str(width) + '</width>\n')
        xml.write('\t\t<height>' + str(height) + '</height>\n')
        xml.write('\t\t<depth>' + str(channels) + '</depth>\n')
        xml.write('\t</size>\n')
        xml.write('\t\t<segmented>0</segmented>\n')
        for multi in json_file["shapes"]:
            points = np.array(multi["points"])
            xmin = min(points[:, 0])
            xmax = max(points[:, 0])
            ymin = min(points[:, 1])
            ymax = max(points[:, 1])
            label = multi["label"]
            if xmax <= xmin:
                pass
            elif ymax <= ymin:
                pass
            else:
                xml.write('\t<object>\n')
                xml.write('\t\t<name>' + label + '</name>\n')
                xml.write('\t\t<pose>Unspecified</pose>\n')
                xml.write('\t\t<truncated>1</truncated>\n')
                xml.write('\t\t<difficult>0</difficult>\n')
                xml.write('\t\t<bndbox>\n')
                xml.write('\t\t\t<xmin>' + str(xmin) + '</xmin>\n')
                xml.write('\t\t\t<ymin>' + str(ymin) + '</ymin>\n')
                xml.write('\t\t\t<xmax>' + str(xmax) + '</xmax>\n')
                xml.write('\t\t\t<ymax>' + str(ymax) + '</ymax>\n')
                xml.write('\t\t</bndbox>\n')
                xml.write('\t</object>\n')
                print(json_file_, xmin, ymin, xmax, ymax, label)
        xml.write('</annotation>')

# 5.split files for txt
train_file = os.path.join(dst_main_dir, 'train.txt')
trainval_file = os.path.join(dst_main_dir, 'trainval.txt')
val_file = os.path.join(dst_main_dir, 'val.txt')
test_file = os.path.join(dst_main_dir, 'test.txt')

ftrain = open(train_file, 'w')
ftrainval = open(trainval_file, 'w')
fval = open(val_file, 'w')
ftest = open(test_file, 'w')

total_annotation_files = glob.glob(os.path.join(dst_annotation_dir, "*.xml"))
total_annotation_names = [i.split("/")[-1].split(".xml")[0] for i in total_annotation_files]

# test_filepath = ""
for file in total_annotation_names:
    ftrainval.writelines(file + '\n')
# test
# for file in os.listdir(test_filepath):
#    ftest.write(file.split(".jpg")[0] + "\n")
# split
train_files, val_files = train_test_split(total_annotation_names, test_size=0.2)
# train
for file in train_files:
    ftrain.write(file + '\n')
# val
for file in val_files:
    fval.write(file + '\n')

ftrainval.close()
ftrain.close()
fval.close()
# ftest.close()

"""


from __future__ import division, print_function
# coding=utf-8
import sys
import os
import glob
import re
import numpy as np
from skimage.color import gray2rgb
from skimage.color import rgb2gray
from keras.models import load_model
import cv2

# Keras
from keras.applications.imagenet_utils import preprocess_input, decode_predictions
from keras.models import load_model
from keras.preprocessing import image

# Flask utils
from flask import Flask, redirect, url_for, request, render_template
from werkzeug.utils import secure_filename
from gevent.pywsgi import WSGIServer

# Define a flask app
app = Flask(__name__)

# Model saved with Keras model.save()
MODEL_PATH = 'C:/Users/HP/Desktop/projinterface/models/aimodelresize.h5'

# Load your trained model
try:
    model = load_model(MODEL_PATH)
    print('-----\n\n\n\n\nModel loaded. Check http://127.0.0.1:5000/-----\n\n\n\n\n')
    model._make_predict_function()          # Necessary
except:
    print("-----\n\n\n\n\ncouldnot load-----\n\n\n\n\n")

def model_predict(img_path, model):
    print("-----\n\n\n\n\img_path   ", img_path)
#    img = image.load_img(img_path)
    from skimage.io import imread
    img = imread(img_path)
    img = cv2.resize(img, (128,128))
    try:
        img =gray2rgb(img)
    except:
        print("-----\n\n\n\n\n Cant not convert into gray scale")

    # Preprocessing the image
    x = image.img_to_array(img)
    # x = np.true_divide(x, 255)
    x = np.expand_dims(x, axis=0)

    preds = model.predict(x)
    print("-----\n\n\n\n\n preds   ", preds)
    return preds

@app.route('/', methods=['GET'])
def index():
    # Main page
    return render_template('index.html')

@app.route('/predict', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST':
        # Get the file from post request
        f = request.files['file']

        # Save the file to ./uploads
        basepath = os.path.dirname(__file__)
        file_path = os.path.join(
            basepath, 'uploads', secure_filename(f.filename))
        f.save(file_path)

        # Make prediction
        preds = model_predict(file_path, model)

        # Process your result for human
        # pred_class = preds.argmax(axis=-1)            # Simple argmax
          # ImageNet Decode
        N_ANIMALS = 3
        ANIMALS = {0: "Bird", 1: "Fish", 2: "Snake"}
        result = ANIMALS[np.argmax(preds)]
        #result = str(pred_class[0][0][1])               # Convert to string
        return result
    return None

if __name__ == '__main__':
    app.run(threaded=False, debug=False)


# ホロライブの画像を集める
import os
import argparse
from icrawler.builtin import BingImageCrawler

def parse_keyword() -> str:
  """入力された引数を取得する

  Returns:
      str: vtuber名
  """
  parser: argparse.ArgumentParser = argparse.ArgumentParser()
  parser.add_argument("-name", required=True)
  args: argparse.Namespace = parser.parse_args()

  return args.name

def crawl(keyword: str):
  """Vtuberの画像を集め、ディレクトリに保存する

  Args:
      keyword (str): 検索ワード
  """
  base_dir: str = "./data"
  if not os.path.exists(base_dir):
    os.mkdir(base_dir)

  if not os.path.exists(os.path.join(base_dir, keyword)):
    os.mkdir(os.path.join(base_dir, keyword))

  crawler: BingImageCrawler = BingImageCrawler(storage={'root_dir': os.path.join(base_dir, keyword)})
  crawler.crawl(keyword=keyword, max_num=1000)

if __name__ == "__main__":
  vtuber_name: str = parse_keyword()
  crawl(vtuber_name)


# Import DB object and models
from .src.app import db
from .src.models.user import User

# Drop all tables
db.drop_all()

# Recreate all tables
db.create_all()

# Add dummy users
admin = User(username='admin', email='admin@example.com')
guest = User(username='guest', email='guest@example.com')

db.session.add(admin)
db.session.add(guest)
db.session.commit()


def quicksort(arr_in):
   # SImple routine for a quicksort
   # input: arr_in - array with numbers, unsorted
   #
   # output: arr - same array, but sorted
   #

   import numpy as np

   if (arr_in.ndim>1):
      print("Array is wrong dimensions")
      return None

   # Make a copy of the array
   arr = np.copy(arr_in)
   n   = arr.size

   for j in range(2,n+1):
      ibreak = 0
      A = arr[j-1]
      for i in range(j-1,0,-1):
         if (arr[i-1] <= A):
            arr[i] = A
            ibreak = 1
            break
         arr[i]=arr[i-1]
      if (ibreak==0):
         arr[0] = A
   return arr


#credit to initial skeleton:webcam-capture-v1.01
# Edited By/ Author: Maryam Azmandian
import cv2

def preProcess():
    key = cv2. waitKey(1)
    webcam = cv2.VideoCapture(0)
    while True:
        try:
            check, frame = webcam.read()
            print(check) #prints true as long as the webcam is running
            print(frame) #prints matrix values of each framecd
            cv2.imshow("Capturing", frame)
            key = cv2.waitKey(1)
            if key == ord('s'):
                cv2.imwrite(filename='saved_img.jpg', img=frame)
                webcam.release()
                img_new = cv2.imread('saved_img.jpg', cv2.IMREAD_GRAYSCALE)
                img_new = cv2.imshow("Captured Image", img_new)
                cv2.waitKey(1650)
                cv2.destroyAllWindows()
                print("Processing image...")
                img_ = cv2.imread('saved_img.jpg', cv2.IMREAD_ANYCOLOR)

                # print("Converting RGB image to grayscale...")
                # gray = cv2.cvtColor(img_, cv2.COLOR_BGR2GRAY)
                # print("Converted RGB image to grayscale...")
                # # print("Resizing image to 28x28 scale...")
                # # img_ = cv2.resize(gray,(28,28))
                # # print("Resized...")
                # # My code here:
                # img_deblured = cv2.medianBlur(gray,11)
                # ret,thresh1 = cv2.threshold(img_deblured,75,255,cv2.THRESH_BINARY)
                # # thresh2_adap_gaus = cv2.adaptiveThreshold(img_deblured,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,\
                # # cv2.THRESH_BINARY,11,2)
                # ###
                # # img_done2 = cv2.imwrite(filename='saved_img-final_thresh2Adaptive.jpg', img=thresh2_adap_gaus)
                gray = cv2.cvtColor(img_, cv2.COLOR_BGR2GRAY)
                cv2.imshow("contours", gray)
                cv2.waitKey(0)
                blurred = cv2.GaussianBlur(gray, (3, 3), 0,cv2.BORDER_DEFAULT)
                cv2.imshow("contours", blurred)
                cv2.waitKey(0)
                edged = cv2.Canny(blurred,100, 200,5)

                # define a (3, 3) structuring element
                kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1, 1))

                # apply the dilation operation to the edged image
                dilate = cv2.dilate(edged, kernel, iterations=1)

                # find the contours in the dilated image
                contours, _ = cv2.findContours(dilate, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
                image_copy =img_.copy()
                # draw the contours on a copy of the original image
                cv2.drawContours(image_copy, contours, -1, (0, 255, 0), 2)
                print(len(contours), "objects were found in this image.")

                cv2.imshow("Dilated image", dilate)
                cv2.imshow("contours", image_copy)
                cv2.waitKey(0)
                dim = (150, 150)
                img_resized = cv2.resize(image_copy, dim, interpolation = cv2.INTER_AREA)
                img_done = cv2.imwrite(filename='saved_img_final.jpg', img=img_resized)
                print("Image saved!")

                break
            elif key == ord('q'):
                print("Turning off camera.")
                webcam.release()
                print("Camera off.")
                print("Program ended.")
                cv2.destroyAllWindows()
                break

        except(KeyboardInterrupt):
            print("Turning off camera.")
            webcam.release()
            print("Camera off.")
            print("Program ended.")
            cv2.destroyAllWindows()
            break
    return contours,image_copy.shape


import random
import numpy as np
import pandas as pd

import tensorflow as tf

from tqdm import tqdm

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MultiLabelBinarizer

# Split dataset into train/test/validation
def split(x, test_size=0.2, val_size=0.0, seed=0):
    if val_size + test_size >= 1:
        return None
    x_train, x_test = train_test_split(
        x, test_size=test_size + val_size, random_state=seed
    )
    x_val = None
    if val_size > 0:
        x_test, x_val = train_test_split(
            x_test,
            test_size=val_size / (test_size + val_size),
            random_state=seed,
        )
    return x_train, x_val, x_test

# Loads an image given its path
def load_image_from_path(path):
    image = tf.io.read_file(path)
    image = tf.io.decode_jpeg(image, channels=3, dct_method="INTEGER_ACCURATE")

    # may need resizing
    #image = tf.image.resize(image, image_shape[:2])
    image = tf.cast(image, dtype=tf.float16)
    image = image / 255.0
    return image

# Loads all features of the given CLIP dataset given the image folder, captions/concepts files and a concept encoder
def load_features(image_folder, captions_file, concepts_file, concept_encoder, filter_percent=1):
    features = []

    # Import CSVs
    csv_caption_dataset = tf.data.experimental.CsvDataset(
        captions_file,
        field_delim='\t',
        record_defaults=[tf.string, tf.string],
        header=True,
        select_cols=[0, 1]
    )
    csv_concept_dataset = tf.data.experimental.CsvDataset(
        concepts_file,
        field_delim='\t',
        record_defaults=[tf.string, tf.string],
        header=True,
        select_cols=[0, 1]
    )

    # We make the assumption that CSV files contain the same key values (image names)
    # following the same ordering

    # Extract features from dataset
    print("Extracting features from CSV file(s)")
    for caption_el, concept_el in tqdm(zip(csv_caption_dataset, csv_concept_dataset)):
        filename_cap, caption = caption_el
        filename_con , concepts = concept_el

        # Sanity check
        assert filename_cap == filename_con

        image_path = image_folder + "/" + filename_cap + ".jpg"

        features.append({
            'caption': caption,
            'image path': image_path,
            'concepts': concept_encoder.transform([concepts.numpy().decode("utf-8").split(";")]),
        })

    # Filter elements
    if filter_percent != 1:
        n_features = int(len(features) * filter_percent)
        features = random.sample(features, n_features)

    return features

# Manual preprocessing of features
def preprocess_features(features, concept_encoder, filter_percent=1):
    print("Preprocessing features")

    # Filter elements
    if filter_percent != 1:
        n_features = int(len(features) * filter_percent)
        features = random.sample(features, n_features)

    return {
        'image paths': tf.convert_to_tensor([x["image path"] for x in tqdm(features)], dtype=tf.string),
        'captions': tf.convert_to_tensor([x["caption"] for x in tqdm(features)], dtype=tf.string),
        'concepts': tf.convert_to_tensor(np.vstack([concept_encoder.transform(x["concepts"]).flatten() for x in tqdm(features)]), dtype=tf.bool),
        # 'images': tf.convert_to_tensor([load_image(x["image path"]) for x in tqdm(features)], dtype=tf.float16),
    }

# Creates a tensorflow dataset given a CLIP dataset and other parameters
def create_dataset(
        features,                       # Dataset features to include in the dataset
        input_features_types,           # Dictionary containing the structure of the dataset features
        feature_shapes,                 # Dictionary containing the shapes of the dataset features
        x_features, y_features=None,    # Lists containing the names for the desired x and y features of the dataset
        x_dict=True, y_dict=True,       # Set to True in order to store x and y features into a dictionary format inside the dataset
        load_images=True,               # Set to True in order to add images to the dataset
        shuffle_buffer_size=1024,       # Shuffle buffer size for the dataset, set to 1 for no shuffling
        batch_size=10,                  # Batch size for the dataset
        cached=False                    # Set to True to enable caching
):
    # Generate dataset following initial input feature types
    dataset = tf.data.Dataset.from_generator(
        lambda: features, { x: input_features_types[x] for x in input_features_types }
    )

    # Preprocessing internal functions
    def setshape(e):
        for (k, v) in feature_shapes.items():
            if k in e:
                e[k].set_shape(v)
        return e
    def add_images(e):
        # Maybe parametrize
        img_from = "image path"
        img_to = "image"
        new_features = list(input_features_types.keys()) + [img_to]
        return {f:e[f] if f != img_to else load_image_from_path(e[img_from]) for f in new_features}
    def split_xy(e):
        e_x = {xf:tf.squeeze(e[xf]) for xf in x_features} if x_dict else tf.squeeze([e[xf] for xf in x_features])
        if y_features:
            e_y = {yf:tf.squeeze(e[yf]) for yf in y_features} if y_dict else tf.squeeze([e[yf] for yf in y_features])
            return (e_x, e_y)
        return e_x

    # Preprocess
    if load_images:
        dataset = dataset.map(add_images)
    dataset = dataset.map(setshape)
    dataset = dataset.map(split_xy)

    # Compile dataset
    if cached:
        dataset = dataset.cache()
    dataset = dataset.shuffle(shuffle_buffer_size).batch(batch_size).prefetch(tf.data.AUTOTUNE)

    return dataset

# Top level function to import datasets
# Returns (concept_list, concept_onehot_encoder), [(train_dataset, val_dataset, test_dataset)], (train_ds_size, val_ds_size, test_ds_size)
def import_datasets(
    image_folder,                   # Folder containing images
    captions_file,                  # File path for the captions file
    concepts_file,                  # File path for the concepts file (concept id <-> concept text)
    concepts_det_file,              # File path for the concepts detection train file (element id <-> concept id)
    input_features_types,           # Dictionary containing the structure of the dataset features
    feature_shapes,                 # Dictionary containing the shapes of the dataset features
    dataset_parameters,             # List containing dictionaries specifying dataset parameters for each desired output dataset collection, parameters are:
                                    # x_features, y_features:None,      Lists containing the names for the desired x and y features of the dataset
                                    # x_dict:True, y_dict:True,         Set to True in order to store x and y features into a dictionary format inside the dataset
                                    # load_images:True,                 Set to True in order to add images to the dataset
                                    # shuffle_buffer_size:1024,         Shuffle buffer size for the dataset, set to 1 for no shuffling
                                    # batch_size:10,                    Batch size for the dataset
                                    # cached:False,                     Set to True to enable caching
    filter_percent=1,               # Percentage of the dataset to retrieve
    test_size=0.2,                  # Percentage of test dataset
    val_size=0.0,                   # Percentage of validation dataset
    seed=0,                         # Random seed
):
    # Loads concepts
    concepts = pd.read_csv(concepts_file, sep='\t')
    concept_list = concepts.set_index('concept')['concept_name'].to_dict()
    # Concept one-hot encoder
    concepts_onehot = MultiLabelBinarizer(classes=list(concept_list.keys()))
    _ = concepts_onehot.fit([list(concept_list.keys())])

    # Load dataset features from csv files, split them and preprocess them
    features = load_features(image_folder, captions_file, concepts_det_file, concepts_onehot, filter_percent=filter_percent)
    feat_train, feat_val, feat_test = split(features, test_size=test_size, val_size=val_size, seed=seed)

    # Calculate dataset sizes
    train_ds_size = len(feat_train) if feat_train else 0
    val_ds_size = len(feat_val) if feat_val else 0
    test_ds_size = len(feat_test) if feat_test else 0

    datasets = []
    # Created datasets
    for parameters in dataset_parameters:
        train_dataset = create_dataset(feat_train, input_features_types, feature_shapes, **parameters) if feat_train else None
        val_dataset = create_dataset(feat_val, input_features_types, feature_shapes, **parameters) if feat_val else None
        test_dataset = create_dataset(feat_test, input_features_types, feature_shapes, **parameters) if feat_test else None
        datasets.append((train_dataset, val_dataset, test_dataset))

    return (concept_list, concepts_onehot), datasets, (train_ds_size, val_ds_size, test_ds_size)


#!/usr/bin/env python3

"""
Create a PDF file (proof) of three predefined images of the head (left, front, right view)
alongside create a CSV file with all pose names by client and px to deliver to the client
"""

import os
import sys
import time
import shlex
import shutil
import logging
import colorama  # https://pypi.org/project/colorama/
import platform
import subprocess
import pandas as pd
import numpy as np
import multiprocessing as mp

from multiprocessing import Pool
from glob import glob
from click import option, command
from colorama import Fore
from queue import PriorityQueue
from fpdf import FPDF

__author__ = "Stephan Osterburg"
__copyright__ = "Copyright 2020, Pixelgun Studio"
__credits__ = ["Stephan Osterburg", "Mauricio Baiocchi"]
__license__ = "MIT"
__version__ = "0.1.5"
__maintainer__ = ""
__email__ = "info@pixelgunstudio.com"
__status__ = "Production"

# Initialise Queue
qi = PriorityQueue()
qo = PriorityQueue()

# Initialise Colorama
colorama.init(autoreset=True)

class ValidationError(ValueError):
    pass

class GlobalDirs:
    """Global default directories"""

    def __init__(self):
        pass

    if platform.system() == 'Darwin':
        incoming = '/Volumes/Bigfoot/_incoming/'
        template = '/Volumes/Bigfoot/Pixelgun_Projects/_XX_XXXX_JobTemplate/Sections/_XX_generic_section'
        projects = '/Volumes/Bigfoot/Pixelgun_Projects'
    elif platform.system() == 'Linux':
        incoming = '/mnt/bigfoot/_incoming/'
        template = '/mnt/bigfoot/Pixelgun_Projects/_XX_XXXX_JobTemplate/Sections/_XX_generic_section'
        projects = '/mnt/bigfoot/Pixelgun_Projects'

    # All the teams in the game available
    teams = {'atl': 'Atlanta Hawks',
             'bkn': 'Brooklyn Nets',
             'bos': 'Boston Celtics',
             'cel': 'Cleveland Cavaliers',
             'cha': 'Charlotte Hornets',
             'chi': 'Chicago Bulls',
             'dal': 'Dallas Mavericks',
             'den': 'Denver Nuggets',
             'det': 'Detroit Pistons',
             'gsw': 'Golden State Warriors',
             'hou': 'Houston Rockets',
             'ind': 'Indiana Pacers',
             'lac': 'Los Angeles Clippers',
             'lal': 'Los Angeles Lakers',
             'mem': 'Memphis Grizzlies',
             'mia': 'Miami Heat',
             'mil': 'Milwaukee Bucks',
             'min': 'Minnesota Timberwolves',
             'nop': 'New Orleans Pelicans',
             'nyk': 'New York Knicks',
             'okc': 'Oklahoma City Thunder',
             'orl': 'Orlando Magic',
             'phi': 'Philadelphia 76ers',
             'phx': 'Phoenix Suns',
             'por': 'Portland Trail Blazers',
             'sac': 'Sacramento Kings',
             'sas': 'San Antonio Spurs',
             'tor': 'Toronto Raptors',
             'uta': 'Utah Jazz',
             'was': 'Washington Wizards'}

def clear_screen():
    """Clear shell"""
    _ = subprocess.run('clear' if os.name == 'posix' else 'cls')

def call_proc(cmd):
    """Execute shell command
    Args:
        cmd: a given shell command
    Returns: None
    """
    subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)

def define_proof_name(pose, game, team):
    """
    Define base output file name for PDF and CSV
    Args:
        pose:
        game:
        team:

    Returns: base file name

    """
    output_dir = os.path.realpath(GlobalDirs.projects + "/" + game + "/Source_Pixelgun/Proof Sheets/" + team)
    if not os.path.exists(output_dir):
        os.mkdir(output_dir)
    proof_name = '_'.join(pose.split('/')[-1].split('_')[0:5])

    return output_dir + '/' + proof_name + '_selects'

def get_placeholder(player_pose, df):
    """Get placeholder string for the images
    Write out PROOF.csv

    Args:
        df: template dataframe
        player_pose: name of players pose

    Returns: Base name of pose, i.e. 'smile'
    """

    # Create "CHUNKS' column by getting all values from 'PX AQUISITION' where there are
    # NaN in 'CLIENT SHAPE NAMES' and combine them
    df['CHUNKS'] = np.where(df['CLIENT SHAPE NAMES'].isnull(), df['PX AQUISITION'], df['CLIENT SHAPE NAMES'])

    try:
        px_pose = '_'.join(player_pose.split('/')[-1].split('_')[5:-1])
        if px_pose not in df['PX AQUISITION'].values:
            _ = pd.DataFrame([[px_pose, px_pose, px_pose]],
                             columns=('PX AQUISITION', 'CLIENT SHAPE NAMES', 'CHUNKS'))
            df = pd.concat([df, _], ignore_index=True)

        # Get pose name
        pose_name = df.loc[df['PX AQUISITION'] == px_pose, 'CHUNKS'].iloc[0]
    except ValueError:
        print(Fore.RED + "Failed to read XLS file")
        return

    return pose_name

def check_tiff_exists(directory, player):
    """Check if the Camera RAW are already converted
       and copy it into the tmp directory

    Args:
        directory: directory name of the team
        player: either the name of a player

    Returns: Boolean (True/False)
    """
    print(Fore.YELLOW + "Checking if TIFF's exists...")

    exists = False
    images = ['A000_POLO', 'AL010_POLO', 'AR010_POLO']

    # Backward compatibility
    if os.path.isdir(directory + '/' + player + '/_acquisition/tiff'):
        poses = glob(directory + '/' + player + '/_acquisition/tiff/*')

        for pose in poses:
            for image in images:
                tif_image = pose + '/' + image + '.tif'
                if os.path.isfile(tif_image):
                    exists = True
                    tmp_image = '/tmp/' + pose.split('/')[-1] + '_' + image + '.tif'
                    shutil.copyfile(tif_image, tmp_image)

    return exists

def convert_images(directory, player):
    """Convert CR2 to TIFF (tmp)

    Args:
        directory: directory name of the team
        player: either the name of a player

    Returns: None
    """

    print(Fore.YELLOW + 'Converting CR2 Images...')

    # Using darktable to convert CR2 to TIFF
    app = '/Applications/darktable.app/Contents/MacOS/darktable-cli'
    opt = ' --core --conf plugins/imageio/format/tiff/bpp=16'

    images = ['A000_POLO', 'AL010_POLO', 'AR010_POLO']
    poses = glob(directory + '/' + player + '/_acquisition/*')

    # HACK/WORKAROUND: Create tmp directories for darktable to e able to run in parallel
    for pose in poses:
        for image in images:
            base_tmp_dir = pose.split('/')[-1]
            if not os.path.isdir('/tmp/' + base_tmp_dir):
                os.mkdir('/tmp/' + base_tmp_dir)

            if not os.path.isdir('/tmp/' + base_tmp_dir + '/' + image):
                os.mkdir('/tmp/' + base_tmp_dir + '/' + image)

    cmd_list = []
    for pose in poses:
        for image in images:
            base_tmp_dir = pose.split('/')[-1]
            in_image = pose + '/' + image + '.CR2'
            out_image = '/tmp/' + pose.split('/')[-1] + '_' + image + '.tif'
            cmd = app + ' ' + in_image + ' ' + out_image + opt + \
                  ' --configdir /tmp/' + str(base_tmp_dir) + '/' + str(image)
            cmd_list.append(cmd)

    start_time = time.time()
    agents = mp.cpu_count()
    pool = Pool(processes=agents)
    results = pool.map(call_proc, cmd_list)
    pool.close()
    pool.join()

    end_time = time.time()
    t = int((end_time - start_time) / 60)
    print(Fore.LIGHTYELLOW_EX + 'Process took: {} min'.format(t))

    # Cleaning up - removing temp directories
    if results:
        for pose in poses:
            base_tmp_dir = pose.split('/')[-1]
            if os.path.isdir('/tmp/' + base_tmp_dir):
                shutil.rmtree('/tmp/' + base_tmp_dir)

def create_proof(directory, team, player):
    """Create a proof of a given player using Nuke

    Args:
        directory: directory name of the team
        team: Name of the team
        player: either the name of a player or "all"

    Returns: None
    """
    print(Fore.YELLOW + "Creating JPEG's...")

    # Location of Nuke
    app = '/Applications/Nuke12.0v3/Nuke12.0v3.app/Contents/MacOS/Nuke12.0 -x -F 1 '

    # List all poses
    proof_output = directory + '/' + player
    poses = glob(proof_output + '/_acquisition/*')
    item = [x for x in poses if 'tiff' in x]
    if len(item) != 0:
        idx = poses.index(item[0])
        poses.pop(idx)

    # Open default CSV file (XLS)
    game = ''.join(directory.rsplit(GlobalDirs.projects)).split('/')[1]
    csv_file = GlobalDirs.projects + '/' + game + '/Source_Pixelgun/Settings/chunk_mappings.csv'
    in_df = pd.read_csv(csv_file)

    # Create delivery CSV file
    out_csv = define_proof_name(poses[0], game, team) + '.csv'
    out_df = pd.DataFrame(columns=['take name', 'take', 'px take name', 'order'])

    for pose in poses:
        # Define replacement text for shot string
        shot_string = 'Px: ' + pose.split('/')[-1]

        # Copy Nuke template file to tmp directory with player and pose name
        nuke_template = '/Users/px/Projects/pxproofs/proof_comp_template.nk'
        render_filename = '/tmp/' + pose.split('/')[-1] + '.nk'
        if os.path.exists(render_filename):
            os.remove(render_filename)
        shutil.copy(nuke_template, render_filename)

        # Get placeholder string and put it into a CSV file
        pose_name = get_placeholder(pose, in_df)
        placeholder = pose_name + ' ' + pose.split('/')[-1].split('_')[-1]

        # Search and replace placeholder text in nuke template file
        placeholder_text = ('PATH_TO_PLAYERS_HEAD', 'PATH_TO_PLAYERS_PROOF', '##_##_####_########_########_####',
                            'SHOTINFORMATIONSTRING', 'PROOF_OUTPUT')
        replace_text = ('/tmp/' + pose.split('/')[-1], proof_output, placeholder, shot_string, pose.split('/')[-1])

        find_replace = dict(zip(placeholder_text, replace_text))
        with open(nuke_template, 'r') as tmp_file:
            with open(render_filename, 'w') as new_file:
                for line in tmp_file:
                    for key in find_replace:
                        if key in line:
                            line = line.replace(key, find_replace[key])
                    new_file.write(line)

        # Create command and submit it
        cmd = app + render_filename
        subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        # -----------------------------------
        # CSV PART - should be somewhere else

        # Put more stuff into the CSV file
        tk_name = pose.split('/')[-1]
        tk = tk_name.split('_')[-1]
        out_df = out_df.append({'take name': pose_name, 'take': tk, 'px take name': tk_name}, ignore_index=True)

    # Write csv file
    with open(out_csv, 'w') as f:
        out_df.to_csv(f, index=False)

def create_pdf(game, team, player):
    """Create pdf for the proof

    Args:
        game: name of the game
        team: name of team
        player: Name of player

    Returns: None
    """
    print(Fore.YELLOW + 'Creating PDF...')

    # Create log
    log_file = '/tmp/' + team + '.log'
    logging.basicConfig(filename=log_file, filemode='a', level=logging.INFO)

    # Define page title
    if team in GlobalDirs.teams:
        team_name = GlobalDirs.teams[team]
    else:
        team_name = team

    if len(player.split()) == 1:
        player_name = ' '.join(map(str, player.split('_')[::-1])).title()
    title = team_name + ' --- ' + player_name

    proof_input = os.path.realpath(GlobalDirs.projects + "/" + game + "/Sections/" + team + '/' + player)

    # Backward compatibility
    if os.path.isdir(proof_input + '/_acquisition/tiff'):
        poses = glob(proof_input + '/_acquisition/tiff/*' + player + '*')
    else:
        poses = glob(proof_input + '/_acquisition/*' + player + '*')

    # Move neutral into first place
    item = [x for x in poses if 'neutral' in x]
    if len(item) != 0:
        idx = poses.index(item[0])
        poses.insert(0, poses.pop(idx))

    # Just in case - remove unwanted parts
    _ = [x for x in poses if 'Thumbs.db' in x]
    if len(_) > 0:
        poses.remove(_[0])

    # Create PDF
    pdf = FPDF('L', 'pt', (1080, 1920))
    pdf.set_margins(0, 0, 0)
    pdf.add_page()
    pdf.set_title(title)
    pdf.set_font('Arial', '', 64)
    pdf.cell(1200, 800, title, 0, 0, 'C')
    pdf.ln()
    for pose in poses:
        render_filename = '/tmp/' + pose.split('/')[-1] + '.jpg'
        # Bail/Write log if one of the images done exist
        if not os.path.isfile(render_filename):
            print(Fore.RED + f"Missing pose: {render_filename.split('/')[-1].split('.')[0]}")
            logging.info(f"Missing pose: {render_filename.split('/')[-1].split('.')[0]}")
            pass
        else:
            pdf.image(render_filename)

    # Define output directory and name of PDF
    proof = define_proof_name(poses[0], game, team) + '.pdf'
    pdf.output(proof, 'F')

def cleanup(game, team, player):
    """Move jpeg images to _thumbs and removing all nuke files

    Args:
        game: Name of the game
        team: Name of the team
        player: Name of player

    Returns: None
    """

    # Get _thumbs directory and if it doesn't exists create
    thumb_dir = GlobalDirs.projects + "/" + game + "/Sections/" + team + '/' + player + '/_acquisition/_thumbs'
    if not os.path.exists(thumb_dir):
        os.mkdir(thumb_dir)

    # Copy neutral to proof directory and move the rest of the jpegs to _thumbs
    output_dir = os.path.realpath(GlobalDirs.projects + "/" + game + "/Source_Pixelgun/Proof Sheets/" + team)
    jpegs = glob('/tmp/*' + player + '*.jpg')
    for jpeg in jpegs:
        if "neutral" in jpeg:
            shutil.copyfile(jpeg, jpeg.replace('/tmp', output_dir))
        else:
            shutil.move(jpeg, jpeg.replace('/tmp', thumb_dir))

    # Remove all files
    temps = glob('/tmp/*' + player + '*')
    for temp in temps:
        os.remove(temp)

def each_player(path, game, team, player):
    """Iterate thur all or just the one player"""
    if len(player.split()) == 1:
        player_name = ' '.join(map(str, player.split('_')[::-1])).title()

    print(Fore.BLUE + "Player:\t\t{}".format(player_name))

    # put the next three steps into a PriorityQueue to make sure that moving the data
    # has finished first before we clean up the naming
    tiff_exists = check_tiff_exists(path, player)
    if not tiff_exists:
        qi.put(1, convert_images(path, player))

    qi.put(2, create_proof(path, team, player))
    qi.put(3, create_pdf(game, team, player))
    qi.put(4, cleanup(game, team, player))

    while not qi.empty():
        qi.get()

@command()
@option('--game', '-g', default='2K_1018_NBA2K21', help='Game name', type=str, required=True)
@option('--team', '-t', help='Team name', type=str, required=True)
@option('--player', '-p', help='Player name or all', type=str, required=True)
def main(game, team, player):
    """
    Create a proof cheat of a given player using Nuke

    \b
    game:      Game name, i.e. 2K_1018_NBA2K21 [Default]
    team:      Team name, i.e. 'det' for the 'Detroit Pistons'
    player:    A players name, i.e. 'king_louis' or you can pass in 'all' to run thru all players
    """

    # Call function to clear screen
    clear_screen()

    # Check if given directory is valid, i.e.: /Pixelgun_Projects/2K_1018_NBA2K21/Sections/orl/birch_khem
    path = os.path.realpath(GlobalDirs.projects + "/" + game + "/Sections/" + team)
    if not os.path.isdir(path):
        print(Fore.RED + 'Error: Path is invalid!')
        sys.exit(1)

    if team in GlobalDirs.teams:
        team_name = GlobalDirs.teams[team]
    else:
        team_name = team

    print('\n')
    print(Fore.BLUE + "Project:\t{}".format(game))
    print(Fore.BLUE + "Team:\t\t{}".format(team_name))

    if player.lower() == 'all':
        players = glob(path + '/*')
        for counter, value in enumerate(players):
            player = value.split('/')[-1]
            qo.put(counter, each_player(path, game, team, player))

        while not qo.empty():
            qo.get()
    else:
        each_player(path, game, team, player)

    print(Fore.GREEN + 'DONE')
    print('\n')

    # Stop using colorama to restore 'stdout' and 'stderr' to their original values.
    colorama.deinit()

if __name__ == '__main__':
    main()


# Training logistic regresison with Scikit-Learn

from ohe import X_train, X_val
from validation_framework import y_train, y_val, df_val, df_train

import numpy as np
import pandas as pd

from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction import DictVectorizer

# set the model
model = LogisticRegression()

# fit the model
model.fit(X_train , y_train)

# print the co-efficients(weights)
# print(model.coef_[0].round(3))

#print the intercept - w0
# print(model.intercept_)

# soft prediction -> get prediction values of the class
# print(model.predict_proba(X_train))

# hard prediction -> get the predicted value (class with higher probability)
# print(model.predict(X_train))

# get soft prediction
y_pred = model.predict_proba(X_val)[:, 1]

churn_decision = (y_pred >= 0.5)

# check how accurate our models are

# print((y_val == churn_decision).mean())

# print(y_val)
# print(churn_decision.astype(int))

df_pred = pd.DataFrame()
df_pred["probability"] = y_pred
df_pred["prediction"] = churn_decision.astype(int)
df_pred["actual"] = y_val

# print(df_pred)

df_pred["correct"] = y_val == churn_decision.astype(int)

# print(df_pred)
# print(df_pred.correct.mean())

# MODEL INTERPRETATION

# print(model.coef_[0].round(3))

small = ["contract", "tenure", "monthlycharges"]

# print(df_train[small])

dicts_train_small = df_train[small].to_dict("records")
dicts_val_small = df_val[small].to_dict("records")

dv_small = DictVectorizer(sparse = False)

dv_small.fit(dicts_train_small)

# print(dv_small.get_feature_names())

X_train_small = dv_small.transform(dicts_train_small)
X_val_small = dv_small.transform(dicts_val_small)

model_small = LogisticRegression()

model_small.fit(X_train_small, y_train)

w0 = model_small.intercept_[0]
w = model_small.coef_[0]

# print(dict(zip(dv_small.get_feature_names(), w.round(3))))

def logistic_regression(X_val_small, w0, w):
    y_pred = w0 + X_val_small.dot(w)
    sigmoid = 1/ (1+np.exp(-y_pred))
    decision_value = sigmoid >= 0.5
    return decision_value.astype(int)

y_val_pred = logistic_regression(X_val_small, w0, w)

# print(y_val_pred)


import json
from unittest import skip
from playwright.sync_api import sync_playwright
import re

# Create a dictionary of course requirements for each program major
def create_majors(major_dict):
    final_majors = {}
    final_majors["programs"] = {}

    # Iterate through program majors
    for major in major_dict["entry"]:
        try:
            table_text = major["table_obj"]

            # Find all courses in this program major
            courses = re.findall(r'[A-Z]{3,4}\*[0-9]{4}', table_text)

            # Parse out extra credit requirements
            total_extra_requirements = {}
            bad_words = ["*", "Select", "Semester", "work", "Exchange", "Option", "Stream", "following", "semesters", "admission"]
            for i in table_text.split('\n'):
                # Smallest length I found was around 14, this len check is used to cull non elective text
                if len(i) > 13:

                    # Skip the line if it contains information signifying its not a credit requirement
                    skip_line = False
                    for j in bad_words:
                        if j in i:
                            skip_line = True
                            break

                    if not skip_line:
                        # Strip whitespace, split into credit number and requirement
                        extra_requirements = i.strip()

                        # Remove duplicate credit value from the start, then extract credit weight of electives from the end of the string
                        extra_requirements = re.sub(r'^\d+\.\d+\s*', r'', extra_requirements)
                        weight = extra_requirements[len(extra_requirements) - 4:]

                        # Remove duplicated credit value on the end, and restrip whitespace
                        extra_requirements = re.sub(r'\d+\.\d+$', r'', extra_requirements)
                        extra_requirements = extra_requirements.strip()

                        # Case where text is like up to X total credits in the rest of the semester
                        if "total credits in this semester" in extra_requirements or "to a maximum of" in extra_requirements:
                            extra_requirements = "Electives or Restricted Electives"

                        # Capitalize first letter
                        if len(extra_requirements) > 0 and extra_requirements[0].isalpha:
                            extra_requirements = extra_requirements[0].upper() + extra_requirements[1:]

                        total_extra_requirements[extra_requirements] = weight

            # Build program major
            final_cur_major = {}
            final_cur_major["program_name"] = major["program_name"]
            final_cur_major["program_link"] = major["program_link"]
            final_cur_major["courses"] = sorted(list(set(courses)))
            final_cur_major["extra_requirements"] = total_extra_requirements

            code = re.sub(r'.*\((.*)\)$', r'\1',  major["program_name"])
            final_majors["programs"][code] = final_cur_major
        except Exception as e:
            print(e)
            pass
    return final_majors

def main():
    with sync_playwright() as scrape:
        browser = scrape.chromium.launch()
        page = browser.new_page()

        # Leaving 30 second default timeout while testing (i.e. not timeout = 0)
        page.goto('https://calendar.uoguelph.ca/undergraduate-calendar/degree-programs/')

        # Extracting all degree links and storing in degree_links
        links = page.query_selector('[id="textcontainer"]').query_selector_all("a")
        degree_links = []
        for link in links:
            degree_links.append(link.get_attribute("href"))

        # All links extracted will be appended to this
        root_url = 'https://calendar.uoguelph.ca/'

        # Loop through all the degrees, extract the major links, and store in major_links
        major_links = []

        for link in degree_links:
            page.goto(root_url + link)
            # print("Checking Degree Link: " + root_url + link)

            # Checking if page has a requirements tab
            try:
                requirements_tab = page.query_selector('[id="requirementstextcontainer"]')
                if(requirements_tab):
                    major_links.append(root_url + link)
            except:
                pass

            # Checking if page has a programs tab
            try:
                programs_tab = page.query_selector('[id="programstextcontainer"]')
                if(programs_tab):
                    temp_links = programs_tab.query_selector_all("a")
                    for n in temp_links:
                        major_links.append(root_url + n.get_attribute("href"))
            except:
                pass

            # Checking if page has a schedule of studies tab
            try:
                sched_of_stud = page.query_selector('[id="scheduleofstudiestextcontainer"]')
                if(sched_of_stud):
                    major_links.append(root_url + link)
            except:
                pass

        # Getting the list of pages that have majors
        # NOTE: major_links_table_dict is an array of programs, where each has attribute "program_name", "program_link", and "table_obj"
        major_links_table_dict = {}
        major_links_table_dict["entry"] = []
        for link in major_links:
            print("Visiting: " + link)
            page.goto(link)

            try:
                major_links_table_entry_dict = {}
                # Checking if the current tab is a requirements, programs, or schedule of studies tab
                current_tab = page.query_selector('[id="requirementstextcontainer"]')
                if(current_tab is None):
                    current_tab = page.query_selector('[id="programstextcontainer"]')
                if(current_tab is None):
                    current_tab = page.query_selector('[id="scheduleofstudiestextcontainer"]')
                if(current_tab is None):
                    continue

                # Checking if the entry contains a major or schedule of studies
                check_minor = current_tab.query_selector_all('h2,h3')
                for cur_degree in check_minor:
                    if("major" in cur_degree.inner_text().lower() or "schedule of studies" in cur_degree.inner_text().lower()):

                        # Getting the name of the major and link
                        major_title = page.query_selector('[class="page-title"]')
                        major_links_table_entry_dict["program_name"] = major_title.text_content()
                        major_links_table_entry_dict["program_link"] = link

                # Check to see if the page contains a Major header and a Schedule of Studies header
                check_headers = current_tab.query_selector_all('h2,h3')
                found_major = False
                found_sos = False
                for header in check_headers:
                    if "major" in header.inner_text().lower():
                        found_major = True
                    elif "schedule of studies" in header.inner_text().lower():
                        found_sos = True
                skip_sos = (found_major and found_sos)

                # Looking for all h2, h3, and tables since a table will almost always have a major prior to it
                current_tab_items = current_tab.query_selector_all('h2,h3,table');
                for i in range(len(current_tab_items)):
                    cur_item = current_tab_items[i].inner_text().lower()
                    cur_item_html = current_tab_items[i].inner_html().lower()

                    # Checking if an element that contains major is found, or schedule of studies (only if major isn't found on the page)
                    # After that check if its a table or not
                    if ("major" in cur_item or ("schedule of studies" in cur_item and not skip_sos)) and "thead" not in cur_item_html:
                        table_obj = current_tab_items[i+1]
                        major_links_table_entry_dict["table_obj"] = table_obj.inner_text()

                        # Check if there are any courses present in the inner text
                        if "*" in table_obj.text_content():
                            major_links_table_dict["entry"].append(major_links_table_entry_dict)
                        break
            except:
                pass

        # Dumping the parsed JSON data
        with open("src/scraper/data/scraped_majors_data.json", "w") as f:
            json.dump(create_majors(major_links_table_dict), f, sort_keys=True, indent=4)

        browser.close()
if __name__ == '__main__':
    main()


#! /usr/bin/env python3

vowels ="aeiouAEIOU"
count = 0

def vowel_recognition(s):
    global count
    count =0

    def vowel_count_char(examine):
        global count
        global vowels

        for i in range(len(examine)):
            if examine[i] in vowels:
                count+=1

    def vowel_count_thro(ns):
        global count
        global vowels
        times= len(ns)
        x=0

        for j in range(times):
            for i in range(len(ns)-1):
                check= ns[0:(i+2)]
                vowel_count_char(check)
            ns= ns[(x+1):]

    vowel_count_char(s)
    vowel_count_thro(s)

    return count

a= vowel_recognition("bbbb")
b= vowel_recognition("baceb")
c= vowel_recognition("aeiou")
d= vowel_recognition("aeiouAEIOU")
e= vowel_recognition("Programfinishesquiggledwithexitcode1PressENTERtsquiggletconsolehedwithexitcode1PressENTEfinsquiggledwithexitcode1PressENTERtoexitconsquigglesolehe")
print(a)
print(b)
print(c)
print(d)
print(e)

# end


class Solution:
    def shuffleArray(self, arr, n):
        m = n//2
        for i in range(m):
            arr[i + m] += arr[i]*1001
        for i in range(m):
            arr[2*i] = arr[i + m]//1001
            arr[2*i + 1] = arr[i + m]%1001

#{
 # Driver Code Starts
if __name__ == '__main__':

    t=int(input())
    for _ in range(0,t):
        n=int(input())
        a = list(map(int,input().split()))
        ob = Solution()
        ob.shuffleArray(a,n)
        print(*a)
# } Driver Code Ends


import display
from colorama import Back
import numpy as np

# Y =  1
# R = -1

class Connect4:
	plyTurn = -1
	win = 0
	plyStart = 0
	board = np.zeros((7, 6), dtype="int8")

	visual_board = None

	def __init__(self, show_board=True):
		self.board = np.zeros((7, 6), dtype="int8")
		# self.board = np.matrix([
		#     [ 0,  0,  0,  0, -1, -1],
		#     [ 0,  0,  1,  1,  1,  1],
		#     [ 0,  1, -1,  1,  1, -1],
		#     [ 0,  0,  0, -1, -1, -1],
		#     [ 0,  0, -1, -1,  1,  1],
		#     [ 0,  0,  0,  0,  1,  1],
		#     [ 0,  0,  0,  0, -1, -1],
		# ])
		self.win = 0
		self.plyStart = -1 if np.random.rand() > 0.5 else 1
		self.plyTurn = self.plyStart
		if show_board:
			self.visual_board = display.VisualBoard()
			self.visual_board.update_all(self.board)

	def drop_coin(self, x, color):
		for i in range(6):
			if self.board[x, i + 1] != 0 or i == 4:
				y = 5 if self.board[x, i + 1] == 0 else i
				self.board[x, y] = color
				if self.visual_board is not None:
					# self.visualBoard.updateCell(x , y, color)
					self.visual_board.update_all(self.board)
				break

	def play(self, play_column):
		if self.board[play_column, 0] != 0 and self.win == 0:
			self.win = self.plyTurn * -1
			return self.observation(), -10, True
		self.drop_coin(play_column, self.plyTurn)
		win = win_play(self.board, play_column)
		if win:
			self.win = self.plyTurn
		else:
			self.plyTurn *= -1
		return self.observation(), 10 if win and self.plyTurn != self.plyStart else 3 if win else 0, win

	def close_game(self):
		if self.visual_board is not None:
			self.visual_board.close()

	def observation(self):
		return self.board.copy().reshape((42, 1))

	def print(self):
		print()
		print_board(self.board)

def win_play(board, play_col):
	x = play_col
	y = 0
	for _y in range(6):
		if board[x, _y] != 0:
			y = _y
			break
	if board[x, y] != 0:
		dirs = [[1, 1], [0, 1], [-1, 1], [1, 0]]
		for curDir in dirs:
			align = 0
			for i in range(-3, 4):
				_x = x + (curDir[0] * i)
				_y = y + (curDir[1] * i)
				if _x < 0 or _y < 0 or _x > 6 or _y > 5:
					continue
				if board[_x, _y] == board[x, y]:
					align += 1
				else:
					align = 0
				if align == 4:
					return True
	return False

def print_board(board):
	for y in range(6):
		for i in range(2):
			pr = ""
			for x in range(7):
				color = Back.YELLOW if board[x, y] == 1 else Back.RED if board[x, y] == -1 else Back.WHITE
				pr = pr + f"{color}    {Back.RESET} "
			print(pr)
		print()


import sys

def extrage_linie(lista_linii,fisier):
    for linie in lista_linii:
        extrage_propozitie(linie.strip(),fisier)

def extrage_propozitie(linie,fisier):
    global propozitii1
    global propozitii2
    propozitie = ""
    for i in range(0, len(linie)):
        if linie[i] != "." and linie[i] !='?' and linie[i]!= '!':
            propozitie += linie[i]
        else:
            propozitie += linie[i]
            new_sentence = linie[i + 1:]
            if fisier == "file1":
                propozitii1.append(propozitie)
                extrage_propozitie(new_sentence,"file1")

            elif fisier =="file2":
                propozitii2.append(propozitie)
                extrage_propozitie(new_sentence,"file2")

def compara_propozitii(propozitie1, propozitie2):
    prop1 = propozitie1.replace(" ", "")
    prop2 = propozitie2.replace(" ", "")
    if len(prop1) != len(prop2):
        return False
    else:
         i = 0
         while i < len(prop1):
             if prop1[i] != prop2[i]:
                 return False
             i = i + 1
    return True

def creeaza_lista_comuna(propozitii1,propozitii2,final_list):
    for prop1 in propozitii1:
        for prop2 in propozitii2:
            if compara_propozitii(prop1,prop2):
                if prop2 not in final_list:
                    final_list.append(prop2)

def lista_fisier(fisier):
    file = open(fisier,'r')
    lista = file.readlines()
    file.close()
    return lista

if len(sys.argv) != 3:
    print("Wrong number of arguments !")
else:
    file1 = sys.argv[1]
    file2 = sys.argv[2]

    lista_linii_f1 = lista_fisier(file1)
    lista_linii_f2 = lista_fisier(file2)

    propozitii1 =[]
    propozitii2=[]
    final_list = []

    extrage_linie(lista_linii_f1,"file1")
    extrage_linie(lista_linii_f2,"file2")

    creeaza_lista_comuna(propozitii1,propozitii2,final_list)
    print(final_list)


'''
' This file contains functions that will extract the HOG/HOF/MBH features from input frames
'
' The code is adapted from the Matlab code used in the papers:
' - J.R.R. Uijlings, I.C. Duta, E. Sangineto, and N. Sebe
' "Video Classification with Densely Extracted HOG/HOF/MBH Features: An Evaluation of the Accuracy/Computational Efficiency Trade-off"
' In International Journal of Multimedia Information Retrieval (IJMIR), 2015.
' - I.C. Duta, J.R.R. Uijlings, T.A. Nguyen, K. Aizawa, A.G. Hauptmann, B. Ionescu, N. Sebe
' "Histograms of Motion Gradients for Real-time Video Classification"
' In International Workshop on Content-based Multimedia Indexing (CBMI), 2016
'
' As the code is to be used on live videos on a per frame basis, it is slightly altered
' And does not take into account the videos as whole blocks
'''

import numpy as np
from numpy.lib.stride_tricks import as_strided
from numpy import einsum
import cv2 as cv
import queue
from scipy.ndimage.filters import convolve as filter2

'''
' This has been hard coded to provide the marix required for the dimensions we are using to save time
'
' Otherwise, it translates a diagonal matrix to a rectangular one, such that all the rows still add to one
' and the columns add to 1/shape[1]
'''
def diag_matrix_linear():
	matrix = np.load('diag_matrix.npy')

	return matrix

'''
' Strides the data, making block as in the paper
' Input:
'	data - data to make into a block
'	A - rectangular diagonal
'	B - rectangular diagonal
' Output:
'	block - the data strided into blocks
'
'''
def make_block(data, A, B):
	block = einsum('ab,bef,eh->fah',A,data,B)

	block = block.T.reshape(64,8)

	inds = np.arange(64).reshape(8,8)
	inds = as_strided(inds,(2,2,7,7),(64,8,8,64))
	inds = inds.transpose(0,1,3,2).reshape(-1)

	block = block[inds]

	block = block.T.reshape(32,-1).T.reshape(-1)

	return block

'''
' Calculate haar gradients of an image
' Input:
' 	im - image
' Output:
'	gr - haar gradients
'''
def haar_gradients(im):
	ogIm = np.zeros((im.shape[0],im.shape[1],8))

	rowIm = im[:-2, 1:-1] - im[2:,1:-1]
	colIm = im[1:-1,:-2] - im[1:-1,2:]

	magIm = np.sqrt(rowIm * rowIm + colIm * colIm)
	angleIm = np.arctan2(rowIm, colIm)

	binReal = (angleIm) * (8 / (2 * np.pi))
	binLow = np.floor(binReal)
	weightHigh = binReal - binLow
	weightLow = 1 - weightHigh
	binLow = (binLow % 8).astype(int)
	binHigh = (binLow + 1).astype(int)
	binHigh[binHigh == 8] = 0

	[colI, rowI] = np.meshgrid(range(1,im.shape[1]-1), range(1,im.shape[1]-1))
	ogIm[(rowI,colI,binLow)] += magIm * weightLow

	ogIm[(rowI,colI,binHigh)] += magIm * weightHigh

	return ogIm

'''
' Calculate histogram of gradients, repeatedly from a queue
' Input:
'	i - input queue
' Output:
'	o - output queue
'''
def HOG(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	if True:
		frame = i

		ogIm = haar_gradients(frame)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put(hogs)

'''
' Calculate horn-schunck optical flow between two images
' Input:
' 	im1 - image
' Output:
'	of - horn-schunck optical flow
'''
def horn_schunck(im1,im2,Niter=10):
	 #set up initial velocities
	U = np.zeros([im1.shape[0],im1.shape[1]])
	V = np.zeros([im1.shape[0],im1.shape[1]])

	sobelX = np.array([[-1,-2,-1],[0,0,0],[-1,-2,-1]])
	sobelY = sobelX.T

	Ix = cv.filter2D(im1,-1,sobelX)
	Iy = cv.filter2D(im1,-1,sobelY)

	It = im2 - im1

	avgK = np.array([[0,1,0],[1,0,1],[0,1,0]])

	for i in range(Niter):
		uAvg = cv.filter2D(U,-1,avgK)
		vAvg = cv.filter2D(V,-1,avgK)

		der = (Ix*uAvg + Iy*vAvg + It) / (1 + Ix**2 + Iy**2)

		U = uAvg - Ix * der
		V = vAvg - Iy * der

	return U, V

def HS(im1, im2, alpha=1, Niter=10):
	"""
	im1: image at t=0
	im2: image at t=1
	alpha: regularization constant
	Niter: number of iteration
	"""

	#set up initial velocities
	uInitial = np.zeros([im1.shape[0],im1.shape[1]])
	vInitial = np.zeros([im1.shape[0],im1.shape[1]])

	# Set initial value for the flow vectors
	U = uInitial
	V = vInitial

	# Estimate derivatives
	[fx, fy, ft] = computeDerivatives(im1, im2)

	# Averaging kernel
	kernel=np.array([[1/12, 1/6, 1/12],
	                  [1/6,    0, 1/6],
	                  [1/12, 1/6, 1/12]],float)

	# Iteration to reduce error
	for _ in range(Niter):
	#Compute local averages of the flow vectors
		uAvg = filter2(U,kernel)
		vAvg = filter2(V,kernel)
	#common part of update step
		der = (fx*uAvg + fy*vAvg + ft) / (alpha**2 + fx**2 + fy**2)
	#iterative step
		U = uAvg - fx * der
		V = vAvg - fy * der

	return U,V

def computeDerivatives(im1, im2):
#%% build kernels for calculating derivatives
	kernelX = np.array([[-1, 1],[-1, 1]]) * .25 #kernel for computing d/dx
	kernelY = np.array([[-1,-1],[ 1, 1]]) * .25 #kernel for computing d/dy
	kernelT = np.ones((2,2))*.25

	fx = filter2(im1,kernelX) + filter2(im2,kernelX)
	fy = filter2(im1,kernelY) + filter2(im2,kernelY)

	#ft = im2 - im1
	ft = filter2(im1,kernelT) + filter2(im2,-kernelT)

	return fx,fy,ft

'''
' Calculate histogram of optical flow, repeatedly from a queue
' Input:
'	i - input queue
' Output:
'	o - output queue
'''
def HOF(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		v1, v2 = HS(prev_frame,frame)

		mag = np.sqrt(v1**2 + v2**2)
		ang = np.arctan2(v1,v2)

		oM = np.zeros((mag.shape[0],mag.shape[1],8))

		binReal = (ang) * (8 / (2 * np.pi))
		binLow = np.floor(binReal)
		weightHigh = binReal - binLow
		weightLow = 1 - weightHigh
		binLow = (binLow % 8).astype(int)
		binHigh = (binLow + 1).astype(int)
		binHigh[binHigh == 8] = 0

		[colI, rowI] = np.meshgrid(range(mag.shape[1]), range(mag.shape[0]))
		oM[(rowI,colI,binLow)] += mag * weightLow

		oM[(rowI,colI,binHigh)] += mag * weightHigh

		hof = make_block(oM,arrayA,arrayB)

		o.put((m,hof))

def MBHr(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		v1, _ = HS(prev_frame,frame)

		ogIm = haar_gradients(v1)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put((m,hogs))

def MBHc(i, o):
	arrayA = diag_matrix_linear()
	arrayB = arrayA.T
	prev_frame = None
	if True:
		n,frame = i.get()

		if prev_frame is None:
			prev_frame = frame
			m = n
			n,frame = i.get()

		_, v2 = HS(prev_frame,frame)

		ogIm = haar_gradients(v2)

		hogs = make_block(ogIm,arrayA,arrayB)

		o.put((m,hogs))


# coding: utf8
from __future__ import unicode_literals

# currently considering for Ekavian-Jekavian normalization

_exc = {
    # jekavian to ekavian
    "млијеко": "млеко",
    "мјесто": "место",
    "мјеста": "места",
    "мијесити": "месити"

}

NORM_EXCEPTIONS = {}

for string, norm in _exc.items():
    NORM_EXCEPTIONS[string] = norm
    NORM_EXCEPTIONS[string.title()] = norm