# Databricks notebook source
# MAGIC %md
# MAGIC ## Mount the Blob storages

# COMMAND ----------

# MAGIC %md
# MAGIC ### Mount the Landing Zone Blob

# COMMAND ----------

# MAGIC %run Users/akshay.shinde@d2iconsulting.com.au/common_utilities/connect_landing_zone

# COMMAND ----------

# MAGIC %md
# MAGIC ### Mount the Silver Layer Blob

# COMMAND ----------

# MAGIC %run Users/akshay.shinde@d2iconsulting.com.au/common_utilities/connect_silver_layer

# COMMAND ----------

# MAGIC %md
# MAGIC ### Mount the Golden Layer Blob

# COMMAND ----------

# MAGIC %run Users/akshay.shinde@d2iconsulting.com.au/common_utilities/connect_golden_layer

# COMMAND ----------

# MAGIC %md
# MAGIC ## Import the Libraries

# COMMAND ----------

### PySpark ###
from pyspark.sql import functions as F
from pyspark.sql.functions import col
from pyspark.sql.types import *
from pyspark.sql import Window
from pyspark.sql.functions import *

# COMMAND ----------

# MAGIC %md
# MAGIC ## Making sure the Delta write is optimised

# COMMAND ----------

spark.sql("set spark.databricks.delta.autoCompact.enabled = true")
spark.sql("set spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite = true")

# COMMAND ----------

# MAGIC %md
# MAGIC ## I have many small files. Why is auto optimize not compacting them?
# MAGIC By default, auto optimize does not begin compacting until it finds more than 50 small files in a directory. You can change this behavior by setting
# MAGIC * spark.databricks.delta.autoCompact.minNumFiles.
# MAGIC
# MAGIC  Having many small files is not always a problem, since it can lead to better data skipping, and it can help minimize rewrites during merges and deletes. However, having too many small files might be a sign that your data is over-partitioned.

# COMMAND ----------

#### Changing the autoCompact to start considering compact after having 1 files. The default to start autocompact to work is 50 files.

spark.sql("set spark.databricks.delta.autoCompact.minNumFiles = 1")

# COMMAND ----------

# MAGIC %md
# MAGIC ### Reading the records from the Landing Zone

# COMMAND ----------

df = spark.read.option("header", "True").option("inferSchema", "True").csv("/mnt/landingzone/5mrecords.csv")

# COMMAND ----------

display(df)

# COMMAND ----------

# MAGIC %md
# MAGIC ### Perform various transformations such as below for the Silver table:
# MAGIC * Having the abbrevations of the Order Priority
# MAGIC * Getting the Day from the Order Date

# COMMAND ----------

df_silver_sales_table_python =  df.withColumn('day_of_week',date_format(col("Order_Date"), "EEEE"))\
                                  .withColumn('Order_Priority',when(F.col('Order_Priority')=='H', 'High')\
                                                               .when(F.col('Order_Priority')=='M', 'Medium')\
                                                               .when(F.col('Order_Priority')=='L', 'Low')
                                                               .otherwise('Clear')\
                                                               )

# COMMAND ----------

display(df_silver_sales_table_python)

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC CREATE TEMPORARY VIEW landing_zone_table
# MAGIC USING csv
# MAGIC OPTIONS (
# MAGIC   path '/mnt/landingzone/5mrecords.csv',
# MAGIC   header 'true',
# MAGIC   inferSchema 'true'
# MAGIC );
# MAGIC
# MAGIC SELECT *
# MAGIC FROM landing_zone_table

# COMMAND ----------

df_silver_sales_table_sql = spark.sql("""
SELECT
  *,
  date_format(Order_Date, 'EEEE') as day_of_week,
  CASE Order_Priority
    WHEN 'H' THEN 'High'
    WHEN 'M' THEN 'Medium'
    WHEN 'L' THEN 'Low'
    ELSE 'Clear'
  END as Order_Priority
FROM landing_zone_table
""")

# COMMAND ----------

display(df_silver_sales_table_sql)

# COMMAND ----------

# MAGIC %md
# MAGIC ### Writing the final dataframe in the Silver layer blob location
# MAGIC * One can use the Python or the SQL dataframes to write the data in the silver layer (I will be using the Python dataframe)

# COMMAND ----------

silver_blob_location = "dbfs:/mnt/silver/sales_table/"

df_silver_sales_table_python.write.mode('append').format('delta').save(silver_blob_location)
spark.sql("CREATE TABLE IF NOT EXISTS silver_sales_fact USING DELTA LOCATION '" + silver_blob_location + "'")

# COMMAND ----------

# MAGIC %md
# MAGIC ### Perform various transformations such as below for the Silver table:
# MAGIC * Grouping the data by Country, Date and Day of week

# COMMAND ----------

df_gold_sales_table_python = spark.read.format("delta").load(silver_blob_location)\
                                       .groupBy('Order_Date','Country','day_of_week').agg(F.sum('Total_Revenue').alias('Total_Revenue'),F.sum('Total_Profit').alias('Total_Profit'))

# COMMAND ----------

display(df_gold_sales_table_python)

# COMMAND ----------

df_gold_sales_table_sql = spark.sql("""SELECT order_date,
                                        country,
                                        day_of_week,
                                        Sum(total_revenue) AS Total_Revenew,
                                        Sum(total_profit)  AS Total_Profit
                                    FROM   silver_sales_fact
                                    GROUP  BY order_date,
                                            country,
                                            day_of_week""")

# COMMAND ----------

# MAGIC %md
# MAGIC ### Writing the final dataframe in the Golden layer blob location
# MAGIC * One can use the Python or the SQL dataframes to write the data in the silver layer (I will be using the Python dataframe)

# COMMAND ----------

gold_blob_location = "dbfs:/mnt/gold/sales_by_country/"

df_gold_sales_table_python.write.mode('overwrite').format('delta').save(gold_blob_location)
spark.sql("CREATE TABLE IF NOT EXISTS gold_sales_fact USING DELTA LOCATION '" + gold_blob_location + "'")

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC UPDATE  gold_sales_fact
# MAGIC SET  Total_Revenue = round(Total_Revenue,2),
# MAGIC      Total_Profit =  round(Total_Profit,2)

# COMMAND ----------

# MAGIC %md
# MAGIC ## Show the version history

# COMMAND ----------

# MAGIC %sql
# MAGIC
# MAGIC DESCRIBE HISTORY gold_sales_fact

# COMMAND ----------

# MAGIC %md
# MAGIC ## Command used to check how many files are available for a written detla table

# COMMAND ----------

# MAGIC %sql
# MAGIC describe detail sales_fact

notebooks/Users/akshay.shinde@d2iconsulting.com.au/data modelling/Databricks versioning sample.py


import sqlite3
import hashlib
import os.path
import shutil

class data_dedup_func(object):
    def __init__(self):
        self.temp_path = r"E:/Arpit/Personal n Clg/4th yr project dedupliction/Main Project/DataDedup/server/temp/"

    def hash_cal(self, filename):
        h = hashlib.sha1()
        f_path = r"E:/Arpit/Personal n Clg/4th yr project dedupliction/Main Project/DataDedup/server/temp/"
        with open(f_path + filename, "rb") as file:
            chunk = 0
            while chunk != b'':
                chunk = file.read(1024)
                h.update(chunk)
        print(f"Hash value successfully created: {filename}")
        return h.hexdigest()

    def file_delete(self, fname):
        file_name = fname
        file_path = r"E:\Arpit\Personal n Clg\4th yr project dedupliction\Main Project\DataDedup\server\temp"
        complete_fname = os.path.join(file_path, file_name)
        try:
            os.remove(complete_fname)
        except Exception as e:
            return f"Unable to delete the file due to Error: {e}"

    def file_move(self, fname):
        file_temp_path = r"E:\Arpit\Personal n Clg\4th yr project dedupliction\Main Project\DataDedup\server\temp"
        complete_src_path = os.path.join(file_temp_path, fname)
        file_db_path = r"E:\Arpit\Personal n Clg\4th yr project dedupliction\Main Project\DataDedup\server\database\db_files"
        try:
            shutil.move(complete_src_path, file_db_path)
        except Exception as e:
            print(e)

    def database(self, file_n, hash_v):
        conn = sqlite3.connect(r'E:\Arpit\Personal n Clg\4th yr project dedupliction\Main Project\DataDedup\server\database\hash_index.db')

        self.c = conn.cursor()

        self.c.execute("""CREATE  TABLE IF NOT EXISTS hash_index (
                                    id  INTEGER PRIMARY KEY NOT NULL,
                                    filename    TEXT,
                                    hashv   CHAR(50))""")

        # Comparing the hash values with the existing ones in the hash index
        statment = f"SELECT filename FROM hash_index WHERE hashv='{hash_v}'"
        self.c.execute(statment)
        if not self.c.fetchone():
            self.c.execute("insert into hash_index (filename, hashv) values (?,?)", (f'{file_n}', f'{hash_v}'))
            conn.commit()
            conn.close()
            self.file_move(file_n)
            return f'No Duplicate file detected.\n {file_n} Saved Successfully.'
        else:
            raw_fn = self.c.execute(f"SELECT filename FROM hash_index WHERE hashv='{hash_v}'")
            raw_fn = self.c.fetchone()
            fname = ' '.join([str(elem) for elem in raw_fn])
            self.file_delete(file_n)
            msg = f"Duplicate Hash Value is detected: {hash_v} \n\nOriginal File: {fname} \n[DELETING]  Duplicate file: {file_n}"
            conn.close()
            return msg

    def hash_db(self, file_n):
        filename = file_n
        msg = self.database(filename, self.hash_cal(filename))
        return msg

def data_dedup_handler():
    temp_path = r"E:/Arpit/Personal n Clg/4th yr project dedupliction/Main Project/DataDedup/server/temp/"
    files = os.listdir(temp_path)
    no_files = len(files)
    if no_files != 0:
        for i in range(no_files):
            file_name = files[i]
            print("***************************************************")
            hash_handler = data_dedup_func()
            msg = hash_handler.hash_db(file_name)

        return msg
    else:
        return f"{no_files} Duplicate File is found in the server."


# Initial exploration of the data to see what's up
# This is for the first set of data we ended up using in the project

import sys
import pandas as pd
import matplotlib.pyplot as plt

data_dir = 'data_tscorrect'
plot_dir = 'plots'
num_sensors = 8
size = 'PM1'

plt.figure()

# First, read the ground truth sensor, taking into account its peculiarities
start_time = '04/11/2021 01:21:17 PM'
start_time = pd.to_datetime(start_time, infer_datetime_format=True)
data = pd.read_csv(f'{data_dir}/GroundTruth.csv')
times = pd.to_timedelta(data['Elapsed Time [s]'], unit='s')
times = start_time + times

particle_density = data[f'{size} [mg/m3]']*333 # 1000 should be right, but something is weird
particle_density_gt = particle_density.rolling(20).mean()
times = times - start_time # convert to seconds since start
times_gt = times.dt.total_seconds()/3600

size = size.replace('.', '')

# Next, read the cheap sensor readings
for i in range(1, num_sensors+1):
    data = pd.read_csv(f'{data_dir}/Sensor{i}.csv')

    # times are written as timestamps instead of seconds since starting
    times = data['DeviceTimeStamp']
    times = pd.to_datetime(times, infer_datetime_format=True)
    times = times - pd.DateOffset(hours=4)
    times = times - start_time
    times = times.dt.total_seconds()/3600

    particle_density = data[size]
    particle_density = particle_density.rolling(20).mean()

    plt.plot(times, particle_density, label=f'Sensor{i}')

plt.plot(times_gt, particle_density_gt, label='GroundTruth')

# Misc plotting things
plt.xlabel('Time (hours)')
plt.ylabel(f'{size} (ug/m3)')
plt.xlim([0, 11.5])
plt.ylim([0, 50])
plt.legend()
plt.savefig(f'{plot_dir}/all_data_{size}.png')

plt.ylim([0, 500])
plt.savefig(f'{plot_dir}/all_data_{size}_largescale.png')


# https://app.codility.com/c/run/trainingUGAXJ9-VFA/

# Failed this

# An array A consisting of N different integers is given. The array contains integers in the range [1..(N + 1)], which means that exactly one element is missing.
#
# Your goal is to find that missing element.
#
# Write a function:
#
#     def solution(A)
#
# that, given an array A, returns the value of the missing element.
#
# For example, given array A such that:
#   A[0] = 2
#   A[1] = 3
#   A[2] = 1
#   A[3] = 5
#
# the function should return 4, as it is the missing element.
#
# Write an efficient algorithm for the following assumptions:
#
#         N is an integer within the range [0..100,000];
#         the elements of A are all distinct;
#         each element of array A is an integer within the range [1..(N + 1)].

def getMissingNumber(list):
    if len(list) == 1:
        return list[-1]+1
    elif len(list) == 0:
        return 1
    else:
        list.sort()
        missingNumber = None
        newList = [i for i in range(list[0], list[-1] + 1)]
        for number in newList:
            if number not in list:
                missingNumber = number

        if missingNumber != None:
            return missingNumber
        else:
            return list[-1] + 1

print(getMissingNumber([2,3,1,5]))


from threading import Thread
from time import sleep
from os import getenv
import socket
import sys
import re

class BackgroundServer(object):

  def __init__(self, bound, port = 10000, debug = False):
    self.port = port
    self.debug = debug

    thread = Thread(target = self.run, args = ())
    thread.daemon = bound
    thread.start()

  def run(self):
    server = Server(self.port, self.debug)

class Server(object):

  def __init__(self, port , debug = False):
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    server_address = ('0.0.0.0', port )
    if debug:
      print ('starting up on {} port {}'.format(server_address[0],server_address[1]))

    sock.bind(server_address)
    sock.listen(1)

    while True:
      connection, client_address = sock.accept()
      if debug:
        print('accepted a connection from {}'.format(client_address))

      data = connection.recv(4096)
      # echo to caller message
      if len(data):
        connection.sendall(data)
        text = data.decode('utf-8')
        if debug:
          print('received {}'.format(text))
        if re.match(r'QUIT', text):
          break
      connection.close()

if getenv('BOUND')!= None and getenv('BOUND').lower() in ['true', '1', 't', 'y', 'yes']:
  bound = True
else:
  bound = False

if getenv('DEBUG')!= None and getenv('DEBUG').lower() in ['true', '1', 't', 'y', 'yes']:
  debug = True
else:
  debug = False

port = getenv('SERVICE_PORT')

if port == None :
  port = 10000

example = BackgroundServer(bound, int(port), debug)
print('Main exit')


def identify_locations(outputs):
    overlaps = [] # overlapping paths for part 1
    path = [] # all scaffold locations, for part 2

    for i in range(len(outputs)):
        for j in range(len(outputs[i])):
            characters = []

            #center
            characters.append(outputs[i][j]=='#')

            try:
                # north
                assert i-1>=0
                characters.append(outputs[i-1][j]=='#')
            except:
                characters.append(False)
            try:
                # south
                assert i+1<=len(outputs)
                characters.append(outputs[i+1][j]=='#')
            except:
                characters.append(False)
            try:
                # east
                assert j+1<=len(outputs[i])
                characters.append(outputs[i][j+1]=='#')
            except:
                characters.append(False)
            try:
                # west
                assert j-1>=0
                characters.append(outputs[i][j-1]=='#')
            except:
                characters.append(False)

            if outputs[i][j] not in['#','.']:
                start = (i,j)

            if sum(characters)==5:
                # it's 2 paths overlapping!
                overlaps.append((i,j))

            # store everything in path
            if characters[0]:
                path.append((i,j))

    return overlaps, path, start

def build_instructions(path, start):
    directions = [[0,1], [1,0], [0,-1], [-1,0]]

    # start drone facing down (inverted coordinates)
    start_direction = directions[3]

    # keep track of all the positions the drone has visited
    already_visited = []

    # initialize drone's position and direction
    current_y, current_x = start
    direction = start_direction

    # start the instruction list ([direction_0, length_0, direction_1, length_1, ...])
    instructions = [direction, 0]

    while len(set(path)-set(already_visited))>0:
        try:
            next_location = path.index((current_y+direction[0], current_x+direction[1]))
            # ^ if this passed, we can keep going straight

            # update distance (instructions[-1])
            instructions[-1] += 1

            # update locations
            current_y, current_x = path[next_location]
            already_visited.append((current_y,current_x))

        except:
            # need to turn drone
            if instructions[-1]==0:
                # drone hasn't moved in current direction - no need to store it
                instructions = instructions[:-2]

            try:
                # turn left 90 degrees
                direction = directions[directions.index(direction)-1]
                path.index((current_y+direction[0], current_x+direction[1]))
                instructions.append('L')
                instructions.append(direction)
                instructions.append(0)

                # instructions now looks like [...., [new_y_slope,new_x_slope], 0]
                continue
            except:
                try:
                    # turn right 90 degrees
                    direction = directions[directions.index(direction)-2]
                    path.index((current_y+direction[0], current_x+direction[1]))
                    instructions.append('R')
                    instructions.append(direction)
                    instructions.append(0)

                    # instructions now looks like [...., [new_y_slope,new_x_slope], 0]
                    continue
                except:
                    break

    return ','.join([str(x) for x in instructions if type(x)!=list])

def functionize(instructions):
    options = instructions.split(',')
    options = [",".join(options[i:i+2]) for i in range(0, len(options), 2)]

    # assemble set of all patterns in instructions, that could be made into function
    function_options = []
    for i in range(len(options)):
        for length in range(1,len(options)-i):
            slic = ','.join(options[i:i+length])
            if slic not in function_options:
                function_options.append(slic)

    # iterate through options, stop when we find one function combo that works
    from itertools import permutations

    for a,b,c in permutations(function_options, 3):
        cpy = instructions.replace(a, 'A').replace(b,'B').replace(c,'C')
        if 'R' not in cpy and 'L' not in cpy:
            break

    main = [ord(x) for x in cpy+'\n']
    a = [ord(x) for x in a+'\n']
    b = [ord(x) for x in b+'\n']
    c = [ord(x) for x in c+'\n']

    # check function lengths
    assert len(main)<=21 and len(a)<=21 and len(b)<=21 and len(c)<=21

    return main, a, b, c

def main():
    # read in data
    with open('day17.txt', 'r') as f:
        data = np.array(f.read().strip().split(','), dtype=np.int64)
    f.close()

    from intComputer import IntcodeComputer
    brain = IntcodeComputer(dt=data)

    # assemble map of scaffolds
    outputs = ''
    while True:
        output = brain.process_code()
        if output is not None:
            outputs += chr(output)
        else:
            break

    # split by newline character to form list of strings
    outputs = [output for output in outputs.split('\n') if len(output)>0]

    overlaps, path, start = identify_locations(outputs)

    # part 1
    print(sum([x*y for x,y in overlaps]))

    instructions = build_instructions(path, start)
    main, a, b, c = functionize(instructions)

    # wake up robot
    brain.dt[0] = 2
    # add calculated inputs to move the bot
    brain.inputs = main+a+b+c+[ord('n'), ord('\n')]

    # run robot until program terminates
    vals = [brain.process_code()]
    while vals[-1] is not None:
        vals.append(brain.process_code())
    # part 2
    print(vals[-2])

if __name__=='__main__':
    import numpy as np
    main()


"""Implementation of advanced generators: .send(), .throw(), .close()"""

def is_palindrome(num):
    # skip single-digit inputs
    if num // 10 == 0:
        return False
    temp = num
    reversed_num = 0
    while temp != 0:
        reversed_num = (reversed_num * 10) + (temp % 10)
        temp = temp // 10
    if num == reversed_num:
        return True
    return False

def infinite_palindromes():
    num = 0
    while True:
        if is_palindrome(num):
            i = (yield num)
            if i is not None:
                num = i
        num += 1

pal_gen = infinite_palindromes()
for i in pal_gen:
    print(i)
    DIGITS = len(str(i))
    if DIGITS == 5:
        # .close() causes the generator to stop
        pal_gen.close()
        # .throw() is used to throw an exception
        pal_gen.throw(ValueError("We don't like large palindromes"))
    # .send() is used to add a digit to the last state
    pal_gen.send(10 ** (DIGITS))


# инициализация класса узел
class Node:
    def __init__(self, value=None, colour=1):
        self.value = value  # значение
        self.left = None  # левый потомок
        self.right = None  # правый потомок
        self.colour = colour  # 0 - чёрный, 1 - красный

    # перевод узла в строку
    def __str__(self):
        return f'{self.value, self.colour, self.left, self.right}'

    '''
    Красно-черное дерево имеет следующие критерии:
    • Каждая нода имеет цвет (красный или черный)
    • Корень дерева всегда черный
    • Новая нода всегда красная
    • Красные ноды могут быть только левым ребенком
    • У красной ноды все дети черного цвета
    '''

class RedBlackTree:
    # при инициализации создаётся root, который будет являться корнем дерева
    def __init__(self):
        self.root = None

    # метод поиска элемента по значению
    def find(self, value: int) -> Node:
        return self._find_assist(self.root, value)

    '''
    Вспомогательная функция поиска элемента по значению нужна, чтобы
    поиск был возможен при условии принятия только значения искомого элемента
    то есть чтобы не требовалось дополнительных данных для получения результата
    '''

    # вспомогательная рекурсивная функция
    def _find_assist(self, current: Node, value: int) -> Node:
        if current is None:  # если нет такого элемента, возвращаем None
            return None
        elif current.value == value:  # если элемент нашёлся, возвращаем его
            return current
        elif current.value > value:  # если элемент больше значения, то идём
            return self._find_assist(current.left, value)  # влево по дереву
        elif current.value < value:  # если элемент больше значения, то идём
            return self._find_assist(current.right, value)  # вправо по дереву

    '''
    При реализации метода вставки я хотел, чтобы функция принимала только
    значение, а затем уже сама добавляла узел там, где это нужно исходя из
    критериев дерева (меньшие значения слева, большие - справа)
    В одной функции не удаётся удовлетворить все требования из-за особенности
    рекурсии, поэтому она делится на 2 функции, 1 из которых рекурсивна

    Параллельно при скрутке стека рекурсии балансируем дерево на основе
    заложенных в балансировку условий
    '''

    # метод вставки узла по значению
    def insert(self, value: int) -> bool:
        if self.root is not None:
            result = self._insert_assist(self.root, value)
            self.root = self._balance(self.root)
            self.root.colour = 0
            return result
        else:
            self.root = Node(value)
            self.root.colour = 0
            return True

    # вспомогательная рекурсивная функция вставки узла
    def _insert_assist(self, node: Node, value: int) -> bool:
        if node.value > value:  # если добавляемое значение меньше узла
            if node.left is not None:  # и если место занято, то ищем место
                result = self._insert_assist(node.left, value)
                node.left = self._balance(node.left)
                return result
            else:  # если место не занято, то инициализируем узел с нашим
                node.left = Node(value)  # значением и добавляем его
                return True
        if node.value < value:  # если добавляемое значение больше узла
            if node.right is not None:  # и если место занято, то ищем место
                result = self._insert_assist(node.right, value)
                node.right = self._balance(node.right)  # балансировка
                return result
            else:  # если место не занято, то инициализируем узел с нашим
                node.right = Node(value)  # значением и добавляем его
                return True
        else:  # если не удалось найти место, то возвращаем False
            return False

    '''
    К удалению у меня были те же требования - дай value, получи result.
    Удаление происходит по следующему принципу:

    1) Если у удаляемого элемента 0 или 1 потомок, то элемент перенимает либо
    None, либо значение имеющегося потомка соответственно.
    2) Если у удаляемого элемента 2 потомка, то он перенимает либо значение
    максимального элемента слева от себя, либо значение минимального элемента
    справа от себя (на выбор), а затем элемент с заимствованными значением
    рекурсивно удаляется
    '''

    # функция удаления узла по значению
    def delete(self, value: int) -> None:
        if self.root.value == value:
            if self.root.left is None:
                self.root = self.root.right
            elif self.root.right is None:
                self.root = self.root.left
            else:
                node_max_left = self._min_find(self.root.right)
                self.root.value = node_max_left.value
                self.root.right = self._delete_assist(self.root.right,
                                                      node_max_left.value)
        else:
            return self._delete_assist(self.root, value)
        # передаём во вспомогательную рекурсию root и значение для удаления

    def _delete_assist(self, del_node: Node, value: int):
        if del_node is None:  # если такого узла нет, возвращается None
            return None
        # алгоритм поиска удаляемого узла аналогичный _find_assist()
        elif del_node.value > value:
            del_node.left = self._delete_assist(del_node.left, value)
        elif del_node.value < value:
            del_node.right = self._delete_assist(del_node.right, value)
        else:
            if del_node.left is None:  # если нет левого потомка, то на место
                del_node = del_node.right  # узла встаёт его правый потомок
            elif del_node.right is None:  # и наоборот
                del_node = del_node.left
            else:
                """
                Если у узла есть 2 потомка, то (в данном примере) находим
                максимальный элемент слева от узла, который встанет на его
                место.
                Передаём значение.
                Рекурсивно удаляем элемент, от которого перенималось значение.
                """
                node_max_left = self._min_find(del_node.right)
                del_node.value = node_max_left.value
                del_node.right = self._delete_assist(del_node.right,
                                                     node_max_left.value)
        return del_node

    # функция поиска минимального значения
    def _min_find(self, node: Node) -> Node:
        if node is None:
            return None
        if node.left is None:
            return node
        return self._min_find(node.left)

    # функция поиска максимального значения
    def _max_find(self, node: Node) -> Node:
        if node is None:
            return None
        if node.right is None:
            return node
        return self._max_find(node.right)

    '''
    Балансировка дерева.
    Для того, чтобы наше дерево было сбалансированным (чтобы сложность
    поиска, вставки и удаления всегда была O(log(n))), надо при каждом добавлении
    элемента обрабатывать несколько возникаемых ситуаций.

    Балансировка будет происходить до тех пор, пока все 3 условия не будут
    удовлетворены.
    '''

    def _balance(self, node: Node) -> Node:
        result = node
        need_balance = True
        while need_balance:
            need_balance = False
            """
            если правый потомок существует, его цвет - красный и либо
            отсутствует левый потомок, либо цвет левого потомка - чёрный, то
            производится правый поворот и следующие действия для балансировки
            """
            if result.right is not None and result.right.colour == 1 and \
                    (result.left is None or result.left.colour == 0):
                need_balance = True
                result = self._right_rotate(result)
            """
            если левый потомок существует и его цвет - красный и левый
            потомок левого потомка существует и его цвет тоже красный, то
            производится левый поворот и следующие действия для балансировки
            """
            if result.left is not None and \
                    result.left.colour == 1 and \
                    result.left.left is not None and \
                    result.left.left.colour == 1:
                need_balance = True
                result = self._left_rotate(result)
            """
            если левый потомок существует и его цвет - красный и правый
            потомок также существует и его цвет тоже красный,
            то производится смена цвета, при которой родитель становится
            красным, а его потомки - чёрными
            """
            if result.left is not None and result.left.colour == 1 and \
                    result.right is not None and result.right.colour == 1:
                need_balance = True
                self._colour_swap(result)
        return result

    # изменение цвета
    @staticmethod
    def _colour_swap(node: Node) -> None:
        node.left.colour = 0  # левый потомок становится чёрным
        node.right.colout = 0  # правый потомок становится чёрным
        node.colour = 1  # родитель становится красным

    # правый поворот
    @staticmethod
    def _right_rotate(node: Node) -> Node:
        right = node.right  # сохраняем правого потомка в буфер
        between = right.left  # в качестве промежуточного потомка выступает
        # левый потомок
        right.left = node  # меняем местами родителя и его левого потомка
        node.right = between  # на место правого потомка ставим промежуточного
        right.colour = node.colour  # правый потомок перенимает цвет родителя
        node.colour = 1  # родитель становится красным
        return right

    # левый поворот
    @staticmethod
    def _left_rotate(node: Node) -> Node:
        left = node.left  # сохраняем левого потомка в буфер
        between = left.right  # в качестве промежуточного потомка выступает
        # правый потомок
        left.right = node  # меняем местами родителя и его правого потомка
        node.left = between  # на место левого потомка ставим промежуточного
        left.colour = node.colour  # левый потомок перенимает цвет родителя
        node.colour = 1  # родитель становится красным
        return left

    '''
    Для трансформации дерева в строку используется метод __str__, вызывающий
    рекурсивную функцию _string и возвращающий элементы от меньшего к большему
    (согласно правилам возрастающего обхода бинарного дерева)
    Это позволит посмотреть, в правильной ли последовательности организованы
    все элементы
    '''

    def __str__(self):  # функция начинает с корня root
        return self._string(self.root)

    def _string(self, node: Node) -> str:  # функция применяется к каждому узлу
        if node is None:  # дерева и возвращает пустую строку
            return ''  # если такого узла нет (None)
        return (
            f'{self._string(node.left)}'  # каждый левый потомок каждой ноды
            f'{str(node.value)} '  # значение ноды
            f'{self._string(node.right)}')  # каждый правый потомок
        # результат переводится в строку, иначе выпадет ошибка


import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

def get_filters():
    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs

    city=None
    month=None
    day=None

    while (not city):
        city = str(input('Please select a number to choose your city for the study city : \
         \n 1-Chicago \n 2-New york city \n 3-Washington \n'))

    # get user input for month (all, january, february, ... , june)

    while (not month):
        print('select your desired month to investigate all, january, february, ... , june')
        print('You can also select all to cover all the months covered by the study \n')
        month = str(input('Please select your desired month through typing the provided numbers to \
        complete the study'))

    # get user input for day of week (all, monday, tuesday, ... sunday)
    while (not day):
        print('select your desired day of the week to investigate all, monday, tuesday, ... sunday')
        print('You can also select all to cover the whole week covered by the study')
        day = str(input('Please select the day of the week which you would like to consider in your study'))

    print('-'*40)
    return city, month, day

get_filters()

def load_data(city, month, day):
    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # display the most common month

    # display the most common day of week

    # display the most common start hour

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # display most commonly used start station

    # display most commonly used end station

    # display most frequent combination of start station and end station trip

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # display total travel time

    # display mean travel time

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df):
    """Displays statistics on bikeshare users."""

    print('\nCalculating User Stats...\n')
    start_time = time.time()

    # Display counts of user types

    # Display counts of gender

    # Display earliest, most recent, and most common year of birth

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def main():
    while True:
        city, month, day = get_filters()
        df = load_data(city, month, day)

        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df)

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
	main()


###############################
#	   quickscmp imports
###############################

from quickscms.bitstream import basic
from quickscms.sockets.node import Node
from quickscms.linker import linkerJSON
from quickscms.utils import errors, enums, containers

###############################
#		   main code
###############################
class NodeBalancer(Node, linkerJSON.Handler):
	def __init__(self, container_addresses, container_paths, directory_file_logging, directory_entry_nodes):
		'''(Balancer, list of strings) -> None
			:constuctor for the Balancer class takes a list of ip-addresses
			 representing the available entry Nodes.
		'''
		Node.__init__(self, container_addresses, container_paths, containers.PRESET_SETTINGS_BALANCER)

		linkerJSON.Handler.__init__(self, directory_entry_nodes)

		self.nodes_entry = self.data[0]
		self.trackers = [0] * len(self.nodes_entry)

	def track(self, ip):
		'''(Balancer, string) -> (int)
			:find the number of times traffic has been re-directed to a specific
			 ip

			@returns an integer representing the number of re-directs
		'''
		index = self.nodes_entry.index(ip)
		return self.trackers[index]

	def redirect(self):
		'''(Balancer) -> (string)
			:find the entry node with the least re-directs and return it

			@returns a string of the ip-address with the least re-directs
		'''
		index = self.trackers.index(min( self.trackers ))
		self.trackers[index] = self.trackers[index] + 1
		return self.nodes_entry[index]

	def synatxValidator(self, message):
		'''(Balancer, string) -> None
			:validates whether the basic markup request is valid

			@returns nothing indicating it is valid
			@exception raises MismatchedSyntax() error if it is invalid
		'''
		check = basic.Parser(message)

	def specialFunctionality(self, message, connectingAddress):
		'''(Balancer, string, string) -> (boolean)
			:
		'''
		try:
			self.synatxValidator(message) #will throw an error if invalid (hence, using try/catch)
			entry = self.redirect()
			#if the syntax is valid and we have a new entry node, send the message into the network
			message_modified = message + f'~{connectingAddress}'
			self.send(entry, message_modified)
		except:
			print(f'Console: Received bad reciept from {connectingAddress}')

		#we always want this to return False, there is NO need to have any other functionality
		return (False, 'Successfuly Handled')


import youtube_dl

def getVid(link):
    ydl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s%(ext)s'})

    with ydl:
        result = ydl.extract_info(
            link,
            download=False # We just want to extract the info
        )

    if 'entries' in result:
        # Can be a playlist or a list of videos
        video = result['entries'][0]
    else:
        # Just a video
        video = result

    print(video)
    print(video_url)

def checkLink(link):
    print(link)

if __name__ == "__main__":
    getVid("https://www.youtube.com/watch?v=rU0f4kmBpX8")


import pandas as pd
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from torch.utils.data import DataLoader, TensorDataset

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Load the dataset
df = pd.read_csv("diabetes.csv")
column_names = df.columns.tolist()

X = df[df.columns[:-1]].values
y = df[df.columns[-1]].values
#scaler = StandardScaler()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)

X_train = torch.FloatTensor(X_train)
y_train = torch.LongTensor(y_train)
X_val = torch.FloatTensor(X_val)
y_val = torch.LongTensor(y_val)
X_test = torch.FloatTensor(X_test)
y_test = torch.LongTensor(y_test)

# Step 3: Define your neural network model
class DiabetesModel(nn.Module):
    def __init__(self):
        super(DiabetesModel, self).__init__()
        self.fc1 = nn.Linear(8, 64)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(64, 2)  # 2 output classes (0 or 1)

    def forward(self, x):
        x = self.fc1(x)
        x = self.relu(x)
        x = self.fc2(x)
        return x

# Step 4: Train the model on the training set

model = DiabetesModel()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
train_dataset = TensorDataset(X_train, y_train)
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
epochs = 2000

# Lists to store training and validation loss and accuracy
train_losses = []
train_accuracies = []
val_losses = []
val_accuracies = []

for epoch in range(epochs):
    model.train()  # Set the model to training mode
    total_loss = 0
    correct = 0
    total = 0

    for inputs, labels in train_loader:
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        total_loss += loss.item()
        _, predicted = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predicted == labels).sum().item()

    # Calculate training loss and accuracy for the epoch
    train_loss = total_loss / len(train_loader)
    train_accuracy = correct / total
    train_losses.append(train_loss)
    train_accuracies.append(train_accuracy)

    print(f'Epoch [{epoch + 1}/{epochs}] Training Loss: {train_loss:.4f} Training Accuracy: {train_accuracy * 100:.2f}%')

# Step 5: Evaluate the model on the validation set
model.eval()  # Set the model to evaluation mode
with torch.no_grad():
    y_pred = model(X_val)
    _, predicted = torch.max(y_pred, 1)
    val_loss = criterion(y_pred, y_val)
    val_accuracy = (predicted == y_val).sum().item() / y_val.size(0)
    val_losses.append(val_loss.item())
    val_accuracies.append(val_accuracy)

print(f'Validation Loss: {val_loss.item():.4f} Validation Accuracy: {val_accuracy * 100:.2f}%')

# Step 6: Evaluate the model on the test set
model.eval()  # Set the model to evaluation mode
with torch.no_grad():
    y_pred = model(X_test)
    _, predicted = torch.max(y_pred, 1)
    test_loss = criterion(y_pred, y_test)
    test_accuracy = (predicted == y_test).sum().item() / y_test.size(0)

print(f'Test Loss: {test_loss.item():.4f} Test Accuracy: {test_accuracy * 100:.2f}%')

# Save the trained model
torch.save(model.state_dict(), 'diabetes_model.pt')
print("Model saved.")

# # Load the saved model
# loaded_model = DiabetesModel()
# loaded_model.load_state_dict(torch.load('diabetes_model.pt'))
# loaded_model.to(device)
# loaded_model.eval()
# print("Loaded Model: ",loaded_model)

def predict_diabetes(input_data):

    model.eval()  # Set the model to evaluation mode

    # Convert input_data to a PyTorch tensor
    input_data_tensor = torch.FloatTensor(input_data)

    # Ensure input_data has the same shape as the model expects (8 features)
    if input_data_tensor.shape != (8,):
        raise ValueError("Input data must have 8 features.")

    with torch.no_grad():
        # Forward pass to get predictions
        predictions = model(input_data_tensor)

        # Assuming class 1 corresponds to index 1
        sigmoid = torch.nn.Sigmoid()
        probability_class_1 = sigmoid(predictions[1]).item()

        # Get the predicted class (0 or 1) by rounding the probability
        predicted_class = torch.round(torch.tensor(probability_class_1))
        print(predicted_class)
        result = 'Diabetic' if predicted_class == 1 else 'Not Diabetic'

    return result

# # Example new inputs for prediction
# new_inputs = [6,148,72,35,0,33.6,0.627,50]
# # non - 0
# # yes - 1
# # Predict the outcome using the loaded model
# predicted_outcome = predict_diabetes(new_inputs)
# print(f"According to your data you are: {predicted_outcome}")


# Author : Catveloper
# Problem : https://www.acmicpc.net/problem/9012
# 01/04 18:00

# 1. '(' 의 개수 == ')'의 개수
# 2. for-else
# 3. cnt >= 0 이여야 함
import sys

def input():
  return sys.stdin.readline().rstrip()

N = int(input())

result = ""

for i in range(N):
  testcase = input()
  cnt = 0
  for c in testcase:
    cnt += 1 if c == '(' else -1
    if cnt < 0:
      result += "NO\n"
      break
  else:
    result += "YES\n" if cnt == 0 else "NO\n"

print(result)


"""
Lab 3
Author: Sofi Vinas
Date: 21 February 2021
Description: BST Implementation
Notes:
    1) Created a helper function _find to return the node with the desired ticketID
    2) Other functions implemented with the help of Jared Hall's pseudocode from office hours
"""
from mealticket import *

#============================== Aux Classes ====================================
class Sentinel():
    """This class builds the Sentinel nodes"""

    def __init__(self):
        """The constructor for the Sentinel class"""
        self._key = None
        self._value = None
        self._leftChild = None
        self._rightChild = None
        self._parent = None

    def isSentinel(self):
        """ This method makes it easy to check if a given node is a Sentinel"""
        return True

class Node():
    """ This class implements a node for the BST. """
    def __init__(self, ticket):
        """
        Description: The constructor for the Node class.
        Inputs: A valid MealTicket (input validation should be done by insert)
        """
        self._parent = Sentinel() #was Sentinel()
        self._leftChild = Sentinel()
        self._rightChild = Sentinel()
        self._value = ticket
        self._key = ticket.ticketID

    def __str__(self):
        """ Returns a string rep of the node (for debugging ^,^) """
        returnValue = "Node: {}\n".format(self._key)
        returnValue += "Parent: {}\n".format(self._parent._key)
        returnValue += "Left Child: {}\n".format(self._leftChild._key)
        returnValue += "Right Child: {}\n".format(self._rightChild._key)
        return returnValue

    def isSentinel(self):
        """ A helper method for figureing out if a node is a Sentinel """
        return False

    #Accessor Methods
    def getParent(self):
        """
        Description: Accessor method for the Node. Returns parent.
        """
        return self._parent

    def getRChild(self):
        """
        Description: Accessor method for the Node. Returns right child.
        """
        return self._rightChild

    def getLChild(self):
        """
        Description: Accessor method for the Node. Returns left child.
        """
        return self._leftChild

    def getValue(self):
        """
        Description: Accessor method for the Node. Returns the MealTicket.
        """
        return self._value

    # Mutator methods
    def setParent(self, node):
        """
        Description: Mutator method. Sets the parent reference.
        Input: A Node() reference.
        """
        self._parent = node

    def setLChild(self, node):
        """
        Description: Mutator method. Sets the lchild reference.
        Input: A Node() reference.
        """
        self._leftChild = node

    def setRChild(self, node):
        """
        Description: Mutator method. Sets the rchild reference.
        Input: A Node() reference.
        """
        self._rightChild = node

    #comparison operators
    def __gt__(self, other):
        """
        Description: Overloads the > operator to allow direct comparison of
                     nodes. Now we can do node1 > node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key > other._key
        return returnValue

    def __lt__(self, other):
        """
        Description: Overloads the < operator to allow direct comparison of
                     nodes. Now we can do node1 < node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key < other._key
        return returnValue

    def __eq__(self, other):
        """
        Description: Overloads the == operator to allow direct comparison of
                     nodes. Now we can do node1 == node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key == other._key
        return returnValue

    def __ne__(self, other):
        """
        Description: Overloads the != operator to allow direct comparison of
                     nodes. Now we can do node1 != node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key != other._key
        return returnValue

    def __le__(self, other):
        """
        Description: Overloads the <= operator to allow direct comparison of
                     nodes. Now we can do node1 <= node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key <= other._key
        return returnValue

    def __ge__(self, other):
        """
        Description: Overloads the >= operator to allow direct comparison of
                     nodes. Now we can do node1 >= node2.
        Input: Another instance of the node class.
        """
        returnValue = False
        if(not self.isSentinel() and not other.isSentinel()):
            returnValue = self._key >= other._key
        return returnValue

    #Some helper methods to make things easy in the BST
    def hasLeftChild(self):
        """
        Description: This method returns true if the current node
                     has a left child
        """
        returnValue = False
        cond1 = not self._leftChild.isSentinel()
        cond2 = self._leftChild._parent is self
        if(cond1 and cond2):
                returnValue = True
        return returnValue

    def hasRightChild(self):
        """ This method returns true|false depending on if the current
            node has a right child or not."""
        returnValue = False
        cond1 = not self._rightChild.isSentinel()
        cond2 = self._rightChild._parent is self
        if(cond1 and cond2):
                returnValue = True
        return returnValue

    def hasOnlyOneChild(self):
        """ Returns True if the current node has only one child."""
        LC = self.hasLeftChild()
        RC = self.hasRightChild()
        return (LC and not RC) or (not LC and RC)

    def hasBothChildren(self):
        """ Returns True if the current node has both children"""
        return self.hasLeftChild() and self.hasRightChild()

    def isLeaf(self):
        """ Returns true if the current node is a leaf node."""
        returnValue = False
        if(self._rightChild.isSentinel() and self._leftChild.isSentinel()):
            returnValue = True
        return returnValue

    def isLeftChild(self):
        """Returns true if the current node is a left child"""
        cond1 = not self._parent.isSentinel()
        cond2 = self._parent._leftChild is self
        cond3 = self._parent._rightChild is not self
        return cond1 and cond2 and cond3

    def isRightChild(self):
        """Returns true if the current node is a right child"""
        cond1 = not self._parent.isSentinel()
        cond2 = self._parent._rightChild is self
        cond3 = self._parent._leftChild is not self
        return cond1 and cond2 and cond3

    def isRoot(self):
        """ Returns true if the current node is the root"""
        return self._parent.isSentinel()
#===============================================================================

#================================ BST Class ====================================
class BinarySearchTree:
    """
    Description: A Binary Search Tree (BST).
    Note: Algorithms for the BST can be found in ch. 12 of the book.
    """

    def __init__(self):
        """ The constructor for our BST """
        self._root = Sentinel()
        #Add any other instance variables you need.
        self._currentSize = 0

    def _isValid(self, ticket):
        """
        Description: A method for checking if the given ticket is a valid
                     mealticket.
        Inputs: Some object in the variable ticket.
        Outputs: Boolean (True|False) depending on if it is a valid mealticket.
        """
        return type(ticket) == MealTicket

    def _findMinimum(self, node):
        """
        Description: Finds the minimum child of a tree when given a node.
        Inputs: A node from the BST.
        Outputs: The minumum node from the sub-tree (e.g the left-most child).
        """
        returnValue = False
        if not node.isSentinel():
            returnValue = node
            while not returnValue._leftChild.isSentinel():
                returnValue = returnValue._leftChild
        return returnValue

    def _findSuccessor(self, node):
        """
        Description: Given a node, returns the successor of that node,
                     or False if there is no successor.
        """
        successor = False
        # if node has a right child
        if(node.hasRightChild()):
            # then successor is the min of the right subtree
            successor = self._findMinimum(node._rightChild)
        elif(node._parent): # node has no right child, but has a parent
            if(node.isLeftChild()): # node is a left child
                successor = node._leftChild._parent #self._parent # then succ is the parent
            else: # node is right child, and has not right child
                successor = node._parent
                while not successor._parent.isSentinel() and node.isRightChild():
                    node = successor
                    successor = successor._parent
        return successor

    def _transplant(self, uNode, vNode):
        """
        Description: Replaces subtree at uNode with subtree at vNode.
        Note: See pg. 296 for description of the transplant routine.
        """
        if uNode == self._root:
            self._root = vNode
            uNode._leftChild = None
            uNode._rightChild = None
        elif uNode.isLeftChild():
            uNode._parent._leftChild = vNode
        else:
            uNode._parent._rightChild = vNode
        if vNode != None:
            vNode._parent = uNode._parent

    def _transplantR(self, cNode):
        """
        Description: This transplant attaches the currentNodes right child
                     to the current nodes parent.
        Notes:
                1. Do not call this method when cNode is the root.
                2. Don't forget to handle the cNodes references in your func.
        """
        parent = cNode.getParent()
        child = cNode.getRChild()
        if(cNode.isLeftChild()):
            parent.setLChild(child)
            child.setParent(parent)
        else:
            parent.setRChild(child)
            child.setParent(parent)

    def _transplantL(self, cNode):
        """
        Description: This transplant attaches the currentNodes right child
                     to the current nodes parent.
        Notes:
                1. Do not call this method when cNode is the root.
                2. Don't forget to handle the cNodes references in your func.
        """
        parent = cNode.getParent()
        child = cNode.getLChild()
        if(cNode.isLeftChild()):
            parent.setLChild(child)
            child.setParent(parent)
        else:
            parent.setRChild(child)
            child.setParent(parent)

    def traverse(self, mode):
        """
        Description: The traverse method returns a string rep
                     of the tree according to the specified mode
        """
        self.output = ""
        if(type(mode) == str):
            if(mode == "in-order"):
                self.inorder(self._root)
            elif(mode == "pre-order"):
                self.preorder(self._root)
            elif(mode == "post-order"):
                self.postorder(self._root)
        else:
            self.output = "  "
        return self.output[:-2]

    def inorder(self, node):
        """ computes the inorder traversal """
        if(not node.isSentinel()):
            self.inorder(node.getLChild())
            self.output += str(node._key) + ", "
            self.inorder(node.getRChild())

    def preorder(self, node):
        """computes the pre-order traversal"""
        if(not node.isSentinel()):
            self.output += str(node._key) + ", "
            self.preorder(node.getLChild())
            self.preorder(node.getRChild())

    def postorder(self, node):
        """ compute postorder traversal"""
        if(not node.isSentinel()):
            self.postorder(node.getLChild())
            self.postorder(node.getRChild())
            self.output += str(node._key) + ", "

    def insert(self, ticket):
        """
        Description: Inserts given MealTicket into the tree while
                     preserving binary tree property.
                     Returns True if successful, False otherwise
        """
        ret = False
        if self._isValid(ticket): #check for valid ticket
            ret = True #if that passes, set ret to True
            node = Node(ticket) #create a node with the MealTicket
            root = self._root #create a variable for the root

            if root.isSentinel():
                self._root = node

            else:
                while ret: #while True
                    if root._key < node._key:
                        if root.hasRightChild(): #if it has rightChild
                            root = root._rightChild  #set the root to rightChild
                        else:
                            node.setParent(root)
                            root.setRChild(node)
                            break
                    else:
                        if root.hasLeftChild():
                            root = root._leftChild
                        else:
                            node.setParent(root)
                            root.setLChild(node)
                            break
        return ret

    def delete(self, ticketID):
        """
        Description: Deletes node from tree with given ticketID;
                     restructures binary tree. Returns True if successful,
                     False otherwise
        """
        ret = False
        if(type(ticketID) is int and ticketID > 0): #ONLY CONTINUE ON VALID INPUT
            node = self._find(ticketID) #_find returns a node with that ticketID
            if node is False: #node with THAT ticketID does not exist
                return False
            else:
                ret = True

            if node.isLeaf(): #if it has no children, its a leaf
                parent = node._parent #get the parent of current node

                if node is self._root: #if it is at the root
                    self._root = Sentinel() #set root to sentinel
                elif (node is not self._root) and node.isLeftChild():
                    parent.setLChild(Sentinel()) #set parent LC to sentinel
                elif (node is not self._root) and node.isRightChild():
                    parent.setRChild(Sentinel()) #set parent RC to sentinel

            if node.hasOnlyOneChild(): #CASE 2: IF current has only one child
                if node.hasLeftChild(): #if it has left child
                    if node is self._root: #subcase
                        self._root = node._leftChild #set root to current.leftChild
                    else: #if NOT ROOT
                        self._transplantL(node) #transplantL(current node)
                if node.hasRightChild(): #if it has right child
                    if node is self._root: #if the current node if the root
                        self._root = node._rightChild #set root to current.rightChild
                    else: #otherwise
                        self._transplantR(node) #transplatR(current node)

            #SWAP
            if node.hasBothChildren(): #if it has both children
                successor = self._findSuccessor(node) #find the successor
                self.delete(successor._key) #delete the successor with the key (recursive call)
                temporary = node._key #set up two temporary variables
                temporary2 = node._value

                node._key = successor._key #do the swapping
                successor._key = temporary
                node._value = successor._value
                successor._value = temporary2

        return ret

    def find(self, ticketID):
        """
        Description: Finds node in tree with given ticketID,
                     returns corresponding ticket. Returns False if unsuccessful.
        """
        """
        ret = False
        if type(ticketID) is int and ticketID > 0:
            rootPtr = self._root

            if rootPtr.isSentinel():
                pass
            else:
                while True:
                    if (ticketID is not rootPtr._key) and rootPtr.isLeaf():
                        break
                    elif ticketID is rootPtr._key:
                        ret = rootPtr.getValue()
                        break
                    else:
                        ret = False
                        if rootPtr._key < ticketID:
                            if rootPtr.hasRightChild():
                                rootPtr = rootPtr._rightChild
                            else:
                                break
        return ret
        """
        answer = False
        if type(ticketID) is int and ticketID > 0: #is valid ticketID
            current = self._root #grab the root of the tree
            while True: #LOOP
                if current._key == ticketID: #CASE 1
                    answer = current._value #CHANGE ANSWER VARIABLE TO TRUE
                    break
                if current.isSentinel(): #CASE 2
                    break #answer = False remains
                if ticketID < current._key: #if the ticketID is less than the key, we know its the leftChild
                    if current.hasLeftChild():
                        current = current._leftChild #update current variable
                    else:
                        break
                if ticketID > current._key: #if the ticketID is less than the key, we know its the rightChild
                    if current.hasRightChild():
                        current = current._rightChild #update current variable
                    else:
                        break
        return answer

    def _find(self, ticketID):
        """Almost exactly the same as the above function, but returns a
        node instead of corresponding ticket. False otherwise"""
        ret = False
        if type(ticketID) is int and ticketID > 0:
            current = self._root #current equal to root
            while True:
                if current._key == ticketID:
                    ret = current #THIS IS THE ONLY LINE THAT CHANGES
                    break
                if current.isSentinel():
                    break
                if ticketID < current._key:
                    if current.hasLeftChild(): #check for left
                        current = current._leftChild
                    else:
                        break
                if ticketID > current._key:
                    if current.hasRightChild():
                        current = current._rightChild
                    else:
                        break
        return ret


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from flask import *
import os, time, random, json, socket
from utils.chars import saveCharacter, switchCharacter, saveUnsavedPlayers
app = Flask(__name__)
app.secret_key = os.urandom(24)

@app.route('/save/<identifier>/<disconnect>')
def save(identifier, disconnect):
	if disconnect == 'false':
		disconnect = False
	elif disconnect == 'true':
		disconnect = True

	print('Saving {} ...'.format(identifier))
	millis = int(round(time.time() * 1000))
	resp = saveCharacter(identifier, disconnect)
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis

	if resp['success']:
		resp['time'] = str(res) + ' ms'
		print("Saved " + identifier)
	else:
		print("Error saving " + identifier)

	return jsonify(resp)

@app.route("/switch/<identifier>/<networkID>")
def switch(identifier, networkID):
	networkID = int(networkID)
	millis = int(round(time.time() * 1000))
	resp = switchCharacter(identifier, networkID)
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis
	resp['time'] = str(res) + ' ms'

	return jsonify(resp)

@app.route('/saveUnsaved')
def saveUnsaved():
	millis = int(round(time.time() * 1000))
	resp = saveUnsavedPlayers()
	millis2 = int(round(time.time() * 1000))
	res = millis2 - millis
	resp['time'] = str(res) + ' ms'

	return jsonify(resp)

if __name__ == "__main__":
	app.run(host="localhost", port=5004, threaded=True)


str = input()
strList = str.split()

def isVowel(ch):
    if ch in 'ueoai':
        return True
    return False

result = []

for i in range (0, len(strList)):
    a = 0
    for j in strList[i]:
        if isVowel(j):
            a += 1
    if a >= 3:
        result.append(strList[i])
        result.append(a)
print(result)


import math

while True:

    epsilon = 0.0001
    x = int(input("Qual o valor de n? "))
    estimativa = int(input("Qual o valor da estimativa inicial? "))

    while True:
        raiz_Newton = (estimativa + x/estimativa)/2

        if abs(raiz_Newton - estimativa) < epsilon:
            break
        estimativa = raiz_Newton

    math_Python = math.sqrt(x)
    diferença = estimativa - math_Python

    print("Raiz_Newton =", raiz_Newton,"\nMath.sqrt =", math_Python, "\nDiferença =",diferença )

Programação 1/Pyhton exercises/Ficha 3/Exercicio 6 ficha 3.py


import argparse
import logging
import os
import os.path as osp
import time
import re
from collections import deque

import torch
import torch.nn.functional as F
import torch_geometric.transforms as T
from torch_geometric.data import DataLoader

from DatasetLoader.loader import LoadDataset
from Utils.helper import time_to_hms_string
from Utils.generics import saveModelCheckpoint, loadFromCheckpoint
from Utils.lr_schedulers import limitedExponentialDecayLR as customExpDecayLambda
from Normalization.normalization import Normalize
from Sampling.poisson_disk_sampling import PoissonDiskSampling

from Models.PointNet2.pointnet2_cls_ssg import PointNet2Class as PointNet2
from Models.PointNet2MSG.pointnet2_cls_msg import PointNet2MSGClass as PointNet2MSG
from Models.PointNet2MRG.pointnet2_cls_mrg import PointNet2MRGClass as PointNet2MRG
from Models.PointNet2MRGSortPool.pointnet2_cls_mrg_sort_pool import PointNet2MRGSortPoolClass as PointNet2MRGSortPool
from Models.PointNet2MSGSortPool.pointnet2_cls_msg_sort_pool import PointNet2MSGSortPoolClass as PointNet2MSGSortPool
from Models.PointNet2MSGFPSortPool.pointnet2_cls_msgfp_sort_pool import PointNet2MSGFPSortPoolClass as PointNet2MSGFPSortPool
from Models.PointNet2MRGLight.pointnet2_cls_mrg_light import PointNet2MRGLightClass as PointNet2MRGLight
from Models.PointNetVanilla.pointnet_cls_vanilla import PointNetVanillaClass  as PointNetVanilla
from Models.PointNetInputEnhanced.pointnet_cls_ie import PointNetInputEnhanced  as PointNetInputEnhanced

parser = argparse.ArgumentParser()
parser.add_argument('--model', default=None, help='Model name (PointNet2, PointNet2MSG, PointNet2MSGSortPool, PointNet2MRG, PointNet2MRGSortPool, PointNet2MSGFPSortPool)')
parser.add_argument('--num_point', type=int, default=1024, help='Point Number sampled from each object [default: 1024]')
parser.add_argument('--epoch', type=int, default=251, help='Epoch to run [default: 251]')
parser.add_argument('--batch_size', type=int, default=32, help='The size of the batch to use [default: 16]')
parser.add_argument('--weight_decay', type=float, default=1e-5, help='The weight decay to use for the training, the original PointNet2 1e-5 [default:1e-5]')
parser.add_argument('--lr_decay', type=float, default=0.7, help='The weight decay used as to start the exponential LR decay, the step is defined from the Exponential class of pytorch.[default:0.7]')
parser.add_argument('--lr_decay_step', type=float, default=2e5, help='Learning rate decay step [default: 2e5]');
parser.add_argument('--train_dataset', default='ModelNet40', help='The name of the dataset to use to train the model (ModelNet10, ModelNet40, ShapeNet)[default: ModelNet40]')
parser.add_argument('--test_dataset', default='ModelNet40', help='The name of the dataset to use to test the model (ModelNet10, ModelNet40, ShapeNet)[default: ModelNet40]')
parser.add_argument('--checkpoint', default=None, help='The path to the checkpoint file from which continue the train')
parser.add_argument('--log_file', default='./train_log.txt', help='The file where to print the logs [default: ./train_log.txt]')
parser.add_argument('--use_normals', type=bool, default=False, help='Specufy if train the model using point normals from the data sets[default: False]')
parser.add_argument('--sort_pool_k', type=int, default=32, help='The number of point the sort_pool should keep <only needed with the sort_pool model> [default: 32]')
parser.add_argument('--sampling_method_train', default='ImportanceSampling', help='The type of sampling to use. [PoissonDiskSampling or ImportanceSampling]')
parser.add_argument('--sampling_method_test', default='ImportanceSampling', help='The type of sampling to use. [PoissonDiskSampling or ImportanceSampling]')
ARGS = parser.parse_args()

BATCH_SIZE = ARGS.batch_size
NUM_POINT = ARGS.num_point
EPOCH = ARGS.epoch
DATASET_TRAIN = ARGS.train_dataset
DATASET_TEST = ARGS.train_dataset
MODEL_NAME = ARGS.model
CHECKPOINT = ARGS.checkpoint
WEIGHT_DECAY = ARGS.weight_decay
LR_DECAY = ARGS.lr_decay
LR_DECAY_STEP = ARGS.lr_decay_step
LOG_FILE = ARGS.log_file
USE_NORMALS = ARGS.use_normals
N_FEATURES = 6 if ARGS.use_normals else 3
SORT_POOL_K = ARGS.sort_pool_k
SAMPLING_TRAIN = ARGS.sampling_method_train
SAMPLING_TEST = ARGS.sampling_method_test

def train(model, train_loader, optimizer, device):
    model.train()

    for data in train_loader:
        data = data.to(device)
        if data.norm is not None:
            data.x = data.norm
        optimizer.zero_grad()
        test_values = data.category if DATASET_TRAIN == 'ShapeNet' else data.y
        loss = F.nll_loss(model(data), test_values)
        loss.backward()
        optimizer.step()

def test(model, test_loader, device):
    model.eval()

    correct = 0
    for data in test_loader:
        data = data.to(device)
        if data.norm is not None:
            data.x = data.norm
        test_result = data.category if DATASET_TEST == 'ShapeNet' else data.y
        with torch.no_grad():
            pred = model(data).max(1)[1]
        correct += pred.eq(test_result).sum().item()
    return correct / len(test_loader.dataset)

def setupTrain(model_name, n_features, class_number, device, w_decay, lr_decay, batch_size, decay_step):
    model = getModel(model_name, n_features, class_number).to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=w_decay)
    start_epoch = 1

    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer,
                    lr_lambda=customExpDecayLambda(lr_decay, batch_size, decay_step))

    return model, optimizer, start_epoch , scheduler

def getModel(name, input_features, class_count):
    model = None
    if (MODEL_NAME == 'PointNetVanilla'):
        model = PointNetVanilla(class_count, nfeatures=input_features, nPoints=NUM_POINT)

    elif (MODEL_NAME == 'PointNetInputEnhanced'):
        model = PointNetInputEnhanced(class_count, nfeatures=input_features, batch_size=BATCH_SIZE, nPoints=NUM_POINT)

    elif (MODEL_NAME == 'PointNet2'):
        model = PointNet2(class_count, bn_momentum=0.1)

    elif (MODEL_NAME == 'PointNet2MSG'):
        model = PointNet2MSG(class_count, nfeatures=input_features)

    elif (MODEL_NAME == 'PointNet2MRG'):
        model = PointNet2MRG(class_count, nfeatures=input_features)

    elif (MODEL_NAME == 'PointNet2MRGLight'):
        model = PointNet2MRGLight(class_count, n_features=input_features)

    elif (MODEL_NAME == 'PointNet2MSGSortPool'):
        model = PointNet2MSGSortPool(class_count, n_feature=input_features, sort_pool_k=SORT_POOL_K)

    elif (MODEL_NAME == 'PointNet2MSGFPSortPool'):
        model = PointNet2MSGFPSortPool(class_count, n_feature=input_features, sort_pool_k=SORT_POOL_K)

    elif (MODEL_NAME == 'PointNet2MRGSortPool'):
        model = PointNet2MRGSortPool(class_count, n_features=input_features, sort_pool_k=SORT_POOL_K)

    return model

def getSampler(name, dataset_name):
    transform = None

    if(dataset_name == 'ShapeNet'):
        transform = T.FixedPoints(NUM_POINT)

    elif(name == 'ImportanceSampling'):
        transform = T.SamplePoints(NUM_POINT, remove_faces=True, include_normals=USE_NORMALS)

    elif (name == 'PoissonDiskSampling'):
        transform = PoissonDiskSampling(NUM_POINT, remove_faces=True)

    return transform

if __name__ == '__main__':
    logging.basicConfig(filename=LOG_FILE, level=logging.INFO)

    modelWeightsPath = osp.join(
        osp.dirname(osp.realpath(__file__)), '.', 'weights/', MODEL_NAME + '_cls/')
    if not os.path.exists(modelWeightsPath):
        os.makedirs(modelWeightsPath)
        os.chmod(modelWeightsPath, 0o777)

    # DATASET preprocessing
    if(USE_NORMALS and ('Poisson' in DATASET_TRAIN or 'Poisson' in DATASET_TEST)):
        print("Poisson sampled data do not support normals(at the moment).")
        exit(1)

    pre_transform = Normalize(),
    transform_train = getSampler(SAMPLING_TRAIN, DATASET_TRAIN)  # POINT SAMPLING
    transform_test = getSampler(SAMPLING_TEST, DATASET_TEST)
    (train_dataset, test_dataset) = LoadDataset(
        DATASET_TRAIN, DATASET_TEST,
        transform_train=transform_train, pre_transform_train=pre_transform,
        transform_test=transform_test, pre_transform_test=pre_transform,
    )

    train_loader = DataLoader(
        train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=6)
    test_loader = DataLoader(
        test_dataset, batch_size=BATCH_SIZE, shuffle=False, num_workers=6)

    # TRAIN SETUP
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    last_10_loss_values = deque([], 10)

    (model, optimizer, start_epoch, optimizer_scheduler) = setupTrain(
        model_name = MODEL_NAME,
        n_features = N_FEATURES,
        class_number = train_dataset.num_classes,
        w_decay = WEIGHT_DECAY,
        lr_decay = LR_DECAY,
        decay_step=LR_DECAY_STEP,
        batch_size = BATCH_SIZE,
        device = device
	)

    train_start_time = time.time()

    if (CHECKPOINT != None):
        (model, optimizer, optimizer_scheduler, start_epoch, train_checkpoint_time) = loadFromCheckpoint(
            CHECKPOINT, model, optimizer, optimizer_scheduler, device)
        train_start_time-=train_checkpoint_time

    # TRAIN CYCLE
    for epoch in range(start_epoch, start_epoch + EPOCH):
        train(model, train_loader, optimizer, device)
        test_acc = test(model, test_loader, device)
        optimizer_scheduler.step()

        # the older item are automatically removed from the deque when full
        last_10_loss_values.append(test_acc)
        loss_avg = sum(last_10_loss_values) / float(len(last_10_loss_values))
        current_train_time = time.time() - train_start_time

        epoch_to_print = '{} :: Epoch: {:03d}, Test: {:.4f}, Last 10 AVG: {:.4f}, LR: {:.8f}'.format(
            time_to_hms_string(current_train_time) ,epoch, test_acc, loss_avg, optimizer_scheduler.get_lr()[0])

        print(epoch_to_print)
        logging.info(epoch_to_print)

        # TODO add better system to save checkpoint, and maybe a better naming (with start and end feature count)
        if (epoch % 10 == 0):

            saveModelCheckpoint(
                modelWeightsPath, model, optimizer, test_acc, epoch, optimizer_scheduler, loss_avg, current_train_time)

    # Final trained Model store for inference
    inferenceReadyModel = osp.join( modelWeightsPath, 'trained/model_inference.pt')

    torch.save(model.state_dict(), inferenceReadyModel)


# -*- coding: utf-8 -*-
"""
Created on Fri Mar  1 15:17:42 2019

@author: Jayesh Jain
"""

import os
os.chdir ="C:/Users/Jayesh Jain/Downloads"

with open('anagram.txt', 'r') as fp:
    line = fp.readlines()

def make_anagram_dict(line):
    d = {}  # avoid using 'dict' as variable name

    for word in line:
        word = word.lower()  # call lower() only once
        key = ''.join(sorted(word))
        if key in d:  # no need to call keys()
            d[key].append(word)
        else:
            d[key] = [word]  # you can initialize list with the initial value

    return d  # just return the mapping to process it later

if __name__ == '__main__':
    d = make_anagram_dict(line)

    for words in d.values():
        if len(words) > 1:  # several anagrams in this group
            print('Anagrams: {}'.format(', '.join(words)))


import cv2
import numpy as np

img = np.zeros((600, 600, 3), np.uint8)

cv2.line(img, (0, 0), (img.shape[1], img.shape[0]), (0, 255, 0), 8)
# cv2.line(影像, 开始座标, 结束座标, 颜色, 线条宽度)
# 在图像上画线
# img.shape[0] = 600 、 img.shape[1] = 600 、 img.shape[2] = 3

cv2.rectangle(img, (0, 0), (400, 300), (0, 0, 255), cv2.FILLED)
# cv2.rectangle(影像, 顶点座标, 对向顶点座标, 颜色, 线条宽度)

cv2.circle(img, (300, 400), 30, (255, 0, 0), cv2.FILLED)
# cv2.circle(影像, 圆心座标, 半径, 颜色, 线条宽度)
# 线条宽度用负值，则会变成实心，也可以用函式来表示

cv2.ellipse(img, (500, 200), (25, 55), 45, 0, 360, (0, 0, 255), 5)
# cv2.ellipse(影像, 中心座标, 轴长, 旋转角度, 起始角度, 结束角度, 颜色, 线条宽度)

cv2.putText(img, 'Eris', (100, 500), cv2.FONT_HERSHEY_PLAIN, 2, (255, 255, 255), 2 , cv2.LINE_AA )
# cv2.putText(影像, 文字, 座标, 字型, 大小, 颜色, 线条宽度, 线条种类)
# 线条种类可以省略不写，预设值为cv2.LINE_8，也可以设定反锯齿的线条效果:cv2.LINE_AA

cv2.imshow('img', img)
cv2.waitKey(0)


import sys
import warnings
import numpy as np
import matplotlib.pyplot as plt
from skimage.segmentation import (morphological_geodesic_active_contour,
                                 inverse_gaussian_gradient)
import skimage
import cv2
import os
import SimpleITK as sitk
import glob
import napari

def viewVolume(volume):
    """
        Shows the 3D image using the Napari library.
        Parameters:
            volume (numpy.ndarray): Numpy array 3D image.
        Returns:
            void
    """
    viewer = napari.view_image(volume, contrast_limits=[0, 1])
    napari.run()

def store_evolution_in(lst):
   """
   Returns a callback function to store the evolution of the level sets in
   the given list.
   """

   def _store(x):
       lst.append(np.copy(x))

   return _store

def despeckle(image,option):
    """
        Despeckles the 3D SimpleITK image using one of the options.
        The options are 'bilateral901', 'median110', 'median111'
        and 'median221'.
        Parameters:
            image (sitk.Image): 3D image that needs to be despeckled
            option: despeckling option
        Returns
            sitk.Image: The despeckled image

    """

    if option == 'bilateral901':
       image = sitk.GetArrayFromImage(image)
       for slice in range(0, image.shape[0]):
           image[slice] = cv2.bilateralFilter(image[slice], 9, 0, 1)  # cv2.medianBlur(image_1bone[slice],3)
       image = sitk.GetImageFromArray(image)

    if option == 'median110':
       image = sitk.BinaryMedian(image,(1,1,0))

    if option == 'median111':

       image = sitk.Median(image,(1,1,1))

    if option == 'median221':
       image = sitk.BinaryMedian(image, (2, 2, 1))

    return image

if __name__ == "__main__":
    try:
        bmp_folder_path_PCCT = sys.argv[1]
        thresh_val = int(sys.argv[2])
        kernel_size = 15
        if thresh_val > 110:
            kernel_size = 35

        # getting the filenames for the output
        filenames = [f for f in os.listdir(bmp_folder_path_PCCT) if
                       os.path.isfile(os.path.join(bmp_folder_path_PCCT, f)) and f.endswith('.bmp')]
        output_path_PCCT = os.path.join(bmp_folder_path_PCCT, 'bonemask')
        if not os.path.exists(output_path_PCCT):
           os.makedirs(output_path_PCCT)

        # reading the image
        image = [cv2.cvtColor(cv2.imread(file), cv2.COLOR_BGR2GRAY) for file in glob.glob(bmp_folder_path_PCCT + "\*.bmp")]
        image_original = np.asarray(image)
        shape_1slice = image_original.shape

        # converting image to SimpleITK format
        image = sitk.GetImageFromArray(image_original)

        # thresholding the image
        image = sitk.Threshold(image,lower=thresh_val,upper=255)
        thresholded_image = sitk.GetArrayFromImage(image)
        image = despeckle(image, 'bilateral901')

        # Despeckle
        image = sitk.ConnectedComponent(image, False)
        image = sitk.RelabelComponent(image, sortByObjectSize=True)
        image = sitk.GetArrayFromImage(image)

        for slice in range(0,shape_1slice[0]):
           print(str(slice))
           image_1slice = image[slice,:,:]
           output_path_file = os.path.join(output_path_PCCT, filenames[slice])
           output_image = np.zeros((shape_1slice[1], shape_1slice[2]), np.uint8)
           for bone in range(1,5):
               one_bone_image_1slice = (image_1slice == bone).astype(np.uint8)

               if thresh_val < 100:
                   one_bone_image_1slice = cv2.bilateralFilter(one_bone_image_1slice, 9, 0, 1)

               empty = np.zeros_like(one_bone_image_1slice)
               one_bone_image_1slice = cv2.normalize(one_bone_image_1slice, empty, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)

               if np.mean(one_bone_image_1slice) > 0.0001:

                   # create mask for active contours
                   kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
                   trial_image = np.copy(one_bone_image_1slice)
                   mask_VOI = cv2.dilate(trial_image,kernel)
                   mask_VOI = cv2.bitwise_not(mask_VOI)
                   num_labels, labels_im = cv2.connectedComponents(mask_VOI)
                   mask_VOI = (labels_im == 1).astype(np.uint8)
                   empty = np.zeros_like(one_bone_image_1slice)
                   mask_VOI = cv2.normalize(mask_VOI, empty, 0, 255, cv2.NORM_MINMAX,dtype=cv2.CV_8U)
                   mask_VOI = cv2.bitwise_not(mask_VOI)

                   output = thresholded_image[slice,:,:]
                   output = cv2.bitwise_and(output, output, mask=one_bone_image_1slice)
                   image_preprocessed = cv2.bitwise_not(output)
                   image_preprocessed = skimage.img_as_float(image_preprocessed)

                   # Shrink wrap
                   # Initial level set
                   init_ls = mask_VOI

                   evolution = []
                   callback = store_evolution_in(evolution)

                   gimage = inverse_gaussian_gradient(image_preprocessed,alpha=100.0,sigma=4.0)

                   ls = morphological_geodesic_active_contour(gimage,num_iter=80,init_level_set = init_ls,
                                                              smoothing=1,balloon=-1,iter_callback=callback)

                   #code for showing the evolution of the active contours
                   # fig, axes = plt.subplots(1, 2, figsize=(16, 16))
                   # ax = axes.flatten()
                   #
                   # ax[0].imshow(image_original[slice,:,:], cmap="gray")
                   # ax[0].set_axis_off()
                   # ax[0].contour(ls, [0.5], colors='r')
                   # ax[0].set_title("Morphological GAC segmentation", fontsize=12)
                   #
                   # ax[1].imshow(image_preprocessed, cmap="gray") #ls ipv image_preprocessed
                   # ax[1].set_axis_off()
                   # contour = ax[1].contour(evolution[2], [0.5], colors='g')
                   # contour.collections[0].set_label("Iteration 2")
                   # contour = ax[1].contour(evolution[30], [0.5], colors='y')
                   # contour.collections[0].set_label("Iteration 30")
                   # contour = ax[1].contour(evolution[-1], [0.5], colors='r')
                   # contour.collections[0].set_label("Iteration 80")
                   # ax[1].legend(loc="upper right")
                   # title = "Morphological GAC evolution"
                   # ax[1].set_title(title, fontsize=12)
                   # fig.tight_layout()
                   # plt.show()

                   # add the different bones to one image
                   output_image = output_image | ls

           # write image
           out = np.zeros_like(one_bone_image_1slice)
           output_image = cv2.normalize(output_image, out, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
           cv2.imwrite(output_path_file, output_image)

    except:
        if len(sys.argv) != 3:
            warnings.warn('Not the right amount of arguments were given.'
            'As a reminder, the input should be '
            'python Bone_Mask.py <"link to input folder"> <threshold>')
        else:
            warnings.warn('An error occurred during the STB segmentation.'
            'Please check the input. As a reminder, the input should be '
            'python Bone_Mask.py <"link to input folder"> <threshold>')


# -*- coding: utf-8 -*-
"""
@author: Yifan Cheng, Skylar Du, Yashash Gaurav, Mark He
"""

# import os
import time
import random
import requests
import numpy as np
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from urllib.request import urlopen
from pandas.core.frame import DataFrame
from user_agent import DESKTOP_USER_AGENTS
from selenium.common.exceptions import NoSuchElementException
import csv

edge_driver_path = (
    "path/to/webdriver"
)

#######################ROI DATA#######################
# Crawl the ROI data with beautifulsoup
# Time Warning: about 10 minutes
def get_roi():
    school_name = []
    rank = []
    twenty_year_roi = []
    total_4_year_cost = []
    graduation_rate = []
    typical_years_to_graduate = []
    average_loan_amount = []
    for page in range(1, 199, 1):
        html = "https://www.payscale.com/college-roi/page/" + str(page)
        # get HTML
        html = urlopen(html)
        bsyc = BeautifulSoup(html.read(), "html.parser")
        fout = open("payscale_temp.txt", "wt", encoding="utf-8")
        fout.write(str(bsyc))
        fout.close()
        # get lists
        tc_table = list(bsyc.body.div.div)
        table = list(tc_table[1].children)
        body = table[4].tbody
        for i in body.children:
            li = list(
                i.find_all("span", {"class": "roi-grid__schoolname--text"})
            )
            for j in li:
                school_name.append(str(j).split(">")[2][:-3])
        for i in body.children:
            li = list(i.find_all("span", {"class": "roi-grid__rank--text"}))
            for j in li:
                rank.append(str(j).split(">")[1][:-6])
        datas = []
        for i in body.children:
            li = list(i.find_all("span", {"class": "data-table__value"}))
            for j in li:
                datas.append(str(j).split(">")[1][:-6])
        for i in range(len(datas)):
            if i % 7 == 2:
                twenty_year_roi.append(datas[i])
            if i % 7 == 3:
                total_4_year_cost.append(datas[i])
            if i % 7 == 4:
                graduation_rate.append(datas[i])
            if i % 7 == 5:
                typical_years_to_graduate.append(datas[i])
            if i % 7 == 6:
                average_loan_amount.append(datas[i])
    # convert to dictionary
    dataf = {
        "Rank": rank,
        "School Name": school_name,
        "20 Year Net ROI": twenty_year_roi,
        "Total 4 Year Cost": total_4_year_cost,
        "Graduation Rate": graduation_rate,
        "Typical Years to Graduate": typical_years_to_graduate,
        "Average Loan Amount": average_loan_amount,
    }
    # Convert to dataFrame and output as Excel
    dataframe = DataFrame(dataf, index=rank)
    dataframe.to_excel("output.xlsx")

# read ROI data
# since the time to crawl is long, I stored them in an excel, and now I will read from the excel
def clean_roi():
    roi = pd.read_excel("output.xlsx")
    roi = roi.drop_duplicates(
        subset=[
            "School Name",
            "20 Year Net ROI",
            "Total 4 Year Cost",
            "Graduation Rate",
            "Typical Years to Graduate",
        ]
    )
    roi["Rank"] = range(len(roi))
    roi["School Name"] = [
        (s.split("-")[0] + " - " + s.split("-")[1]).strip()
        if s.find("-") != -1
        else s
        for s in roi["School Name"]
    ]
    #os.remove("output.xlsx")
    return roi

#######################NICHE DATA#######################
# Crawl the NICHE SAT data with beautifulsoup
# Time Warning: about 10 minutes
def send_request(link):
    time.sleep(random.choice(range(2, 10)))
    headers = {"user-agent": random.choice(DESKTOP_USER_AGENTS)}
    res = requests.get(link, headers=headers)
    if res == None:
        print("res is empty")
        return res
    return res

def get_niche():
    # create a list with the url of all 82 pages of the ranking
    allurl = ["https://www.niche.com/colleges/search/best-value-colleges/",]
    nexturl = "https://www.niche.com/colleges/search/best-value-colleges/?page="
    for i in range(2,82):
        myurl = nexturl+str(i)
        allurl.append(myurl)

    school_name = []
    fact = []
    location = []
    # iterate and scrape through each link
    for url in allurl:
        #define the link to be scrape and sent request
        res =send_request(url)
        #parse data from each page
        newsoup = BeautifulSoup(res.text,'html.parser')
        #find the info on each school
        #for each school, create a new dict that contain its name and fact
        for div in newsoup.find_all("div", class_="card"):
            # remove the sponsered colleges
            if len(div.find_all("div", class_="search-result__sponsered-bar"))==0:
                school_name.append(div.find("h2", class_="search-result__title"))
                fact.append(div.find_all("span", class_= "search-result-fact__value"))
                if len(div.find_all("li", class_= "search-result-tagline__item"))!=0:
                    location.append(div.find_all("li", class_= "search-result-tagline__item")[1])
                else:
                    location.append('null')

    cleaned_school_name = []
    for i in school_name:
        new = str(i).split('>')
        if (len(new)<2):
            cleaned_school_name.append("null")
        else:
            cleaned_school_name.append(new[1].split('<')[0])

    cleaned_fact = []
    for list in fact:
        temp = []
        for i in list:
            new = str(i).split('>')
            if (len(new)<2):
                temp.append("null")
            else:
                temp.append(new[1].split('<')[0])
        cleaned_fact.append(temp)

    cleaned_location = []
    for i in location:
        new = str(i).split('>')
        if (len(new)<2):
            cleaned_location.append("null")
        else:
            cleaned_location.append(new[1].split('<')[0])

    city = []
    state = []
    for i in cleaned_location:
        new = i.split(', ')
        if (len(new)<2):
            city.append("null")
            state.append("null")
        else:
            city.append(new[0])
            state.append(new[1])

    Acceptance_Rate = []
    Net_Price = []
    SAT_Range = []

    for eachfact in cleaned_fact:
        if len(eachfact)>0:
            Acceptance_Rate.append(eachfact[0]), Net_Price.append(eachfact[1])
            if len(eachfact)==3:
                if len(eachfact[2])>1:
                    SAT_Range.append(eachfact[2])
                else:
                    SAT_Range.append("null")
            else:
                SAT_Range.append("null")
        else:
            Acceptance_Rate.append("null"), Net_Price.append("null"), SAT_Range.append("null")

    file = open("cleaned_niche.csv", "w", newline = "")
    writer = csv.writer(file)
    # field names
    fields = ['School Name', 'City', 'State', 'Acceptance Rate', 'Net Price', 'SAT Range']
    writer.writerow(fields)
    for i in range(len(cleaned_school_name)):
        if cleaned_school_name[i] != "null":
            writer.writerow([cleaned_school_name[i], city[i], state[i], Acceptance_Rate[i], Net_Price[i], SAT_Range[i]])
    file.close()

#######################Merge the data#######################
def add_calculation_columns(merged_data):
    def convert_currency_to_int(currency):
        try:
            return int(currency.replace(",", "").replace("$", ""))
        except:
            return np.NaN

    def get_sat_range_min(sat_range):
        if type(sat_range) is float and np.isnan(sat_range):
            return np.nan
        else:
            return int(sat_range.split("-")[0])

    def get_sat_range_max(sat_range):
        if type(sat_range) is float and np.isnan(sat_range):
            return np.nan
        else:
            return int(sat_range.split("-")[1])

    merged_data["SAT Min"] = merged_data["SAT Range"].map(
        lambda cell: get_sat_range_min(cell)
    )
    merged_data["SAT Max"] = merged_data["SAT Range"].map(
        lambda cell: get_sat_range_max(cell)
    )
    merged_data["Total 4 Year Cost (Integer)"] = merged_data[
        "Total 4 Year Cost"
    ].map(lambda x: convert_currency_to_int(x))

def merge_data():
    roi = clean_roi()
    roi["Rank"] = roi["Rank"] + 1
    niche = pd.read_csv("cleaned_niche.csv", encoding='ISO-8859-1')
    niche = niche.drop_duplicates()
    merged_data = pd.merge(roi, niche, how="left", on="School Name")
    merged_data = merged_data.drop(labels="Unnamed: 0", axis=1)
    add_calculation_columns(merged_data)
    merged_data.to_excel("merged_data.xlsx")

##################### Combining Fetch ###############
def refresh_all_data():
    get_roi()
    get_niche()
    merge_data()
    # get_careers_data() - uncomment if you have webdriver path set
    print("Success! All data is refreshed.")

#################### Best Colleges Data ####################
# this portion of the script is to scrape bestcollege.com
# for information on career data

def get_careers_data():
    # Call your browser
    driver = webdriver.Edge(executable_path=edge_driver_path)

    # Get all career links
    driver.get("https://www.bestcolleges.com/careers/")
    career_anchors = driver.find_elements_by_css_selector(
        "div.swiper-slide a[data-wpel-link='internal']"
    )
    career_links = [career.get_attribute("href") for career in career_anchors]

    career_data_list = []

    # Loop through urls collected and collect data
    for career in career_links:
        driver.get(career)
        career_name = driver.find_element_by_css_selector(
            "section.hero h1"
        ).text
        career_info = driver.find_element_by_css_selector(
            "section.container.content>p:first-child"
        ).text

        try:
            why_career = driver.find_element_by_css_selector(
                'a[id^="why-pursue"]+h2+p'
            ).text
            why_career += (
                " "
                + driver.find_element_by_css_selector(
                    'a[id^="why-pursue"]+h2+p+div+p'
                ).text
            )
        except NoSuchElementException:
            why_career = ""

        try:
            how_to_start = driver.find_element_by_css_selector(
                "a#advancing-your-career+h2+p"
            ).text
            how_to_start += driver.find_element_by_css_selector(
                "a#advancing-your-career+h2+p+div+p"
            ).text
        except NoSuchElementException:
            how_to_start = ""

        career_data_list.append(
            {
                "career_name": career_name,
                "career_info": career_info,
                "why_career": why_career,
                "how_to_start": how_to_start,
            }
        )

    # Create a data frame of the data collected
    career_data = pd.DataFrame(
        career_data_list,
        columns=["career_name", "career_info", "why_career", "how_to_start"],
    )

    # Save the data frame created for future use.
    career_data.to_csv(
        "bestcolleges_careers.csv", index=False, encoding="utf-8"
    )

    driver.close()

merge_data()


def main():

    x = "AAAABBBCCDAABBB"
    y = 1103735

    print(unique_in_order(x))

def unique_in_order(iterable):

    result = []

    result.append(iterable[0])

    for i in range(1, len(iterable), 1):
        if iterable[i - 1] != iterable[i]:
            result.append(iterable[i])

    return result

if __name__ == "__main__":
    main()