from cloudman.gcp.utils import run, GCPError
from cloudman.gcp.constants import MACHINE_TYPES, GPU_TYPES, GPU_DEFAULT_MACHINE

def get_gpu(name):
    """Get full GPU name from a valid shorthand.

    See https://cloud.google.com/compute/docs/gpus/#introduction for a full list.
    """
    if name == 'nogpu':
        return name
    if name in GPU_TYPES:
        return "nvidia-tesla-" + name
    raise GCPError("Invalid GPU type '" + name +
                   "'! Choose one of:\n\t" + ', '.join(GPU_TYPES) + "\n\n" +
                   "See https://cloud.google.com/compute/docs/gpus/#introduction " +
                   "for more details about available GPUs.")

def get_machine(gpu, machine):
    """Get the full machine type from a valid gpu+machine shorthand

    See https://cloud.google.com/compute/docs/machine-types for a full list.
    """
    # Manually specified machine
    if machine != 'auto':
        if machine in MACHINE_TYPES:
            return machine
        # Show error message with machine types
        raise GCPError("Invalid machine type '" + machine + "'! " +
                       "See https://cloud.google.com/compute/docs/machine-types " +
                       "for a list of valid machine types.")
    # Automatic choice
    if gpu in GPU_TYPES:
        return GPU_DEFAULT_MACHINE[gpu]

    # Show error message with GPU types
    raise GCPError("Invalid GPU type '" + gpu +
                   "'! Choose one of:\n\t" + ', '.join(GPU_TYPES) + "\n\n" +
                   "See https://cloud.google.com/compute/docs/gpus/#introduction " +
                   "for more details about available GPUs.")

def resolve_gpu_machine(gpu, machine):
    """Translate the gpu and machine shorthand to GCP types"""
    gpu_str = get_gpu(gpu)
    machine_str = get_machine(gpu, machine)
    return gpu_str, machine_str


# Databricks notebook source
# MAGIC %md
# MAGIC #### Initialization

# COMMAND ----------

###############################################################################
# Create an Azure Service Principal and give it access to our storage account #
###############################################################################
# vedio 25,26,27
# step 1: Create Service Principal account
# - Goto Azure Active Directory > App Registration > New Registration: databricks-course-service-app
# > Copy the 'Application (client) ID' & 'Directory (tenant) ID' into a NOTEPAD
# - Click on Certificates & secrets > New client secret > Description: databricks-course-access-secret
# > Copy the secret value into a NOTEPAD
# step 2: Give storage account access to the Service Principal account
# - Goto Storage accounts > Select it: adfcourseanyistaccdl > Access Control (IAM)
#   > Add role assignment > Storage Blob Data Contributor > User, group, or service principal > Select members
#   > search for the Service Principal account you just created:databricks-course-service-app > select > create

####Crdentials are exposed hence we should use Azure Key vaults to keep our secrets
### Vedio 29

# COMMAND ----------

# MAGIC %md
# MAGIC ### Calling Key Vault Secrete

# COMMAND ----------

# - after creating key vault
#  > copy the client-id, tenant-id, client-secret into the key-vault
# - click on the databricks home page
# - append 'secrets/createScope' to the URL 'https://adb-7576904290643392.12.azuredatabricks.net/?o=7576904290643392#'
#   > https://adb-7576904290643392.12.azuredatabricks.net/?o=7576904290643392#secrets/createScope
# - this will open a form
# Fill into the form as follows
# * Scope Name: adfcourseanyistaccdl-scope
# * Manage Principal: Select > All Users (Select 'Creator' option if it is premium.)
# * DNS Name: Goto Key vaults > course-test-key-vault > properties > copy 'Vault URI'
# * Resource ID: Goto Key vaults > course-test-key-vault > properties > copy 'Resource ID'

# COMMAND ----------

# MAGIC %md
# MAGIC ##### Use the dbutils secrets to manipulate the key vaults

# COMMAND ----------

dbutils.secrets.help()

# COMMAND ----------

# lists the seceret scopes to the sorage accounts
dbutils.secrets.listScopes()

# COMMAND ----------

# we can list the secerets in the 'adfcourseanyistaccdl-scope'
dbutils.secrets.list('adfcourseanyistaccdl-scope')

# COMMAND ----------

# We can also get the secerets like this
# Hence, we see that its tries to hide the secerets so that it does not show in the output
dbutils.secrets.get(scope="adfcourseanyistaccdl-scope", key="databricks-app-client-id")

# COMMAND ----------

# using this can also reviel secrets hence be careful who has access to your notebook
#for x in dbutils.secrets.get(scope="adfcourseanyistaccdl-scope", key="databricks-app-client-id"):
#    print(x)

# COMMAND ----------

## Now the secretes are no longer hard coded
storage_account_name = "adfcourseanyistaccdl"
client_id = dbutils.secrets.get(scope="adfcourseanyistaccdl-scope", key="databricks-app-client-id")
tenant_id = dbutils.secrets.get(scope="adfcourseanyistaccdl-scope", key="databricks-app-tenant-id")
client_secret = dbutils.secrets.get(scope="adfcourseanyistaccdl-scope", key="databricks-app-client-secret")

# COMMAND ----------

# MAGIC %md
# MAGIC #### Setup Configurations

# COMMAND ----------

configs = {
    "fs.azure.account.auth.type": "OAuth",
    "fs.azure.account.oauth.provider.type": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
    "fs.azure.account.oauth2.client.id": f"{client_id}",
    "fs.azure.account.oauth2.client.secret": f"{client_secret}",
    "fs.azure.account.oauth2.client.endpoint": f"https://login.microsoftonline.com/{tenant_id}/oauth2/token"
}

# COMMAND ----------

# MAGIC %md
# MAGIC #### Mount our storage

# COMMAND ----------

# function to mount the containers
def mount_adls(container_name):
    # mount the container
    dbutils.fs.mount(
        source = f'abfss://{container_name}@{storage_account_name}.dfs.core.windows.net/',
        mount_point = f'/mnt/{storage_account_name}/{container_name}',
        extra_configs = configs
    )

# COMMAND ----------

raw_ctnr_name = "raw-ctnr"
proc_ctnr_name = "processed-ctnr"
pres_ctnr_name = "presentation-ctnr"
demo_ctnr_name = "demo-ctnr"

# COMMAND ----------

mount_adls(raw_ctnr_name)

# COMMAND ----------

mount_adls(proc_ctnr_name)

# COMMAND ----------

mount_adls(pres_ctnr_name)

# COMMAND ----------

mount_adls(demo_ctnr_name)

# COMMAND ----------

# MAGIC %md
# MAGIC #### List all files in our mount

# COMMAND ----------

# list the files in our mount
# /mnt/adfcourseanyistaccdl/raw-ctnr
dbutils.fs.ls("/mnt/{}/{}".format(storage_account_name, raw_ctnr_name))

# COMMAND ----------

dbutils.fs.ls("/mnt/{}/{}".format(storage_account_name, proc_ctnr_name))

# COMMAND ----------

dbutils.fs.ls("/mnt/{}/{}".format(storage_account_name, pres_ctnr_name))

# COMMAND ----------

dbutils.fs.ls("/mnt/{}/{}".format(storage_account_name, demo_ctnr_name))

# COMMAND ----------

## you can unmount containers if you want
#dbutils.fs.unmount("/mnt/{}/{}".format(storage_account_name, raw_ctnr_name))
#dbutils.fs.unmount("/mnt/{}/{}".format(storage_account_name, proc_ctnr_name))
#dbutils.fs.unmount("/mnt/{}/{}".format(storage_account_name, pres_ctnr_name))
#dbutils.fs.unmount("/mnt/{}/{}".format(storage_account_name, demo_ctnr_name))

# COMMAND ----------

# MAGIC %md
# MAGIC #### List all the mounts in our workspace

# COMMAND ----------

# list all the mounts in our workspace
display(dbutils.fs.mounts())

# COMMAND ----------


# Programming Exercises
# Question 1 Bug Collector

def main():
	total = 0

	for days in range(1, 8):
		print('Day', days)
		bugs = int(input('How many bugs today: '))
		total += bugs
	print('You have collected', total, 'bugs!')
main()


from __future__ import unicode_literals, print_function
from music21 import corpus
from music21.expressions import Fermata
import audiolazy as lz
import random, operator, sys, time
from functools import reduce

def ks_synth(freq):
  """
  Synthesize the given frequency into a Stream by using a model based on
  Karplus-Strong.
  """
  ks_mem = (sum(lz.sinusoid(x * freq) for x in [1, 3, 9]) +
            lz.white_noise() + lz.Stream(-1, 1)) / 5
  return lz.karplus_strong(freq, memory=ks_mem)

def get_random_choral(log=True):
  """ Gets a choral from the J. S. Bach chorals corpus (in Music21). """
  choral_file = corpus.getBachChorales()[random.randint(0, 399)]
  choral = corpus.parse(choral_file)
  if log:
    print("Chosen choral:", choral.metadata.title)
  return choral

def m21_to_stream(score, synth=ks_synth, beat=90, fdur=2., pad_dur=.5,
                  rate=lz.DEFAULT_SAMPLE_RATE):
  """
  Converts Music21 data to a Stream object.

  Parameters
  ----------
  score :
    A Music21 data, usually a music21.stream.Score instance.
  synth :
    A function that receives a frequency as input and should yield a Stream
    instance with the note being played.
  beat :
    The BPM (beats per minute) value to be used in playing.
  fdur :
    Relative duration of a fermata. For example, 1.0 ignores the fermata, and
    2.0 (default) doubles its duration.
  pad_dur :
    Duration in seconds, but not multiplied by ``s``, to be used as a
    zero-padding ending event (avoids clicks at the end when playing).
  rate :
    The sample rate, given in samples per second.

  """
  # Configuration
  s, Hz = lz.sHz(rate)
  step = 60. / beat * s

  # Creates a score from the music21 data
  score = reduce(operator.concat,
                 [[(pitch.frequency * Hz, # Note
                    note.offset * step, # Starting time
                    note.quarterLength * step, # Duration
                    Fermata in note.expressions) for pitch in note.pitches]
                                                 for note in score.flat.notes]
                )

  # Mix all notes into song
  song = lz.Streamix()
  last_start = 0
  for freq, start, dur, has_fermata in score:
    delta = start - last_start
    if has_fermata:
      delta *= 2
    song.add(delta, synth(freq).limit(dur))
    last_start = start

  # Zero-padding and finishing
  song.add(dur + pad_dur * s, lz.Stream([]))
  return song

# Play the song!
if __name__ == "__main__":
  api = next(arg for arg in sys.argv[1:] + [None] if arg != "loop")
  lz.chunks.size = 1 if api == "jack" else 16
  rate = 44100
  while True:
    with lz.AudioIO(True, api=api) as player:
      player.play(m21_to_stream(get_random_choral(), rate=rate), rate=rate)
    if not "loop" in sys.argv[1:]:
      break
    time.sleep(3)


#!/usr/bin/env python
#-*- coding: utf-8 -*-

urls = (
    '/' 						,	'Index',
    '/studentlist/(.+)/(.+)'	,	'GetStudentList',
    '/class/(.+)'				,	'GetAClassStudent',
    '/deletestudent/(.+)'		,	'DeleteStudent',
    '/studentinfo/(.+)'			,	'GetStudentInfo',
    '/highlevelinfo/(.+)'       ,   'GetHighLevelInfo',
    '/editstudentinfo/(.+)'		,	'EditStudentInfo',
    '/savestudentinfo'			,	'SaveStudentInfo',
    '/addstudent'				,	'AddStudent',
    '/savestudent'				,	'SaveStudent',
    '/search'					,	'Search',
    '/addmeeting'               ,   'AddMeeting',
    '/savemeeting'              ,   'SaveMeeting',
    '/meetinglist'              ,   'GetMeetingList',
    '/addnote'                  ,   'AddNote',
    '/savenote'                 ,   'SaveNote',
    '/notelist'                 ,   'GetNoteList',
    '/sendmailto/(.+)'          ,   'NewEmail'  ,
    '/sendmail'                 ,   'SendEmail',
    '/login'                    ,   'Login',
    '/logout'                   ,   'Logout'

)


from click import Context, style

from samcli.cli.core.command import CoreCommand
from samcli.cli.row_modifiers import RowDefinition, ShowcaseRowModifier
from samcli.commands.logs.core.formatters import LogsCommandHelpTextFormatter
from samcli.commands.logs.core.options import OPTIONS_INFO

COL_SIZE_MODIFIER = 38

class LogsCommand(CoreCommand):
    class CustomFormatterContext(Context):
        formatter_class = LogsCommandHelpTextFormatter

    context_class = CustomFormatterContext

    @staticmethod
    def format_examples(ctx: Context, formatter: LogsCommandHelpTextFormatter):
        with formatter.indented_section(name="Examples", extra_indents=1):
            with formatter.indented_section(
                name="Fetch logs with Lambda Function Logical ID and Cloudformation Stack Name"
            ):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(f"$ {ctx.command_path} -n HelloWorldFunction --stack-name mystack"),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )
            with formatter.indented_section(name="View logs for specific time range"):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(
                                f"$ {ctx.command_path} -n HelloWorldFunction --stack-name mystack -s "
                                f"'10min ago' -e '2min ago'"
                            ),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )
            with formatter.indented_section(name="Tail new logs"):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(f"$ {ctx.command_path} -n HelloWorldFunction --stack-name " f"mystack --tail"),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )
            with formatter.indented_section(name="Fetch from Cloudwatch log groups"):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(
                                f"$ {ctx.command_path} --cw-log-group /aws/lambda/myfunction-123 "
                                f"--cw-log-group /aws/lambda/myfunction-456"
                            ),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )

            with formatter.indented_section(name="Fetch logs from supported resources in Cloudformation stack"):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(f"$ {ctx.command_path} ---stack-name mystack"),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )

            with formatter.indented_section(name="Fetch logs from resource defined in nested Cloudformation stack"):
                formatter.write_rd(
                    [
                        RowDefinition(
                            text="\n",
                        ),
                        RowDefinition(
                            name=style(
                                f"$ {ctx.command_path} ---stack-name mystack -n MyNestedStack/HelloWorldFunction"
                            ),
                            extra_row_modifiers=[ShowcaseRowModifier()],
                        ),
                    ]
                )

    def format_options(self, ctx: Context, formatter: LogsCommandHelpTextFormatter) -> None:  # type:ignore
        # `ignore` is put in place here for mypy even though it is the correct behavior,
        # as the `formatter_class` can be set in subclass of Command. If ignore is not set,
        # mypy raises argument needs to be HelpFormatter as super class defines it.

        self.format_description(formatter)
        LogsCommand.format_examples(ctx, formatter)

        CoreCommand._format_options(
            ctx=ctx,
            params=self.get_params(ctx),
            formatter=formatter,
            formatting_options=OPTIONS_INFO,
            write_rd_overrides={"col_max": COL_SIZE_MODIFIER},
        )


## Modules
import iio
import numpy as np
import os
from skimage.metrics import structural_similarity
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

##Functions

def build_variance_map(u1, u2, u3, u4, u5, u6, u7, u8, s1, s2, s3, s4, s5, s6, s7, s8):
    variance_map = s1*torch.tensor(u1) + s2*torch.tensor(u2) + s3*torch.tensor(u3) + s4*torch.tensor(u4) + s5*torch.tensor(u5) + s6*torch.tensor(u6) + s7*torch.tensor(u7) + s8*torch.tensor(u8)
    variance_map = torch.unsqueeze(variance_map, 0)
    variance_map = torch.unsqueeze(variance_map, 0)
    variance_map = variance_map.cuda()
    return variance_map

check_string = lambda string : all(c in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] for c in string)

def create_parameter(value):
    param = Variable(torch.tensor([value], requires_grad = True).cuda())
    param.requires_grad = True
    param = torch.nn.Parameter(param)
    return param

def find_brightness(u):
    brightness = give_brightness(u)
    u1 = (brightness < 0.125 )
    u2 = (brightness >= 0.125) * (brightness < 0.25 )
    u3 = (brightness >= 0.25 ) * (brightness < 0.375)
    u4 = (brightness >= 0.375) * (brightness < 0.5  )
    u5 = (brightness >= 0.5  ) * (brightness < 0.625)
    u6 = (brightness >= 0.625) * (brightness < 0.75 )
    u7 = (brightness >= 0.75 ) * (brightness < 0.875)
    u8 = (brightness >= 0.875)
    return(u1, u2, u3, u4, u5, u6, u7, u8)

def gives_flow(path_flow, H, W):
    flow = read_flow(path_flow)[:H, :W]
    flow = np.expand_dims(flow, 0)
    flow = Variable(torch.Tensor(flow))
    flow = flow.permute(0, 3, 1, 2)
    return flow
give_brightness = lambda x : torch.mean(x, 0) #average on the three channels

def gives_masks(path_collition, path_warping_error_based, H, W):
    if check_string(path_collition):
        mask1 = np.ones((H, W, 1))
        exclusive_mask1 = np.ones((H, W, 1))
    else:
        mask0 = iio.read(path_collition)[:H, :W]
        mask1 = 1. * (mask0 < 1.5)*(mask0 > 0.5)
        mask1 = 1. - (mask1 > 0)
        exclusive_mask1 = 1. * (mask0 >= 1.5)
        exclusive_mask1 = 1. - exclusive_mask1
    if check_string(path_warping_error_based):
        mask0 = np.ones((H, W, 1))
        exclusive_mask_warping_res = np.ones((H, W, 1))
    else:
        mask_warping_res = iio.read(path_warping_error_based)[:H, :W]
        mask0 = 1. * (mask_warping_res < 1.5)*(mask_warping_res > 0.5)
        exclusive_mask_warping_res = 1. * (mask_warping_res >= 1.5)
        exclusive_mask_warping_res = 1. - exclusive_mask_warping_res

    exclusive_mask = exclusive_mask1 * exclusive_mask_warping_res
    mask = mask1 * mask0

    mask = mask.transpose(2,0,1)
    exclusive_mask = exclusive_mask.transpose(2,0,1)
    mask = np.expand_dims(mask, 0)
    mask = torch.Tensor(mask).view(1,1,H,W).repeat(1,3,1,1).cuda()
    exclusive_mask = np.expand_dims(exclusive_mask, 0)
    exclusive_mask = torch.Tensor(exclusive_mask).view(1,1,H,W).repeat(1,3,1,1).cuda()

    return mask, exclusive_mask

learning_rate = lambda x : 10**-5 * np.exp(-5*x)

def psnr(img1, img2, peak=1):
    '''
    Computes the PSNR 'metric' between two images assumed to be in the range [0,1]
    '''
    x = ((np.array(img1).squeeze() - np.array(img2).squeeze()).flatten())
    return (10*np.log10(peak**2 / np.mean(x**2)))

def read_flow(file):
    assert type(file) is str, "file is not str %r" % str(file)
    assert os.path.isfile(file)is True, "file does not exist %r" % str(file)
    assert file[-4:] == '.flo', "file ending is not .flo %r" % file[:-4]
    f = open(file, 'rb')
    flo_number = np.fromfile(f, np.float32, count = 1)[0]
    assert flo_number == 202021.25, "Flow number %r incorrect. Invalid .flo file" % flo_number
    w = np.fromfile(f, np.int32, count = 1)
    h = np.fromfile(f, np.int32, count = 1)
    data = np.fromfile(f, np.float32, count = 2*w[0]*h[0])
    flow = np.resize(data, (int(h), int(w), 2))
    f.close()
    return flow

def reads_image(path, H, W):
    image = iio.read(path)[:H, :W]
    image = image / 255.
    image = image.transpose(2,0,1)
    image = torch.Tensor(image).cuda()
    image = Variable(image)
    return image

def save_image(path, img):
    img = image.detach().cpu().numpy.squeeze()
    img = img.transpose(1,2,0)
    img = (img*255).astype(np.uint8)
    iio.write(path, img)

ssim = lambda x, y : structural_similarity(x.astype(float),y.astype(float),multichannel=True, data_range=255)

def temp_denoise(model, noisyframe, sigma_noise):
    '''Encapsulates call to denoising model and handles padding.
        Expects noisyframe to be normalized in [0., 1.]
    '''
    # make size a multiple of four (we have two scales in the denoiser)
    sh_im = noisyframe.size()
    expanded_h = sh_im[-2]%4
    if expanded_h:
        expanded_h = 4-expanded_h
    expanded_w = sh_im[-1]%4
    if expanded_w:
        expanded_w = 4-expanded_w
    padexp = (0, expanded_w, 0, expanded_h)
    noisyframe = F.pad(input=noisyframe, pad=padexp, mode='reflect')
    sigma_noise = F.pad(input=sigma_noise, pad=padexp, mode='reflect')

    # denoise
    out = model(noisyframe, sigma_noise)

    if expanded_h:
        out = out[:, :, :-expanded_h, :]
    if expanded_w:
        out = out[:, :, :, :-expanded_w]

    return out

def temp_denoise_8_sigmas(model, noisyframe, sigma_noise1, sigma_noise2, sigma_noise3):
    '''Encapsulates call to denoising model and handles padding.
        Expects noisyframe to be normalized in [0., 1.]
    '''
    # make size a multiple of four (we have two scales in the denoiser)
    sh_im = noisyframe.size()
    expanded_h = sh_im[-2]%4
    if expanded_h:
        expanded_h = 4-expanded_h
    expanded_w = sh_im[-1]%4
    if expanded_w:
        expanded_w = 4-expanded_w
    padexp = (0, expanded_w, 0, expanded_h)
    noisyframe = F.pad(input=noisyframe, pad=padexp, mode='reflect')
    sigma_noise1 = F.pad(input=sigma_noise1, pad=padexp, mode='reflect')
    sigma_noise2 = F.pad(input=sigma_noise2, pad=padexp, mode='reflect')
    sigma_noise3 = F.pad(input=sigma_noise3, pad=padexp, mode='reflect')

    # denoise
    out = model(noisyframe, sigma_noise1, sigma_noise2, sigma_noise3)

    if expanded_h:
        out = out[:, :, :-expanded_h, :]
    if expanded_w:
        out = out[:, :, :, :-expanded_w]

    return out

def tensor_to_image(tensor):
    image = torch.clamp(tensor, 0, 1)
    image = np.array(image.cpu())
    image = np.squeeze(image)
    image = image.transpose(1,2,0)
    return image


import os

def get_doc_html(id):
    try:
        path = os.environ.get('DOCS_PATH', '../data/docs/') + id + '.html'
        with open(path, 'r') as f:
            return f.read()
    except IOError:
        return None


from flask_app import app
from flask import Flask, render_template, redirect, session, request, flash
from flask_app.models.dislike import Dislikes
from flask_app.models.user import User
from flask_app.models.userDislike import User_Dislikes

@app.route('/dislike/create/', methods=['POST'])
def createDislike():
    data = {
        'name': request.form['name'],
        'description': request.form['description'],
        'user_id': request.form['user_id'],
    }
    Dislikes.save(data)
    flash('Dislike created')
    print("printing data save from note controller: ", data)
    return redirect('/dashboard/')

@app.route('/dislike/agree/', methods=['POST'])
def userDislikeAgree():
    data = {
        'user_id': request.form['user_id'],
        'dislikes_id': request.form['dislikes_id'],
        'agreeCount': request.form['agreeCount']
    }
    User_Dislikes.saveAgree(data)
    flash('You agreed with something someone dislikes')
    print('printing data from userDislikeAgree controller: ', data)
    return redirect('/dashboard/')

@app.route('/dislike/disagree/', methods=['POST'])
def userDislikeDisagree():
    data = {
        'user_id': request.form['user_id'],
        'dislikes_id': request.form['dislikes_id'],
        'disagreeCount': request.form['disagreeCount']
    }
    User_Dislikes.saveDisagree(data)
    flash('You disagreed with something someone dislikes')
    print('printing data from userDislikeDisagree controller: ', data)
    return redirect('/dashboard/')


# Extension: use complete sentences + padding as input size
import doc_methods

doc = doc_methods.load_doc("shakespeare_input.txt")
# clean document and cut down size
tokens = doc_methods.clean_doc(doc[:int(len(doc)/4)])
print('Total Tokens: %d' % len(tokens))
print('Unique Tokens: %d' % len(set(tokens)))
# organize into sequences of tokens
length = 50 + 1
sequences = list()
for i in range(length, len(tokens)):
    # select sequence of tokens
    seq = tokens[i - length:i]
    # convert into a line
    line = ' '.join(seq)
    # store
    sequences.append(line)
print('Total Sequences: %d' % len(sequences))

# save sequences to file
out_filename = 'ss_words.txt'
doc_methods.save_doc(sequences, out_filename)


import cv2
import numpy as np
import face_recognition
import os

path = "IMGD"
img_Prifix = []
chek = []
images = []
img_List = os.listdir(path)
for x in img_List:

    ImgP = cv2.imread(f'{path}/{x}')
    images.append(ImgP)
    img_Prifix.append(os.path.splitext(x)[0])

# Finding Arrays

def encode(imgenco):
    encode_list = []
    for imgs in imgenco:
        img2_bgr = cv2.cvtColor(imgs, cv2.COLOR_BGR2RGB)
        encoded = face_recognition.face_encodings(img2_bgr)[0]
        encode_list.append(encoded)
        return encode_list

encoded_L = encode(images)
print("Encoding Completed")

# RecogniZer
cap = cv2.VideoCapture(0)

while True:
    success, img = cap.read()
    imgSsize = cv2.resize(img, (0, 0), None, 0.25, 0.25)
    imgSsize = cv2.cvtColor(imgSsize, cv2.COLOR_BGR2RGB)

    face_detection_Live = face_recognition.face_locations(imgSsize)
    encoded_Live_img = face_recognition.face_encodings(imgSsize, face_detection_Live)

    for encodeFace, faceLoc in zip(encoded_Live_img, face_detection_Live):

        matches = face_recognition.compare_faces(encoded_L, encodeFace)
        face_Dis = face_recognition.face_distance(encoded_L, encodeFace)
        print(face_Dis)
        matchIndex = np.argmin(face_Dis)

        if matches[matchIndex]:
            chek.append(1)
            name = img_Prifix[matchIndex].upper()[0]
            print(name)

            y1, x2, y2, x1 = faceLoc
            y1, x2, y2, x1 = y1 * 4, x2 * 4, y2 * 4, x1 * 4
            cv2.rectangle(img, (x1, y1), (x2, y2), (0, 255, 0), 2)
            cv2.rectangle(img, (x1, y2 - 35), (x2, y2), (0, 255, 0), cv2.FILLED)
            cv2.putText(img, name, (x1 + 6, y2 - 6), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)

        else:
            chek.append(0)

    cv2.imshow('Recognising...', img)
    k = cv2.waitKey(1)
    # press ESC to exit the camera view

    if k % 256 == 27:
        break

    elif any(chek) == 1:
        os.system('Application root')
        print("App started")
        break
    elif len(chek) == 5:
        break

cap.release()
cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
# @Time : 2021/10/18 下午1:55
# @Author: yl
# @File: dataset.py

import os

import torch
import torch.utils.data as data
from typing import Any, Callable, List, Optional, Tuple

from torchvision.datasets import ImageFolder
from PIL import Image

from typing import Any, Callable, cast, Dict, List, Optional, Tuple

# Copying from torchvision
class VisionDataset(data.Dataset):
    _repr_indent = 4

    def __init__(
            self,
            root: str,
            transforms: Optional[Callable] = None,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
    ) -> None:
        if isinstance(root, torch._six.string_classes):
            root = os.path.expanduser(root)
        self.root = root

        has_transforms = transforms is not None
        has_separate_transform = transform is not None or target_transform is not None
        if has_transforms and has_separate_transform:
            raise ValueError("Only transforms or transform/target_transform can "
                             "be passed as argument")

        # for backwards-compatibility
        self.transform = transform
        self.target_transform = target_transform

        if has_separate_transform:
            transforms = StandardTransform(transform, target_transform)
        self.transforms = transforms

    def __getitem__(self, index: int) -> Any:
        raise NotImplementedError

    def __len__(self) -> int:
        raise NotImplementedError

    def __repr__(self) -> str:
        head = "Dataset " + self.__class__.__name__
        body = ["Number of datapoints: {}".format(self.__len__())]
        if self.root is not None:
            body.append("Root location: {}".format(self.root))
        body += self.extra_repr().splitlines()
        if hasattr(self, "transforms") and self.transforms is not None:
            body += [repr(self.transforms)]
        lines = [head] + [" " * self._repr_indent + line for line in body]
        return '\n'.join(lines)

    def _format_transform_repr(self, transform: Callable, head: str) -> List[str]:
        lines = transform.__repr__().splitlines()
        return (["{}{}".format(head, lines[0])] +
                ["{}{}".format(" " * len(head), line) for line in lines[1:]])

    def extra_repr(self) -> str:
        return ""

class StandardTransform(object):
    def __init__(self, transform: Optional[Callable] = None, target_transform: Optional[Callable] = None) -> None:
        self.transform = transform
        self.target_transform = target_transform

    def __call__(self, input: Any, target: Any) -> Tuple[Any, Any]:
        if self.transform is not None:
            input = self.transform(input)
        if self.target_transform is not None:
            target = self.target_transform(target)
        return input, target

    def _format_transform_repr(self, transform: Callable, head: str) -> List[str]:
        lines = transform.__repr__().splitlines()
        return (["{}{}".format(head, lines[0])] +
                ["{}{}".format(" " * len(head), line) for line in lines[1:]])

    def __repr__(self) -> str:
        body = [self.__class__.__name__]
        if self.transform is not None:
            body += self._format_transform_repr(self.transform,
                                                "Transform: ")
        if self.target_transform is not None:
            body += self._format_transform_repr(self.target_transform,
                                                "Target transform: ")

        return '\n'.join(body)

IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')

def has_file_allowed_extension(filename: str, extensions: Tuple[str, ...]) -> bool:
    """Checks if a file is an allowed extension.

    Args:
        filename (string): path to a file
        extensions (tuple of strings): extensions to consider (lowercase)

    Returns:
        bool: True if the filename ends with one of given extensions
    """
    return filename.lower().endswith(extensions)

def is_image_file(filename: str) -> bool:
    """Checks if a file is an allowed image extension.

    Args:
        filename (string): path to a file

    Returns:
        bool: True if the filename ends with a known image extension
    """
    return has_file_allowed_extension(filename, IMG_EXTENSIONS)

def make_dataset(
        directory: str,
        class_to_idx: Dict[str, int],
        extensions: Optional[Tuple[str, ...]] = None,
        is_valid_file: Optional[Callable[[str], bool]] = None,
) -> List[Tuple[str, int]]:
    """Generates a list of samples of a form (path_to_sample, class).

    Args:
        directory (str): root dataset directory
        class_to_idx (Dict[str, int]): dictionary mapping class name to class index
        extensions (optional): A list of allowed extensions.
            Either extensions or is_valid_file should be passed. Defaults to None.
        is_valid_file (optional): A function that takes path of a file
            and checks if the file is a valid file
            (used to check of corrupt files) both extensions and
            is_valid_file should not be passed. Defaults to None.

    Raises:
        ValueError: In case ``extensions`` and ``is_valid_file`` are None or both are not None.

    Returns:
        List[Tuple[str, int]]: samples of a form (path_to_sample, class)
    """
    instances = []
    directory = os.path.expanduser(directory)
    both_none = extensions is None and is_valid_file is None
    both_something = extensions is not None and is_valid_file is not None
    if both_none or both_something:
        raise ValueError("Both extensions and is_valid_file cannot be None or not None at the same time")
    if extensions is not None:
        def is_valid_file(x: str) -> bool:
            return has_file_allowed_extension(x, cast(Tuple[str, ...], extensions))
    is_valid_file = cast(Callable[[str], bool], is_valid_file)
    for target_class in sorted(class_to_idx.keys()):
        class_index = class_to_idx[target_class]
        target_dir = os.path.join(directory, target_class)
        if not os.path.isdir(target_dir):
            continue
        for root, _, fnames in sorted(os.walk(target_dir, followlinks=True)):
            for fname in sorted(fnames):
                path = os.path.join(root, fname)
                if is_valid_file(path):
                    item = path, class_index
                    instances.append(item)
    return instances

def pil_loader(path: str) -> Image.Image:
    # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835)
    with open(path, 'rb') as f:
        img = Image.open(f)
        return img.convert('RGB')

# TODO: specify the return type
def accimage_loader(path: str) -> Any:
    import accimage
    try:
        return accimage.Image(path)
    except IOError:
        # Potentially a decoding problem, fall back to PIL.Image
        return pil_loader(path)

def default_loader(path: str) -> Any:
    from torchvision import get_image_backend
    if get_image_backend() == 'accimage':
        return accimage_loader(path)
    else:
        return pil_loader(path)

class DatasetFolder(VisionDataset):
    """A generic data loader where the samples are arranged in this way: ::

        root/class_x/xxx.ext
        root/class_x/xxy.ext
        root/class_x/[...]/xxz.ext

        root/class_y/123.ext
        root/class_y/nsdf3.ext
        root/class_y/[...]/asd932_.ext

    Args:
        root (string): Root directory path.
        loader (callable): A function to load a sample given its path.
        extensions (tuple[string]): A list of allowed extensions.
            both extensions and is_valid_file should not be passed.
        transform (callable, optional): A function/transform that takes in
            a sample and returns a transformed version.
            E.g, ``transforms.RandomCrop`` for images.
        target_transform (callable, optional): A function/transform that takes
            in the target and transforms it.
        is_valid_file (callable, optional): A function that takes path of a file
            and check if the file is a valid file (used to check of corrupt files)
            both extensions and is_valid_file should not be passed.

     Attributes:
        classes (list): List of the class names sorted alphabetically.
        class_to_idx (dict): Dict with items (class_name, class_index).
        samples (list): List of (sample path, class_index) tuples
        targets (list): The class_index value for each image in the dataset
    """

    def __init__(
            self,
            root: str,
            loader: Callable[[str], Any],
            extensions: Optional[Tuple[str, ...]] = None,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            is_valid_file: Optional[Callable[[str], bool]] = None,
    ) -> None:
        super(DatasetFolder, self).__init__(root, transform=transform,
                                            target_transform=target_transform)
        classes, class_to_idx = self._find_classes(self.root)
        samples = self.make_dataset(self.root, class_to_idx, extensions, is_valid_file)
        if len(samples) == 0:
            msg = "Found 0 files in subfolders of: {}\n".format(self.root)
            if extensions is not None:
                msg += "Supported extensions are: {}".format(",".join(extensions))
            raise RuntimeError(msg)

        self.loader = loader
        self.extensions = extensions

        # 0xx_name is a class, when save in class variable, only name are preserved.

        # self.classes = classes
        self.classes = ['_'.join(cls.split('_')[1:]) if '_' in cls else cls for cls in classes]
        self.class_to_idx = class_to_idx
        self.samples = samples
        self.targets = [s[1] for s in samples]

    @staticmethod
    def make_dataset(
            directory: str,
            class_to_idx: Dict[str, int],
            extensions: Optional[Tuple[str, ...]] = None,
            is_valid_file: Optional[Callable[[str], bool]] = None,
    ) -> List[Tuple[str, int]]:
        return make_dataset(directory, class_to_idx, extensions=extensions, is_valid_file=is_valid_file)

    def _find_classes(self, dir: str) -> Tuple[List[str], Dict[str, int]]:
        """
        Finds the class folders in a dataset.

        Args:
            dir (string): Root directory path.

        Returns:
            tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary.

        Ensures:
            No class is a subdirectory of another.
        """
        classes = [d.name for d in os.scandir(dir) if d.is_dir()]
        classes.sort()
        unique_classes = [s.split('_')[0] for s in classes]
        unique_classes = list(set(unique_classes))
        unique_classes.sort()
        unique_classes_to_idx = {unique_classes[i]: i for i in range(len(unique_classes))}
        # class_to_idx = {cls_name: i for i, cls_name in enumerate(classes)}
        class_to_idx = {}
        j = 0
        k = 0
        main_classes = []
        for i in range(len(classes)):
            if not unique_classes[j] in classes[i]:
                j += 1
                k = 0
            class_to_idx[classes[i]] = unique_classes_to_idx[unique_classes[j]]
            k += 1
            if k == 1:  # the next class name
                tmp = classes[i]
                main_classes.append(classes[i])
            else:  # k >1, same class with different directory, keep the smallest class name
                tmp = classes[i] if len(classes[i]) < len(tmp) else tmp
                main_classes[-1] = tmp

        return main_classes, class_to_idx

    def __getitem__(self, index: int) -> Tuple[Any, Any]:
        """
        Args:
            index (int): Index

        Returns:
            tuple: (sample, target) where target is class_index of the target class.
        """
        path, target = self.samples[index]
        sample = self.loader(path)
        if self.transform is not None:
            sample = self.transform(sample)
        if self.target_transform is not None:
            target = self.target_transform(target)

        return sample, target

    def __len__(self) -> int:
        return len(self.samples)

class ImageFolder(DatasetFolder):
    """A generic data loader where the images are arranged in this way: ::

        root/dog/xxx.png
        root/dog/xxy.png
        root/dog/[...]/xxz.png

        root/cat/123.png
        root/cat/nsdf3.png
        root/cat/[...]/asd932_.png

    Args:
        root (string): Root directory path.
        transform (callable, optional): A function/transform that  takes in an PIL image
            and returns a transformed version. E.g, ``transforms.RandomCrop``
        target_transform (callable, optional): A function/transform that takes in the
            target and transforms it.
        loader (callable, optional): A function to load an image given its path.
        is_valid_file (callable, optional): A function that takes path of an Image file
            and check if the file is a valid file (used to check of corrupt files)

     Attributes:
        classes (list): List of the class names sorted alphabetically.
        class_to_idx (dict): Dict with items (class_name, class_index).
        imgs (list): List of (image path, class_index) tuples
    """

    def __init__(
            self,
            root: str,
            transform: Optional[Callable] = None,
            target_transform: Optional[Callable] = None,
            loader: Callable[[str], Any] = default_loader,
            is_valid_file: Optional[Callable[[str], bool]] = None,
    ):
        super(ImageFolder, self).__init__(root, loader, IMG_EXTENSIONS if is_valid_file is None else None,
                                          transform=transform,
                                          target_transform=target_transform,
                                          is_valid_file=is_valid_file)
        self.imgs = self.samples


import geopandas as gpd
import pickle

def getPolyCoords(row, geom, coord_type, boundaries):
    """Returns the coordinates ('x' or 'y') of edges of a Polygon exterior and checks that it is in boundary"""
    if row[geom]:
        # Parse the exterior of the coordinate
        if row[geom].type == 'Polygon':
            exterior = row[geom].exterior
        elif row[geom].type == 'MultiPolygon':
            exterior = row[geom][0].exterior

        x_range = exterior.coords.xy[0]
        y_range = exterior.coords.xy[1]

        if coord_type == 'x':
            if max(x_range) > boundaries[2] or min(x_range) < boundaries[3]:
                return 0
            else:
                # Return 1 if in range
                return 1
        elif coord_type == 'y':
            if max(y_range) > boundaries[0] or min(y_range) < boundaries[1]:
                return 0
            else:
                # Return 1 if in range
                return 1

state = 'vic'
shp = 'SSC_2016_AUST.shp'

# Defining boundary: http://boundingbox.klokantech.com
if state == 'vic':
    # boundary = [-37.54, -38.27, 145.37, 144.63]
    boundary = [-37.24, -38.71, 146.72, 143.63]
elif state == 'nsw':
    boundary = [-33.663, -34.006, 151.342, 150.785]
elif state == 'sa':
    boundary = [-34.470, -35.056, 139.019, 138.236]
else:
    raise NameError('Boundary Conditions not defined for state ' + state)

out = state + ' filtered.p'

try:
    g_df = pickle.load(open(out, "rb"))
except Exception:
    # Read files
    grid = gpd.read_file(shp)

    # Get the Polygon x and y coordinates
    grid['x'] = grid.apply(getPolyCoords, geom='geometry', coord_type='x', boundaries=boundary, axis=1)
    grid['y'] = grid.apply(getPolyCoords, geom='geometry', coord_type='y', boundaries=boundary, axis=1)

    grid['val'] = grid.x * grid.y
    filtered_df = grid.loc[grid.val == 1]
    pickle.dump(filtered_df, open(out, 'wb'))


import re
import requests
import json
from datetime import datetime

try:
    import rb_exceptions
    import soup as soup_helper
except ImportError:  # No implicit package imports in py3.
    from ratebeer import rb_exceptions
    from ratebeer import soup as soup_helper

class Beer(object):
    """The Beer object. Contains information about an individual beer.

    Args:
        url (string): the URL of the beer you're looking for.

    Returns:
        abv (float): percentage alcohol
        brewery (Brewery object): the beer's brewery
        brewed_at (Brewery object): actual brewery if contract brewed
        calories (float): estimated calories for the beer
        description (string): the beer's description
        id (int): id of the beer
        img_url (string): a url to an image of the beer
        mean_rating (float): the mean rating for the beer (out of 5)
        name (string): the full name of the beer (may include the brewery name)
        num_ratings (int): the number of reviews*
        overall_rating (int): the overall rating (out of 100)
        seasonal (string): Summer, Winter, Autumn, Spring, Series, Special, None
        style (string): beer style
        style_url (string): beer style URL
        style_rating (int): rating of the beer within its style (out of 100)
        tags (list of strings): tags given to the beer
        url (string): the beer's url
        weighted_avg (float): the beer rating average, weighted using some unknown
            algorithm (out of 5)

        Any attributes not available will be returned as None

    """
    def __init__(self, url, fetch=None, id=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        self.id = id
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Beer('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _format(self, value):
        """Sets what is now a blank string or int to None, otherwise returns value"""
        if value == "" or value == 0:
            return None
        else:
            return value

    def _populate(self):
        if not self.id:
            self.id = self.url.split('/')[-2]

        data = [
                 {"operationName":"beer",
                  "variables":{"beerId":self.id},
                  "query":"query beer($beerId: ID!) { \n info: beer(id: $beerId) { \n id \n name \n description \n style { \n id \n name \n glasses { \n id \n name \n __typename \n } \n __typename \n } \n styleScore \n overallScore \n averageRating \n abv \n ibu \n calories \n brewer { \n id \n name \n __typename \n } \n ratingCount \n isRetired \n isUnrateable \n seasonal \n labels \n availability { \n bottle \n tap \n distribution \n __typename \n } \n __typename \n } \n} \n"},
                 # {"operationName":"beerReviews",
                 #  "variables":
                 #   {"beerId":self.id,
                 #    "order":"RECENT",
                 #    "first":10
                 #   },
                 #  "query":"query beerReviews($beerId: ID!, $authorId: ID, $order: ReviewOrder, $after: ID) { \n beerReviewsArr: beerReviews(beerId: $beerId, authorId: $authorId, order: $order, after: $after) { \n items { \n id \n comment \n score \n scores { \n appearance \n aroma \n flavor \n mouthfeel \n overall \n __typename \n } \n author { \n id \n username \n reviewCount \n __typename \n } \n checkin { \n id \n place { \n name \n city \n state { \n name \n __typename \n } \n country { \n name \n __typename \n } \n __typename \n } \n __typename \n } \n createdAt \n updatedAt \n __typename \n } \n totalCount \n last \n __typename \n } \n} \n"},
                 {"operationName":"beerByAlias",
                  "variables":{"aliasId":self.id},
                  "query":"query beerByAlias($aliasId: ID!) {\n beerByAlias(aliasId: $aliasId) {\n id\n name \n overallScore \n __typename \n } \n } \n"},
                 {"operationName":"tagDisplay",
                  "variables":{"beerId":self.id},
                  "query":"query tagDisplay($beerId: ID!, $first: Int) { \n tagDisplayArr: beerTags(beerId: $beerId, first: $first) { \n items { \n id \n urlName: plain \n __typename \n } \n __typename \n } \n} \n"
                 }
                ]

        request = requests.post(
            "https://beta.ratebeer.com/v1/api/graphql/"
           ,data=json.dumps(data)
           ,headers={"content-type": "application/json"}
        )

        try:
            results = json.loads(request.text)
        except:
            raise rb_exceptions.JSONParseException(self.id)

        beer_data = results[0]['data']['info']

        if beer_data == None:
            raise rb_exceptions.PageNotFound(self.id)

        alias_data = results[1]['data']['beerByAlias']

        if alias_data != None:
            raise rb_exceptions.AliasedBeer(self.id, alias_data['id'])

        tag_data = results[2]['data']['tagDisplayArr']['items']

        self.name = beer_data['name']
        self.brewery = Brewery('/brewers/{0}/{1}/'.format(re.sub('[/ ]','-',beer_data['brewer']['name'].lower()),beer_data['brewer']['id']))
        self.brewery.name = beer_data['brewer']['name']
        self.brewed_at = None #no longer supported
        self.overall_rating = self._format(beer_data['overallScore'])
        self.style_rating = self._format(beer_data['styleScore'])
        self.style = beer_data['style']['name']
        self.style_url = "/beerstyles/{0}/{1}/".format(re.sub('/','-',self.style.lower()), beer_data['style']['id'])
        self.img_url = "https://res.cloudinary.com/ratebeer/image/upload/w_152,h_309,c_pad,d_beer_img_default.png,f_auto/beer_{0}".format(self.id)
        self.num_ratings = self._format(beer_data['ratingCount'])
        self.mean_rating = self._format(beer_data['averageRating'])
        self.weighted_avg = None # does not appear to exist anymore
        if(beer_data['seasonal'] != 'UNKNOWN'):
            self.seasonal = beer_data['seasonal']
        else:
            self.seasonal = None
        self.ibu = self._format(beer_data['ibu'])
        self.calories = self._format(beer_data['calories'])
        self.abv = self._format(beer_data['abv'])
        self.retired = beer_data['isRetired']
        self.description = re.sub(r'\x92', '\'', beer_data['description'])
        if tag_data:
            self.tags = [t['urlName'] for t in tag_data]
        else:
            self.tags = None

        self._has_fetched = True

        return self

    def get_reviews(self, review_order="most recent"):
        """Returns reviews for a specific beer.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/beer/deschutes-inversion-ipa/55610/"
            review_order (string): How to sort reviews. Three inputs:
                most recent: Newer reviews appear earlier.
                top raters: RateBeer.com top raters appear earlier.
                highest score: Reviews with the highest overall score appear
                earlier.

        Returns:
            A generator of dictionaries, containing the information about the review.
        """

        if not self._has_fetched:
            self._populate()

        review_order = review_order.lower()
        url_codes = {
            "most recent": 1,
            "top raters": 2,
            "highest score": 3
        }
        url_flag = url_codes.get(review_order)
        if not url_flag:
            raise ValueError("Invalid ``review_order``.")

        page_number = 1
        while True:
            complete_url = u'{0}{1}/{2}/'.format(self.url, url_flag, page_number)
            soup = soup_helper._get_soup(complete_url)
            content = soup.find('div', class_='reviews-container')
            reviews = content.find_all('div', style='padding: 0px 0px 0px 0px;')
            if len(reviews) < 1:
                raise StopIteration

            for review_soup in reviews:
                yield Review(review_soup)

            page_number += 1

class Review(object):
    """
    Args:
        review_soup (soup): the soup of the review

    Returns:
        appearance (int): rating for appearance (out of 5)
        aroma (int): aroma rating (out of 10)
        date (datetime): review date
        overall (int): overall rating (out of 20, for some reason)
        palate (int): palate rating (out of 5)
        rating (float): another overall rating provided in the review. Not sure how this different from overall.
        taste (int): taste rating (out of 10)
        text (string): actual text of the review.
        user_location (string): writer's location
        user_name (string): writer's username
    """

    def __init__(self, review_soup):
        # gets every second entry in a list
        review_title_attr = review_soup.find_all('div')[1].get('title')

        # some ratings may now just contain the x/5.0 rating, with no sub-ratings
        if '' in review_title_attr:
            raw_ratings = re.search(r'(.+?)', review_title_attr).group(1).split('
')
            # strip html and everything else
            for rating_text in raw_ratings:
                parts = rating_text.split(' ')
                # only set a rating if all of the information exists
                if rating_text:
                    label = parts[0]
                    rating_int = int(parts[1][:parts[1].find("/")])
                    setattr(
                        self,
                        label.lower().strip(),
                        rating_int
                    )
        self.rating = float(review_soup.find_all('div')[1].text)

        # get user information
        userinfo = review_soup.next_sibling
        self.text = userinfo.next_sibling.next_sibling.text.strip()
        self.user_name = re.findall(r'(.*?)\xa0\(\d*?\)', userinfo.a.text)[0]
        self.user_location = re.findall(r'-\s(.*?)\s-', userinfo.a.next_sibling)[0]

        # get date it was posted
        date = re.findall(r'-(?:\s.*?\s-)+\s(.*)', userinfo.a.next_sibling)[0]
        self.date = datetime.strptime(date.strip(), '%b %d, %Y').date()

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.text

class Brewery(object):
    def __init__(self, url, fetch=None):
        """Initialize with URL and do not fetch"""
        self.url = url
        if fetch is None:
            fetch = False
        if fetch:
            self._populate()
        self._has_fetched = fetch

    def __getattr__(self, item):
        """Return the value of the `attr` attribute."""
        if item in self.__dict__.keys():
            return self.__dict__[item]
        elif not self._has_fetched:
            self._populate()
            return getattr(self, item)
        raise AttributeError('{0} has no attribute {1}'.format(type(self), item))

    def __setattr__(self, name, value):
        """Set the `name` attribute to `value."""
        object.__setattr__(self, name, value)

    def __getstate__(self):
        """Provide state information for pickling"""
        result = self.__dict__.copy()
        return result

    def __setstate__(self, statedata):
        """Reset the state after pickling"""
        self.__dict__ = statedata

    def __repr__(self):
        """Unambiguous representation to recreate object"""
        return "<Brewery('{0}')>".format(self.url)

    def __str__(self):
        """Provide a nicely formatted representation"""
        return self.name

    def _populate(self):
        """Returns information about a specific brewery.

        Args:
            url (string): The specific url of the beer. Looks like:
                "/brewers/new-belgium-brewing-company/77/"

        Returns:
            A dictionary of attributes about that brewery."""

        soup = soup_helper._get_soup(self.url)
        s_contents = soup.find_all('div', {'itemtype':'http://schema.org/LocalBusiness'})
        if not s_contents:
            raise rb_exceptions.PageNotFound(self.url)

        self.name = soup.h1.text
        self.type = s_contents[0].find_all('div')[1].text.strip()
        website = s_contents[0].find_all('div',{'class':'media-links'})[0].find_all('a')[0]
        if website:
            self.web = website['href']
        self.telephone = Brewery._find_span(s_contents[0], 'telephone')
        self.street = Brewery._find_span(s_contents[0], 'streetAddress')
        self.city = Brewery._find_span(s_contents[0], 'addressLocality')
        self.state = Brewery._find_span(s_contents[0], 'addressRegion')
        self.country = Brewery._find_span(s_contents[0], 'addressCountry')
        self.postal_code = Brewery._find_span(s_contents[0], 'postalCode')
        self._has_fetched = True

        return self

    @staticmethod
    def _find_span(search_soup, item_prop):
        output = search_soup.find('span', attrs={'itemprop': item_prop})
        output = output.text.strip() if output else None
        return output

    def get_beers(self):
        """Generator that provides Beer objects for the brewery's beers"""
        if not self._has_fetched:
            self._populate()

        _id = self.url.split('/')[-2]
        complete_url = u'/Ratings/Beer/ShowBrewerBeers.asp?BrewerID={0}'.format(_id)
        soup = soup_helper._get_soup(complete_url)
        soup_beer_rows = soup.find('table', id='brewer-beer-table').findAll('tr')

        for row in soup_beer_rows[1:]:
            url = row.a.get('href')
            # Only return rows that are ratable
            if not row.find('a',title="Rate this beer"):
                continue
            # Remove any whitespace characters. Rare, but possible.
            url = re.sub(r"\s+", "", url, flags=re.UNICODE)
            beer = Beer(url)
            beer.name = row.a.text.strip()
            # Add attributes from row
            abv = row.findAll('td')[1].text
            weighted_avg = row.findAll('td')[4].text.strip()
            style_rating = row.findAll('td')[5].text.strip()
            num_ratings = row.findAll('td')[6].text.strip()
            if abv and abv != '-':
                beer.abv = float(abv)
            if weighted_avg:
                beer.weighted_avg = float(weighted_avg)
            if style_rating:
                beer.style_rating = int(style_rating)
            if num_ratings:
                beer.num_ratings = int(num_ratings)
            yield beer


"""
The integration for grocy.
"""
import logging
import os
from datetime import timedelta

import voluptuous as vol

import homeassistant.helpers.config_validation as cv
from homeassistant.const import CONF_API_KEY, CONF_URL
from homeassistant.helpers import discovery
from homeassistant.util import Throttle

from .const import (CONF_ENABLED, CONF_NAME, CONF_SENSOR, DEFAULT_NAME, DOMAIN,
                    DOMAIN_DATA, ISSUE_URL, PLATFORMS, REQUIRED_FILES, STARTUP,
                    VERSION)

MIN_TIME_BETWEEN_UPDATES = timedelta(seconds=30)

_LOGGER = logging.getLogger(__name__)

SENSOR_SCHEMA = vol.Schema(
    {
        vol.Optional(CONF_ENABLED, default=True): cv.boolean,
        vol.Optional(CONF_NAME, default=DEFAULT_NAME): cv.string,
    }
)

CONFIG_SCHEMA = vol.Schema(
    {
        DOMAIN: vol.Schema(
            {
                vol.Required(CONF_URL): cv.string,
                vol.Required(CONF_API_KEY): cv.string,
                vol.Optional(CONF_SENSOR): vol.All(cv.ensure_list, [SENSOR_SCHEMA]),
            }
        )
    },
    extra=vol.ALLOW_EXTRA,
)

async def async_setup(hass, config):
    """Set up this component."""
    # Import client from a external python package hosted on PyPi
    from pygrocy import Grocy,TransactionType

    # Print startup message
    startup = STARTUP.format(name=DOMAIN, version=VERSION, issueurl=ISSUE_URL)
    _LOGGER.info(startup)

    # Check that all required files are present
    file_check = await check_files(hass)
    if not file_check:
        return False

    # Create DATA dict
    hass.data[DOMAIN_DATA] = {}

    # Get "global" configuration.
    url = config[DOMAIN].get(CONF_URL)
    api_key = config[DOMAIN].get(CONF_API_KEY)

    # Configure the client.
    grocy = Grocy(url, api_key)
    hass.data[DOMAIN_DATA]["client"] = GrocyData(hass, grocy)

    # Load platforms
    for platform in PLATFORMS:
        # Get platform specific configuration
        platform_config = config[DOMAIN].get(platform, {})

        # If platform is not enabled, skip.
        if not platform_config:
            continue

        for entry in platform_config:
            entry_config = entry
            _LOGGER.critical(entry_config)

            # If entry is not enabled, skip.
            if not entry_config[CONF_ENABLED]:
                continue

            hass.async_create_task(
                discovery.async_load_platform(
                    hass, platform, DOMAIN, entry_config, config
                )
            )

    def handle_add_product(call):
        product_id = call.data['product_id']
        amount = call.data.get('amount', 0)
        price = call.data.get('price', None)
        grocy.add_product(product_id, amount, price)

    hass.services.async_register(DOMAIN, "add_product", handle_add_product)

    def handle_consume_product(call):
        product_id = call.data['product_id']
        amount = call.data.get('amount', 0)
        spoiled = call.data.get('spoiled', False)

        transaction_type_raw = call.data.get('transaction_type', None)
        transaction_type = TransactionType.CONSUME

        if transaction_type_raw is not None:
            transaction_type = TransactionType[transaction_type_raw]
        grocy.consume_product(product_id, amount, spoiled=spoiled, transaction_type=transaction_type)

    hass.services.async_register(DOMAIN, "consume_product", handle_consume_product)

    return True

class GrocyData:
    """This class handle communication and stores the data."""

    def __init__(self, hass, client):
        """Initialize the class."""
        self.hass = hass
        self.client = client

    @Throttle(MIN_TIME_BETWEEN_UPDATES)
    async def update_data(self):
        """Update data."""
        # This is where the main logic to update platform data goes.
        try:
            stock = self.client.stock(get_details=True)
            self.hass.data[DOMAIN_DATA]["stock"] = stock
        except Exception as error:  # pylint: disable=broad-except
            _LOGGER.error("Could not update data - %s", error)

async def check_files(hass):
    """Return bool that indicates if all files are present."""
    # Verify that the user downloaded all files.
    base = "{}/custom_components/{}/".format(hass.config.path(), DOMAIN)
    missing = []
    for file in REQUIRED_FILES:
        fullpath = "{}{}".format(base, file)
        if not os.path.exists(fullpath):
            missing.append(file)

    if missing:
        _LOGGER.critical("The following files are missing: %s", str(missing))
        returnvalue = False
    else:
        returnvalue = True

    return returnvalue


import os
from math import pi

import numpy as NP

import aster
from libaster import Function, FunctionComplex

from ..Utilities import accept_array, injector
from .function_py import t_fonction, t_fonction_c
from .table_graph import Graph

@injector(Function)
class ExtendedFunction(object):
    cata_sdj = "SD.sd_fonction.sd_fonction_aster"

    setValues = accept_array(Function.setValues)

    def abs(self):
        """Return the absolute value of the function.

        Returns:
            (Function): Absolute value of the function.
        """
        new = Function()
        absc, ordo = self.Valeurs()
        new.setValues(absc, NP.abs(ordo))
        return new

    def getValuesAsArray(self):
        """Return the values of the function as a `numpy.array` of shape (N, 2).

        Returns:
            numpy.array: Array containing the values.
        """
        size = self.size()
        values = NP.array(self.getValues())
        values.shape = (2, size)
        return values.transpose()

    def convert(self, arg='real'):
        """
        Retourne un objet de la classe t_fonction
        représentation python de la fonction
        """
        class_fonction = t_fonction
        if arg == 'complex':
            class_fonction = t_fonction_c
        absc, ordo = self.Valeurs()
        return class_fonction(absc, ordo, self.Parametres(), nom=self.getName())

    def Valeurs(self):
        """
        Retourne deux listes de valeurs : abscisses et ordonnees
        """
        values = self.getValuesAsArray()
        return values[:, 0], values[:, 1]

    def Absc(self):
        """Retourne la liste des abscisses"""
        return self.Valeurs()[0]

    def Ordo(self):
        """Retourne la liste des ordonnées"""
        return self.Valeurs()[1]

    def __call__(self, val, tol=1.e-6):
        """Evaluate a function at 'val'. If provided, 'tol' is a relative
        tolerance to match an abscissa value."""
        try:
            val = val.valeur
        except AttributeError:
            val = float(val)
        __ff = self.convert()
        return __ff(val, tol=tol)

    def Parametres(self):
        """
        Retourne un dictionnaire contenant les parametres de la fonction ;
        le type jeveux (FONCTION, FONCT_C, NAPPE) n'est pas retourne,
        le dictionnaire peut ainsi etre fourni a CALC_FONC_INTERP tel quel.
        """
        TypeProl = {'E': 'EXCLU', 'L': 'LINEAIRE', 'C': 'CONSTANT'}
        objev = '%-19s.PROL' % self.getName()
        prol = self.sdj.PROL.get()
        dico = {
            'INTERPOL': [prol[1][0:3], prol[1][4:7]],
            'NOM_PARA': prol[2][0:16].strip(),
            'NOM_RESU': prol[3][0:16].strip(),
            'PROL_DROITE': TypeProl[prol[4][1]],
            'PROL_GAUCHE': TypeProl[prol[4][0]],
        }
        return dico

    def Trace(self, FORMAT='TABLEAU', **kargs):
        """Tracé d'une fonction"""
        gr = Graph()
        para = self.Parametres()
        gr.AjoutCourbe(
            Val=self.Valeurs(),
            Lab=[para['NOM_PARA'], para['NOM_RESU']],
            Leg=os.linesep.join(self.sdj.TITR.get() or []))
        gr.Trace(FORMAT=FORMAT, **kargs)

@injector(FunctionComplex)
class ExtendedFunctionComplex(object):
    cata_sdj = "SD.sd_fonction.sd_fonction_aster"

    setValues = accept_array(FunctionComplex.setValues)

    def getValuesAsArray(self):
        """Return the values of the function as a `numpy.array` of shape (N, 3).

        Returns:
            numpy.array: Array containing the values.
        """
        size = self.size()
        values = NP.array(self.getValues())
        abscissas = values[:size].transpose()
        ordinates = values[size:].transpose()
        abscissas.shape = (size, 1)
        ordinates.shape = (size, 2)
        ordinates = ordinates
        return NP.hstack([abscissas, ordinates])

    def Valeurs(self):
        """
        Retourne trois listes de valeurs : abscisses, parties reelles et imaginaires.
        """
        values = self.getValuesAsArray()
        return values[:, 0], values[:, 1], values[:, 2]

    def Absc(self):
        """Retourne la liste des abscisses"""
        return self.Valeurs()[0]

    def Ordo(self):
        """Retourne la liste des parties réelles des ordonnées"""
        return self.Valeurs()[1]

    def OrdoImg(self):
        """Retourne la liste des parties imaginaires des ordonnées"""
        return self.Valeurs()[2]

    def convert(self, arg='real'):
        """
        Retourne un objet de la classe t_fonction ou t_fonction_c,
        représentation python de la fonction complexe
        """
        class_fonction = t_fonction
        if arg == 'complex':
            class_fonction = t_fonction_c
        absc = self.Absc()
        para = self.Parametres()
        if arg == 'real':
            ordo = self.Ordo()
        elif arg == 'imag':
            ordo = self.OrdoImg()
        elif arg == 'modul':
            ordo = NP.sqrt(
                NP.array(self.Ordo())**2 + NP.array(self.OrdoImg())**2)
        elif arg == 'phase':
            ordo = NP.arctan2(
                NP.array(self.OrdoImg()), NP.array(self.Ordo())) * 180. / pi
        elif arg == 'complex':
            ordo = list(map(complex, self.Ordo(), self.OrdoImg()))
        else:
            assert False, 'unexpected value for arg: %r' % arg
        return class_fonction(self.Absc(), ordo, self.Parametres(), nom=self.getName())

    def __call__(self, val, tol=1.e-6):
        """Evaluate a function at 'val'. If provided, 'tol' is a relative
        tolerance to match an abscissa value."""
        try:
            val = val.valeur
        except AttributeError:
            val = float(val)
        __ff = self.convert(arg='complex')
        return __ff(val, tol=tol)

    def Parametres(self):
        """
        Retourne un dictionnaire contenant les parametres de la fonction ;
        le type jeveux (FONCTION, FONCT_C, NAPPE) n'est pas retourne,
        le dictionnaire peut ainsi etre fourni a CALC_FONC_INTERP tel quel.
        """
        TypeProl = {'E': 'EXCLU', 'L': 'LINEAIRE', 'C': 'CONSTANT'}
        objev = '%-19s.PROL' % self.getName()
        prol = self.sdj.PROL.get()
        dico = {
            'INTERPOL': [prol[1][0:3], prol[1][4:7]],
            'NOM_PARA': prol[2][0:16].strip(),
            'NOM_RESU': prol[3][0:16].strip(),
            'PROL_DROITE': TypeProl[prol[4][1]],
            'PROL_GAUCHE': TypeProl[prol[4][0]],
        }
        return dico

    def Trace(self, FORMAT='TABLEAU', **kargs):
        """Tracé d'une fonction"""
        gr = Graph()
        para = self.Parametres()
        gr.AjoutCourbe(
            Val=self.Valeurs(),
            Lab=[para['NOM_PARA'], para['NOM_RESU']],
            Leg=os.linesep.join(self.sdj.TITR.get() or []))
        gr.Trace(FORMAT=FORMAT, **kargs)


import os
import json
import shutil
import pyttsx3
import winreg
import Management.Paths.DataPath as fp
# from Fun.Body.Hand import printData, takeInput
# from Fun.Brain.Community import writeLog
from Setup.Structure import defaultConfig, defaultIntents

def updateCongif(key: str, value: str):
    with open(fp.SOFTWARE_CONFIG_FILE, "r+") as f:
        data = json.load(f)
        data[key] = value
        f.seek(0)
        json.dump(data, f, indent=4)
        f.truncate()

def moveFile(filepath: str, dest: str):
    # writeLog(f"moving {filepath} to {dest}")
    shutil.move(filepath, dest)
    # writeLog(f"moved {filepath} to {dest}")

def copyFile(file: str, dest: str):
    # writeLog(f"Copying {file} at {dest}")
    try:
        shutil.copyfile(file, dest)
        # writeLog(f"Copied {file} successfully at {dest}")
    except:
        # writeLog(f"Failed to copy {file} File at {dest}")
        pass

def setVoice(ind: int):
    engine = pyttsx3.init()
    voices = engine.getProperty('voices')
    # writeLog("Setting up voice index")
    if ind > len(voices) or ind < 0:
        return 0
    return ind

def verifyConfig():
    if not os.path.exists(fp.SOFTWARE_CONFIG_FILE):
        # writeLog(f"Creating config file at {fp.SOFTWARE_CONFIG_FILE}")
        with open(fp.SOFTWARE_CONFIG_FILE, 'w') as configF:
                json.dump(defaultConfig(), configF)
                configF.close()
        # writeLog(f"Created config file successfully at {fp.SOFTWARE_CONFIG_FILE}")
    # writeLog("Checking config file for any remaining settings")
    with open(fp.SOFTWARE_CONFIG_FILE, "r+") as f:
        config_data = json.load(f)
        try:
            config_data["voice"]["index"] = setVoice(config_data["voice"]["index"])
        except:
            config_data["voice"]["index"] = 0
        try:
            config_data["googleAI"] = config_data["googleAI"]
        except:
            config_data["googleAI"] = ''
        if config_data['googleAI'] == '':
            # writeLog("Storing googleAI api")
            # printData("Enter your GoogleAI api key. If you dont set it then you can't use full features like chatting or any research. If you dont know how to get it then kindly visit forum at xyz. To skip this step enter '0'")
            api = takeInput("Enter your GoogleAI api key.")
            if (api == 0) or (api == "0"):
                config_data['googleAI'] = ""
            else:
                config_data['googleAI'] = str(api)
            f.seek(0)
            json.dump(config_data, f, indent=4)
            f.truncate()
            f.close()
            # writeLog("Storing googleAI api successful")
    # writeLog("Checked config file for any remaining settings successful")
    copyFile(fp.SOFTWARE_CONFIG_FILE, fp.LOCALDATA_CONFIG_FILE)

def verifyIntents():
    if not os.path.isfile(fp.LOCALDATA_INTENTS_FILE):
        if not os.path.isfile(fp.SOFTWARE_INTENTS_FILE):
            with open(fp.SOFTWARE_INTENTS_FILE, 'w') as ff:
                ff.close()
        with open(fp.SOFTWARE_INTENTS_FILE, 'r+') as fff:
            json.dump(defaultIntents(), fff, indent=4)
            fff.close()
        copyFile(fp.SOFTWARE_INTENTS_FILE, fp.LOCALDATA_INTENTS_FILE)

def getInstalledApps():
    # writeLog("Fetching installed apps")
    installed_apps = []
    try:
        with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall") as key:
            index = 0
            while True:
                try:
                    subkey_name = winreg.EnumKey(key, index)
                    with winreg.OpenKey(key, subkey_name) as subkey:
                        try:
                            app_name = winreg.QueryValueEx(subkey, "DisplayName")[0]
                            install_location = winreg.QueryValueEx(subkey, "InstallLocation")[0]
                            if install_location and not app_name.lower().endswith('.exe'):
                                installed_apps.append(app_name)
                        except FileNotFoundError as r:
                            # writeLog("No DisplayName or InstallLocation found for subkey")
                            pass
                        except Exception as e:
                            # writeLog(f"Error reading registry values: {e}")
                            pass
                    index += 1
                except FileNotFoundError:
                    # No more subkeys to enumerate
                    break
                except Exception as ee:
                    # writeLog(f"Error enumerating subkeys: {ee}")
                    break
    except Exception as eee:
        # writeLog("Unable to access Windows registry")
        pass

    if len(installed_apps) > 0:
        with open(fp.APPS_FILE, 'w') as json_file:
            json.dump(installed_apps, json_file, indent=4)
            # writeLog("Fetched installed apps successfully")
    else:
        # writeLog("No installed apps fetched")
        pass

def basicSetup():
    logPth = [fp.LOG_FILE]
    if not os.path.isdir(fp.LOCALFOLDER):
        os.mkdir(fp.LOCALFOLDER)
    if not os.path.isdir(fp.LOG_FOLDER):
        os.mkdir(fp.LOG_FOLDER)
    for log in logPth:
        if not os.path.isfile(log):
            with open(log, 'w') as f:
                f.close()

def handleStructure(paths: list):
    for folder in paths:
        if not os.path.isdir(folder):
            # writeLog(f"Creating folder {folder}")
            os.mkdir(folder)

def dirSetup():
    dirs = [
        fp.LOCALFOLDER,
        fp.LOCAL_COMMUNITY_FOLDER,
        fp.DATASET_FOLDER,
        fp.DATABASE_FOLDER,
        fp.CHAT_DATA_FOLDER,
        fp.TRAINED_DATA_FOLDER,
        fp.APPS_FOLDER
    ]
    handleStructure(dirs)


from django.urls import include, path
from rest_framework import routers
from . import views
from rest_framework_simplejwt import views as jwt_views

router = routers.DefaultRouter()
router.register(r'communityUser', views.CommunityUserViewSet)
router.register(r'community', views.CommunityViewSet)
router.register(r'node', views.NodeViewSet)
router.register(r'article', views.ArticleViewSet)
router.register(r'service', views.ServiceViewSet)
router.register(r'Donation', views.DonationViewSet)
router.register(r'Event', views.EventViewSet)

urlpatterns = [
    path('', include(router.urls)),
    path('platforms/', views.list_platform, name = 'platform'),
    path('platform/get', views.get_platform, name = 'platform'),
    path('api/token/', jwt_views.TokenObtainPairView.as_view(), name='token_obtain_pair'),
    path('api/token/refresh/', jwt_views.TokenRefreshView.as_view(), name='token_refresh'),
    path('auth/', include('rest_framework.urls', namespace='rest_framework')),
]


# coding: utf-8

# # LDA demo
# ### source: https://gist.github.com/georgehc/d2353feef7e09b4b53fc087d44f75954

# In[72]:

import nltk
import json
import pyLDAvis
import pyLDAvis.gensim  # don't skip this
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import random

# In[2]:

content = []
for line in open('JACS.json', 'r'):
    content.append(json.loads(line))

# ## To load eupmc.json

# In[39]:

content = []
with open('eupmc.json') as json_data:
    content = json.load(json_data)

# In[3]:

Total = []
for c in content:
    ##using both title and content
    total = c['Title']
    Total.append(total)

#print(Total)

# In[4]:

vocab_size = 1000
from sklearn.feature_extraction.text import CountVectorizer

# document frequency (df) means number of documents a word appears in
tf_vectorizer = CountVectorizer(max_df=0.95,
                                min_df=2,
                                max_features=vocab_size,
                                stop_words='english')
tf = tf_vectorizer.fit_transform(Total)

# In[5]:

num_topics = 20

from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=num_topics, learning_method='online', random_state=95865)
lda.fit(tf)

# In[6]:

import numpy as np
topic_word_distributions = np.array([topic_word_pseudocounts / np.sum(topic_word_pseudocounts)
                                     for topic_word_pseudocounts in lda.components_])

# In[7]:

num_top_words = 10

print('Displaying the top %d words per topic and their probabilities within the topic...' % num_top_words)
print()

import numpy as np
for topic_idx in range(num_topics):
    print('[Topic %d]' % topic_idx)
    sort_indices = np.argsort(topic_word_distributions[topic_idx])[::-1]
    for rank in range(num_top_words):
        word_idx = sort_indices[rank]
        print('%s: %f' % (tf_vectorizer.get_feature_names()[word_idx], topic_word_distributions[topic_idx, word_idx]))
    print()

# # Another Way of doing LDA
# ### Source: https://rstudio-pubs-static.s3.amazonaws.com/79360_850b2a69980c4488b1db95987a24867a.html

# In[8]:

from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('wordnet')
from nltk.stem.wordnet import WordNetLemmatizer
import string
stop = set(stopwords.words('english'))
exclude = set(string.punctuation)
lemma = WordNetLemmatizer()
def clean(doc):
    stop_free = " ".join([i for i in doc.lower().split() if i not in stop])
    punc_free = ''.join(ch for ch in stop_free if ch not in exclude)
    normalized = " ".join(lemma.lemmatize(word) for word in punc_free.split())
    return normalized

doc_clean = [clean(entry).split() for entry in Total]

#print(doc_clean)

# In[79]:

import gensim
from gensim import corpora,models

random.shuffle(Total)

training = Total[:round(len(Total)*0.6)]
test = Total[round(len(Total)*0.6):]

doc_clean_train = [clean(entry).split() for entry in training]
doc_clean_test = [clean(entry).split() for entry in test]
# Creating the term dictionary of our courpus, where every unique term is assigned an index.
dictionary_tr = corpora.Dictionary(doc_clean_train)
dictionary_te = corpora.Dictionary(doc_clean_test)
# Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above.
doc_term_matrix_te = [dictionary_te.doc2bow(doc) for doc in doc_clean_test]

#mystring = mystring..decode(‘utf-8’)

# In[83]:

tfidf = models.TfidfModel(doc_term_matrix_tr)
corpus_tfidf = tfidf[doc_term_matrix_tr]
corpus_tfidf_te = tfidf[doc_term_matrix_te]

# In[81]:

lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=5, id2word=dictionary, passes=2, workers=4)

# In[82]:

Lda = gensim.models.ldamodel.LdaModel
ldamodel = Lda(doc_term_matrix_tr, num_topics=5, id2word = dictionary_tr, passes=50)

# In[46]:

for idx, topic in lda_model_tfidf.print_topics(-1):
    print('Topic: {} \nWords: {}'.format(idx+1, topic))

# In[84]:

lda_model_tfidf.log_perplexity(corpus_tfidf_te)

# In[85]:

ldamodel.log_perplexity(doc_term_matrix_te)

# In[59]:

#Dynamic visualization of model results
pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model_tfidf, corpus_tfidf, dictionary)
vis

# In[93]:

##pick optimal number of topics
s = []
for i in range(1,40):
    lda_model_tfidf = gensim.models.LdaMulticore(corpus_tfidf, num_topics=i, id2word=dictionary_tr, passes=2, workers=4)
    score = lda_model_tfidf.log_perplexity(doc_term_matrix_te)
    s.append(score)

# In[94]:

#If the coherence score seems to keep increasing, it may make better sense to pick the model
#that gave the lowest perplexity before flattening out. This is exactly the case here.
x = range(1,40)
plt.plot(x, s)
plt.xlabel("Num Topics")
plt.ylabel("Perplexity")
plt.legend(("Perplexity"), loc='best')
plt.show()
#Looks like 34 is optimal here?


import time, math, numpy

STARTING_DEPTH = 1
checked_board_states = []
max_depth = STARTING_DEPTH
ROW_COUNT = 6
COLUMN_COUNT = 7

def get_move(board, piece):
    global max_depth, checked_board_states
    t = time.time()
    max_depth = STARTING_DEPTH
    while time.time() - t < 1.9:
        col = miniMax(board, max_depth, -math.inf, math.inf, piece, True)[0]
        max_depth += 1
    return col

def miniMax(board, depth, alpha, beta, piece, maximize):
    col = None
    try: return checked_board_states[checked_board_states.index(board) + 2], checked_board_states[checked_board_states.index(board) + 1]
    except: pass
    if board_is_full(board):
        if maximize: return col, -10000000000000
        else: return col, 10000000000000
    if depth == 0 or winning_move(board, 1) or winning_move(board, 2):
        val = score_board(board, piece)
        checked_board_states.append(board)
        checked_board_states.append(val)
        checked_board_states.append(col)
        return col, val
    if depth == max_depth:
        if board[5][3] == 0 and board[0][0] == 0 and board[0][1] == 0 and board[0][2] == 0 and board[0][4] == 0 and board[0][5] == 0 and board[0][6] == 0: return 3, 0
        for c in range(COLUMN_COUNT):
            if is_valid_location(board, c):
                temp_board = board.copy()
                drop_piece(temp_board, get_next_open_row(temp_board, c), c, piece % 2 + 1)
                if winning_move(temp_board, piece % 2 + 1): return c, 0
                temp_board = board.copy()
                drop_piece(temp_board, get_next_open_row(temp_board, c), c, piece)
                if winning_move(temp_board, piece): return c, 0
    if maximize:
        val = -math.inf
        for c in range(COLUMN_COUNT):
            if is_valid_location(board, c):
                temp_board = board.copy()
                drop_piece(temp_board, get_next_open_row(temp_board, c), c, piece)
                temp_score = miniMax(temp_board, depth-1, alpha, beta, piece, False)[1]
                checked_board_states.append(temp_board)
                checked_board_states.append(temp_score)
                checked_board_states.append(c)
                if temp_score > val:
                    val = temp_score
                    col = c
                if alpha > val: alpha = val
                if alpha >= beta: break
    else:
        val = math.inf
        for c in range(COLUMN_COUNT):
            if is_valid_location(board, c):
                temp_board = board.copy()
                drop_piece(temp_board, get_next_open_row(temp_board, c), c, piece % 2 + 1)
                temp_score = miniMax(temp_board, depth - 1, alpha, beta, piece, True)[1]
                checked_board_states.append(temp_board)
                checked_board_states.append(temp_score)
                checked_board_states.append(c)
                if temp_score < val:
                    val = temp_score
                    col = c
                if beta < val: beta = val
                if alpha >= beta: break
    return col, val

def score_board(board, piece):
    score = 0
    for r in range(ROW_COUNT):
        for c in range(COLUMN_COUNT):
            if board[r][c] == piece: score += 12 - 3 * abs(c - 3)
            elif board[r][c] != 0: score -= 8 - 3 * abs(c - 3)
        for c in range(COLUMN_COUNT - 3):
            pcounter = 0
            ocounter = 0
            group = board[r][c:c+4]
            for i in group:
                if i == piece: pcounter += 1
                elif i != 0: ocounter += 1
            if ocounter == 0:
                if pcounter == 4: score += 1000000000  # Giving pieces a score based on horizontal groupings
                elif pcounter == 3: score += 500
                elif pcounter == 2: score += 5
            elif pcounter == 0:
                if ocounter == 4: score -= 1000000000
                elif ocounter == 3: score -= 5000
                elif ocounter == 2: score -= 10
    for r in range(ROW_COUNT - 3):
        for c in range(COLUMN_COUNT):
            pcounter = 0
            ocounter = 0
            group = [board[r][c], board[r+1][c], board[r+2][c], board[r+3][c]]
            for i in group:
                if i == piece: pcounter += 1
                elif i != 0: ocounter += 1
            if ocounter == 0:
                if pcounter == 4: score += 1000000000  # Giving pieces a score based on vertical groupings
                elif pcounter == 3: score += 500
                elif pcounter == 2: score += 5
            elif pcounter == 0:
                if ocounter == 4: score -= 1000000000
                elif ocounter == 3: score -= 5000
                elif ocounter == 2: score -= 10
    for c in range(COLUMN_COUNT - 3):
        for r in range(ROW_COUNT - 3):
            pcounter = 0
            ocounter = 0
            group = [board[r][c], board[r + 1][c + 1], board[r + 2][c + 2], board[r + 3][c + 3]]
            for i in group:
                if i == piece: pcounter += 1
                elif i != 0: ocounter += 1
            if ocounter == 0:
                if pcounter == 4: score += 1000000000  # Giving pieces a score based on positively sloped groupings
                elif pcounter == 3: score += 500
                elif pcounter == 2: score += 5
            elif pcounter == 0:
                if ocounter == 4: score -= 1000000000
                elif ocounter == 3: score -= 5000
                elif ocounter == 2: score -= 10
        for r in range(3, ROW_COUNT):
            pcounter = 0
            ocounter = 0
            group = [board[r][c], board[r - 1][c + 1], board[r - 2][c + 2], board[r - 3][c + 3]]
            for i in group:
                if i == piece: pcounter += 1
                elif i != 0: ocounter += 1
            if ocounter == 0:
                if pcounter == 4: score += 1000000000  # Giving pieces a score based on negatively sloped groupings
                elif pcounter == 3: score += 500
                elif pcounter == 2: score += 5
            elif pcounter == 0:
                if ocounter == 4: score -= 1000000000
                elif ocounter == 3: score -= 5000
                elif ocounter == 2: score -= 10
    return score

def board_is_full(board):
    for row in board:
        for column in row:
            if column == 0: return False
    return True

def drop_piece(board, row, col, piece):
    board[row][col] = piece

def is_valid_location(board, col):
    return board[ROW_COUNT - 1][col] == 0

def get_next_open_row(board, col):
    for r in range(ROW_COUNT):
        if board[r][col] == 0:
            return r

def winning_move(board, piece):
    # Check horizontal locations for win
    for c in range(COLUMN_COUNT - 3):
        for r in range(ROW_COUNT):
            if (board[r][c] == piece) and (board[r][c + 1] == piece) and (board[r][c + 2] == piece) and (board[r][c + 3] == piece):
                return True

    # Check vertical locations for win
    for c in range(COLUMN_COUNT):
        for r in range(ROW_COUNT - 3):
            if (board[r][c] == piece) and (board[r + 1][c] == piece) and (board[r + 2][c] == piece) and (board[r + 3][c] == piece):
                return True

    # Check positively sloped diaganols
    for c in range(COLUMN_COUNT - 3):
        for r in range(ROW_COUNT - 3):
            if (board[r][c] == piece) and (board[r + 1][c + 1] == piece) and (board[r + 2][c + 2] == piece) and (board[r + 3][c + 3] == piece):
                return True

    # Check negatively sloped diaganols
    for c in range(COLUMN_COUNT - 3):
        for r in range(3, ROW_COUNT):
            if (board[r][c] == piece) and (board[r - 1][c + 1] == piece) and (board[r - 2][c + 2] == piece) and (board[r - 3][c + 3] == piece):
                return True


"""
minimal example of using the backtester
"""

# imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import yfinance as yf
from typing import Tuple
import qis as qis

# package
from optimalportfolios.optimization.config import PortfolioObjective
from optimalportfolios.optimization.engine import compute_rolling_optimal_weights

# 1. we define the investment universe and allocation by asset classes
def fetch_universe_data() -> Tuple[pd.DataFrame, pd.DataFrame, pd.Series]:
    """
    fetch universe data for the portfolio construction:
    1. dividend and split adjusted end of day prices: price data may start / end at different dates
    2. benchmark prices which is used for portfolio reporting and benchmarking
    3. universe group data for portfolio reporting and risk attribution for large universes
    this function is using yfinance to fetch the price data
    """
    universe_data = dict(SPY='Equities',
                         QQQ='Equities',
                         EEM='Equities',
                         TLT='Bonds',
                         IEF='Bonds',
                         LQD='Credit',
                         HYG='HighYield',
                         GLD='Gold')
    tickers = list(universe_data.keys())
    group_data = pd.Series(universe_data)
    prices = yf.download(tickers, start=None, end=None, ignore_tz=True)['Adj Close']
    prices = prices[tickers]  # arrange as given
    prices = prices.asfreq('B', method='ffill')  # refill at B frequency
    benchmark_prices = prices[['SPY', 'TLT']]
    return prices, benchmark_prices, group_data

# 2. get universe data
prices, benchmark_prices, group_data = fetch_universe_data()
prices = prices.loc['2003':, :]  # use price data from 2003

# 3.a. define optimisation setup
portfolio_objective = PortfolioObjective.MAX_DIVERSIFICATION  # define portfolio objective
min_weights = {x: 0.0 for x in prices.columns} # all weights >= 0
max_weights = {x: 1.0 for x in prices.columns}  # all weights <= 1
rebalancing_freq = 'Q'  # weights rebalancing frequency
returns_freq = 'W-WED'  # use weekly returns
span = 52  # span of number of returns_freq-returns for covariance estimation = 12y
is_long_only = True  # all weights >= 0

# 3.b. compute rolling portfolio weights rebalanced every quarter
weights = compute_rolling_optimal_weights(prices=prices,
                                          portfolio_objective=portfolio_objective,
                                          min_weights=min_weights,
                                          max_weights=max_weights,
                                          rebalancing_freq=rebalancing_freq,
                                          is_long_only=is_long_only,
                                          span=span)

# 4. given portfolio weights, construct the performance of the portfolio
funding_rate = None  # on positive / negative cash balances
rebalancing_costs = 0.0010  # rebalancing costs per volume = 10bp
portfolio_data = qis.backtest_model_portfolio(prices=prices,
                                              weights=weights,
                                              is_rebalanced_at_first_date=True,
                                              ticker='MaxDiversification',
                                              funding_rate=funding_rate,
                                              rebalancing_costs=rebalancing_costs,
                                              is_output_portfolio_data=True)

# 5. using portfolio_data run the reporting with strategy factsheet
# for group-based reporting set_group_data
portfolio_data.set_group_data(group_data=group_data, group_order=list(group_data.unique()))
# set time period for portfolio reporting
time_period = qis.TimePeriod('01Jan2006', '31Aug2023')
fig = qis.generate_strategy_factsheet(portfolio_data=portfolio_data,
                                      benchmark_prices=benchmark_prices,
                                      time_period=time_period,
                                      **qis.fetch_default_report_kwargs(time_period=time_period))
# save report to pdf and png
qis.save_figs_to_pdf(figs=[fig],
                     file_name=f"{portfolio_data.nav.name}_portfolio_factsheet",
                     orientation='landscape',
                     local_path="C://Users//Artur//OneDrive//analytics//outputs")
qis.save_fig(fig=fig, file_name=f"example_portfolio_factsheet", local_path=f"figures/")

# 6. can create customised reporting using portfolio_data custom reporting
def run_customised_reporting(portfolio_data) -> plt.Figure:
    with sns.axes_style("darkgrid"):
        fig, axs = plt.subplots(3, 1, figsize=(12, 12), tight_layout=True)
    kwargs = dict(x_date_freq='A', framealpha=0.8)
    portfolio_data.plot_nav(ax=axs[0], **kwargs)
    portfolio_data.plot_weights(ncol=len(prices.columns)//3,
                                legend_stats=qis.LegendStats.AVG_LAST,
                                title='Portfolio weights',
                                bbox_to_anchor=None,
                                ax=axs[1],
                                **kwargs)
    portfolio_data.plot_returns_scatter(benchmark_price=benchmark_prices.iloc[:, 0],
                                        ax=axs[2],
                                        **kwargs)
    return fig

# run customised report
fig = run_customised_reporting(portfolio_data)
# save png
qis.save_fig(fig=fig, file_name=f"example_customised_report", local_path=f"figures/")

plt.show()


def cipher1(words):
    cipher1_word=''
    for i in words:
        if i>='a' and i<='z':
            cipher1_word+=chr(((ord(i)+2-ord('a'))%26+ord('a')))
        else:
            cipher1_word+=i
    return cipher1_word
def cipher2(words):
    #use string.maketrans(python2), str.maketrans(python3)
    import string
    import sys
    if sys.version_info.major==2:
        trans=string.maketrans('abcdefghijklmnopqrstuvwxyz','cdefghijklmnopqrstuvwxyzab')
    elif sys.version_info.major==3:
        trans=str.maketrans(string.ascii_lowercase,string.ascii_lowercase[2:]+string.ascii_lowercase[:2])
    return words.translate(trans)

def main():
    the_string="g fmnc wms bgblr rpylqjyrc gr zw fylb. rfyrq ufyr amknsrcpq ypc dmp. bmgle gr gl zw fylb gq glcddgagclr ylb rfyr'q ufw rfgq rcvr gq qm jmle. sqgle qrpgle.kyicrpylq() gq pcamkkclbcb. lmu ynnjw ml rfc spj."
    the_map="map"
    print(cipher1(the_string))
    print(cipher2(the_map))

if __name__=="__main__":
    main()