from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import enum
from googlecloudsdk.api_lib.notebooks import util
from googlecloudsdk.command_lib.util.apis import arg_utils
from googlecloudsdk.core import log
from googlecloudsdk.core import resources

def CreateRuntime(args, messages):
  """Creates the Runtime message for the create request.

  Args:
    args: Argparse object from Command.Run
    messages: Module containing messages definition for the specified API.

  Returns:
    Runtime of the Runtime message.
  """

  def GetRuntimeVirtualMachineFromArgs():
    machine_type = 'n1-standard-4'
    if args.IsSpecified('machine_type'):
      machine_type = args.machine_type
    virtual_machine_config = messages.VirtualMachineConfig(
        machineType=machine_type, dataDisk=messages.LocalDisk())
    return messages.VirtualMachine(virtualMachineConfig=virtual_machine_config)

  def GetRuntimeAccessConfigFromArgs():
    runtime_access_config = messages.RuntimeAccessConfig
    type_enum = None
    if args.IsSpecified('runtime_access_type'):
      type_enum = arg_utils.ChoiceEnumMapper(
          arg_name='runtime-access-type',
          message_enum=runtime_access_config.AccessTypeValueValuesEnum,
          include_filter=lambda x: 'UNSPECIFIED' not in x).GetEnumForChoice(
              arg_utils.EnumNameToChoice(args.runtime_access_type))
    return runtime_access_config(
        accessType=type_enum, runtimeOwner=args.runtime_owner)

  def GetPostStartupScriptBehavior():
    type_enum = None
    if args.IsSpecified('post_startup_script_behavior'):
      runtime_software_config_message = messages.RuntimeSoftwareConfig
      type_enum = arg_utils.ChoiceEnumMapper(
          arg_name='post-startup-script-behavior',
          message_enum=(runtime_software_config_message
                        .PostStartupScriptBehaviorTypeValueValuesEnum),
          include_filter=lambda x: 'UNSPECIFIED' not in x).GetEnumForChoice(
              arg_utils.EnumNameToChoice(args.post_startup_script_behavior))
    return type_enum

  def GetRuntimeSoftwareConfigFromArgs():
    runtime_software_config = messages.RuntimeSoftwareConfig()
    if args.IsSpecified('idle_shutdown_timeout'):
      runtime_software_config.idleShutdownTimeout = args.idle_shutdown_timeout
    if args.IsSpecified('install_gpu_driver'):
      runtime_software_config.installGpuDriver = args.install_gpu_driver
    if args.IsSpecified('custom_gpu_driver_path'):
      runtime_software_config.customGpuDriverPath = args.custom_gpu_driver_path
    if args.IsSpecified('post_startup_script'):
      runtime_software_config.postStartupScript = args.post_startup_script
    if args.IsSpecified('post_startup_script_behavior'):
      runtime_software_config.postStartupScriptBehavior = (
          GetPostStartupScriptBehavior())
    return runtime_software_config

  runtime = messages.Runtime(
      name=args.runtime,
      virtualMachine=GetRuntimeVirtualMachineFromArgs(),
      accessConfig=GetRuntimeAccessConfigFromArgs(),
      softwareConfig=GetRuntimeSoftwareConfigFromArgs(),
  )
  return runtime

def CreateRuntimeCreateRequest(args, messages):
  parent = util.GetParentForRuntime(args)
  runtime = CreateRuntime(args, messages)
  return messages.NotebooksProjectsLocationsRuntimesCreateRequest(
      parent=parent, runtime=runtime, runtimeId=args.runtime)

def CreateRuntimeListRequest(args, messages):
  parent = util.GetParentFromArgs(args)
  return messages.NotebooksProjectsLocationsRuntimesListRequest(parent=parent)

def GetRuntimeResource(args):
  return args.CONCEPTS.runtime.Parse()

def GetSwitchRuntimeRequest(args, messages):
  """Create and return switch runtime request."""
  machine_type = 'n1-standard-4'
  if args.IsSpecified('machine_type'):
    machine_type = args.machine_type
  runtime_accelerator_config = messages.RuntimeAcceleratorConfig()
  if args.IsSpecified('accelerator_core_count'):
    runtime_accelerator_config.coreCount = args.accelerator_core_count
  if args.IsSpecified('accelerator_type'):
    type_enum = arg_utils.ChoiceEnumMapper(
        arg_name='accelerator-type',
        message_enum=runtime_accelerator_config.TypeValueValuesEnum,
        include_filter=lambda x: 'UNSPECIFIED' not in x).GetEnumForChoice(
            arg_utils.EnumNameToChoice(args.accelerator_type))
    runtime_accelerator_config.type = type_enum
  return messages.SwitchRuntimeRequest(
      machineType=machine_type, acceleratorConfig=runtime_accelerator_config)

def CreateRuntimeDeleteRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  return messages.NotebooksProjectsLocationsRuntimesDeleteRequest(
      name=runtime)

def CreateRuntimeResetRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  reset_request = messages.ResetRuntimeRequest()
  return messages.NotebooksProjectsLocationsRuntimesResetRequest(
      name=runtime, resetRuntimeRequest=reset_request)

def CreateRuntimeStartRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  start_request = messages.StartRuntimeRequest()
  return messages.NotebooksProjectsLocationsRuntimesStartRequest(
      name=runtime, startRuntimeRequest=start_request)

def CreateRuntimeStopRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  stop_request = messages.StopRuntimeRequest()
  return messages.NotebooksProjectsLocationsRuntimesStopRequest(
      name=runtime, stopRuntimeRequest=stop_request)

def CreateRuntimeSwitchRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  switch_request = GetSwitchRuntimeRequest(args, messages)
  return messages.NotebooksProjectsLocationsRuntimesSwitchRequest(
      name=runtime, switchRuntimeRequest=switch_request)

def CreateRuntimeDescribeRequest(args, messages):
  runtime = GetRuntimeResource(args).RelativeName()
  return messages.NotebooksProjectsLocationsRuntimesGetRequest(name=runtime)

def CreateRuntimeDiagnoseRequest(args, messages):
  """"Create and return Diagnose request."""
  runtime = GetRuntimeResource(args).RelativeName()
  diagnostic_config = messages.DiagnosticConfig(
      gcsBucket=args.gcs_bucket,
  )
  if args.IsSpecified('relative_path'):
    diagnostic_config.relativePath = args.relative_path
  if args.IsSpecified('enable-repair'):
    diagnostic_config.repairFlagEnabled = True
  if args.IsSpecified('enable-packet-capture'):
    diagnostic_config.packetCaptureFlagEnabled = True
  if args.IsSpecified('enable-copy-home-files'):
    diagnostic_config.copyHomeFilesFlagEnabled = True

  timeout_minutes = None
  if args.IsSpecified('timeout_minutes'):
    timeout_minutes = int(args.timeout_minutes)

  return messages.NotebooksProjectsLocationsRuntimesDiagnoseRequest(
      name=runtime, diagnoseRuntimeRequest=messages.DiagnoseRuntimeRequest(
          diagnosticConfig=diagnostic_config, timeoutMinutes=timeout_minutes))

def GetRuntimeURI(resource):
  instance = resources.REGISTRY.ParseRelativeName(
      resource.name, collection='notebooks.projects.locations.runtimes')
  return instance.SelfLink()

class OperationType(enum.Enum):
  CREATE = (log.CreatedResource, 'created')
  DELETE = (log.DeletedResource, 'deleted')
  UPDATE = (log.UpdatedResource, 'updated')
  RESET = (log.ResetResource, 'reset')

def HandleLRO(operation,
              args,
              runtime_service,
              release_track,
              operation_type=OperationType.CREATE):
  """Handles Long Running Operations for both cases of async.

  Args:
    operation: The operation to poll.
    args: ArgParse instance containing user entered arguments.
    runtime_service: The service to get the resource after the long running
      operation completes.
    release_track: base.ReleaseTrack object.
    operation_type: Enum value of type OperationType indicating the kind of
      operation to wait for.

  Raises:
    apitools.base.py.HttpError: if the request returns an HTTP error

  Returns:
    The Runtime resource if synchronous, else the Operation Resource.
  """
  logging_method = operation_type.value[0]
  if args.async_:
    logging_method(
        util.GetOperationResource(operation.name, release_track),
        kind='notebooks runtime {0}'.format(args.runtime),
        is_async=True)
    return operation
  else:
    response = util.WaitForOperation(
        operation,
        'Waiting for operation on Runtime [{}] to be {} with [{}]'.format(
            args.runtime, operation_type.value[1], operation.name),
        service=runtime_service,
        release_track=release_track,
        is_delete=(operation_type.value[1] == 'deleted'))
    logging_method(
        util.GetOperationResource(operation.name, release_track),
        kind='notebooks runtime {0}'.format(args.runtime),
        is_async=False)
    return response


import unittest
import sys
import os

sys.path.append(os.path.dirname(__file__) + "/../../..")

simple_cfg = '''
from sts.control_flow.replayer import Replayer
from sts.simulation_state import SimulationConfig

simulation_config = SimulationConfig()
control_flow = Replayer(simulation_config, "%s")
'''

class ReplayerTest(unittest.TestCase):
  tmpsuperlog = '/tmp/superlog.tmp'
  tmpcfg = 'config/replayer_simple_test.py'
  tmpcfgpyc = 'config/replayer_simple_test.pyc'
  tmpcfgmodule = 'config.replayer_simple_test'

  def write_simple_superlog(self):
    ''' Returns the file. Make sure to close afterwards! '''
    superlog = open(self.tmpsuperlog, 'w')
    e1 = str('''{"dependent_labels": ["e2"], "start_dpid": 8, "class": "LinkFailure",'''
             ''' "start_port_no": 3, "end_dpid": 15, "end_port_no": 2, "label": "e1", "time": [0,0], "round": 0}''')
    superlog.write(e1 + '\n')
    e2 = str('''{"dependent_labels": [], "start_dpid": 8, "class": "LinkRecovery",'''
             ''' "start_port_no": 3, "end_dpid": 15, "end_port_no": 2, "label": "e2", "time": [0,0], "round": 0}''')
    superlog.write(e2 + '\n')
    superlog.close()

  def write_simple_cfg(self):
    cfg = open(self.tmpcfg, 'w')
    cfg.write(simple_cfg % self.tmpsuperlog)
    cfg.close()

  def basic_test(self):
    try:
      self.write_simple_superlog()
      self.write_simple_cfg()
      ret = os.system("./simulator.py -c %s" % self.tmpcfgmodule)
      self.assertEqual(0, ret)
    finally:
      os.unlink(self.tmpsuperlog)
      os.unlink(self.tmpcfg)
      if os.path.exists(self.tmpcfgpyc):
        os.unlink(self.tmpcfgpyc)

if __name__ == '__main__':
  unittest.main()


# USING THIS
# https://www.kaggle.com/mirlei/image-classification-of-dice-with-neural-network

import random
import os
import pickle

import numpy as np
import matplotlib.pyplot as plt
import cv2

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, Dense, Dropout, Activation, Flatten, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D
from tensorflow.keras.optimizers import Adam

DATADIR = '../ml-data/dice/train'
MODELDIR = '../gen/models/'
CATEGORIES = ['d4', 'd6', 'd8', 'd10', 'd12', 'd20']

train_path = ('../ml-data/dice/train')
valid_path = ('../ml-data/dice/valid')

batch_size_train = 10
batch_size_valid = 10
targetsize = 48

train_batches = ImageDataGenerator().flow_from_directory(train_path, target_size=(targetsize, targetsize), classes=['d4', 'd6', 'd8', 'd10', 'd12', 'd20'], batch_size=batch_size_train)
valid_batches = ImageDataGenerator().flow_from_directory(valid_path, target_size=(targetsize, targetsize), classes=['d4', 'd6', 'd8', 'd10', 'd12', 'd20'], batch_size=batch_size_valid)

train_num = len(train_batches)
val_num = len(valid_batches)

def plots(ims, figsize=(20,10), rows=1, interp= False, titles= None):
    if type(ims[0]) is np.ndarray:
        ims = np.array(ims).astype(np.uint8)

        if ims.shape[-1] != 3:
            ims = ims.transpose((0,1,2,3))

    f = plt.figure(figsize=figsize)
    cols = len(ims)//rows if len(ims) %2 == 0 else len(ims)//rows + 1

    for i in range(len(ims)):
        sp = f.add_subplot(rows, cols, i+1)
        sp.axis('Off')

        if titles is not None:
            sp.set_title(titles[i], fontsize=12)

        cv2.imshow('im', ims[i])#, interpolation=None if interp else 'none')
        cv2.waitKey(0)

# imgs, labels = next(train_batches)
# plots(imgs, titles=labels)

model = Sequential([
    Conv2D(32, (3, 3), padding='same', activation='relu', input_shape=(targetsize, targetsize, 3)),
    Conv2D(32, (3, 3), activation='relu'),
    MaxPooling2D(pool_size=(2, 2)),
    Dropout(0.25),

    Conv2D(64, (3, 3), padding='same', activation='relu'),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D(pool_size=(2, 2)),
    Dropout(0.25),

    Conv2D(64, (3, 3), padding='same', activation='relu'),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D(pool_size=(2, 2)),
    Dropout(0.25),

    Flatten(),
    Dense(512, activation='relu'),
    Dropout(0.5),
    Dense(6, activation='softmax'),

])

model.compile(Adam(lr=.0001), loss='categorical_crossentropy', metrics= ['accuracy'])
history = model.fit_generator(train_batches, steps_per_epoch= train_num , validation_data=valid_batches, validation_steps= val_num, epochs=15, verbose=2)

# model.fit(X, y, batch_size=16, epochs=10, verbose=1, validation_split=0.1)

model.save(MODELDIR + 'test-guys.model')


#!/usr/bin/python
import cv2
import sys
import os
if __name__ == "__main__" :

            imagename="contrast.jpg"
            img=cv2.imread(imagename)
            # if (img==None):
            # print("File not found") # Or not readable?
            # sys.exit()

        #create haar cascade classifier
            path="haarcascade_frontalface_alt.xml"
        #path = "/usr/share/opencv/haarcascades/haarcascade_frontalface_alt2.xml"
            hc=cv2.CascadeClassifier(path)
            # if (hc.empty()):
            #   print("CascadeClassifer could not be loaded")
            #   sys.exit()

        #detect faces
            faces=hc.detectMultiScale(img)
            i=1
            for face in faces:
              imgROI = img[face[1]:face[1]+face[3],face[0]:face[0]+face[2]]
            #imgROI = img[face[1]-25:face[1]+face[3]+25,
            #             face[0]-20:face[0]+face[2]+20]
              s=imagename
              cv2.imwrite(s,imgROI)
              i=i+1

            print(str(i-1)+" face(s) extracted")


import copy
import torch as th
import numpy as np
import torch.nn.functional as F

from modules.mixers.dmaq_general import DMAQer
from components.episode_buffer import EpisodeBatch
from modules.mixers.dmaq_qatten import DMAQ_QattenMixer
from torch.optim import RMSprop
from torch.utils.data.sampler import BatchSampler, SubsetRandomSampler
from modules.CDS.predict_net import Predict_Network, Predict_Network_WithID, Predict_ID_obs_tau

class CDS_QPLEX:
    def __init__(self, mac, scheme, logger, args):
        self.args = args
        self.mac = mac
        self.logger = logger

        self.params = list(mac.parameters())
        self.last_target_update_episode = 0

        self.mixer = None
        if args.mixer is not None:
            if args.mixer == "dmaq":
                self.mixer = DMAQer(args)
            elif args.mixer == 'dmaq_qatten':
                self.mixer = DMAQ_QattenMixer(args)
            else:
                raise ValueError("Mixer {} not recognised.".format(args.mixer))
            self.params += list(self.mixer.parameters())
            self.target_mixer = copy.deepcopy(self.mixer)

        self.eval_predict_id = Predict_ID_obs_tau(
            args.rnn_hidden_dim, args.predict_net_dim, args.n_agents)
        self.target_predict_id = Predict_ID_obs_tau(
            args.rnn_hidden_dim, args.predict_net_dim, args.n_agents)

        if args.ifaddobs:
            self.eval_predict_withoutid = Predict_Network(
                args.rnn_hidden_dim + args.obs_shape + args.n_actions, args.predict_net_dim, args.obs_shape)
            self.target_predict_withoutid = Predict_Network(
                args.rnn_hidden_dim + args.obs_shape + args.n_actions, args.predict_net_dim, args.obs_shape)

            self.eval_predict_withid = Predict_Network_WithID(args.rnn_hidden_dim + args.obs_shape + args.n_actions + args.n_agents, args.predict_net_dim,
                                                              args.obs_shape, args.n_agents)
            self.target_predict_withid = Predict_Network_WithID(args.rnn_hidden_dim + args.obs_shape + args.n_actions + args.n_agents, args.predict_net_dim,
                                                                args.obs_shape, args.n_agents)
        else:
            self.eval_predict_withoutid = Predict_Network(
                args.rnn_hidden_dim + args.n_actions, args.predict_net_dim, args.obs_shape)
            self.target_predict_withoutid = Predict_Network(
                args.rnn_hidden_dim + args.n_actions, args.predict_net_dim, args.obs_shape)

            self.eval_predict_withid = Predict_Network_WithID(args.rnn_hidden_dim + args.n_actions + args.n_agents, args.predict_net_dim,
                                                              args.obs_shape, args.n_agents)
            self.target_predict_withid = Predict_Network_WithID(args.rnn_hidden_dim + args.n_actions + args.n_agents, args.predict_net_dim,
                                                                args.obs_shape, args.n_agents)

        if self.args.use_cuda:

            self.eval_predict_withid.to(th.device(self.args.GPU))
            self.target_predict_withid.to(th.device(self.args.GPU))

            self.eval_predict_withoutid.to(th.device(self.args.GPU))
            self.target_predict_withoutid.to(th.device(self.args.GPU))

            self.eval_predict_id.to(th.device(self.args.GPU))
            self.target_predict_id.to(th.device(self.args.GPU))

        self.target_predict_withid.load_state_dict(
            self.eval_predict_withid.state_dict())
        self.target_predict_withoutid.load_state_dict(
            self.eval_predict_withoutid.state_dict())
        self.target_predict_id.load_state_dict(
            self.eval_predict_id.state_dict())

        self.optimiser = RMSprop(
            params=self.params, lr=args.lr, alpha=args.optim_alpha, eps=args.optim_eps)

        # a little wasteful to deepcopy (e.g. duplicates action selector), but should work for any MAC
        self.target_mac = copy.deepcopy(mac)
        self.log_stats_t = -self.args.learner_log_interval - 1

        self.n_actions = self.args.n_actions
        self.list = [(np.arange(args.n_agents - i) + i).tolist() + np.arange(i).tolist()
                     for i in range(args.n_agents)]

    def train_predict(self, batch: EpisodeBatch, t_env: int):
        # Get the relevant quantities
        terminated = batch["terminated"][:, :-1].float()
        mask = batch["filled"][:, :-1].float()
        mask[:, 1:] = mask[:, 1:] * (1 - terminated[:, :-1])
        actions_onehot = batch["actions_onehot"][:, :-1]
        last_actions_onehot = th.cat([th.zeros_like(
            actions_onehot[:, 0].unsqueeze(1)), actions_onehot], dim=1)

        # Calculate estimated Q-Values
        self.mac.init_hidden(batch.batch_size)
        initial_hidden = self.mac.hidden_states.clone().detach()
        initial_hidden = initial_hidden.reshape(
            -1, initial_hidden.shape[-1]).to(self.args.device)
        input_here = th.cat((batch["obs"], last_actions_onehot),
                            dim=-1).permute(0, 2, 1, 3).to(self.args.device)

        _, hidden_store, _ = self.mac.agent.forward(
            input_here.clone().detach(), initial_hidden.clone().detach())
        hidden_store = hidden_store.reshape(
            -1, input_here.shape[1], hidden_store.shape[-2], hidden_store.shape[-1]).permute(0, 2, 1, 3)

        obs = batch["obs"][:, :-1]
        obs_next = batch["obs"][:, 1:]

        h_cat = hidden_store[:, :-1]
        add_id = th.eye(self.args.n_agents).to(obs.device).expand(
            [obs.shape[0], obs.shape[1], self.args.n_agents, self.args.n_agents])
        mask_reshape = mask.unsqueeze(-1).expand_as(
            h_cat[..., 0].unsqueeze(-1))

        _obs = obs.reshape(-1, obs.shape[-1]).detach()
        _obs_next = obs_next.reshape(-1, obs_next.shape[-1]).detach()
        _h_cat = h_cat.reshape(-1, h_cat.shape[-1]).detach()
        _add_id = add_id.reshape(-1, add_id.shape[-1]).detach()
        _mask_reshape = mask_reshape.reshape(-1, 1).detach()
        _actions_onehot = actions_onehot.reshape(
            -1, actions_onehot.shape[-1]).detach()

        if self.args.ifaddobs:
            h_cat_r = th.cat(
                [th.zeros_like(h_cat[:, 0]).unsqueeze(1), h_cat[:, :-1]], dim=1)
            intrinsic_input = th.cat(
                [h_cat_r, obs, actions_onehot], dim=-1)
            _inputs = intrinsic_input.detach(
            ).reshape(-1, intrinsic_input.shape[-1])
        else:
            _inputs = th.cat([_h_cat, _actions_onehot], dim=-1)

        loss_withid_list, loss_withoutid_list, loss_predict_id_list = [], [], []
        # update predict network
        for _ in range(self.args.predict_epoch):
            for index in BatchSampler(SubsetRandomSampler(range(_obs.shape[0])), 256, False):
                loss_withoutid = self.eval_predict_withoutid.update(
                    _inputs[index], _obs_next[index], _mask_reshape[index])
                loss_withid = self.eval_predict_withid.update(
                    _inputs[index], _obs_next[index], _add_id[index], _mask_reshape[index])

                if loss_withoutid:
                    loss_withoutid_list.append(loss_withoutid)
                if loss_withid:
                    loss_withid_list.append(loss_withid)

        self.logger.log_stat("predict_loss_noid", np.array(
            loss_withoutid_list).mean(), t_env)
        self.logger.log_stat("predict_loss_withid", np.array(
            loss_withid_list).mean(), t_env)

        if self.args.ifaver:
            pass
        else:
            ID_for_predict = th.tensor(self.list[0]).type_as(
                hidden_store).unsqueeze(0).unsqueeze(0)

            ID_for_predict = ID_for_predict.expand_as(hidden_store[..., 0])
            _ID_for_predict = ID_for_predict.reshape(-1)

            for _ in range(self.args.predict_epoch):
                for index in BatchSampler(SubsetRandomSampler(range(_obs.shape[0])), 256, False):
                    loss_predict_id = self.eval_predict_id.update(
                        _h_cat[index], _ID_for_predict[index], _mask_reshape[index].squeeze())
                    if loss_predict_id:
                        loss_predict_id_list.append(loss_predict_id)

            self.logger.log_stat("predict_loss_forid", np.array(
                loss_predict_id_list).mean(), t_env)

    def sub_train(self, batch: EpisodeBatch, t_env: int, episode_num: int, mac, mixer, optimiser, params,
                  show_demo=False, save_data=None):
        # Get the relevant quantities
        rewards = batch["reward"][:, :-1]
        actions = batch["actions"][:, :-1]
        terminated = batch["terminated"][:, :-1].float()
        mask = batch["filled"][:, :-1].float()
        mask[:, 1:] = mask[:, 1:] * (1 - terminated[:, :-1])
        avail_actions = batch["avail_actions"]
        actions_onehot = batch["actions_onehot"][:, :-1]
        last_actions_onehot = th.cat([th.zeros_like(
            actions_onehot[:, 0].unsqueeze(1)), actions_onehot], dim=1)

        # Calculate estimated Q-Values
        mac.init_hidden(batch.batch_size)
        initial_hidden = mac.hidden_states.clone().detach()
        initial_hidden = initial_hidden.reshape(
            -1, initial_hidden.shape[-1]).to(self.args.device)
        input_here = th.cat((batch["obs"], last_actions_onehot),
                            dim=-1).permute(0, 2, 1, 3).to(self.args.device)

        mac_out, hidden_store, local_qs = mac.agent.forward(
            input_here.clone().detach(), initial_hidden.clone().detach())
        hidden_store = hidden_store.reshape(
            -1, input_here.shape[1], hidden_store.shape[-2], hidden_store.shape[-1]).permute(0, 2, 1, 3)

        # Pick the Q-Values for the actions taken by each agent
        chosen_action_qvals = th.gather(
            mac_out[:, :-1], dim=3, index=actions).squeeze(3)  # Remove the last dim

        x_mac_out = mac_out.clone().detach()
        x_mac_out[avail_actions == 0] = -9999999
        max_action_qvals, max_action_index = x_mac_out[:, :-1].max(dim=3)

        max_action_index = max_action_index.detach().unsqueeze(3)
        is_max_action = (max_action_index == actions).int().float()

        if show_demo:
            q_i_data = chosen_action_qvals.detach().cpu().numpy()
            q_data = (max_action_qvals -
                      chosen_action_qvals).detach().cpu().numpy()
            # self.logger.log_stat('agent_1_%d_q_1' % save_data[0], np.squeeze(q_data)[0], t_env)
            # self.logger.log_stat('agent_2_%d_q_2' % save_data[1], np.squeeze(q_data)[1], t_env)

        # Calculate the Q-Values necessary for the target
        self.target_mac.init_hidden(batch.batch_size)
        initial_hidden_target = self.target_mac.hidden_states.clone().detach()
        initial_hidden_target = initial_hidden_target.reshape(
            -1, initial_hidden_target.shape[-1]).to(self.args.device)
        target_mac_out, _, _ = self.target_mac.agent.forward(
            input_here.clone().detach(), initial_hidden_target.clone().detach())
        target_mac_out = target_mac_out[:, 1:]

        # Mask out unavailable actions
        target_mac_out[avail_actions[:, 1:] == 0] = -9999999

        # Max over target Q-Values
        if self.args.double_q:
            # Get actions that maximise live Q (for double q-learning)
            mac_out_detach = mac_out.clone().detach()
            mac_out_detach[avail_actions == 0] = -9999999
            cur_max_actions = mac_out_detach[:, 1:].max(dim=3, keepdim=True)[1]
            target_chosen_qvals = th.gather(
                target_mac_out, 3, cur_max_actions).squeeze(3)
            target_max_qvals = target_mac_out.max(dim=3)[0]
            target_next_actions = cur_max_actions.detach()

            cur_max_actions_onehot = th.zeros(
                cur_max_actions.squeeze(3).shape + (self.n_actions,)).to(th.device(self.args.GPU))
            cur_max_actions_onehot = cur_max_actions_onehot.scatter_(
                3, cur_max_actions, 1)
        else:
            # Calculate the Q-Values necessary for the target
            target_mac_out = []
            self.target_mac.init_hidden(batch.batch_size)
            for t in range(batch.max_seq_length):
                target_agent_outs = self.target_mac.forward(batch, t=t)
                target_mac_out.append(target_agent_outs)
            # We don't need the first timesteps Q-Value estimate for calculating targets
            target_mac_out = th.stack(
                target_mac_out[1:], dim=1)  # Concat across time
            target_max_qvals = target_mac_out.max(dim=3)[0]

        # Mix
        if mixer is not None:
            if self.args.mixer == "dmaq_qatten":
                ans_chosen, q_attend_regs, head_entropies = \
                    mixer(chosen_action_qvals, batch["state"][:, :-1], is_v=True,
                          obs=batch["obs"][:, :-1])  # changed for cds_gfootball
                ans_adv, _, _ = mixer(chosen_action_qvals, batch["state"][:, :-1], actions=actions_onehot,
                                      max_q_i=max_action_qvals, is_v=False,
                                      obs=batch["obs"][:, :-1])  # changed for cds_gfootball
                chosen_action_qvals = ans_chosen + ans_adv
            else:
                ans_chosen = mixer(chosen_action_qvals,
                                   batch["state"][:, :-1], is_v=True)
                ans_adv = mixer(chosen_action_qvals, batch["state"][:, :-1], actions=actions_onehot,
                                max_q_i=max_action_qvals, is_v=False)
                chosen_action_qvals = ans_chosen + ans_adv

            if self.args.double_q:
                if self.args.mixer == "dmaq_qatten":
                    target_chosen, _, _ = self.target_mixer(target_chosen_qvals, batch["state"][:, 1:], is_v=True,
                                                            obs=batch["obs"][:, 1:])  # changed for cds_gfootball
                    target_adv, _, _ = self.target_mixer(target_chosen_qvals, batch["state"][:, 1:],
                                                         actions=cur_max_actions_onehot,
                                                         max_q_i=target_max_qvals, is_v=False,
                                                         obs=batch["obs"][:, 1:])  # changed for cds_gfootball
                    target_max_qvals = target_chosen + target_adv
                else:
                    target_chosen = self.target_mixer(
                        target_chosen_qvals, batch["state"][:, 1:], is_v=True)
                    target_adv = self.target_mixer(target_chosen_qvals, batch["state"][:, 1:],
                                                   actions=cur_max_actions_onehot,
                                                   max_q_i=target_max_qvals, is_v=False)
                    target_max_qvals = target_chosen + target_adv
            else:
                target_max_qvals = self.target_mixer(
                    target_max_qvals, batch["state"][:, 1:], is_v=True)

        # Intrinsic

        with th.no_grad():

            obs = batch["obs"][:, :-1]
            obs_next = batch["obs"][:, 1:]
            h_cat = hidden_store[:, :-1]
            add_id = th.eye(self.args.n_agents).to(obs.device).expand(
                [obs.shape[0], obs.shape[1], self.args.n_agents, self.args.n_agents])

            if self.args.ifaddobs:
                h_cat_reshape = th.cat(
                    [th.zeros_like(h_cat[:, 0]).unsqueeze(1), h_cat[:, :-1]], dim=1)
                intrinsic_input = th.cat(
                    [h_cat_reshape, obs, actions_onehot], dim=-1)
            else:
                intrinsic_input = th.cat([h_cat, actions_onehot], dim=-1)

            log_p_o = self.target_predict_withoutid.get_log_pi(
                intrinsic_input, obs_next)

            add_id = th.eye(self.args.n_agents).to(obs.device).expand(
                [obs.shape[0], obs.shape[1], self.args.n_agents, self.args.n_agents])
            log_q_o = self.target_predict_withid.get_log_pi(
                intrinsic_input, obs_next, add_id)
            obs_diverge = self.args.beta1 * log_q_o - log_p_o

            # estimate p(a|o)
            mac_out_c_list = []
            for item_i in range(self.args.n_agents):
                mac_out_c, _, _ = self.mac.agent.forward(
                    input_here[:, self.list[item_i]], initial_hidden)
                mac_out_c_list.append(mac_out_c)

            mac_out_c_list = th.stack(mac_out_c_list, dim=-2)
            mac_out_c_list = mac_out_c_list[:, :-1]

            if self.args.ifaver:
                mean_p = th.softmax(mac_out_c_list, dim=-1).mean(dim=-2)
            else:
                weight = self.target_predict_id(h_cat)
                weight_expend = weight.unsqueeze(-1).expand_as(mac_out_c_list)
                mean_p = (weight_expend *
                          th.softmax(mac_out_c_list, dim=-1)).sum(dim=-2)

            q_pi = th.softmax(self.args.beta1 * mac_out[:, :-1], dim=-1)

            pi_diverge = th.cat([(q_pi[:, :, id] * th.log(q_pi[:, :, id] / mean_p[:, :, id])).sum(
                dim=-1, keepdim=True) for id in range(self.args.n_agents)], dim=-1).unsqueeze(-1)

            intrinsic_rewards = obs_diverge + self.args.beta2 * pi_diverge
            intrinsic_rewards = intrinsic_rewards.mean(dim=2)

        # Calculate 1-step Q-Learning targets
        targets = rewards + self.args.beta * intrinsic_rewards + \
            self.args.gamma * (1 - terminated) * target_max_qvals

        if show_demo:
            tot_q_data = chosen_action_qvals.detach().cpu().numpy()
            tot_target = targets.detach().cpu().numpy()
            print('action_pair_%d_%d' % (save_data[0], save_data[1]), np.squeeze(q_data[:, 0]),
                  np.squeeze(q_i_data[:, 0]), np.squeeze(tot_q_data[:, 0]), np.squeeze(tot_target[:, 0]))
            self.logger.log_stat('action_pair_%d_%d' % (save_data[0], save_data[1]),
                                 np.squeeze(tot_q_data[:, 0]), t_env)
            return

        # Td-error
        td_error = (chosen_action_qvals - targets.detach())

        mask = mask.expand_as(td_error)

        # 0-out the targets that came from padded data
        masked_td_error = td_error * mask

        # Normal L2 loss, take mean over actual data
        if self.args.mixer == "dmaq_qatten":
            loss = (masked_td_error ** 2).sum() / mask.sum() + q_attend_regs
        else:
            loss = (masked_td_error ** 2).sum() / mask.sum()
        update_prior = (masked_td_error ** 2).squeeze().sum(dim=-1,
                                                            keepdim=True) / mask.squeeze().sum(dim=-1, keepdim=True)

        masked_hit_prob = th.mean(is_max_action, dim=2) * mask
        hit_prob = masked_hit_prob.sum() / mask.sum()

        norm_loss = F.l1_loss(local_qs, target=th.zeros_like(
            local_qs), reduction='none')[:, :-1]
        mask_expand = mask.unsqueeze(-1).expand_as(norm_loss)
        norm_loss = (norm_loss * mask_expand).sum() / mask_expand.sum()
        loss += 0.1 * norm_loss

        # Optimise
        optimiser.zero_grad()
        loss.backward()
        grad_norm = th.nn.utils.clip_grad_norm_(
            params, self.args.grad_norm_clip)
        optimiser.step()

        if t_env - self.log_stats_t >= self.args.learner_log_interval:
            self.logger.log_stat("loss", loss.item(), t_env)
            self.logger.log_stat("hit_prob", hit_prob.item(), t_env)
            self.logger.log_stat("grad_norm", grad_norm, t_env)
            mask_elems = mask.sum().item()
            self.logger.log_stat(
                "td_error_abs", (masked_td_error.abs().sum().item() / mask_elems), t_env)
            self.logger.log_stat("q_taken_mean",
                                 (chosen_action_qvals * mask).sum().item() / (mask_elems * self.args.n_agents), t_env)
            self.logger.log_stat("target_mean", (targets * mask).sum().item() / (mask_elems * self.args.n_agents),
                                 t_env)

            intrinsic_rewards_mask = th.abs(intrinsic_rewards.detach()) * mask
            intrinsic_rewards_mask_max = intrinsic_rewards_mask.max().to('cpu').item()
            intrinsic_rewards_mask_mean = (
                intrinsic_rewards_mask.sum() / mask.sum()).to('cpu').item()

            self.logger.log_stat("intrinsic_reward_max",
                                 intrinsic_rewards_mask_max, t_env)
            self.logger.log_stat("intrinsic_reward_mean",
                                 intrinsic_rewards_mask_mean, t_env)
            self.log_stats_t = t_env

        return update_prior.squeeze().detach()

    def train(self, batch: EpisodeBatch, t_env: int, episode_num: int, show_demo=False, save_data=None):
        update_prior = self.sub_train(batch, t_env, episode_num, self.mac, self.mixer, self.optimiser, self.params,
                                      show_demo=show_demo, save_data=save_data)
        if (episode_num - self.last_target_update_episode) / self.args.target_update_interval >= 1.0:
            self._update_targets()
            self.last_target_update_episode = episode_num

        return update_prior

    def _update_targets(self):
        self.target_mac.load_state(self.mac)
        if self.mixer is not None:
            self.target_mixer.load_state_dict(self.mixer.state_dict())

        self.target_predict_withid.load_state_dict(
            self.eval_predict_withid.state_dict())
        self.target_predict_withoutid.load_state_dict(
            self.eval_predict_withoutid.state_dict())
        self.target_predict_id.load_state_dict(
            self.eval_predict_id.state_dict())
        self.logger.console_logger.info("Updated target network")

    def cuda(self):
        self.mac.cuda()
        self.target_mac.cuda()
        if self.mixer is not None:
            self.mixer.to(th.device(self.args.GPU))
            self.target_mixer.to(th.device(self.args.GPU))

    def save_models(self, path):
        self.mac.save_models(path)
        if self.mixer is not None:
            th.save(self.mixer.state_dict(), "{}/mixer.th".format(path))
        th.save(self.optimiser.state_dict(), "{}/opt.th".format(path))

    def load_models(self, path):
        self.mac.load_models(path)
        # Not quite right but I don't want to save target networks
        self.target_mac.load_models(path)
        if self.mixer is not None:
            self.mixer.load_state_dict(
                th.load("{}/mixer.th".format(path), map_location=lambda storage, loc: storage))
            self.target_mixer.load_state_dict(th.load("{}/mixer.th".format(path),
                                                      map_location=lambda storage, loc: storage))
        self.optimiser.load_state_dict(
            th.load("{}/opt.th".format(path), map_location=lambda storage, loc: storage))


from pprint import pprint
from time import sleep
from selenium.webdriver.chrome.service import Service
from selenium import webdriver
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import requests

class DataEntryJobAutomation():
    def __init__(self) -> None:
        self.captura_completa = False

    def get_research_houses(self, url_zillow):
        self.url_zillow = url_zillow
        headers_list = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
            "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 "
            "Safari/537.36 OPR/83.0.4254.27",
            "Accept-Language": "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7"
        }

        response = requests.get(self.url_zillow, headers=headers_list)
        response.raise_for_status()

        data = response.text
        soup = BeautifulSoup(data, "html.parser")

        print()
        # Getting all the links elements
        links_element = soup.select(".list-card-info a")

        all_links = []
        for link in links_element:
            href = link["href"]
            if "http" not in href:
                href = f"https://www.zillow.com{href}"
            print(href)
            all_links.append(href)
        print(f"Tamanho de links: {len(all_links)}\n")

        # Getting all the rent values
        rent_values_elements = soup.select("div .list-card-price")

        all_rent_values = []
        for rent_value in rent_values_elements:
            value = rent_value.getText().split("/")[0].split("+")[0]
            print(value)
            all_rent_values.append(value)
        print(f"Tamanho de alugueis: {len(all_rent_values)}\n")

        # Getting all the address
        all_address_elements = soup.select(".list-card-addr")

        all_address = []
        for address in all_address_elements:
            address_text = address.getText()
            print(address_text)
            all_address.append(address_text)
        print(f"Tamanho de Enderecos: {len(all_address)}\n")

        if len(all_address) == len(all_links) and \
                len(all_links) == len(all_rent_values):
            self.tamanho = len(all_links)
            self.lista = [
                {
                    "address": all_address[i],
                    "rent-value": all_rent_values[i],
                    "link": all_links[i],
                }
                for i in range(self.tamanho)
            ]
            pprint(self.lista)
            self.captura_completa = True
        else:
            print("Houve problemas na captura dos dados.")
            self.captura_completa = False

    def inicializar_driver(self):
        chrome_driver_path =\
            "C:\\github\\100_days_of_code_python\\chromedriver\\chromedriver.exe"

        ser = Service(chrome_driver_path)
        op = webdriver.ChromeOptions()
        self.driver = webdriver.Chrome(service=ser, options=op)

    def send_forms(self, url_form):
        self.url_form = url_form
        if self.captura_completa:
            self.inicializar_driver()
            self.driver.get(self.url_form)

            for indice in range(self.tamanho):
                sleep(2)

                input_address = self.driver.find_element(
                    By.CSS_SELECTOR,
                    "#mG61Hd > div.freebirdFormviewerViewFormCard.exportFormCard > div > div.freebirdFormviewerViewItemList > div:nth-child(1) > div > div > div.freebirdFormviewerComponentsQuestionTextRoot > div > div.quantumWizTextinputPaperinputMainContent.exportContent > div > div.quantumWizTextinputPaperinputInputArea > input"
                )
                input_address.send_keys(self.lista[indice]["address"])

                input_rent = self.driver.find_element(
                    By.CSS_SELECTOR,
                    "#mG61Hd > div.freebirdFormviewerViewFormCard.exportFormCard > div > div.freebirdFormviewerViewItemList > div:nth-child(2) > div > div > div.freebirdFormviewerComponentsQuestionTextRoot > div > div.quantumWizTextinputPaperinputMainContent.exportContent > div > div.quantumWizTextinputPaperinputInputArea > input"
                )
                input_rent.send_keys(self.lista[indice]["rent-value"])

                input_link = self.driver.find_element(
                    By.CSS_SELECTOR,
                    "#mG61Hd > div.freebirdFormviewerViewFormCard.exportFormCard > div > div.freebirdFormviewerViewItemList > div:nth-child(3) > div > div > div.freebirdFormviewerComponentsQuestionTextRoot > div > div.quantumWizTextinputPaperinputMainContent.exportContent > div > div.quantumWizTextinputPaperinputInputArea > input"
                )
                input_link.send_keys(self.lista[indice]["link"])

                button_send = self.driver.find_element(
                    By.XPATH,
                    '//*[@id="mG61Hd"]/div[2]/div/div[3]/div[1]/div[1]/div/span'
                )
                button_send.click()

                button_resend = self.driver.find_element(
                    By.LINK_TEXT,
                    "Enviar outra resposta"
                )
                button_resend.click()

            self.driver.close()
        else:
            print("Não foi possivel enviar os dados para o formulario.")


import numpy as np

class User:
    def __init__(self, cpu_speed, avg_job_cycle, server_cpu_speed,
                 job_arrive_prob, rate, position, time_weight, energy_weight):
        self.trans_power = 100
        self.SNR = -40
        self.path_loss_exp = 3.5
        self.data_offload_rate = 100

        self.cpu_speed = cpu_speed
        self.avg_job_cycles = avg_job_cycle
        self.SERVER_CPU_SPEED = server_cpu_speed
        self.job_arrive_prob = job_arrive_prob
        self.job_arrive_rate = rate
        self.position = position
        self.time_weight = time_weight
        self.energy_weight = energy_weight

        self.buffer_using = 0
        self.trans_speed = 360/8 * np.log2(1 + (self.trans_power * np.power(self.position, -1 * self.path_loss_exp)) / np.power(10, self.SNR / 10))

    def get_load(self):
        return self.avg_job_cycles * self.job_arrive_rate

    def local_power(self):
        return 1e-27 * self.cpu_speed**2 * self.avg_job_cycles

    def local_time(self):
        return self.avg_job_cycles / self.cpu_speed + self.buffer_using / self.cpu_speed

    def edge_time(self, delay):
        return (self.avg_job_cycles / self.SERVER_CPU_SPEED) + (self.data_offload_rate / self.trans_speed) + delay

    def edge_power(self):
        return self.trans_power / 1000 * self.data_offload_rate / self.trans_speed

    def make_offload_decision(self, price, delay):
        if self.buffer_using - 10 * self.cpu_speed > 0:
            self.buffer_using -= 10 * self.cpu_speed
        else:
            self.buffer_using = 0

        if np.random.rand() > self.job_arrive_prob:
            return 'NO_TASK'
        else:
            if self.get_user_profit(price, delay) > 0:
                return 'EDGE'
            else:
                self.buffer_using += self.get_load()
                return 'LOCAL'

    def get_user_profit(self, price, delay):
        local_power = self.local_power()
        local_time = self.local_time()
        edge_time = self.edge_time(delay)
        edge_power = self.edge_power()
        user_profit = self.time_weight * (local_time - edge_time) + self.energy_weight * (local_power - edge_power) - price
        return user_profit

    def get_sum_cost(self, delay, judge):
        if judge == 'LOCAL':
            return -1 * (self.time_weight * (self.local_time()) + self.energy_weight * (self.local_power()))
        elif judge == 'EDGE':
            return -1 * (self.time_weight * (self.edge_time(delay)) + self.energy_weight * (self.edge_power()))


from data_extract import *

fiction = pd.Series(df_fiction.groupby('Title')['ratings'].mean())
non_fiction = pd.Series(df_non_fiction.groupby('Title')['ratings'].mean())

fiction = np.array(fiction)
non_fiction = np.array(non_fiction)

#handle any NaNs and covnert to zero
fiction = np.nan_to_num(fiction)
non_fiction = np.nan_to_num(non_fiction)

#permutation sampling to simulate the hypothesis that fiction book ratings and non-fiction book ratings have identical probability distributions.
def permutation_sample(data1, data2):
    """Generate a permutation sample from two data sets."""

    # Concatenate the data sets: data
    data = np.concatenate((data1, data2))

    # Permute the concatenated array: permuted_data
    permuted_data = np.random.permutation(data)

    # Split the permuted array into two: perm_sample_1, perm_sample_2
    perm_sample_1 = permuted_data[:len(data1)]
    perm_sample_2 = permuted_data[len(data1):]

    return perm_sample_1, perm_sample_2

#generate a permutation replicate from the permutation sample
def draw_perm_reps(data_1, data_2, func, size=1):
    """Generate multiple permutation replicates."""

    # Initialize array of replicates: perm_replicates
    perm_replicates = np.empty(size)

    for i in range(size):
        # Generate permutation sample
        perm_sample_1, perm_sample_2 = permutation_sample(data_1, data_2)
        # Compute the test statistic
        perm_replicates[i] = func(perm_sample_1, perm_sample_2)

    return perm_replicates

def diff_of_means(data_1, data_2):
    """Difference in means of two arrays."""
    # The difference of means of data_1, data_2: diff
    diff = np.mean(data_1) - np.mean(data_2)

    return diff

#Compute difference of mean ratings
empirical_diff_means = diff_of_means(fiction, non_fiction)

# Draw 10,000 permutation replicates: perm_replicates
perm_replicates = draw_perm_reps(fiction, non_fiction,
                                 diff_of_means, size=10000)

# Compute p-value: p
p = np.sum(perm_replicates >= empirical_diff_means) / len(perm_replicates)

# Print the result
print('same distribution p-value =', p)
#p-value = 0.493

#test the hypothesis that Frog A and Frog B have the same mean impact force

def bootstrap_replicate_1d(data, func):
    bs_sample = np.random.choice(data, len(data))
    return func(bs_sample)

#function to generate many bootstrap replicates from the data set
def draw_bs_reps(data, func, size=1):
    """Draw bootstrap replicates."""

    # Initialize array of replicates: bs_replicates
    bs_replicates = np.empty(size)

    # Generate replicates
    for i in range(size):
        bs_replicates[i] = bootstrap_replicate_1d(data, func)

    return bs_replicates

#concatenate the datasets
ratings_concat = np.concatenate((fiction, non_fiction))
#compute mean for all ratings
mean_ratings = np.mean(ratings_concat)

#shift both arrays to have the same mean as we are simulating the hypothesis that their means are equal
fiction_shifted = fiction - np.mean(fiction) + mean_ratings
non_fiction_shifted = non_fiction - np.mean(non_fiction) + mean_ratings

#draw bootstrap samples out of the shifted arrays and compute the difference in means
bs_replicates_fiction = draw_bs_reps(fiction_shifted, np.mean, size=10000)
bs_replicates_non_fiction = draw_bs_reps(non_fiction_shifted, np.mean, size=10000)
#calculate the bootstrap replicate
bs_replicates = bs_replicates_fiction - bs_replicates_non_fiction

#p-value is the fraction of replicates with a difference in means greater than or equal to what was observed
p = np.sum(bs_replicates >= empirical_diff_means) / len(bs_replicates)
print('same mean p-value =', p)
#p-value = 0.5087


from django.shortcuts import render
from rest_framework import status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from .models import Branch_Office
from .serializers import *

@api_view(['GET'])
def branch_office_list(request):
    """
    List  products, or create a new product.
    """
    if request.method == 'GET':
        data = []
        nextPage = 1
        previousPage = 1
        products = Branch_Office.objects.all()
        page = request.GET.get('page', 1)
        paginator = Paginator(products, 10)
        try:
            data = paginator.page(page)
        except PageNotAnInteger:
            data = paginator.page(1)
        except EmptyPage:
            data = paginator.page(paginator.num_pages)

        serializer = Branch_OfficeSerializer(data,context={'request': request} ,many=True)
        if data.has_next():
            nextPage = data.next_page_number()
        if data.has_previous():
            previousPage = data.previous_page_number()

        return Response({'data': serializer.data , 'count': paginator.count, 'numpages' : paginator.num_pages, 'nextlink': '/api/branch_office/?page=' + str(nextPage), 'prevlink': '/api/branch_office/?page=' + str(previousPage)})


# from pudb import set_trace; set_trace()
from typing import List
from collections import Counter, deque
import heapq

class Solution1:
    def maxResult(self, nums: List[int], k: int) -> int:
        """LeetCode 1696

        The DP idea is fairly straightforward. In fact it is exactly the same
        as the one we handled on the past Monday. The tricky part is that k is
        also a variable, which means if we iterate through all k values in the
        DP table for each new n, the runtime would be O(NK). This will time out
        given the size of N and K. Therefore, the problem to solve is how to
        acquire the max value in a sliding window of size k in less than linear
        time.

        My solution is to use a heap, which tracks the max value in logarithm
        time. The trick is to prune the heap when the heap size grows larger
        than the allowed k. Since we can only prune the max value in the heap,
        we need to also record the indices of each value pushed into the heap.
        Once the heap size is larger than k, AND the top of the heap has index
        outside the current sliding window range, we pop it. This way, it is
        guaranteed that each value in the heap is only popped once.

        O(Nlog(K)), 1652 ms, very bad runtime.
        """
        heap = []
        N = len(nums)
        for i in range(N - 1):
            # remove the top of heap if its index suggests that it is no longer
            # in range
            while len(heap) > k and i - heap[0][1] > k:
                heapq.heappop(heap)
            cur = nums[i] + (-heap[0][0] if heap else 0)
            heapq.heappush(heap, (-cur, i))
        # Must prune the heap one more time
        while len(heap) > k and N - 1 - heap[0][1] > k:
            heapq.heappop(heap)
        return nums[-1] + (-heap[0][0] if heap else 0)

class Solution2:
    def maxResult(self, nums: List[int], k: int) -> int:
        """O(N) solution, 1676 ms. Although the runtime seems bad, but this is
        a brilliant solution. While I used a heap to solve the problem of
        maintaining the max value in a sliding window, this solution uses a
        non increasing (monotonic) deque to solve the same problem. The idea is to always
        keep the max value on the left of the deque. This requires that each
        time a new value gets pushed into the deque, we pop out, from the right
        side, all the values that are smaller than the new one. Also, we check
        the left side to see if the index is out of range. If it is, we pop from
        the left.

        The runtime is bad probably because I am using a tuple for the deque.
        Or it could be because LeetCode has increased test cases.
        """
        scores = deque([(nums[0], 0)])
        for i in range(1, len(nums)):
            while i - scores[0][1] > k:
                scores.popleft()
            cur = nums[i] + scores[0][0]
            while scores and scores[-1][0] < cur:
                scores.pop()
            scores.append((cur, i))
        return scores[-1][0]

class Solution3:
    def maxResult(self, nums: List[int], k: int) -> int:
        """Speed up version of solution2

        Courtesy: https://leetcode.com/problems/jump-game-vi/discuss/978462/C%2B%2B-DP-%2B-Monoqueue-O(n)

        1228 ms.
        """
        indices = deque([0])
        scores = [0] * len(nums)
        scores[0] = nums[0]
        for i in range(1, len(nums)):
            while i - indices[0] > k:
                indices.popleft()
            cur = nums[i] + scores[indices[0]]
            while indices and scores[indices[-1]] < cur:
                indices.pop()
            indices.append(i)
            scores[i] = cur
        return scores[-1]

sol = Solution3()
tests = [
    ([1, -1, -2, 4, -7, 3], 2, 7),
    ([10, -5, -2, 4, 0, 3], 3, 17),
    ([1, -5, -20, 4, -1, 3, -6, -3], 2, 0),
    ([1], 1, 1),
    ([100, -1, -100, -1, 100], 2, 198),
]

for i, (nums, k, ans) in enumerate(tests):
    res = sol.maxResult(nums, k)
    if res == ans:
        print(f'Test {i}: PASS')
    else:
        print(f'Test {i}; Fail. Ans: {ans}, Res: {res}')


from __future__ import print_function
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD
from keras.utils import np_utils
from keras.layers.normalization import BatchNormalization
import numpy
import random
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping
from sklearn.metrics import accuracy_score
import theano

from keras import backend as K

import numpy as np
import scipy as sp
from scipy import signal
import os
from PIL import Image

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import time

from scipy.misc import imread

def get_kernel(shape=(3,3),sigma=0.5):
    """
    2D gaussian mask - should give the same result as MATLAB's
    fspecial('gaussian',[shape],[sigma])
    """
    m,n = [(ss-1.)/2. for ss in shape]
    y,x = np.ogrid[-m:m+1,-n:n+1]
    h = np.exp( -(x*x + y*y) / (2.*sigma*sigma) )
    h[ h < np.finfo(h.dtype).eps*h.max() ] = 0
    sumh = h.sum()
    if sumh != 0:
        h /= sumh
    return h

def add_gnoise_util(image):
    kernel = get_kernel(shape=(10,10),sigma=3)
    image   = signal.convolve2d(image, kernel, boundary='fill', fillvalue=0,mode='same')
    #print(image.shape)
    return image

def convert_to_image(image, name):

	#plt.imshow(image, interpolation='none')
	#plt.savefig(name)
    a = (image*255).astype('uint8')
    im = Image.fromarray(a)
    im.save(name)

im = imread("boat.jpg")

im = numpy.swapaxes(im,1,2)
im = numpy.swapaxes(im,0,1)

im = im[numpy.newaxis,:,:,:]
print(im.shape)

im = im.astype('float32')
#im /= 255

batch_size = 256
nb_classes = 10
nb_epoch = 200
data_augmentation = True

# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3

# the data, shuffled and split between train and test sets
(X_train, y_train), (X_test, y_test) = cifar10.load_data()

model = Sequential()

model.add(Convolution2D(64, 3, 3, border_mode='same',
                        input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2,2)))

model.add(Convolution2D(128, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Convolution2D(256, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(256, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Convolution2D(1024, 4, 4, border_mode='valid'))
model.add(Activation('relu'))
model.add(Convolution2D(1024, 1, 1, border_mode='valid'))
model.add(Activation('relu'))

model.add(Convolution2D(nb_classes, 1, 1, border_mode='valid'))
vc = BatchNormalization()
model.add(vc)
model.add(Flatten())
#model.add(Dense(nb_classes))
model.add(Activation('softmax'))

model.load_weights("/data/lisatmp4/sarath/data/output/conv/1/weights.hdf5")#/data/lisatmp4/chinna/data/ift6268/temp/1/weights.hdf5")

# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

convout = theano.function([model.get_input(train=False)], vc.get_output(train=False))
t0 = time.clock()
[layer_output] = convout(im)
print(layer_output.shape)

dpath = "/data/lisatmp4/chinna/data/ift6268/temp/1/"

for i in range(0,10):
	convert_to_image(layer_output[i],dpath+str(i)+"old.jpg")
	layer_output[i] = add_gnoise_util(layer_output[i])
	print(max(layer_output[i].flatten()))
	convert_to_image(layer_output[i],dpath+str(i)+".jpg")
print ("Time")
print (time.clock() - t0)


import numpy as np
import torch
import torch.nn.functional as f

class Conv:
    def __init__(self, c_in, c_out, kernel, stride=1, padding=0, dilation=1):
        self.kernel = kernel
        self.stride = stride
        self.padding = padding
        self.dilation = dilation
        self.c_in = c_in
        self.c_out = c_out

        self.weight = np.random.rand(self.c_out, self.c_in, self.kernel, self.kernel)
        self.bias = np.random.rand(self.c_out)

    def forward(self, X):
        """
        X is of shape BxC_inxHxW
        """

        B, cx_in, H, W = X.shape
        assert cx_in == self.c_in

        H_out = (
            H + 2 * self.padding - ((self.kernel - 1) * self.dilation + 1)
        ) // self.stride + 1
        W_out = (
            W + 2 * self.padding - ((self.kernel - 1) * self.dilation + 1)
        ) // self.stride + 1

        X = np.pad(
            X,
            (
                (0, 0),
                (0, 0),
                (self.padding, self.padding),
                (self.padding, self.padding),
            ),
        )
        out = np.zeros((B, self.c_out, H_out, W_out))

        for i in range(H_out):
            for j in range(W_out):
                w2 = (self.kernel // 2) * self.dilation
                in_i = w2 + i * self.stride
                in_j = w2 + j * self.stride
                patch = X[
                    :,
                    :,
                    in_i - w2 : in_i + w2 + 1 : self.dilation,
                    in_j - w2 : in_j + w2 + 1 : self.dilation,
                ]
                out[:, :, i, j] = (patch[:, None] * self.weight[None]).sum(
                    (2, 3, 4)
                ) + self.bias[None]

        return out

    def forward_torch(self, X):
        out = f.conv2d(
            torch.from_numpy(X),
            torch.from_numpy(self.weight),
            torch.from_numpy(self.bias),
            self.stride,
            self.padding,
            self.dilation,
        )
        return out.numpy()

def main():
    img = np.random.randn(1, 3, 10, 10)
    conv = Conv(3, 2, 3, 2, 1, 2)
    out2 = conv.forward_torch(img)
    out1 = conv.forward(img)
    assert (out1 - out2).sum() < 1e-10

if __name__ == "__main__":
    main()


# Дан список сотрудников:
staff = [
    {
        'name': 'Григорий',
        'surname': 'Петров',
        'salary': 124300
    },
    {
        'name': 'Николай',
        'surname': 'Петров',
        'salary': 120000
    },
    {
        'name': 'Иван',
        'surname': 'Павлов',
        'salary': 34500
    },
    {
        'name': 'Василий',
        'surname': 'Кукушкин',
        'salary': 162500
    },
    {
        'name': 'Василий',
        'surname': 'Павлов',
        'salary': 34500
    },
]

# 1. Выведите список сотрудников, отсортированный по уменьшению их заработной платы.
# Если у нескольких сотрудников одинаковая ЗП, то добавьте сортировку по Фамилии
print("Список сотрудников отсортированный по уменьшению ЗП:")

# 2. Найдите сумму зарплат трех самых низкооплачиваемых сотрудников:


import os

from celery import Celery

from configs.default.cron import task_schedule

# set the default Django settings module for the 'celery' program.
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs")

app = Celery("sspanel")

# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object("django.conf:settings", namespace="CELERY")

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()

app.conf.beat_schedule = {
    task: dict(task=task, schedule=schedule) for task, schedule in task_schedule.items()
}


import argparse

parser = argparse.ArgumentParser(description='read in message')
parser.add_argument('n', help='public key n', type=str)
parser.add_argument('d', help='private key d', type=str)
parser.add_argument('message', help='message', nargs=argparse.REMAINDER, default=[])
args = parser.parse_args()

def decrypt(encr_m, d, n):
    message = [chr(pow(int(t), d, n)) for t in encr_m]
    return ''.join(message)

if __name__ == '__main__':
    n = int(args.n)
    d = int(args.d)
    message = args.message

    print('*Decrypted Message*')
    print(decrypt(message, d, n))


#!/usr/bin/env python3
"""
usage: gendummydata.py outputfilename.sql
"""
#
# This script seeds the AUR database with dummy data for
# use during development/testing.  It uses random entries
# from /usr/share/dict/words to create user accounts and
# package names.  It generates the SQL statements to
# insert these users/packages into the AUR database.
#
import hashlib
import logging
import os
import random
import sys
import time

LOG_LEVEL = logging.DEBUG  # logging level. set to logging.INFO to reduce output
SEED_FILE = "/usr/share/dict/words"
USER_ID = 5            # Users.ID of first bogus user
PKG_ID = 1             # Packages.ID of first package
MAX_USERS = 76000      # how many users to 'register'
MAX_DEVS = .1          # what percentage of MAX_USERS are Developers
MAX_TUS = .2           # what percentage of MAX_USERS are Trusted Users
MAX_PKGS = 64000       # how many packages to load
PKG_DEPS = (1, 15)     # min/max depends a package has
PKG_RELS = (1, 5)      # min/max relations a package has
PKG_SRC = (1, 3)       # min/max sources a package has
PKG_CMNTS = (1, 5)     # min/max number of comments a package has
CATEGORIES_COUNT = 17  # the number of categories from aur-schema
VOTING = (0, .001)     # percentage range for package voting
OPEN_PROPOSALS = 5     # number of open trusted user proposals
CLOSE_PROPOSALS = 15   # number of closed trusted user proposals
RANDOM_TLDS = ("edu", "com", "org", "net", "tw", "ru", "pl", "de", "es")
RANDOM_URL = ("http://www.", "ftp://ftp.", "http://", "ftp://")
RANDOM_LOCS = ("pub", "release", "files", "downloads", "src")
FORTUNE_FILE = "/usr/share/fortune/cookie"

# setup logging
logformat = "%(levelname)s: %(message)s"
logging.basicConfig(format=logformat, level=LOG_LEVEL)
log = logging.getLogger()

if len(sys.argv) != 2:
    log.error("Missing output filename argument")
    raise SystemExit(1)

# make sure the seed file exists
#
if not os.path.exists(SEED_FILE):
    log.error("Please install the 'words' Arch package")
    raise SystemExit(1)

# make sure comments can be created
#
if not os.path.exists(FORTUNE_FILE):
    log.error("Please install the 'fortune-mod' Arch package")
    raise SystemExit(1)

# track what users/package names have been used
#
seen_users = {}
seen_pkgs = {}
user_keys = []

# some functions to generate random data
#
def genVersion():
    ver = []
    ver.append("%d" % random.randrange(0, 10))
    ver.append("%d" % random.randrange(0, 20))
    if random.randrange(0, 2) == 0:
        ver.append("%d" % random.randrange(0, 100))
    return ".".join(ver) + "-%d" % random.randrange(1, 11)

def genCategory():
    return random.randrange(1, CATEGORIES_COUNT)

def genUID():
    return seen_users[user_keys[random.randrange(0, len(user_keys))]]

def genFortune():
    return fortunes[random.randrange(0, len(fortunes))].replace("'", "")

# load the words, and make sure there are enough words for users/pkgs
#
log.debug("Grabbing words from seed file...")
fp = open(SEED_FILE, "r", encoding="utf-8")
contents = fp.readlines()
fp.close()
if MAX_USERS > len(contents):
    MAX_USERS = len(contents)
if MAX_PKGS > len(contents):
    MAX_PKGS = len(contents)
if len(contents) - MAX_USERS > MAX_PKGS:
    need_dupes = 0
else:
    need_dupes = 1

# select random usernames
#
log.debug("Generating random user names...")
user_id = USER_ID
while len(seen_users) < MAX_USERS:
    user = random.randrange(0, len(contents))
    word = contents[user].replace("'", "").replace(".", "").replace(" ", "_")
    word = word.strip().lower()
    if word not in seen_users:
        seen_users[word] = user_id
        user_id += 1
user_keys = list(seen_users.keys())

# select random package names
#
log.debug("Generating random package names...")
num_pkgs = PKG_ID
while len(seen_pkgs) < MAX_PKGS:
    pkg = random.randrange(0, len(contents))
    word = contents[pkg].replace("'", "").replace(".", "").replace(" ", "_")
    word = word.strip().lower()
    if not need_dupes:
        if word not in seen_pkgs and word not in seen_users:
            seen_pkgs[word] = num_pkgs
            num_pkgs += 1
    else:
        if word not in seen_pkgs:
            seen_pkgs[word] = num_pkgs
            num_pkgs += 1

# free up contents memory
#
contents = None

# developer/tu IDs
#
developers = []
trustedusers = []
has_devs = 0
has_tus = 0

# Just let python throw the errors if any happen
#
out = open(sys.argv[1], "w", encoding="utf-8")
out.write("BEGIN;\n")

# Begin by creating the User statements
#
log.debug("Creating SQL statements for users.")
for u in user_keys:
    account_type = 1  # default to normal user
    if not has_devs or not has_tus:
        account_type = random.randrange(1, 4)
        if account_type == 3 and not has_devs:
            # this will be a dev account
            #
            developers.append(seen_users[u])
            if len(developers) >= MAX_DEVS * MAX_USERS:
                has_devs = 1
        elif account_type == 2 and not has_tus:
            # this will be a trusted user account
            #
            trustedusers.append(seen_users[u])
            if len(trustedusers) >= MAX_TUS * MAX_USERS:
                has_tus = 1
        else:
            # a normal user account
            #
            pass

    h = hashlib.new('md5')
    h.update(u.encode())
    s = ("INSERT INTO Users (ID, AccountTypeID, Username, Email, Passwd)"
         " VALUES (%d, %d, '%s', '%s@example.com', '%s');\n")
    s = s % (seen_users[u], account_type, u, u, h.hexdigest())
    out.write(s)

log.debug("Number of developers: %d" % len(developers))
log.debug("Number of trusted users: %d" % len(trustedusers))
log.debug("Number of users: %d" % (MAX_USERS-len(developers)-len(trustedusers)))
log.debug("Number of packages: %d" % MAX_PKGS)

log.debug("Gathering text from fortune file...")
fp = open(FORTUNE_FILE, "r", encoding="utf-8")
fortunes = fp.read().split("%\n")
fp.close()

# Create the package statements
#
log.debug("Creating SQL statements for packages.")
count = 0
for p in list(seen_pkgs.keys()):
    NOW = int(time.time())
    if count % 2 == 0:
        muid = developers[random.randrange(0, len(developers))]
        puid = developers[random.randrange(0, len(developers))]
    else:
        muid = trustedusers[random.randrange(0, len(trustedusers))]
        puid = trustedusers[random.randrange(0, len(trustedusers))]
    if count % 20 == 0:  # every so often, there are orphans...
        muid = "NULL"

    uuid = genUID()  # the submitter/user

    s = ("INSERT INTO PackageBases (ID, Name, FlaggerComment, SubmittedTS, ModifiedTS, "
         "SubmitterUID, MaintainerUID, PackagerUID) VALUES (%d, '%s', '', %d, %d, %d, %s, %s);\n")
    s = s % (seen_pkgs[p], p, NOW, NOW, uuid, muid, puid)
    out.write(s)

    s = ("INSERT INTO Packages (ID, PackageBaseID, Name, Version) VALUES "
         "(%d, %d, '%s', '%s');\n")
    s = s % (seen_pkgs[p], seen_pkgs[p], p, genVersion())
    out.write(s)

    count += 1

    # create random comments for this package
    #
    num_comments = random.randrange(PKG_CMNTS[0], PKG_CMNTS[1])
    for i in range(0, num_comments):
        now = NOW + random.randrange(400, 86400*3)
        s = ("INSERT INTO PackageComments (PackageBaseID, UsersID,"
             " Comments, RenderedComment, CommentTS) VALUES (%d, %d, '%s', '', %d);\n")
        s = s % (seen_pkgs[p], genUID(), genFortune(), now)
        out.write(s)

# Cast votes
#
track_votes = {}
log.debug("Casting votes for packages.")
for u in user_keys:
    num_votes = random.randrange(int(len(seen_pkgs)*VOTING[0]),
                                 int(len(seen_pkgs)*VOTING[1]))
    pkgvote = {}
    for v in range(num_votes):
        pkg = random.randrange(1, len(seen_pkgs) + 1)
        if pkg not in pkgvote:
            s = ("INSERT INTO PackageVotes (UsersID, PackageBaseID)"
                 " VALUES (%d, %d);\n")
            s = s % (seen_users[u], pkg)
            pkgvote[pkg] = 1
            if pkg not in track_votes:
                track_votes[pkg] = 0
            track_votes[pkg] += 1
            out.write(s)

# Update statements for package votes
#
for p in list(track_votes.keys()):
    s = "UPDATE PackageBases SET NumVotes = %d WHERE ID = %d;\n"
    s = s % (track_votes[p], p)
    out.write(s)

# Create package dependencies and sources
#
log.debug("Creating statements for package depends/sources.")
# the keys of seen_pkgs are accessed many times by random.choice,
# so the list has to be created outside the loops to keep it efficient
seen_pkgs_keys = list(seen_pkgs.keys())
for p in seen_pkgs_keys:
    num_deps = random.randrange(PKG_DEPS[0], PKG_DEPS[1])
    for i in range(0, num_deps):
        dep = random.choice(seen_pkgs_keys)
        deptype = random.randrange(1, 5)
        if deptype == 4:
            dep += ": for " + random.choice(seen_pkgs_keys)
        s = "INSERT INTO PackageDepends(PackageID, DepTypeID, DepName) VALUES (%d, %d, '%s');\n"
        s = s % (seen_pkgs[p], deptype, dep)
        out.write(s)

    num_rels = random.randrange(PKG_RELS[0], PKG_RELS[1])
    for i in range(0, num_deps):
        rel = random.choice(seen_pkgs_keys)
        reltype = random.randrange(1, 4)
        s = "INSERT INTO PackageRelations(PackageID, RelTypeID, RelName) VALUES (%d, %d, '%s');\n"
        s = s % (seen_pkgs[p], reltype, rel)
        out.write(s)

    num_sources = random.randrange(PKG_SRC[0], PKG_SRC[1])
    for i in range(num_sources):
        src_file = user_keys[random.randrange(0, len(user_keys))]
        src = "%s%s.%s/%s/%s-%s.tar.gz" % (
                RANDOM_URL[random.randrange(0, len(RANDOM_URL))],
                p, RANDOM_TLDS[random.randrange(0, len(RANDOM_TLDS))],
                RANDOM_LOCS[random.randrange(0, len(RANDOM_LOCS))],
                src_file, genVersion())
        s = "INSERT INTO PackageSources(PackageID, Source) VALUES (%d, '%s');\n"
        s = s % (seen_pkgs[p], src)
        out.write(s)

# Create trusted user proposals
#
log.debug("Creating SQL statements for trusted user proposals.")
count = 0
for t in range(0, OPEN_PROPOSALS+CLOSE_PROPOSALS):
    now = int(time.time())
    if count < CLOSE_PROPOSALS:
        start = now - random.randrange(3600*24*7, 3600*24*21)
        end = now - random.randrange(0, 3600*24*7)
    else:
        start = now
        end = now + random.randrange(3600*24, 3600*24*7)
    if count % 5 == 0:  # Don't make the vote about anyone once in a while
        user = ""
    else:
        user = user_keys[random.randrange(0, len(user_keys))]
    suid = trustedusers[random.randrange(0, len(trustedusers))]
    s = ("INSERT INTO TU_VoteInfo (Agenda, User, Submitted, End,"
         " Quorum, SubmitterID) VALUES ('%s', '%s', %d, %d, 0.0, %d);\n")
    s = s % (genFortune(), user, start, end, suid)
    out.write(s)
    count += 1

# close output file
#
out.write("COMMIT;\n")
out.write("\n")
out.close()
log.debug("Done.")


# Kernel Support Vector Machine for Non-linearly Separable Data
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

df = pd.read_csv('Datasets/Social_Network_Ads.csv')
X = df.iloc[:, 2:4].values
y = df.iloc[:, 4].values

# Split the Dataset into Train and Test Data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, random_state = 0)

# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

# Now Its time to impliment the SVM Classifier
from sklearn.svm import SVC
classifier_rbf = SVC(kernel = 'rbf', random_state = 0)  # Changing the kernel value fron linear to rbf (Gaussion)
classifier_rbf.fit(X_train, y_train)

# Predict the Resultant Value
y_pred_rbf = classifier_rbf.predict(X_test)

# Classifier for Linear kernel
classifier_linear = SVC(kernel = 'linear', random_state = 0)  # Changing the kernel value fron linear to rbf (Gaussion)
classifier_linear.fit(X_train, y_train)

# Predict the Resultant Value
y_pred_linear = classifier_linear.predict(X_test)

# Classifier for SIGMOID kernel
classifier_sigmoid = SVC(kernel = 'sigmoid', random_state = 0)  # Changing the kernel value fron linear to rbf (Gaussion)
classifier_sigmoid.fit(X_train, y_train)

# Predict the Resultant Value
y_pred_sigmoid = classifier_sigmoid.predict(X_test)

# Classifier for SIGMOID kernel
classifier_poly = SVC(kernel = 'poly', random_state = 0)  # Changing the kernel value fron linear to rbf (Gaussion)
classifier_poly.fit(X_train, y_train)

# Predict the Resultant Value
y_pred_sigmoid = classifier_sigmoid.predict(X_test)

# Checking the accuracy of the Traning Set
print('RBF Train : ', classifier_rbf.score(X_train, y_train))
print('RBF Test : ', classifier_rbf.score(X_test, y_test))
print('Linear Train : ', classifier_linear.score(X_train, y_train))
print('Linear Test : ', classifier_linear.score(X_test, y_test))
print('Sigmoid Train : ', classifier_sigmoid.score(X_train, y_train))
print('Sigmoid Test : ', classifier_sigmoid.score(X_test, y_test))
print('Poly Train : ', classifier_poly.score(X_train, y_train))
print('Poly Test : ', classifier_poly.score(X_test, y_test))

# Lest Understand the confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred_rbf)
print(cm)

# Visualising the Training Set Result
from matplotlib.colors import ListedColormap
X_set, y_set = X_train, y_train
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
                     np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier_rbf.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape), alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
    plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1],
                c = ListedColormap(('red', 'green'))(i), label = j)
plt.title('Support Vector Machine (Traning Set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()

# Visualising the Test Set Result
from matplotlib.colors import ListedColormap
X_set, y_set = X_test, y_test
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
                     np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier_rbf.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape), alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
    plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1],
                c = ListedColormap(('red', 'green'))(i), label = j)
plt.title('Support Vector Machine (Test Set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()


# https://i.kym-cdn.com/entries/icons/original/000/020/546/7e7.jpg
import random

def make_poem(n1, n2, n3, v1, v2, v3, a1, a2, a3, p1, p2, a):
    return print(f"A {a1} {n1}\n\nA {a1} {n1} {v1} {p1} the {a2} {n2} {a1}, the {n1} {v2}\nthe {n2} {v3} {p2} a {a3} {n3}")

## Lists
# The lists are created by splitting the strings by the comma, turning them into an IEnumerable
nouns = "fossil, horse, aardvark, judge, chef, mango, extrovert, gorilla".split(", ")
verbs = "kicks, jingles, bounces, slurps, meows, explodes, curdles".split(", ")
adjectives = "furry, balding, incredolous, fragrant, exuberant, glistening".split(", ")
prepositions = "against, after, into, beneath, upon, for, in, like, over, within".split(", ")
adverbs = "curiously, extravagantly, tantalizingly, furiously, sensuously".split(", ")

## Strings
# To ensure that the loops will run at least once.
noun1, noun2, noun3 = "", "", ""
verb1, verb2, verb3 = "", "", ""
adjective1, adjective2, adjective3 = "", "", ""
preposition1, preposition2 = "", ""
adverb = random.choice(adverbs)

## Getting random words
# While some of the words are the same, get random values.
while noun1 == noun2 or noun1 == noun3 or noun2 == noun3:
    noun1 = random.choice(nouns)
    noun2 = random.choice(nouns)
    noun3 = random.choice(nouns)
while verb1 == verb2 or verb1 == verb3 or verb2 == verb3:
    verb1 = random.choice(verbs)
    verb2 = random.choice(verbs)
    verb3 = random.choice(verbs)
while adjective1 == adjective2 or adjective1 == adjective3 or adjective2 == adjective3:
    adjective1 = random.choice(adjectives)
    adjective2 = random.choice(adjectives)
    adjective3 = random.choice(adjectives)
while preposition1 == preposition2:
    preposition1 = random.choice(prepositions)
    preposition2 = random.choice(prepositions)

# print it out as a poem.
make_poem(noun1, noun2, noun3, verb1, verb2, verb3, adjective1, adjective2, adjective3, preposition1, preposition2, adverb)


import socket
import struct
# 该模块执行Python值和用Python字节对象表示的C结构体之间转换
#   1.处理存储在文件, 网络链接, 其他资源中的二进制数据
#   2.使用指定的格式化字符串作为C结构体和Python值的预期转换的布局的紧凑描述

def main():
    udp_socket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)  # 创建UDP socket对象
    pack_data = struct.pack('!h10sb5sb', 1, b'demo04.txt', 0, b'octet', 0)    # TFTP下载请求协议, 网络大端

    dst_addr_port = ('192.168.85.1', 69)  # destination
    udp_socket.sendto(pack_data, dst_addr_port)  # 发

    recv_data = udp_socket.recvfrom(1024)  # 收
    recv_data_bytes = recv_data[0]  # 服务端发送字节数据
    recv_data_addr = recv_data[1]   # 服务端发送数据包IP + 随机port

    option_block_code = struct.unpack('!hh', recv_data_bytes[:4])  # 2个字节表示操作码, 2个字节表示块编号
    option_code = option_block_code[0]   # 操作码
    block_code = option_block_code[1]    # 块编号

    if option_code != 5:
        file = open('demo04.txt', 'wb')
        file.write(recv_data_bytes[4:])

        while len(recv_data_bytes) >= 516:
            ack = struct.pack('!hh', 4, block_code)    # 确认反馈数据包
            udp_socket.sendto(ack, recv_data_addr)

            recv_data_bytes = udp_socket.recv(1024)
            block_code = struct.unpack('!hh', recv_data_bytes[:4])[1]
            file.write(recv_data_bytes[4:])
        file.close()
    else:
        print('出错...')

    udp_socket.close()

if __name__ == '__main__':
    main()

# 问题
# 1.为什么不写入文件, 直接解码输出报错?
#   data = recv_data_bytes[4:].decode('UTF-8')
#   print(data)
# 2.为什么在While循环中设置延时, 下载数据有错?

# 一.内存地址从0X000...开始编号, 低位到高位
#   1.小端: 高低位对应存储
#   2.大端: 高低位相反存储
#     eg.
#       1.个人PC: 小端
#       2.IBM等大型服务器: 大端
#   3.网络协议规定以大端为准


from copy import deepcopy
from json import dumps, loads

def handle_content(is_open, **_):
    return {
        'add_as_content': True,
        'open': not is_open,
    }

def handle_delimiter(has_ending_quotes, has_starting_quotes, is_open, **_):
    if is_open:
        if not has_starting_quotes or has_ending_quotes:
            return {
                'close': True,
                'flags': {
                    'has_ending_quotes': False,
                    'has_starting_quotes': False,
                },
            }
        return {
            'add_as_content': True,
        }
    return {
        'open': True,
        'close': True,
    }

def handle_escape(**_):
    return {'flags': {'escape_next': True, 'escape_occurred': True}}

def handle_quotation(has_starting_quotes, is_open, **_):
    if is_open:
        if not has_starting_quotes:
            return None
        return {'flags': {'has_ending_quotes': True}}
    return {'flags': {'has_starting_quotes': True, 'quotation_occurred': True}, 'open': True}

handlerMap = {
    'content': handle_content,
    'delimiter': handle_delimiter,
    'escape': handle_escape,
    'quotation': handle_quotation,
}

def get_seq_type(config, flags, raw, cursor_position):
    matches = {
        special_name: bool(special_char) and raw.startswith(special_char, cursor_position)
        for special_name, special_char in config.items()
    }
    match_count = len([match_result for match_result in matches.values() if match_result])
    if match_count > 1 and matches['delimiter']:
        raise Exception('Multiple possible interpretations')
    if not match_count:
        return 'content', 1
    matched_type = next(k for k, v in matches.items() if v)
    if match_count > 1:  # escape and quotation both match
        if flags['has_starting_quotes']:
            if raw.startswith(config['quotation'], cursor_position + len(config['escape'])) or \
                    raw.startswith(config['escape'], cursor_position + len(config['escape'])):
                matched_type = 'escape'
            else:
                matched_type = 'quotation'
        else:
            matched_type = 'quotation'
    return matched_type, len(config[matched_type])

def parse_line_character_delimiter(
        raw,
        config,
        handle_possible_quoting_character,
        handle_possible_escape_character,
        max_length,
        cache=None
):
    flags = cache['flags'] if cache else{
        'escape_next': False,
        'has_ending_quotes': False,
        'has_starting_quotes': False,
        'escape_occurred': False,
        'quotation_occurred': False,
        'is_open': False
    }
    row = cache['row'] if cache else []
    current_cell = cache['cell'] if cache else ''
    cursor_position = cache['cursor'] if cache else 0
    while cursor_position < len(raw):
        try:
            seq_type, length = get_seq_type(config, flags, raw, cursor_position)
            if flags['escape_next']:
                if seq_type == 'content':
                    return None
                else:
                    seq_type = 'content'
            if (not ('quotation' in config)
                    and seq_type == 'content'
                    and not raw[cursor_position].isalnum()):
                [
                    handle_possible_quoting_character(raw[cursor_position:cursor_position+i], {'row': [*row], 'cursor': cursor_position, 'flags': {**flags}, 'cell': current_cell})
                    for i in range(1, max_length + 1)
                    if not any(char.isalnum() for char in raw[cursor_position:cursor_position+i])
                ]
            if ('quotation' in config
                    and not ('escape' in config)
                    and seq_type == 'content'
                    and not raw[cursor_position].isalnum()
                    and cursor_position < len(raw) - 1
                    and flags['has_starting_quotes']
            ):
                [
                    handle_possible_escape_character(config['quotation'], raw[cursor_position:cursor_position+i], {'row': [*row], 'cursor': cursor_position, 'flags': {**flags}, 'cell': current_cell})
                    for i in range(1, max_length + 1)
                    if raw[cursor_position:cursor_position+i] and not any(char.isalnum() for char in raw[cursor_position:cursor_position+i])
                ]
            if flags['has_ending_quotes'] and seq_type != 'delimiter':
                # warnings.warn('Illegal character at position ' + str(cursor_position))
                return None
            flags['escape_next'] = False
            if not flags['is_open'] and seq_type == 'content' and raw[cursor_position].isspace():
                # early exit for cases where we have spaces outside of fields
                cursor_position += length
                continue
            update = handlerMap[seq_type](**flags)
            if update is None:
                return None
            flags = {**flags, **update['flags']} if 'flags' in update else flags
            if update.get('open'):
                flags['is_open'] = True
                current_cell = ''
            if update.get('add_as_content'):
                current_cell += raw[cursor_position:cursor_position + length]
            cursor_position += length
            if update.get('close'):
                flags['is_open'] = False
                row.append(current_cell.strip(' '))
                current_cell = ''
        except:  # Exception in get_seq_type
            return None
    if flags['has_starting_quotes'] and not flags['has_ending_quotes'] or flags['escape_next']:
        # warnings.warn('Illegal character at position ' + str(cursor_position))
        return None
    else:
        row.append(current_cell.strip(' '))
    if ('escape' in config and not flags['escape_occurred']) or ('quotation' in config and not flags['quotation_occurred']):
        return None
    return {
        'type': 'character',
        **config,
        'parsed': row
    }

def parse_line(raw, delimiter, max_length):
    possible_quotes = []
    possible_escapes = []
    solutions = []

    def handle_possible_escape_character(quotation, escape, cache):
        if {'quotation': quotation, 'escape': escape} in possible_escapes:
            return
        possible_escapes.append({'quotation': quotation, 'escape': escape})
        solutions.append(parse_line_character_delimiter(
            raw,
            {
                'delimiter': delimiter,
                'quotation': quotation,
                'escape': escape
            },
            handle_possible_quoting_character,
            handle_possible_escape_character,
            max_length,
            deepcopy(cache)
        ))

    def handle_possible_quoting_character(quotation, cache):
        if quotation in possible_quotes:
            return
        possible_quotes.append(quotation)
        solutions.append(parse_line_character_delimiter(
            raw,
            {
                'delimiter': delimiter,
                'quotation': quotation,
            },
            handle_possible_quoting_character,
            handle_possible_escape_character,
            max_length,
            deepcopy(cache)
        ))
        solutions.append(parse_line_character_delimiter(
            raw,
            {
                'delimiter': delimiter,
                'quotation': quotation,
                'escape': quotation  # according to RFC 4180 2 (7)
            },
            handle_possible_quoting_character,
            handle_possible_escape_character,
            max_length,
            deepcopy(cache)
        ))

    solutions.append(parse_line_character_delimiter(raw.strip(' '), {'delimiter': delimiter}, handle_possible_quoting_character,
                                                    handle_possible_escape_character, max_length))
    return [
        loads(dump)
        for dump in set(
            dumps(solution, sort_keys=True)
            for solution in solutions
            if solution
        )
    ]


import sqlite3

class DBHelper:

    def __init__(self, dbname="julio_bot.sqlite"):
        self.dbname = dbname
        self.conn = sqlite3.connect(dbname)

    def setup(self):
        tblstmt = "CREATE TABLE IF NOT EXISTS items (trigger, response text)"
        trig_idx = "CREATE INDEX IF NOT EXISTS triggerIndex ON items (trigger ASC)"
        resp_idx = "CREATE INDEX IF NOT EXISTS responseIndex ON items (response ASC)"
        self.conn.execute(tblstmt)
        self.conn.execute(trig_idx)
        self.conn.execute(resp_idx)
        self.conn.commit()

    def add_item(self, trigger, response):
        stmt = "INSERT INTO items (trigger, response) VALUES (?, ?)"
        args = (trigger, response)
        self.conn.execute(stmt, args)
        self.conn.commit()

    def delete_item(self, trigger, response):
        stmt = "DELETE FROM items WHERE trigger = (?) AND response = (?)"
        args = (trigger, response)
        self.conn.execute(stmt, args)
        self.conn.commit()

    def get_items(self, trigger):
        stmt = "SELECT trigger FROM items"
        # args = (trigger)
        return [x[0] for x in self.conn.execute(stmt)]

    def get_response(self, trigger):
        stmt = "SELECT * FROM items WHERE trigger = (?)"
        args = (trigger, )
        return [x for x in self.conn.execute(stmt, args)]


from django import forms

from .models import PostModel

class PostForm(forms.ModelForm):

    class Meta:
        model = PostModel
        fields = ("post",)
        labels = {
            'post': 'Напишите пост',
        }
        widgets = {
            'post': forms.Textarea(attrs={'class': 'post-input'})
        }


# from django.contrib.auth.models import User
from django.db import models

from rest_framework.serializers import (
        HyperlinkedIdentityField,
        ModelSerializer,
        SerializerMethodField,
        )

from directories.models import  PhoneNumber, Department, Faculty
# from users.api.serializers import UserDetailSerializer

#Creating an ApplyMedical
class PhoneNumberCreateUpdateSerializer(ModelSerializer):
    department  = models.ForeignKey(Department, default=1, on_delete = models.CASCADE)
    class Meta:
        model = PhoneNumber
        fields = [
            'department',
            'phone',
        ]

apply_detail_url = HyperlinkedIdentityField(
        view_name='directories-api:detail',
        lookup_field='pk'#or primary key <pk>
    )

class FacultyDetailSerializer(ModelSerializer):

    class Meta:
        model = Faculty
        fields = [
            'id',
            'shortname',
            'fullname',
            'dean',
            'location',
            'lat',
            'lng',
            'timestamp',
            'updated',
            ]

class DepartmentDetailSerializer(ModelSerializer):
    faculty = FacultyDetailSerializer(read_only=True)
    class Meta:
        model = Department
        fields = [

            'id',
            'shortname',
            'fullname',
            'location',
            'chairman',
            'email',
            'lat',
            'lng',
            'timestamp',
            'updated',
            'faculty',
            ]

class PhoneNumberDetailSerializer(ModelSerializer):
    url        = apply_detail_url
    department = DepartmentDetailSerializer(read_only=True)

    class Meta:
        model = PhoneNumber
        fields = [
            'url',
            'id',
            'phone',
            'timestamp',
            'updated',
            'department',
        ]

class PhoneNumberListSerializer(ModelSerializer):
    url = apply_detail_url
    department = DepartmentDetailSerializer(read_only=True)
    delete_url = HyperlinkedIdentityField(
        view_name='directories-api:delete',
        lookup_field='pk'#or primary key <pk>
    )
    class Meta:
        model = PhoneNumber
        fields = [
            'url',
            'id',
            'phone',
            'timestamp',
            'updated',
            'delete_url',
            'department',
        ]


#   使用朴素贝叶斯算法进行邮件分类
#   非垃圾邮件：2  email/ham
#   垃圾邮件:   -2  email/spam

import re
from os import listdir
import random

def getVocabularyList():
    vocabularySet = set([])
    fileList = listdir('email/ham')  # 正常邮件
    m = len(fileList)
    fileList.extend(listdir('email/spam'))  #非正常邮件
    regEx = re.compile('\\W*')
    for filename in fileList:
        if(m != 0):
            m -= 1
            ifile = open('email/ham' + '/' + filename)
        else:
            ifile = open('email/spam' + '/' + filename)
        lines = ifile.readlines()
        s = ''
        for line in lines:
            s = s + " " + line.split('\n')[0]
        listOfTokens = regEx.split(s)
        vocabularySet = vocabularySet | set(listOfTokens)
        ifile.close()
    return list(vocabularySet)

def readData(vocabularList):
    dataSet = []
    label = []
    dirs = ['email/ham', 'email/spam']
    regEx = re.compile('\\W*')
    for dir in dirs:
        fileList = listdir(dir)
        for filename in fileList:
            data = [0] * len(vocabularList)
            ifile = open(dir + '/' + filename)
            lines = ifile.readlines()
            for line in lines:
                line = line.split('\n')[0]
                listOfTokens = regEx.split(line)
                for token in listOfTokens:
                    index = vocabularList.index(token)
                    data[index] = 1
            dataSet.append(data)
            if(dir == dirs[0]): #非垃圾邮件
                label.append(2)
            else:
                label.append(-2) #垃圾邮件
    #使用相同的规则打乱两个列表
    zipTmp = list(zip(dataSet, label))
    random.shuffle(zipTmp)
    dataSet[:], label[:] = zip(*zipTmp)
    # 拆分成train 和 test数据集
    trainM = int(len(label) * 0.7)  # 前70%用于训练
    trainDataSet = dataSet[:trainM][:]
    trainLabel = label[:trainM]
    testDataSet = dataSet[trainM:][:]
    testLabel = label[trainM:]

    return trainDataSet, trainLabel, testDataSet, testLabel

#训练
# -- 获得先验概率和条件概率
def train(trainDataSet, trainLabel, vocabularyList):
    classSet = set(trainLabel)  # 标签的取值范围
    m = len(trainLabel)  #训练样本数
    k = len(classSet) #类别个数
    lambd = 1   #拉普拉斯平滑系数
    prob_ck = {}
    #计算p(Y = ck)
    for ck in classSet:
        prob_ck[ck] = float(trainLabel.count(ck) + lambd) / (m + k * lambd)
    # 计算条件概率  p(Xi = xi | Y = ck)
    #tmp = [0] * len(vocabularyList)
    prob_condition = {}
    for ck in classSet:
        prob_condition[ck] = {}
    for key in prob_condition.keys():
        sumCk = trainLabel.count(key)
        for j in range(len(trainDataSet[0])):
            #XjSet = set(x[j] for x in trainDataSet)
            XjSet = set([0, 1])  #不能用上面的一句，因为一定要包含所有的！
            Sj = len(XjSet)
            prob_condition[key][j] = {}
            for aji in XjSet:
                res = []
                for i in range(m):
                    if(trainDataSet[i][j] == aji):
                        res.append(i)
                countList = [i for i in res if trainLabel[i] == key]
                sumXji = len(countList)
                prob_condition[key][j][aji] = (float)(sumXji + lambd) / (sumCk + Sj * lambd)
    return prob_ck, prob_condition

#测试
def test(testDataSet, testLabel, vocabularyList, prob_ck, prob_condition):
    m = len(testDataSet)
    n = len(testDataSet[0])
    errorCount = 0
    for i in range(m):
        maxProb = -10
        maxLabel = -1
        for key in prob_ck.keys():
            currentProb = prob_ck[key]
            ckDict = prob_condition[key]
            for j in range(n):
                #print(testDataSet[i][j])
                currentProb = currentProb * ckDict[j][testDataSet[i][j]]
            if(currentProb > maxProb):
                maxLabel = key
                maxProb = currentProb
        if(maxLabel == testLabel[i]):
            print("right! %d" % maxLabel)
        else:
            print("error! realLabel: %d  predictLabel %d: " % (testLabel[i], maxLabel))
            errorCount += 1
    print("total: %d  errorRate: %f" % (m, float(errorCount) / m))

def main():
    vocabularList = getVocabularyList()
    #print(vocabularList)
    trainDataSet, trainLabel, testDataSet, testLabel = readData(vocabularList)
    #训练 - 获得各种概率
    prob_ck, prob_condition = train(trainDataSet, trainLabel, vocabularList)
#    print(prob_ck)
#    print(prob_condition)

    #测试
    test(testDataSet, testLabel, vocabularList, prob_ck, prob_condition)

if __name__ == "__main__":
    main()


import requests
from io import StringIO
import pandas as pd
import matplotlib.pyplot as plt

url = "https://www.twse.com.tw/exchangeReport/STOCK_DAY?response=csv&date=20210301&stockNo=2330"
resp = requests.get(url)
raw_data = resp.text
clean_data = raw_data.split('\r\n')[1:-6]
clean_data[0] = '"date","deal_amount","deal_price","open_price","highest_price","lowest_price","end_price","diff_pirce","deal_num",'

sum_str = ""
for data in clean_data:
    sum_str += data + '\r\n'

df = pd.read_csv(StringIO(sum_str), index_col=0)
df.to_csv("stock.csv")
df['end_price'].plot()
plt.show()


# Python script to convert between data formats.
# It takes as input the json outputs of Alphapose and converts them to the appropriate dictionary form for
# the SLT methods. This script assumes that AlphaPose extraction has already been performed.
# Developed in the context of the aiD project
import gzip
import pickle
import torch
import json

def save(Data,name):

  with gzip.open(name, 'wb') as handle:
    pickle.dump(Data, handle, protocol=pickle.HIGHEST_PROTOCOL)

directory = 'data/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/features/fullFrame-210x260px/test/'
text = 'data/PHOENIX-2014-T-release-v3/PHOENIX-2014-T/annotations/manual/PHOENIX-2014-T.test.corpus.csv'
count = 0
all_data = []
with open(text, 'r') as csvfile:
    for line in csvfile:
        if count == 0:
            count +=1
            continue
        #print(line.split('|'))
        dir, _, _, _, signer,gloss, text = line.split('|')
        outputpath = directory + dir + '/'

        with open(outputpath + 'AlphaPose_out.json') as f:
            data = json.load(f)

        data_tensor = torch.Tensor([ x['keypoints'] for x in data[3:]])

        dict_data = {
            'name': dir,
            'signer': signer,
            'gloss': gloss.strip(),
            'text': text.strip(),
            'sign': data_tensor
            }

        all_data.append(dict_data)
        count += 1

        if count % 200 == 0:
            print(count)

save(all_data, 'alphapose_phoenix.test')