#! usr/local/bin/python
# encoding: utf-8
import os
import sys
from driller_all_nodes import *
from cfg_entry import *
import shutil
import random
import mmap
from python_uart import *
import time
from math import ceil, log2
from multiprocessing.context import Process

class GrowingList(list):
    '''auto-growing list, with random value filled'''
    def __setitem__(self, index, value):
        if index >= len(self):
          glength = 2**ceil(log2(index))-len(self)
          #self.extend([0x00]*glength)
          self.extend([random.randint(0x00,0xff) for _ in range(glength)])
        list.__setitem__(self, index, value)

def read_one(txt_path):
    read_data=[]
    wen_jian = open('%s'%txt_path)
    for linedata in wen_jian.readlines():
        linedata = str(linedata).replace("\n","")
        cut = linedata.find("     G")
        #print(cut)
        linedata = linedata[0:cut]
        #print(linedata)
        if int(linedata.split(":", 1)[1],16) not in read_data: #去除重复数据
          read_data.append(int(linedata.split(":", 1)[1],16))
    return(read_data)

def read_function(inputpath):
    with open(inputpath, 'r', encoding='utf-8') as infile:
         data1 = []
         for line in infile:
                data_line = line.strip("\n").split()  # 去除首尾换行符，并按空格划分
                if len(data_line) != 0:
                  data1.append(data_line)
    infile.close()
    return data1

#def demo_solve_uart(path):
def main(path,info_path,is_custom):
    seedsnum=0
    #path = './Drone.elf' #测试用例
    function_info = read_function(info_path)  #调用txt载入目标函数信息
    entrys=[]
    #entrys = [0x8000423]  #不含axf时需要手动写入需要探索的地址  可以手动指定探索哪个函数，节省时间
    seeds=[]
    if path.find('.bin') != -1:  #为bin文件
      file_path_bin= path #载入的文件地址
      file_path_axf = file_path_bin.replace("bin","axf")
      if os.path.exists(file_path_axf) == False:
          print("[*] WARNING! Need Axf for Bin!")
      if entrys == []:   #对于bin文件，如果没有手动指明哪个函数地址，就遍历所有函数
        print("[*] Entrys Is Empty!")
        #entrys = entry_get_bin(file_path_bin,function_info)
        entrys = entry_get_bin(file_path_bin)#bin文件如果没有axf提供包含函数名bootloader
    #elif path.find('.axf') != -1:  #为axf文件
    else :  #elf文件
         entrys = entry_get(path,1,function_info)
    #print(entrys)

    #full_name = path.split('/')[-1] if path.find('/') != -1 else path
    basename = os.path.basename(path)
    main_name = basename.split('.')[0] if basename.find('.') != -1 else basename
    if os.path.exists('./crashes/%s'%main_name):
      shutil.rmtree('./crashes/%s'%main_name)
    for entry_addr in entrys:
        os.makedirs('./crashes/%s/%s'%(main_name,hex(entry_addr)),exist_ok=True)
        get_nodes(entry_addr,path,main_name)
    for entry_addr in entrys:
      if os.listdir('./crashes/%s/%s'%(main_name,hex(entry_addr))) != []: #delete
         seeds.append(entry_addr)
      else:
         shutil.rmtree('./crashes/%s/%s'%(main_name,hex(entry_addr)))

    #qemu还是手动接线
    qemu_flag = 1 #选择是否是固件还是选用qemu  固件:0 // qemu:1

    if qemu_flag == 0:
      seeds=[0xc29]  #注释掉就选择所有中断为入口，固件需更改接线(qemu从seeds中手动挑选) 目前只支持一个一个探索
      print("[*] Start Fuzzing In UART")
    elif qemu_flag == 1:
      print("[*] Start Fuzzing In QEMU")

    #从crashes队列中读取字符串需要变异的位置以及可供选择变异的值
    all_data=dict()
    testbuf = ''
    logging.debug("If Something ERROR,Please Check cfg_entry.py line31")  #如果缺函数，请修改或核对cfg_entry.py line31
    time.sleep(3)
    for func in seeds:
        print("[*] Target Is ",hex(func))
        list_addr = os.listdir('./crashes/%s/%s'%(main_name,hex(func)))
        for i in list_addr:
            all_data[i]=read_one('./crashes/%s/%s/%s'%(main_name,hex(func),i))
        #random_buf= '21'+"".join([random.choice("0123456789abcde")for i in range(60)])+'ff'  #生成64位随机16进制
        os.makedirs('./crashes/%s/seeds/%s'%(main_name,hex(func)),exist_ok=True)
        #print(all_data.items()) #查看items
        #print(len(os.listdir('./crashes/seeds/%s'%hex(func))))
        while len(os.listdir('./crashes/%s/seeds/%s'%(main_name,hex(func)))) <10:  #qemu测试生成至多10个种子 固件只需1个 固件修改为：while *** == 0:
         #if qemu_flag == 0:
         #  random_buf= '21'+"".join([random.choice("0123456789abcde")for i in range(124)])+'ff'
         #if qemu_flag == 1:  #变异范围为ascii码表
         # random_buf= '21'+"".join([hex(random.randrange(48,57,1)).split('x')[-1]for i in range(31)])
         #random_buf= '21'+"".join([random.choice("0123456789abcde")for i in range(128-4)])+'ff'   #生成64个16进制的数值，如需修改位数，上下需同时修改
         random_buf= "".join([random.choice("0123456789abcde")for i in range(64)])   #生成256个16进制的数值，如需修改位数，上下需同时修改
         #random_buf= "".join(["0" for i in range(128)])
         for i in all_data.items():
           testbuf = ''
           #locate = int(i[0][5:],10) #XXX Update 07-25-23
           locate = (int(i[0][5:],10)+1)*2-1  #u16
           #print(locate)
           data = i[1][random.randint(0,len(i[1])-1)]
           #print(data)
           buf = bytes.fromhex(random_buf)
           #buflist = list(buf) #sometimes crashed by IndexError
           buflist = GrowingList(buf)
           #print(buflist)
           try:
            buflist[locate] = data #变异位置     不再局限于ascii码内
           except IndexError:
            print("[*] 超过种子长度限制.")
           for i in buflist:
              testbuf += str(hex(i)[2:].zfill(2))
           random_buf =  str(testbuf) #保留单次变异结果
         File_address = './crashes/%s/seeds/%s'%(main_name,hex(func))
         Seeds_address = './crashes/%s/seeds/%s/%d'%(main_name,hex(func),seedsnum)
         if qemu_flag == 0:
            uart_main(testbuf,func,main_name)
            print(testbuf)
         elif qemu_flag == 1:
            f = open(Seeds_address,'w+')
            #teststring = "a"*(64-1)       #生成64个16进制的数值，如需修改位数，上下需同时修改
            teststring = "a"*(len(buflist)-1)       #生成256个16进制的数值，如需修改位数，上下需同时修改
            print(teststring,file=f)
            #以下选择输出ascii类型 qemu   不再局限于ascii码内
            #ascii_string=''.join([chr(int(b, 16)) for b in [testbuf[i:i+2] for i in range(0, len(testbuf), 2)]])
            f.close()
            f = open(Seeds_address,'r+')
            m = mmap.mmap(f.fileno(), 0)
            m[:] = bytes.fromhex(random_buf)
            #os.system('./input.sh '+ascii_string)  #发送qemu
            #time.sleep(0.1)
            #os.system('./enter.sh')
            ##print('echo -n "',ascii_string,'" > /dev/pts/1 ',file=f)
            ##print('echo -e "\r\n" > /dev/pts/1',file=f)
            m.close()
            f.close()
         time.sleep(0.5) #每0.5s发送一次
         seedsnum = seedsnum+1
        print("[*] 0x{:x} Seeds Complited!".format(func))
        if is_custom:
         P2IMpath = input("[*] Enter the Path of P2IM INPUTS:")
        else:
         from datetime import datetime
         now = datetime.now()
         timestr = now.strftime("%d-%m-%Y")
         basename = os.path.basename(path)
         namedir = basename.split('.')[0] if basename.find('.') != -1 else basename
         P2IMpath = '/home/uzio/Projects/P2IM/seeds/'+ namedir + '/' + timestr
         for i in range(200):
          seedpath = P2IMpath + f'/s{i}'
          if os.path.exists(seedpath):
           continue
          else:
           os.makedirs(seedpath)
           P2IMpath = seedpath
           break

        cmd='cp -r {}/* {}'.format(File_address,P2IMpath)
        os.system(cmd)
        #for i in all_data:
            #if isinstance(i,str): #判断是否为字符串
            #   print(i)
        #uart_main(testbuf) #发送变异好的字符串
        #time.sleep(1)
    #将值输入uart中
    print("[*] Works Completed!")

if __name__ == "__main__":
    path = sys.argv[1]
    info_path = sys.argv[2]
    tic = time.perf_counter()
    main(path,info_path,False)
    toc = time.perf_counter()
    e_time = toc - tic
    print(f"Elapsed time: {e_time:0.4f} s")


# Fibonacci Numbers

# Step 1 : For "Looping"
# Based on number
# The first value
f0, f1 = 0, 1
n = int(input("Enter Number : "))

# Process
for i in range(n):
    sum = f0
    f0, f1 = f1, f0 + f1
    # Result
    print(sum, end=" ")


# id string for gv tool
gvtool_id_string = 'conn_gvtool'
freebies_id_string = 'conn_freebies'

from .dummy_interface import Dummy_Interface
from .amivapi_interface import AMIV_API_Interface
from .gvtool_interface import GV_Tool_Interface
from .pvkapi_interface import PVK_API_Interface
from .freebies_interface import Freebies_Interface

def create_connectors():
    # instantiate interfaces
    if_1 = AMIV_API_Interface()
    #if_2 = PVK_API_Interface()
    if_3 = GV_Tool_Interface()
    #if_4 = Dummy_Interface()
    if_5 = Freebies_Interface()

    # create list and return
    #connectors = [if_1, if_2, if_3, if_4]
    connectors = [if_1, if_3, if_5]
    return connectors

def get_connector_by_id(connectors: list, idstr: str) -> AMIV_API_Interface:
    for c in connectors:
        if c.id_string == idstr:
            return c

    raise Exception('Could not find connector with id: {}'.format(idstr))


import time
import os
import multiprocessing
from PIL import Image  # pip install pillow if not found
from glob import glob

__author__ = "Mithun"

def create_thumbnail(image_files):
    size = 128, 128

    for image_file in image_files:
        file_name, ext = os.path.splitext(image_file)
        image = Image.open(image_file)
        image.thumbnail(size)

        t_file = file_name + ".t.jpeg"
        image.save(t_file, "jpeg")

        print("Process {}: Thumbnail created for {} as {}".format(multiprocessing.current_process().name, image_file,
                                                                  t_file))

if __name__ == "__main__":
    processor_count = multiprocessing.cpu_count()
    print("Processor Count= {}".format(processor_count))

    image_files = glob(r"C:\happy-face\happy*.jpeg")
    images_per_process = (int)(len(image_files) / processor_count) + 1
    processes = []

    start = time.time()

    for i in range(processor_count):
        if (i + 1) * images_per_process > len(image_files):
            image_file_subset = image_files[i * images_per_process:]
        else:
            image_file_subset = image_files[i * images_per_process:(i + 1) * images_per_process]

        p = multiprocessing.Process(target=create_thumbnail, args=(image_file_subset,))
        processes.append(p)
        p.start()

    for p in processes:
        p.join()

    end = time.time()

    print("Total time taken is for thumbnail generation is {} seconds".format(end - start))

Concurrent Programming in Python - Section 6/pool-1.py


#!/usr/bin/env pybricks-micropython
import turtle
def draw_bar(t, height):
    if height > 0:
        """ Get turtle t to draw one bar, of height. """
        if height >= 200:
            tess.color("blue", "red")
        elif height >= 100:
            tess.color("blue", "yellow")
        elif height <100:
            tess.color("blue", "green")
        t.begin_fill()           # Added this line
        t.left(90)
        t.forward(height)
        t.write("  "+ str(height))
        t.right(90)
        t.forward(40)
        t.right(90)
        t.forward(height)
        t.left(90)
        t.end_fill()
        t.penup()             # Added this line
        t.forward(10)
        t.pendown()

    else:
        tess.color("blue", "green")
        t.begin_fill()           # Added this line
        t.left(90)
        t.forward(height)
        t.penup()
        t.forward(-15)
        t.write("  "+ str(height))
        t.forward(15)
        t.pendown()
        t.right(90)
        t.forward(40)
        t.right(90)
        t.forward(height)
        t.left(90)
        t.end_fill()
        t.penup()             # Added this line
        t.forward(10)
        t.pendown()
wn = turtle.Screen()         # Set up the window and its attributes
wn.bgcolor("lightgreen")

tess = turtle.Turtle()       # Create tess and set some attributes
tess.pensize(3)

xs = [48,117,200,-50,160,260,220]

for a in xs:
    draw_bar(tess, a)

wn.mainloop()


#!/usr/bin/env python3

from flask import Flask, json, request, g
from flask_restplus import Resource, Api
from flask_restplus import Resource, Api
from values import Configuration
from streamLocal import Stream
from recordLocal import Record
from time import sleep
import threading
import datetime

app = Flask(__name__)
api = Api(app, version='1.0', title='PiVideoAPI',
    description='Expose the PiVideo functionality interraction',
)

ns = api.namespace('pivideo', description='Interraction with the PiVideo')

class base(Resource):

  def get(self):
    response = app.response_class(
      response=json.dumps({'routes':{'/':'endpoint list', '/param/record':'change record times', '/param/backend':'change backend address', '/record':'record', '/stream': 'stream'}}),
      status=200,
      mimetype='application/json'
    )
    return response

class paramRecord(Resource):
  """interracting with configuration file"""

  @api.response(200, 'Successing')
  def get(self):
    config = Configuration()
    config.load()
    record_values = {}
    record_values['t_before'] = config.get("record", "t_before")
    record_values['t_after'] = config.get("record", "t_after")

    return json.dumps(record_values), 200

  @api.response(201, 'Key created successfully')
  def post(self):
    config = Configuration()
    config.load()

    # TODO: check sent value is an integer
    try:
      t_before = request.form['t_before']
    except:
      t_before = None
    try:
      t_after = request.form['t_after']
    except:
      t_after = None

    if t_before != None:
      config.set(route='record', key='t_before', value=t_before)
    if t_after != None:
      config.set(route='record', key='t_after', value=t_after)
    config.store()

    return None, 201

class paramBackend(Resource):
  """interracting with configuration file"""

  @api.response(200, 'Successing')
  def get(self):
    config = Configuration()
    config.load()
    record_values = {}
    record_values['backend_address'] = config.get("backend", "address")

    return json.dumps(record_values), 200

  @api.response(201, 'Key created successfully')
  def post(self):
    config = Configuration()
    config.load()

    try:
      backend_address = request.form['backend_address']
    except:
      backend_address = None

    if backend_address != None:
      config.set(route='backend', key='address', value=backend_address)
    config.store()

    return None, 201

class RecordEndpoint(Resource):

  api.response(200, 'record started')
  def get(self):
    global t1
    t1.storeVideo('video_'+str(datetime.datetime.now()))

    return None, 200

class StreamEndpoint(Resource):
  api.response(200, 'stream started')
  def get(self):
    global t1
    if (request.args.get('start') == "true") :
        t1 = Stream()
        t1.start()
        print("starting")
        print(t1)

    elif (request.args.get('start') == "false"):
        print("stopping")
        print(threading.enumerate())
        sleep(3)
        t1.stop()
        t1.join()
        print("done")

    return None, 200

api.add_resource(base, '/')
api.add_resource(RecordEndpoint, '/record', methods=['GET']) # Route_Record
api.add_resource(StreamEndpoint, '/stream')
api.add_resource(paramRecord, '/param/record')
api.add_resource(paramBackend, '/param/backend')

if __name__ == "__main__":
    global t1
    t1 = None
    config = Configuration()
    config.load()
    t_before = config.get('record', 't_before')
    t_after = config.get('record', 't_after')
    t1 = Record(t_before, t_after)
    t1.start()
    app.run(host='0.0.0.0', port='2001')


class User:
    bankName = "First National Dojo"

    def __init__(self,name,email):
        self.name = name
        self.email = email
        self.accountBalance = 0

    def makeDeposit(self, amount):
        self.accountBalance = self.accountBalance + amount

    def makeWithdrawal(self, amount):
        self.accountBalance = self.accountBalance - amount

    def displayAccBal(self):
        print(f"{self.name}, Your Account Balance is {self.accountBalance}")

    def transferMoney(self, amount, user):
        self.accountBalance -= amount
        user.accountBalance += amount
        self.displayAccBal()
        user.displayAccBal()

acc1 = User("Tom R.", "Tomsthebest@everything.com")
acc2 = User("Guido van Rossum", "guido@python.com")
acc3 = User("Monty Python", "monty@python.com")
acc1.makeDeposit(10622)
acc1.makeDeposit(31)
acc1.makeDeposit(2015)
acc1.makeWithdrawal(349)
acc1.displayAccBal()
acc2.makeDeposit(500)
acc2.makeDeposit(275.50)
acc2.makeWithdrawal(30)
acc2.makeWithdrawal(20.10)
acc2.displayAccBal()
acc3.makeDeposit(730.60)
acc3.makeWithdrawal(5.05)
acc3.makeWithdrawal(10.12)
acc3.makeWithdrawal(9.10)
acc3.displayAccBal()

acc1.transferMoney(319,acc3)


# -*- coding:utf-8 -*-
# Author:Alfred

from cocos.menu import *
from cocos.scene import *
from cocos.layer import *
from cocos.director import *
from cocos.sprite import *
from cocos.audio.effect import Effect

# 创建新的背景层
class BackGroud_seting(Layer):

    def __init__(self):
        super(BackGroud_seting, self).__init__()

        # 获取窗口的宽高
        self.width ,self.height = director.get_window_size()

        # 创建两个背景精灵
        backgroud = Sprite('images/setting-bg.png')
        backgroud.position = self.width // 2, self.height // 2

        # 添加进背景
        self.add(backgroud,0)

        # 创建两个开关精灵
        on = Sprite('images/on.png')
        on.position = 818, 280
        self.add(on, 0)
        on = Sprite('images/on.png')
        on.position = 818, 420
        self.add(on, 0)

# 自定义MainMenu层类
class MainMenu(Menu):

    def __init__(self):
        super(MainMenu, self).__init__()

        # 创建菜单字体
        self.font_item['font_size'] = 160
        self.font_item['color'] = (255, 255, 255, 255)
        self.font_item_selected['font_size'] = 160
        self.font_item_selected['color'] = (230, 230, 230, 255)

        # 设置菜单项
        on_up = ImageMenuItem('images/ok-up.png', self.on_on_up_callback)

        # 创建菜单
        self.create_menu([on_up], layout_strategy=fixedPositionMenuLayout([(560,130)]))

    def on_on_up_callback(self):
        effect = Effect('sound/Blip.wav')
        effect.play()
        director.pop()

def create_scene():
    # 创建一个场景，并将新的层实例添加到场景中
    main_scene = Scene(BackGroud_seting())
    main_scene.add(MainMenu())

    return main_scene


import copy

import numpy as np

import pymoo
from pysurrogate.surrogate import Surrogate
from pysurrogate.util.distance import calc_distance_matrix, eucl_distance

class MyRBF(Surrogate):

    def __init__(self, basis, optimize, theta=None) -> None:
        super().__init__()

        # the basis function to calculate the kernel
        self.basis = basis
        self.optimize = optimize
        self.theta = theta

        # data itself
        self.X = None
        self.y = None
        self.n_var = None

        # kernel matrix
        self.K = None

        # fitted weights
        self.w = None

        # the actual hyperparameter
        self.theta = None

    def _fit(self, X, y):

        # fit the values to predict
        self.X = X
        self.y = y
        self.n_var = X.shape[1]

        self.func_basis, theta_length = get_basis_function(self.basis, self.n_var)

        # if a hyperparameter is needed at all
        if theta_length is not None:

            if self.optimize is None:

                # create the kernel matrix
                self.K = self.basis.calc(X, X)

                # find the weights according to kernel and target
                self.w = np.linalg.solve(self.K, y)

            elif self.optimize == 'simple':

                h = np.mean(calc_distance_matrix(X, X, eucl_distance))

                self.basis.set_hyperparameter(h)
                self.refit()

            elif self.optimize == 'scipy':
                X_max = np.max(self.X, axis=0)
                X_min = np.min(self.X, axis=0)
                edges = X_max - X_min
                edges = edges[np.nonzero(edges)]
                self.theta = np.power(np.prod(edges) / X.shape[0], 1.0 / edges.size)

            elif optimize == 'ga':
                optimize(self, 5, 0.001, 20)

            else:
                raise Exception("Unknown optimization method: %s" % optimize)

    def refit(self, **kwargs):
        self.fit(self.X, self.y, **kwargs)

    def _predict(self, X):
        D = self.func_basis(X, self.X, self.theta)
        Y = (self.w.T @ D.T).T
        return Y, None

    @staticmethod
    def get_params():
        val = []
        for basis in ['gaussian']:
            for optimize in ['scipy']:
                val.append({'basis': basis, 'optimize': optimize})
        return val

def optimize(model, n_folds, xl, xu):
    def func_evaluate(x):

        f = np.full((x.shape[0], 1), np.inf)

        # the data to use for crossvalidation
        X = model.X
        y = model.y

        # a copy of the original model to optimize on
        m = copy.deepcopy(model)
        error = np.zeros((x.shape[0], len(n_folds)))

        for i in range(x.shape[0]):

            m.basis.set_hyperparameter(np.exp(x[i, :]))

            for j, (train, test) in enumerate(n_folds):
                m.fit(X[train, :], y[train])
                y_pred = m.predict(X[test, :])

                error[i, j] = np.mean(np.square(y[test] - y_pred))

        f[:, 0] = np.mean(error, axis=1)

    res = pymoo.optimize.minimize(func_evaluate, method="ga", xl=xl, xu=xu, n_var=model.n_var)
    model.basis.set_hyperparameter(np.exp(res.X[0, :]))
    model.refit()

def get_basis_function(name, n_var):
    if name == 'gaussian':
        return func_gaussian, n_var
    else:
        raise Exception("Basis not known.")

def func_gaussian(A, B, theta):
    D = calc_distance_matrix(A, B, func=lambda _A, _B: np.sqrt(np.square((_A - _B) * theta)))
    K = np.exp(-np.square(D))
    return K


#!/usr/bin/env python3

import sys
import os
import threading
import subprocess

from envsubst import envsubst
from fuse import FUSE, FuseOSError, Operations, fuse_get_context

class Presenter(Operations):
    def __init__(self, base):
        self.base = base
        self.fd = 0
        self.filestore = {}
        self.meta = {}

    def _base_path(self, relpath):
        if relpath.startswith("/"):
            relpath = relpath[1:]
        path = os.path.join(self.base, relpath)
        return path

    def getattr(self, path, fh=None):
        orig_path = self._base_path(path)
        st = os.lstat(orig_path)
        attrs = dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime',
                     'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))

        self.meta[path] = attrs

        # Do the actual load and transform here - since all programs call getattr before open
        # The size that getattr returns is used to read in bytes
        # so, this has to be "correct"

        if os.path.isfile(orig_path):
            # Pull out data from the original file
            orig = open(orig_path, "r")
            # read raw data
            data = orig.read()
            orig.close()

            # substitute vars and store file
            self.filestore[path] = envsubst(data).encode('ascii')
            # update size metadata
            self.meta[path]['st_size'] = len(self.filestore[path])

        return attrs

    def readdir(self, path, fh):
        orig_path = self._base_path(path)

        dirents = ['.', '..']
        if os.path.isdir(orig_path):
            dirents.extend(os.listdir(orig_path))
        for r in dirents:
            yield r

    def open(self, path, flags):
        # create and return dummy fd
        self.fd += 1
        return self.fd

    def read(self, path, size, offset, fh):
        return self.filestore[path][offset:offset + size]

def mount(base, view):
    # Check if view is an empty directory and fail if not
    # Create the directory if not present

    FUSE(Presenter(base), view, nothreads=True, foreground=True)

    # Provide a way to clean-up the view dir when we are done

def env_reader():
    ppid = os.getppid()

    ### The BPF command being used to spy on env var updates in the parent process
    # All the escaping is confusing here, so an explanation of what we are doing:
    # We run a uprobe trace on the "bind_variable" function in bash
    # we check the first argument - the key name, is not equal to "_"
    #   - this is because there are a *lot* of fires with that key, something like a processing step
    # We print out the first and second args, which happens to be the key and value
    # We restrict this tracing to only the parent shell

    # Obviously, this only works for bash, that too, one located at `/usr/bin/bash`.

    cmd = "sudo bpftrace -e 'uprobe:/usr/bin/bash:bind_variable /str(arg0) != \"_\"/{ printf(\"%s=%s\\n\", str(arg0), str(arg1)) }' -p " + str(ppid)

    tracer = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
    for line in tracer.stdout:
        entry = line.decode().rstrip()
        try:
            (key, value) = entry.split('=')
            # Only process keys which are all upper case
            if key.isupper():
                os.environ[key] = value
        except ValueError:
            # in case of any extra lines not in key = val format
            pass

if __name__ == "__main__":
    args = sys.argv[1:]
    if len(args) != 2:
        print("Expect 2 args: <base dir> <view dir>")
        exit(1)

    envReader = threading.Thread(target=env_reader)
    envReader.start()

    mount(args[0], args[1])


import os

from shutil import copytree
from backports.tempfile import TemporaryDirectory

from config_app.config_util.config.fileprovider import FileConfigProvider

OLD_CONFIG_SUBDIR = "old/"

class TransientDirectoryProvider(FileConfigProvider):
    """
    Implementation of the config provider that reads and writes the data from/to the file system,
    only using temporary directories, deleting old dirs and creating new ones as requested.
    """

    def __init__(self, config_volume, yaml_filename, py_filename):
        # Create a temp directory that will be cleaned up when we change the config path
        # This should ensure we have no "pollution" of different configs:
        # no uploaded config should ever affect subsequent config modifications/creations
        temp_dir = TemporaryDirectory()
        self.temp_dir = temp_dir
        self.old_config_dir = None
        super(TransientDirectoryProvider, self).__init__(temp_dir.name, yaml_filename, py_filename)

    @property
    def provider_id(self):
        return "transient"

    def new_config_dir(self):
        """
        Update the path with a new temporary directory, deleting the old one in the process.
        """
        self.temp_dir.cleanup()
        temp_dir = TemporaryDirectory()

        self.config_volume = temp_dir.name
        self.temp_dir = temp_dir
        self.yaml_path = os.path.join(temp_dir.name, self.yaml_filename)

    def create_copy_of_config_dir(self):
        """
        Create a directory to store loaded/populated configuration (for rollback if necessary)
        """
        if self.old_config_dir is not None:
            self.old_config_dir.cleanup()

        temp_dir = TemporaryDirectory()
        self.old_config_dir = temp_dir

        # Python 2.7's shutil.copy() doesn't allow for copying to existing directories,
        # so when copying/reading to the old saved config, we have to talk to a subdirectory,
        # and use the shutil.copytree() function
        copytree(self.config_volume, os.path.join(temp_dir.name, OLD_CONFIG_SUBDIR))

    def get_config_dir_path(self):
        return self.config_volume

    def get_old_config_dir(self):
        if self.old_config_dir is None:
            raise Exception("Cannot return a configuration that  was no old configuration")

        return os.path.join(self.old_config_dir.name, OLD_CONFIG_SUBDIR)


import os
import sys
#!/usr/bin/env python

import time
import signal
from starflow import optcomplete
from starflow.logger import log

class CmdBase(optcomplete.CmdComplete):
    parser = None
    opts = None
    gopts = None
    gparser = None
    subcmds_map = None

    @property
    def goptions_dict(self):
        return dict(self.gopts.__dict__)

    @property
    def options_dict(self):
        return dict(self.opts.__dict__)

    @property
    def specified_options_dict(self):
        """ only return options with non-None value """
        specified = {}
        options = self.options_dict
        for opt in options:
            if options[opt] is not None:
                specified[opt] = options[opt]
        return specified

    @property
    def cfg(self):
        return self.goptions_dict.get('CONFIG')

    def cancel_command(self, signum, frame):
        print
        log.info("Exiting...")
        sys.exit(1)

    def catch_ctrl_c(self, handler=None):
        handler = handler or self.cancel_command
        signal.signal(signal.SIGINT, handler)

    def warn_experimental(self, msg):
        for l in msg.splitlines():
            log.warn(l)
        num_secs = 10
        r = range(1,num_secs+1)
        r.reverse()
        print
        log.warn("Waiting %d seconds before continuing..." % num_secs)
        log.warn("Press CTRL-C to cancel...")
        for i in r:
            sys.stdout.write('%d...' % i)
            sys.stdout.flush()
            time.sleep(1)
        print


# Program to create variables storing values for each datatype. print the object identities of all the created variables. Next, convert the types of the each varible into another type. After changing the type, store all the values of all the variables in a list. Finally, Print the list.

place = "Hyderabad"
code = 500032
street = 22/7

print(type(place))
print(type(code))
print(type(street))

place = list(place)
code = float(code)
street = int(street)

Details = [place, code, street]

print("The list is : ", Details)


import numpy as np
from tqdm import tqdm

names = './data/llvm280/names'
wrongat = './data/llvm280/wrongat.txt'
cov_prefix = './data/llvm280/'
ep_file = './passing-testsuite-coverage/testsuite_280_ep.txt'
np_file = './passing-testsuite-coverage/testsuite_280_np.txt'
dis_url = './coverage_dis.npy'

id2level = {}
with open(wrongat, 'r') as f:
    for index, line in enumerate(f):
        line = line.strip()
        levels = line.split(' ')
        res = levels.index('n')
        level = str(res + 1)
        id2level[index] = level

all_covs = []
name2id = {}
id2name = {}
with open(names, 'r') as f:
    for index, line in enumerate(f):
        line = line.strip()
        name = line.split('/')[-1].split('.')[0]
        id = index
        name2id[name] = id
        id2name[id] = name
        if 'llvm' in names:
            if index <= 84:
                cov = cov_prefix + name + '.' + id2level[id] + 'fcov'
                all_covs.append(cov)
            else:
                cov = cov_prefix + name + '.ofcov'
                all_covs.append(cov)
        elif 'gcc450' in names:
            cov = cov_prefix + name + '.' + 'levelfcov'
            all_covs.append(cov)
        else:
            cov = cov_prefix + name + '.' + id2level[id] + 'fcov'
            all_covs.append(cov)

all_func_list = []
for i in all_covs:
    if i.strip().split('/')[-1].split('.')[0] in id2name.values():
        with open(i, 'r') as f:
            for line in f:
                n = line.split(' ')
                all_func_list.append(n[0] + n[1])
    all_func_list = list(set(all_func_list))
all_func_list.sort()

ep_dict = {}
with open(ep_file, 'r') as f:
    for line in f:
        name = line.strip().split(',')[0]
        count = int(line.strip().split(',')[1])
        ep_dict[name] = count

np_dict = {}
with open(np_file, 'r') as f:
    for line in f:
        name = line.strip().split(',')[0]
        count = int(line.strip().split(',')[1])
        np_dict[name] = count

all_cases_set = {}
for single_case in tqdm(all_covs):
    rank_set = {}
    tc_name = single_case.strip().split('/')[-1].split('.')[0]
    if tc_name in id2name.values():
        failures = set()
        ef_dict = {}
        with open(single_case, 'r') as f:
            for line in f:
                n = line.split(' ')
                name = n[0] + ' ' + n[1] + ' ' + n[2]
                ef_dict[name] = ef_dict.setdefault(name, 0) + int(n[-1])
                failures.add(name)

        func_score = dict.fromkeys(all_func_list)  # default value is None
        for statement in failures:
            ef = 1
            ep = ep_dict.get(statement, 0)
            mnp = np_dict.get(statement, 0.0000001)

            # ochiai
            score = ef / (ep + ef) ** .5

            # # kulczynski2
            # score = 0.5 * (1 + (ef / (ef + ep)))

            # # ochiai2
            # score = np / ((ef + ep) * np * (ep + np)) ** .5

            function_name = statement.split(' ')[0] + statement.split(' ')[1]

            if func_score[function_name] is None:
                func_score[function_name] = [score]
            else:
                func_score[function_name].append(score)

        for key, value in func_score.items():
            func_score[key] = max(value) if value is not None and value else 0

        vec = np.array(list(func_score.values()))
        all_cases_set[name2id[tc_name]] = vec

distance = np.full((len(all_cases_set.values()), len(all_cases_set.values())), -1000.0)
for i1, v1 in tqdm(all_cases_set.items()):
    for i2, v2 in all_cases_set.items():
        try:
            if distance[int(i2), int(i1)] != -1000.0:
                distance[int(i1), int(i2)] = distance[int(i2), int(i1)]
                continue
            if int(i1) == int(i2):
                distance[int(i1), int(i2)] = 0.0
                distance[int(i2), int(i1)] = 0.0
                continue
            distance[int(i1), int(i2)] = np.linalg.norm(v1 - v2)
        except Exception as e:
            print('')
            continue
np.save(dis_url, distance)
print(distance)
print(distance.shape)


'''
This tkinter project shows how to create a simple gui
with 'File' and 'Edit' menus with their respective submenus

'''

import tkinter as tk

def doNothing():
    print('Do nothing - alright fine')

root = tk.Tk()

# **** Main Menu ****

menu = tk.Menu(root)
root.config(menu=menu)

subMenu = tk.Menu(menu)
menu.add_cascade(label='File', menu=subMenu)
subMenu.add_command(label='New Project..', command=doNothing)
subMenu.add_command(label='Save..', command=doNothing)
subMenu.add_separator()
subMenu.add_command(label='Exit', command=root.quit)

editMenu = tk.Menu(menu)
menu.add_cascade(label='Edit', menu=editMenu)
editMenu.add_command(label='Redo', command=doNothing)

# **** Toolbar ****

tb = tk.Frame(root, bg='blue')
button1 = tk.Button(tb, text='Insert Image', command=doNothing)
button1.pack(side=tk.LEFT, padx=2, pady=2)
button2 = tk.Button(tb, text='Print', command=doNothing)
button2.pack(side=tk.LEFT, padx=2, pady=2)

tb.pack(side=tk.TOP, fill=tk.X)

# **** Status Bar ****

status = tk.Label(root, text='Preparing to do nothing...', bd=1, relief=tk.SUNKEN, anchor=tk.W)
status.pack(side=tk.BOTTOM, fill=tk.X)

root.mainloop()


import pandas as pd
from sklearn.metrics import cohen_kappa_score
from sklearn.preprocessing import MultiLabelBinarizer
from itertools import combinations

states_path = 'path/to/ECG-states.csv'
reports_path = 'path/to/reports.xls'

all_labels = list(set(list(pd.read_csv(states_path)['Код в SCP'])))
mlt = MultiLabelBinarizer(classes=all_labels)
labels_order = {label: i for i, label in enumerate(all_labels)}

def get_pair_report(reports_df, username1, username2):
    pair_report = reports_df[reports_df['username'].isin([username1, username2])]
    pair_report = pair_report[pair_report.duplicated(subset=['electrocardiogram'], keep=False)]
    return pair_report

def get_agreement(username1, username2, labels=None):
    labels = labels if labels else all_labels
    agreement_per_label = {}
    pair_report = get_pair_report(reports, username1, username2)
    pair_report = pair_report.sort_values(by='electrocardiogram')

    user1_ecg_one_hot_labels = mlt.fit_transform(list(pair_report[pair_report['username'] == username1].diagnoses))
    user2_ecg_one_hot_labels = mlt.fit_transform(list(pair_report[pair_report['username'] == username2].diagnoses))

    for label in labels:
        order = labels_order[label]
        user1_label_decisions = [one_hot_labels[order] for one_hot_labels in user1_ecg_one_hot_labels]
        user2_label_decisions = [one_hot_labels[order] for one_hot_labels in user2_ecg_one_hot_labels]

        support_df = pair_report[pair_report['diagnoses'].apply(lambda x: label in x)]
        support_df = support_df.drop_duplicates(subset='electrocardiogram', keep="last")
        support = len(support_df)

        if support == 0:
            agreement_per_label[label] = None
        else:
            agreement_per_label[label] = cohen_kappa_score(user1_label_decisions, user2_label_decisions), support
    return agreement_per_label

reports = pd.read_excel(reports_path)
reports = reports[reports['username'] != 'user_g1']
reports['diagnoses'] = reports['diagnoses'].apply(lambda x: x.split(','))
reports['createdAt'] = pd.to_datetime(reports.createdAt)
reports = reports.sort_values(by='createdAt')
reports = reports.drop_duplicates(subset=['electrocardiogram', 'username'], keep='last')

user_pairs = list(combinations(list(reports.username.unique()), 2))
user_pair_agreements = {}
for user1, user2 in user_pairs:
    diagnosis_report = {}
    agreement_per_label = get_agreement(user1, user2)
    user_pair_agreements['_'.join([user1, user2])] = agreement_per_label

rep_df = pd.DataFrame.from_dict(user_pair_agreements)
rep_df = rep_df.dropna(how='all', axis=0)
rep_df = rep_df.dropna(how='all', axis=1)
rep_df = rep_df.sort_values(by='_'.join(user_pairs[0]), ascending=False)
rep_df.to_csv("cohen's_kappa_scores.csv")


menuList = []
priceList = []
totalPrice = 0

def showBill():
    print("------ Restaurant ------")
    for number in range(len(menuList)):
        print(menuList[number],priceList[number],"Baht")
while True:
    menuName = input("Please Enter Manu: ")
    if(menuName.lower() == "exit"):
        break
    else:
        menuPrice = float(input("Price: "))
        menuList.append(menuName)
        priceList.append(menuPrice)
        totalPrice += menuPrice
showBill()
print("Total Price is",totalPrice,"Baht")


def encode(message, key):
    message_output = ''
    message_char_length = len(message)
    for i in range(message_char_length):
        num = message[i]
        num = ord(message[i]) + key
        character = chr(num)
        message_output = message_output + character
    return message_output

def encode_better(message, key):
    key_val = key
    key_length = len(key_val)
    message_length = len(message)
    coded_message = ""

    for i in range(message_length):
        character_num = ord(message[i])
        key_num = ord(key_val[i % key_length])
        character_num = character_num - 65
        key_num = key_num - 65
        shift_val = (character_num + key_num) % 58
        coded_val = shift_val + 65
        coded_val = chr(coded_val)
        coded_message = coded_message + coded_val

    return coded_message


N = int(input())
A = list(map(int, input().split()))
evens = []
for i in range(N):
    if A[i] % 2 == 0:
        evens.append(A[i])
for even in evens:
    if even % 3 == 0 or even % 5 == 0:
        continue
    else:
        print("DENIED")
        exit()

print("APPROVED")


#!/usr/bin/env python3
import random

GAME_RULES = 'Find the greatest common divisor of given numbers.'

def euclid_gcd(num1, num2):
    """Euclid algorithm to calculate gcd"""
    while num1 != 0 and num2 != 0:
        if num1 > num2:
            num1 = num1 % num2
        else:
            num2 = num2 % num1
    return num1 + num2

def get_question_and_answer():
    """
    GCD game
    Player get two numbers and need to find greatest common divisor
    """
    num1, num2 = random.randint(0, 100), random.randint(0, 100)
    correct_answer = str(euclid_gcd(num1, num2))
    return f'{str(num1)} {str(num2)}', correct_answer


from __future__ import absolute_import
import os
import sys
import math
import traceback
import bleu
import pickle
import torch
import json
import random
import logging
import argparse
import numpy as np
from copy import deepcopy
from io import open
from itertools import cycle
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.metrics import pairwise_distances
from sklearn.preprocessing import normalize
import torch.nn as nn
from model import Encoder, SimilarityClassifier
from utils import EarlyStopping

from tqdm import tqdm, trange
from torch.utils.data import DataLoader, Dataset, SequentialSampler, RandomSampler,WeightedRandomSampler, TensorDataset
from torch.utils.data.distributed import DistributedSampler
from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup,
                          RobertaConfig, RobertaModel, RobertaTokenizer)

from transformers import AutoTokenizer, AutoModel

from torch.utils.tensorboard import SummaryWriter
from torch.cuda.amp import autocast
from dataset import TripletTensorDataset, TripletTensorDatasetV2, TripletTensorDatasetV3
# from transformers import RobertaForMaskedLM as RobertaModel
MODEL_CLASSES = {'roberta': (RobertaConfig, RobertaModel, RobertaTokenizer)}

logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',
                    datefmt = '%m/%d/%Y %H:%M:%S',
                    level = logging.INFO)
logger = logging.getLogger(__name__)

class Pair(object):
    """A single text-code (nl-pl) pair."""
    def __init__(self,
                 idx,
                 simple_name,
                 thread_id,
                 text,
                 code
                 ):
        self.idx = idx
        self.simple_name = simple_name
        self.thread_id = thread_id
        self.text = text
        self.code = code

class LabeledPair(object):
    """A single text-code (nl-pl) pair."""
    def __init__(self,
                 idx,
                 simple_name,
                 thread_id,
                 text,
                 code,
                 api_comment,
                 api_implementation,
                 label
                 ):
        self.idx = idx
        self.simple_name = simple_name
        self.thread_id = thread_id
        self.text = text
        self.code = code
        self.cmt = api_comment
        self.impl = api_implementation
        self.label = label

def read_pairs(filename, mode="infer"):
    pairs = []
    if mode == "infer":
        with open(filename, "r") as fp:
            nl_pl_pairs = fp.readlines()
        for pair_info in nl_pl_pairs:
            pair_info = json.loads(pair_info)
            text, code = pair_info['pairs']
            pairs.append(
                Pair(
                    pair_info['idx'],
                    pair_info['simple_name'],
                    pair_info['thread_id'],
                    text,
                    code
                    )
            )
    elif mode == "train" or mode == "test" or mode == "eval":
        with open(filename, "r") as fp:
            training_pairs = fp.readlines()
        for training_info in training_pairs:
            training_info = json.loads(training_info)
            text, code = training_info['pairs']
            api_comment, api_implementation = training_info['doc_impl']
            label = training_info['cls_label']
            pairs.append(
                LabeledPair(
                    training_info['idx'],
                    training_info['simple_name'],
                    training_info['thread_id'],
                    text,
                    code,
                    api_comment,
                    api_implementation,
                    label
                )
            )
    return pairs

class PairFeatures(object):
    """
        A pair features combining text and code into source.
        For inference mode.
    """
    def __init__(self,
                 pair_id,
                 source_ids,
                 source_mask
                ):
        self.pair_id = pair_id
        self.source_ids = source_ids
        self.source_mask = source_mask

class LabeledPairFeatures(object):
    """
        A pair features combining text and code into source
        Has label. For training, testing and evaluating mode.
    """
    def __init__(self,
                 pair_id,
                 text_code_ids,
                 text_code_mask,
                 cmt_impl_ids,
                 cmt_impl_mask,
                 label
                ):
        self.pair_id = pair_id
        self.text_code_ids = text_code_ids
        self.text_code_mask = text_code_mask
        self.cmt_impl_ids = cmt_impl_ids
        self.cmt_impl_mask = cmt_impl_mask
        self.label = label

def truncate_pair(longer_one, shorter_one, args=None):
    total_length = len(longer_one) + len(shorter_one)
    delta = len(longer_one) - len(shorter_one)
    if len(shorter_one) < 200:
        # tokenized_test = tokenized_test[:-1*delta]
        longer_one = longer_one[:-1*(total_length-args.max_source_length+3)]
    else:
        longer_one = longer_one[:-1*delta]
        new_total_length = len(longer_one) + len(shorter_one)
        need_remove_tokens_length = new_total_length - args.max_source_length+3
        if need_remove_tokens_length % 2 == 0:
            longer_one = longer_one[:-1*int(need_remove_tokens_length/2)]
            shorter_one = shorter_one[:-1*int(need_remove_tokens_length/2)]
        else:
            longer_one = longer_one[:-1*int(need_remove_tokens_length/2+1)]
            shorter_one = shorter_one[:-1*int(need_remove_tokens_length/2)]
    return longer_one, shorter_one

def convert_pairs_to_features(pairs, tokenizer, args=None):
    features = []
    for pair_i, pair in enumerate(pairs):
        tokenized_text = tokenizer.tokenize(pair.text)
        tokenized_code = tokenizer.tokenize(pair.code)
        # truncate
        total_length = len(tokenized_text) + len(tokenized_code)
        if total_length > args.max_source_length -3:
            if len(tokenized_text) > len(tokenized_code):
                tokenized_text, tokenized_code = truncate_pair(tokenized_text, tokenized_code, args)
            else:
                tokenized_code, tokenized_text = truncate_pair(tokenized_code, tokenized_text, args)

        source_tokens = [tokenizer.cls_token]+tokenized_text+[tokenizer.sep_token]+tokenized_code+[tokenizer.sep_token]
        source_ids =  tokenizer.convert_tokens_to_ids(source_tokens)
        source_mask = [1] * (len(source_tokens))
        padding_length = args.max_source_length - len(source_ids)
        source_ids+=[tokenizer.pad_token_id]*padding_length
        source_mask+=[0]*padding_length
        features.append(
            PairFeatures(
                pair.idx,
                source_ids,
                source_mask
            )
        )
    return features

def get_training_features(pairs, tokenizer, args=None):
    def truncate(tokenized_text, tokenized_code):
        total_length = len(tokenized_text) + len(tokenized_code)
        if total_length > args.max_source_length -3:
            if len(tokenized_text) > len(tokenized_code):
                tokenized_text, tokenized_code = truncate_pair(tokenized_text, tokenized_code, args)
            else:
                tokenized_code, tokenized_text = truncate_pair(tokenized_code, tokenized_text, args)

        source_tokens = [tokenizer.cls_token]+tokenized_text+[tokenizer.sep_token]+tokenized_code+[tokenizer.sep_token]
        source_ids =  tokenizer.convert_tokens_to_ids(source_tokens)
        source_mask = [1] * (len(source_tokens))
        padding_length = args.max_source_length - len(source_ids)
        source_ids+=[tokenizer.pad_token_id]*padding_length
        source_mask+=[0]*padding_length
        return source_ids, source_mask

    features = []
    nrof_positives = 0
    nrof_negatives = 0

    for pair_i, pair in enumerate(pairs):
        tokenized_text = tokenizer.tokenize(pair.text)
        tokenized_code = tokenizer.tokenize(pair.code)
        tokenized_cmt = tokenizer.tokenize(pair.cmt)
        tokenized_impl = tokenizer.tokenize(pair.impl)
        # label = [0, 1] if pair.label else [1, 0] #binary classification if text_code is relevant to cmt_impl
        label = 1 if pair.label else 0 # For nn.CrossEntropyLoss label
        if label == 1:
            nrof_positives += 1
        else:
            nrof_negatives += 1

        # truncate
        text_code_ids, text_code_mask = truncate(tokenized_text, tokenized_code)
        cmt_impl_ids, cmt_impl_mask = truncate(tokenized_cmt, tokenized_impl)

        features.append(
            LabeledPairFeatures(
                pair.idx,
                text_code_ids,
                text_code_mask,
                cmt_impl_ids,
                cmt_impl_mask,
                label
            )
        )
    return features, nrof_positives, nrof_negatives

def set_seed(args):
    """set random seed."""
    random.seed(args.seed)
    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    if args.n_gpu > 0:
        torch.cuda.manual_seed_all(args.seed)

def main():
    parser = argparse.ArgumentParser()

    ## Required parameters
    parser.add_argument("--model_type", default=None, type=str, required=True,
                        help="Model type: e.g. roberta")
    parser.add_argument("--model_name_or_path", default=None, type=str, required=True,
                        help="Path to pre-trained model: e.g. roberta-base" )
    parser.add_argument("--output_dir", default=None, type=str, required=True,
                        help="The output directory where the model predictions and checkpoints will be written.")
    parser.add_argument("--load_model_path", default=None, type=str,
                        help="Path to trained model: Should contain the .bin files" )
    ## Other parameters
    parser.add_argument("--train_filename", default=None, type=str,
                        help="The train filename. Should contain the .jsonl files for this task.")
    parser.add_argument("--dev_filename", default=None, type=str,
                        help="The dev filename. Should contain the .jsonl files for this task.")
    parser.add_argument("--test_filename", default=None, type=str,
                        help="The test filename. Should contain the .jsonl files for this task.")

    parser.add_argument("--config_name", default="", type=str,
                        help="Pretrained config name or path if not the same as model_name")
    parser.add_argument("--tokenizer_name", default="", type=str,
                        help="Pretrained tokenizer name or path if not the same as model_name")
    parser.add_argument("--max_source_length", default=512, type=int,
                        help="The maximum total source sequence length after tokenization. Sequences longer "
                             "than this will be truncated, sequences shorter will be padded.")
    parser.add_argument("--max_target_length", default=32, type=int,
                        help="The maximum total target sequence length after tokenization. Sequences longer "
                             "than this will be truncated, sequences shorter will be padded.")

    parser.add_argument("--do_train", action='store_true',
                        help="Whether to run training.")
    parser.add_argument("--do_eval", action='store_true',
                        help="Whether to run eval on the dev set.")
    parser.add_argument("--do_test", action='store_true',
                        help="Whether to run eval on the dev set.")
    parser.add_argument("--do_infer", action='store_true',
                        help="Whether to run inferencing on the new files.")
    parser.add_argument("--do_lower_case", action='store_true',
                        help="Set this flag if you are using an uncased model.")
    parser.add_argument("--no_cuda", action='store_true',
                        help="Avoid using CUDA when available")

    parser.add_argument("--train_batch_size", default=8, type=int,
                        help="Batch size per GPU/CPU for training.")
    parser.add_argument("--eval_batch_size", default=8, type=int,
                        help="Batch size per GPU/CPU for evaluation.")
    parser.add_argument('--gradient_accumulation_steps', type=int, default=1,
                        help="Number of updates steps to accumulate before performing a backward/update pass.")
    parser.add_argument("--learning_rate", default=5e-5, type=float,
                        help="The initial learning rate for Adam.")
    parser.add_argument("--beam_size", default=10, type=int,
                        help="beam size for beam search")
    parser.add_argument("--weight_decay", default=0.0, type=float,
                        help="Weight deay if we apply some.")
    parser.add_argument("--adam_epsilon", default=1e-8, type=float,
                        help="Epsilon for Adam optimizer.")
    parser.add_argument("--max_grad_norm", default=1.0, type=float,
                        help="Max gradient norm.")
    parser.add_argument("--num_train_epochs", default=3.0, type=float,
                        help="Total number of training epochs to perform.")
    parser.add_argument("--max_steps", default=-1, type=int,
                        help="If > 0: set total number of training steps to perform. Override num_train_epochs.")
    parser.add_argument("--eval_steps", default=-1, type=int,
                        help="")
    parser.add_argument("--train_steps", default=-1, type=int,
                        help="")
    parser.add_argument("--warmup_steps", default=0, type=int,
                        help="Linear warmup over warmup_steps.")
    parser.add_argument("--local_rank", type=int, default=-1,
                        help="For distributed training: local_rank")
    parser.add_argument('--seed', type=int, default=42,
                        help="random seed for initialization")
    # print arguments
    args = parser.parse_args()
    logger.info(args)

    # Setup CUDA, GPU & distributed training
    if args.local_rank == -1 or args.no_cuda:
        device = torch.device("cuda:0" if torch.cuda.is_available() and not args.no_cuda else "cpu")
        # args.n_gpu = torch.cuda.device_count()
        args.n_gpu = 1

    else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
        torch.cuda.set_device(args.local_rank)
        device = torch.device("cuda", args.local_rank)
        torch.distributed.init_process_group(backend='nccl')
        args.n_gpu = 1
    logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s",
                    args.local_rank, device, args.n_gpu, bool(args.local_rank != -1))
    args.device = device
    # Set seed
    set_seed(args)
    # make dir if output_dir not exist
    args.output_dir = os.path.join(args.output_dir, "0")
    if os.path.exists(args.output_dir) is False:
        os.makedirs(args.output_dir)

    # code2nl tokenizer
    config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
    config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path)
    tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path,do_lower_case=args.do_lower_case)
    # print(tokenizer.convert_tokens_to_ids(tokenizer.tokenize("{ input }")) )
    # source_ids =  tokenizer.convert_tokens_to_ids(tokenizer.tokenize("{ input }"))

    # exit()
    # build model
    encoder = model_class.from_pretrained(args.model_name_or_path,config=config)

    # with open("CodeBERT_arch.txt", "w+") as fp:
    #     print("[CodeBERT Encoder Architecture]", file=fp)
    #     print(str(encoder), file=fp)

    model= SimilarityClassifier(encoder=encoder,config=config, max_length=args.max_target_length,
                                sos_id=tokenizer.cls_token_id, eos_id=tokenizer.sep_token_id)
    if args.load_model_path is not None:
        logger.info("reload model from {}".format(args.load_model_path))
        torch.cuda.memory_summary()
        model.load_state_dict(torch.load(args.load_model_path,map_location=args.device))
        # model.load_state_dict(torch.load(args.load_model_path, map_location='cuda:1'))
    model.to(device)

    # with open("dnn_arch.txt", "a") as fp:
    #         print("[Full DNN Architecture]", file=fp)
    #         print(str(model), file=fp)

    if args.local_rank != -1:
        # Distributed training
        try:
            from apex.parallel import DistributedDataParallel as DDP
        except ImportError:
            raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.")

        model = DDP(model)
    elif args.n_gpu > 1:
        # multi-gpu training
        model = torch.nn.DataParallel(model)

    def assign_token_indice(placeholder, list_subwords_indices):
        # f.token_subwords_indices
        subword_indices_vector = deepcopy(placeholder)
        try:
            subword_indices_vector[:len(list_subwords_indices)] = list_subwords_indices
        except Exception as e:
            traceback.print_exc(e)
            exit()
        return subword_indices_vector

    if args.do_train:
        result_file = os.path.join(args.output_dir, "mod4_result_small_data.txt")
        with open(result_file, "w+") as res_fp:
            pass # to clear the old result

        train_file = "data/approach_4/text_code_pairs_train.jsonl"
        train_pairs = read_pairs(args.train_filename, mode="train")
        train_features, nrof_positives, nrof_negatives = get_training_features(train_pairs, tokenizer, args)
        all_pair_id = torch.tensor([f.pair_id for f in train_features], dtype=torch.long)
        all_text_code_ids = torch.tensor([f.text_code_ids for f in train_features], dtype=torch.long)
        all_text_code_mask = torch.tensor([f.text_code_mask for f in train_features], dtype=torch.long)
        all_cmt_impl_ids = torch.tensor([f.cmt_impl_ids for f in train_features], dtype=torch.long)
        all_cmt_impl_mask = torch.tensor([f.cmt_impl_mask for f in train_features], dtype=torch.long)
        all_label = torch.tensor([f.label for f in train_features], dtype=torch.long)
        labels = np.array([f.label for f in train_features])
        print("labels.shape", labels.shape)
        labels = labels.astype(int)

        num_of_majority_class_training_examples = nrof_negatives
        num_of_minority_class_training_examples = nrof_positives
        majority_weight = 1/num_of_majority_class_training_examples
        minority_weight = 1.5/num_of_minority_class_training_examples
        sample_weights = np.array([majority_weight, minority_weight]) # minority has integer 1 as label
        weights = sample_weights[labels]

        train_dataset = TensorDataset(all_pair_id,
                                      all_text_code_ids,
                                      all_text_code_mask,
                                      all_cmt_impl_ids,
                                      all_cmt_impl_mask,
                                      all_label)

        # train_sampler = RandomSampler(train_dataset)
        train_sampler = WeightedRandomSampler(weights=weights, num_samples=len(train_dataset), replacement=True)
        train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)

        num_train_optimization_steps =  args.train_steps

        # Prepare optimizer and schedule (linear warmup and decay)
        no_decay = ['bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
             'weight_decay': args.weight_decay},
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps,
                                                    num_training_steps=num_train_optimization_steps)
        early_stopping = EarlyStopping()
        citeration = nn.CrossEntropyLoss()

        #Start training
        logger.info("***** Running training *****")
        logger.info("  Num examples = %d", len(train_pairs))
        logger.info("  Batch size = %d", args.train_batch_size)
        logger.info("  Num epoch = %d", num_train_optimization_steps*args.train_batch_size//len(train_pairs))

        model.train()

        # Freeze the parameters of CodeBERT
        for child_i, child in enumerate(model.encoder.children()):
            for k,v in child.named_parameters():
                if child_i == 0:
                    v.requires_grad = False

        nb_tr_examples, nb_tr_steps,tr_loss,global_step,best_acc,best_loss = 0, 0,0,0,0,1e6
        bar = tqdm(range(num_train_optimization_steps),total=num_train_optimization_steps)
        train_dataloader=cycle(train_dataloader)
        eval_flag = True
        first_eval = True
        for step in bar:
            # pair_id, text_code_ids, text_code_mask, cmt_impl_ids, cmt_impl_mask, label = next(train_dataloader)
            batch = next(train_dataloader)
            batch = tuple(t.to(device) for t in batch)
            pair_id, text_code_ids, text_code_mask, cmt_impl_ids, cmt_impl_mask, label = batch

            with autocast():
                preds = model(text_code_ids, text_code_mask, cmt_impl_ids, cmt_impl_mask)
                loss = citeration(preds, label)

            if args.n_gpu > 1:
                loss = loss.mean() # mean() to average on multi-gpu.
            if args.gradient_accumulation_steps > 1:
                loss = loss / args.gradient_accumulation_steps

            skip_updating = False
            if not math.isnan(loss):
                tr_loss += loss.item()
                train_loss=round(tr_loss*args.gradient_accumulation_steps/(nb_tr_steps+1),4)
                bar.set_description("loss {}".format(train_loss))
                nb_tr_examples += pair_id.size(0)
                nb_tr_steps += 1
                skip_updating =False
                try:
                    loss.backward()
                except Exception as e:
                    skip_updating = True
            else:
                skip_updating = True

            if (nb_tr_steps + 1) % args.gradient_accumulation_steps == 0:
                global_step += 1
                eval_flag = True
                #Update parameters
                if  not skip_updating:
                    optimizer.step()
                    optimizer.zero_grad()
                    scheduler.step()
            if skip_updating:
                optimizer.zero_grad()
                skip_updating = False

            # if args.do_eval and eval_flag and (global_step+1)% args.eval_steps == 0:
            # do eval
            e_steps = 1000
            if args.do_eval and eval_flag and ((global_step == 1) or (global_step+1)% e_steps == 0):
                eval_pairs = read_pairs(args.dev_filename, mode="train")
                eval_features, _, _ = get_training_features(eval_pairs, tokenizer, args)
                eval_pair_id = torch.tensor([f.pair_id for f in eval_features], dtype=torch.long)
                eval_text_code_ids = torch.tensor([f.text_code_ids for f in eval_features], dtype=torch.long)
                eval_text_code_mask = torch.tensor([f.text_code_mask for f in eval_features], dtype=torch.long)
                eval_cmt_impl_ids = torch.tensor([f.cmt_impl_ids for f in eval_features], dtype=torch.long)
                eval_cmt_impl_mask = torch.tensor([f.cmt_impl_mask for f in eval_features], dtype=torch.long)
                eval_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)

                eval_dataset = TensorDataset(
                                            eval_pair_id,
                                            eval_text_code_ids,
                                            eval_text_code_mask,
                                            eval_cmt_impl_ids,
                                            eval_cmt_impl_mask,
                                            eval_label)

                eval_sampler = SequentialSampler(eval_dataset)
                eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.train_batch_size)

                logger.info("\n***** Running evaluation *****")
                logger.info("  Num examples = %d", len(eval_pairs))
                logger.info("  Batch size = %d", args.eval_batch_size)

                model.eval()
                count = 0
                tp = 0
                fp = 0
                fn = 0
                tn = 0
                softmax = nn.Softmax(dim=1)
                for eval_batch in tqdm(eval_dataloader,total=len(eval_dataloader)):
                    eval_batch = tuple(t.to(device) for t in eval_batch)
                    eval_pair_id, eval_text_code_ids, eval_text_code_mask, eval_cmt_impl_ids, eval_cmt_impl_mask, eval_label = eval_batch
                    eval_preds = model(
                                    eval_text_code_ids,
                                    eval_text_code_mask,
                                    eval_cmt_impl_ids,
                                    eval_cmt_impl_mask
                                    )
                    eval_preds = softmax(eval_preds)
                    # print(eval_preds.cpu().detach().numpy())
                    # print(eval_label)
                    chosen_preds = torch.argmax(eval_preds, dim=1)
                    # print(chosen_preds.cpu().detach().numpy())
                    for eval_i, each_label in enumerate(eval_label):
                        eval_predict_result = chosen_preds[eval_i].cpu().detach().numpy()
                        # print("each_label", each_label.item())
                        # print("eval_predict_result", eval_predict_result)
                        if each_label == 1: # positive
                            if eval_predict_result == each_label.item():
                                tp += 1
                            else:
                                fn += 1
                        else:
                            if eval_predict_result == each_label.item():
                                tn += 1
                            else:
                                fp += 1

                prec = tp/(tp+fp)
                if tp+fn == 0:
                    if tn > 0:
                        recall = 1
                    else:
                        recall = 0
                else:
                    recall = tp/(tp+fn)
                f1 = 2*tp/(2*tp+fp+fn)
                model.train()
                # eval_loss = eval_loss
                result = {#'eval_ppl': round(np.exp(eval_loss),5),
                          'global_step': global_step+1,
                          'train_loss': round(train_loss,5),
                        #   'eval_prec': round(prec, 5),
                        #   'eval_recall': round(recall, 5),
                          'eval_f1': round(f1, 5),
                          'tp': tp, 'fp': fp, 'fn': fn, 'tn': tn}
                with open(result_file, "a") as res_fp:
                    print(json.dumps(result), file=res_fp)

                for key in sorted(result.keys()):
                    logger.info("  %s = %s", key, str(result[key]))
                logger.info("  "+"*"*20)

                epoch_no = round((global_step+1)/ (math.ceil(len(train_pairs)/args.train_batch_size)))
                logger.info(f"Eval at epoch {epoch_no}")
                last_output_dir = os.path.join(args.output_dir, 'checkpoint-last')
                if not os.path.exists(last_output_dir):
                    os.makedirs(last_output_dir)
                model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self
                # output_model_file = os.path.join(last_output_dir, "pytorch_model.bin")
                output_model_file = os.path.join(last_output_dir, f"pytorch_model_{epoch_no:04d}.bin")
                torch.save(model_to_save.state_dict(), output_model_file)
                logger.info(f"Saved last model at: {output_model_file}")

                # if eval_loss<best_loss:
                if (first_eval):
                    first_eval = False
                    continue
                if f1 > best_acc:
                    # logger.info("  Best ppl:%s",round(np.exp(eval_loss),5))
                    logger.info(f"Best acc: {round(f1, 4)}")
                    logger.info("  "+"*"*20)
                    # best_loss=eval_loss
                    best_acc=f1
                    # Save best checkpoint for best ppl
                    output_dir = os.path.join(args.output_dir, 'checkpoint-best-ppl')
                    if not os.path.exists(output_dir):
                        os.makedirs(output_dir)
                    model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self
                    output_model_file = os.path.join(output_dir, "pytorch_model_best.bin")
                    torch.save(model_to_save.state_dict(), output_model_file)
                    logger.info(f"Saved best model at: {output_model_file}")

                early_stopping(f1)
                if early_stopping.early_stop:
                    logger.info(f"Early stop at global steps: {global_step+1}")
                    break

        pass
    if args.do_test:
        pass
    if args.do_infer:
        activation = {}

        for child_i, child in enumerate(model.children()):
            for k,v in child.named_parameters():
                v.requires_grad = False

        files=[]
        infer_out_dir = "data/approach_4/"
        os.makedirs(infer_out_dir, exist_ok=True)

        # Stack Overflow pairs
        infer_file="data/approach_4/text_code_pairs_test.jsonl"
        # infer_file="data/approach_4/sample_train_pairs.jsonl"
        file_path = infer_out_dir+os.sep+"test_127_threads_result.json"

        # API doc implementation pairs
        # infer_file="data/approach_4/doc_impl_pairs.jsonl"
        # file_path = infer_out_dir+os.sep+"api_doc_impl_embs.json"

        eval_examples = read_pairs(infer_file, mode="train")
        # eval_features = convert_pairs_to_features(eval_examples, tokenizer, args)
        eval_features, _, _ = get_training_features(eval_examples, tokenizer, args)
        eval_pair_id = torch.tensor([f.pair_id for f in eval_features], dtype=torch.long)
        eval_text_code_ids = torch.tensor([f.text_code_ids for f in eval_features], dtype=torch.long)
        eval_text_code_mask = torch.tensor([f.text_code_mask for f in eval_features], dtype=torch.long)
        eval_cmt_impl_ids = torch.tensor([f.cmt_impl_ids for f in eval_features], dtype=torch.long)
        eval_cmt_impl_mask = torch.tensor([f.cmt_impl_mask for f in eval_features], dtype=torch.long)
        eval_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)

        eval_dataset = TensorDataset(
                                    eval_pair_id,
                                    eval_text_code_ids,
                                    eval_text_code_mask,
                                    eval_cmt_impl_ids,
                                    eval_cmt_impl_mask,
                                    eval_label)

        eval_sampler = SequentialSampler(eval_dataset)
        eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.train_batch_size)

        logger.info("\n***** Running evaluation *****")
        logger.info("  Num examples = %d", len(eval_examples))
        logger.info("  Batch size = %d", args.eval_batch_size)

        model.eval()
        result_idx = 0
        output_results = {}
        softmax = nn.Softmax(dim=1)
        for eval_batch in tqdm(eval_dataloader,total=len(eval_dataloader)):
            eval_batch = tuple(t.to(device) for t in eval_batch)
            eval_pair_id, eval_text_code_ids, eval_text_code_mask, eval_cmt_impl_ids, eval_cmt_impl_mask, eval_label = eval_batch
            eval_preds = model(
                            eval_text_code_ids,
                            eval_text_code_mask,
                            eval_cmt_impl_ids,
                            eval_cmt_impl_mask
                            )

            eval_preds = softmax(eval_preds)
            chosen_preds = torch.argmax(eval_preds, dim=1)
            for eval_i, each_label in enumerate(eval_label):
                eval_predict_result = chosen_preds[eval_i].cpu().detach().numpy()
                output_results[str(result_idx)] = eval_predict_result.tolist()
                result_idx += 1

        import codecs
        with open(file_path, "w+") as fp:
            json.dump(output_results, fp, indent=2)
if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf8 -*-
from django.test import TestCase

from lists.views import home_page
from lists.models import Item, List

# Create your tests here.

"""Unittests for lists app.

Before you can design a unittest that works on progressing the functional test
you need to confirm the test runner will run the unittest.

functional_test.py was a programmer executed testrunner. The Django class
TestCase is more specialised than unittest.TestCase and even has a magic
testrunner.

Djano testrunner called:
python manage.py test
"""

class HomePageTest(TestCase):
    """ Unit test for '/' URL request and its resolution.

    Can we resolve the requested URL for the root of the site? Can the
    resolution take a view function I've made?
    """

    def test_uses_home_template(self):
        "Unit test for view resolution of '/' URL request."
        response = self.client.get('/')  # View func generetes a response.
        self.assertTemplateUsed(response, 'home.html')

class ListAndItemModelTest(TestCase):
    """Unit tests for interacting with Django ORM.

    ORM is the abstraction for a database of tables.
        Classes map to tables.
        Attributes map to columns.
        An instance of a class is a row(s).
    """

    def test_saving_and_retrieving_items(self):
        """Unit test for ORM saving and retrieving objects."""
        _list = List()
        _list.save()

        first_item = Item()
        first_item.text = 'The first (ever) list item.'
        first_item.list = _list
        first_item.save()

        second_item = Item()
        second_item.text = 'Item the second'
        second_item.list = _list
        second_item.save()

        saved_list = List.objects.first()
        self.assertEqual(saved_list, _list)

        saved_items = Item.objects.all()
        self.assertEqual(saved_items.count(), 2)

        first_saved_item = saved_items[0]
        second_saved_item = saved_items[1]
        self.assertEqual(first_saved_item.text, 'The first (ever) list item.')
        self.assertEqual(first_saved_item.list, _list)
        self.assertEqual(second_saved_item.text, 'Item the second')
        self.assertEqual(second_saved_item.list, _list)

class ListViewTest(TestCase):
    def test_uses_list_template(self):
        _list = List.objects.create()

        response = self.client.get(f'/lists/{_list.id}/')

        self.assertTemplateUsed(response, 'list.html')

    def test_display_only_items_for_that_list(self):
        _list_a = List.objects.create()
        _list_b = List.objects.create()
        string1a, string2a = 'itemey 1', 'itemey 2'
        string1b, string2b = 'other itemey 1', 'other itemey 2'
        Item.objects.create(text=string1a, list=_list_a)
        Item.objects.create(text=string2a, list=_list_a)
        Item.objects.create(text=string1b, list=_list_b)
        Item.objects.create(text=string2b, list=_list_b)

        response = self.client.get(f'/lists/{_list_a.id}/')

        self.assertContains(response, string1a)
        self.assertContains(response, string2a)
        self.assertNotContains(response, string1b)
        self.assertNotContains(response, string2b)

    def test_passes_correct_list_to_template(self):
        prior_list = List.objects.create()  # Dummy list to prevent head/tail indexing.
        correct_list = List.objects.create()
        later_list = List.objects.create()  # Dummy list to prevent head/tail indexing.

        response = self.client.get(f'/lists/{correct_list.id}/')

        self.assertEqual(response.context['list'], correct_list)

class NewItemTest(TestCase):

    def test_can_save_a_POST_request_to_an_exisiting_list(self):
        new_item_txt = 'A new item for an existing list'
        prior_list = List.objects.create()  # Dummy list to prevent head/tail indexing.
        correct_list = List.objects.create()
        later_list = List.objects.create()  # Dummy list to prevent head/tail indexing.

        self.client.post(f'/lists/{correct_list.id}/add_item',
                         data={'item_text': new_item_txt})

        self.assertEqual(Item.objects.count(), 1)
        new_item = Item.objects.first()
        self.assertEqual(new_item.text, new_item_txt)
        self.assertEqual(new_item.list, correct_list)

    def test_redirects_to_list_view(self):
        new_item_txt = 'A new item for an existing list'
        prior_list = List.objects.create()  # Dummy list to prevent head/tail indexing.
        correct_list = List.objects.create()
        later_list = List.objects.create()  # Dummy list to prevent head/tail indexing.

        response = self.client.post(path=f'/lists/{correct_list.id}/add_item',
                                    data={'item_text': new_item_txt})

        self.assertRedirects(response, f'/lists/{correct_list.id}/')

class NewListsTest(TestCase):
    def test_can_save_a_POST_request(self):
        "Unit test for view function checking POST value is added to DB."
        self.client.post('/lists/new', data={'item_text': 'A new list item'})
        self.assertEqual(Item.objects.count(), 1)

        new_item = Item.objects.first()

        self.assertEqual(new_item.text, 'A new list item')

    def test_redirects_after_POST(self):
        "Unit test that checks view function redirects after POST"
        response = self.client.post('/lists/new', data={'item_text': 'A new list item'})
        new_list = List.objects.first()
        self.assertRedirects(response, f'/lists/{new_list.id}/')


import statistics as stats

from code.StockData import StockData

class StockMetrics(StockData):
    def __init__(self, path):
        # call the parent method's constructor
        super(StockMetrics, self).__init__(path)

        # now that we've ran self.load(), we can interact with "self.data" as a
        # list of lists
        self.load()

    def average01(self):
        """pt1
        """
        averages = []
        for row in self.data:
            data_row = row[1:]
            y = []
            for x in data_row:
                try:
                    y.append(float(x))
                except ValueError:
                    continue
            average = stats.mean(y)
            rounded_average = round(average, 3)
            averages.append(rounded_average)
        return averages

    def median02(self):
        """pt2
        """
        ...

    def stddev03(self):
        """pt3
        """
        ...


import os
import sys
# load config.json
import json
with open('config.json') as f:
    conf = json.load(f)

# if config[openai_api_key] is not None, then set the environment variable
if conf['openai_api_key'] is not "YOUR_API_KEY":
    os.environ['OPENAI_API_KEY'] = conf['openai_api_key']

from langchain.agents import load_tools, Tool
from langchain.agents import initialize_agent
from langchain.agents import AgentType
from langchain.llms import OpenAI

llm = OpenAI(temperature=0.7)

from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import Pinecone
from langchain.document_loaders import TextLoader
#loader = TextLoader('/mnt/s/Documents/Collage/university-study-notes/content/20221231174507-fundamentals_of_data_analysis_class_notes.org')
from langchain.document_loaders import UnstructuredPDFLoader
loader = UnstructuredPDFLoader(conf['source'])
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
texts = text_splitter.split_documents(documents)

embeddings = OpenAIEmbeddings()
from langchain.vectorstores import Chroma
docsearch = Chroma.from_documents(texts, embeddings)

from langchain.llms import OpenAI
from langchain.chains import RetrievalQA
qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())

stats_tool = Tool(
    name="stats",
    description="Search notes on statistics and how to use python to calculate them. You can also ask what to do when solving some problem.",
    func=qa.run
)
from langchain.tools import DuckDuckGoSearchTool
ddg = DuckDuckGoSearchTool()
from langchain.utilities import PythonREPL
repl = PythonREPL()
tools = []
tools.append(stats_tool)
tools.append(ddg)
python_calc_tool = Tool(
    name="python",
    description="Run python code in the REPL. Or execute some calculation. Do not search any information here.",
    func=repl.run
)
tools.append(python_calc_tool)

from langchain.memory import ConversationBufferMemory
memory = ConversationBufferMemory(memory_key="chat_history")

agent = initialize_agent(tools, llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True, memory=memory)

template = '''
{problem}
---
Answer the following questions only:
{need_to_know}
---
When using the Python tool, always print all the variables you want to see. For example, if you want to see the value of x, you should print(x) instead of just x.
'''

while True:
    task = input("Task: ")
    if task == "exit":
        break
    if task == 'solve':
        prompt = input("Context of Question: ")
        prompt_2 = input("Goals: ")
        response = agent.run(template.format(problem=prompt, need_to_know=prompt_2))
        print("A: ", response)
    else:
        response = agent.run(task)
        print("A: ", response)


#!/usr/bin/env python3
import genanki
import requests

class ankibh:
    SPARQL_ENDPOINT = 'https://query.wikidata.org/sparql'

    bh_query = """
    SELECT ?bezirk ?bezirkLabel ?lpl WHERE {
    ?bezirk wdt:P31 wd:Q871419;
            wdt:P395 ?lpl;

    SERVICE wikibase:label { bd:serviceParam wikibase:language "de,en". }
    }
    """

    def __init__(self):
        self._genanki_setup()

    def _genanki_setup(self):
        bh_model = genanki.Model(
            1468421455, # keep static, generated with random.randrange(1 << 30, 1 << 31) as per genanki docs
            'BH Model',
            fields=[
                {'name': 'Autokennzeichen'},
                {'name': 'Name'},
            ],
            templates=[
                {
                    'name': 'Kennzeichen -> Name',
                    'qfmt': '
{{Autokennzeichen}}
',
                    'afmt': '
{{Name}}
',
                },
                {
                    'name': 'Name -> Kennzeichen',
                    'qfmt': '
{{Name}}
',
                    'afmt': '
{{Autokennzeichen}}
',
                }
            ]
        )

        self.bh_model = bh_model

        bh_deck = genanki.Deck(
            1468421456,
            'BHs in .at'
        )

        self.bh_deck = bh_deck

    def run(self):
        r = requests.get(self.SPARQL_ENDPOINT, params={'format': 'json', 'query': self.bh_query})
        data = r.json()

        for bezirk in data['results']['bindings']:
            lpl = bezirk['lpl']['value']
            name = bezirk['bezirkLabel']['value']
            if name.startswith('Bezirk '):
                name = name[7:]

            print(lpl, name)
            new_note = genanki.Note(
                self.bh_model,
                fields=[lpl, name]
            )
            self.bh_deck.add_note(new_note)

        genanki.Package(self.bh_deck).write_to_file('bhs.apkg')

if __name__ == '__main__':
    ourankibh = ankibh()
    ourankibh.run()


from django.db.models.signals import post_save,post_delete
from django.dispatch import receiver

from django.contrib.auth.models import User
from .models import Profile

from django.core.mail import send_mail
from django.conf import settings

def create_profile(sender,created,instance, **kwargs):
    """create profile automatically when a user is created.
        Args:
            sender ([User]): User notifies this function that a new user being created in db
            created ([Bool]): If User does NOT exists,create profile. created = True
            instance ([User]): new user associated with the profile.
    """
    if created:
        user = instance
        profile = Profile.objects.create(
            user=user,
            username=user.username,
            email = user.email,
            name=user.first_name,
            )
        subject = 'Welcome to DevSearch'
        message = 'We are glad you are here!'
        send_mail(
            subject,
            message,
            settings.EMAIL_HOST_USER, #sender
            [profile.email],#recipient
            fail_silently=False,
)

def update_user_account(sender,created,instance, **kwargs):
    """ Update and save profile.

    Args:
        sender ([Profile]): [anytime Profile is updated,this function gets triggered]
        created ([Bool]): If User already exists,update profile. created = False
        instance ([Profile]): current profile.
    """
    profile = instance
    user = profile.user
    #validate created == false to avoid recursion error
    if created == False:
        user.first_name = profile.name
        user.username = profile.username
        user.email = profile.email
        user.save()

def delete_profile(sender,instance, **kwargs):
    """ delete User when profile is deleted

    Args:
        sender ([Profile]): [description]
        created ([Bool]): If User exists, delete profile. created = False
        instance ([User]): current user associated with the profile.
    """
    try:
        user = instance.user
        user.delete()
    except:
        print('Error: Unable to delete user')

post_save.connect(create_profile, sender=User)
post_save.connect(update_user_account, sender=Profile)
post_delete.connect(delete_profile, sender=Profile)