# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

import json
import logging
import math
import os
import platform
import random
import threading
import time
import unicodedata

import bpy
from bpy.app.handlers import persistent
from bpy.props import (  # TODO only keep the ones actually used when cleaning
    BoolProperty,
    StringProperty,
)
from bpy.types import Operator

from . import (
    addon_updater_ops,
    asset_bar_op,
    bkit_oauth,
    categories,
    colors,
    comments_utils,
    daemon_lib,
    global_vars,
    image_utils,
    paths,
    ratings_utils,
    reports,
    rerequests,
    resolutions,
    tasks_queue,
    ui,
    utils,
    version_checker,
)
from .daemon import tasks

bk_logger = logging.getLogger(__name__)

def check_errors(rdata):
  if rdata.get('statusCode') and int(rdata.get('statusCode')) > 299:
    utils.p(rdata)
    if rdata.get('detail') == 'Invalid token.':
      bkit_oauth.logout()
      return False, "Invalid token. You've been logged out. Use login panel to connect your profile."
    else:
      return False, rdata.get('detail')
  if rdata.get('statusCode') is None and rdata.get('results') is None:
    return False, 'Connection error'
  return True, ''

search_tasks = {}

def update_ad(ad):
  if not ad.get('assetBaseId'):
    try:
      ad['assetBaseId'] = ad['asset_base_id']  # this should stay ONLY for compatibility with older scenes
      ad['assetType'] = ad['asset_type']  # this should stay ONLY for compatibility with older scenes
      ad['verificationStatus'] = ad[
        'verification_status']  # this should stay ONLY for compatibility with older scenes
      ad['author'] = {}
      ad['author']['id'] = ad['author_id']  # this should stay ONLY for compatibility with older scenes
      ad['canDownload'] = ad['can_download']  # this should stay ONLY for compatibility with older scenes
    except Exception as e:
      bk_logger.error('BlenderKit failed to update older asset data')
  return ad

def update_assets_data():  # updates assets data on scene load.
  '''updates some properties that were changed on scenes with older assets.
  The properties were mainly changed from snake_case to CamelCase to fit the data that is coming from the server.
  '''
  data = bpy.data

  datablocks = [
    bpy.data.objects,
    bpy.data.materials,
    bpy.data.brushes,
  ]
  for dtype in datablocks:
    for block in dtype:
      if block.get('asset_data') != None:
        update_ad(block['asset_data'])

  dicts = [
    'assets used',
  ]
  for s in bpy.data.scenes:
    for bkdict in dicts:

      d = s.get(bkdict)
      if not d:
        continue;

      for asset_id in d.keys():
        update_ad(d[asset_id])
        # bpy.context.scene['assets used'][ad] = ad

@persistent
def undo_post_reload_previews(context):
  load_previews()

@persistent
def undo_pre_end_assetbar(context):
  ui_props = bpy.context.window_manager.blenderkitUI

  ui_props.turn_off = True
  ui_props.assetbar_on = False

@persistent
def scene_load(context):
  """Load categories , check timers registration, and update scene asset data.
  Should (probably) also update asset data from server (after user consent).
  """
  if not bpy.app.timers.is_registered(bkit_oauth.refresh_token_timer) and not bpy.app.background:
    bpy.app.timers.register(bkit_oauth.refresh_token_timer, persistent=True, first_interval=5)
    #bpy.app.timers.register(bkit_oauth.refresh_token_timer, persistent=True, first_interval=36000)
  # if utils.experimental_enabled() and not bpy.app.timers.is_registered(
  #         refresh_notifications_timer) and not bpy.app.background:
  #     bpy.app.timers.register(refresh_notifications_timer, persistent=True, first_interval=5)

  update_assets_data()

def fetch_server_data():
  '''Download profile, and refresh token if needed.'''
  if bpy.app.background:
    return
  user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
  api_key = user_preferences.api_key
  # Only refresh new type of tokens(by length), and only one hour before the token timeouts.
  if api_key != '' and global_vars.DATA.get('bkit profile') == None:
    get_profile()
  # all_notifications_count = comments_utils.count_all_notifications()
  # comments_utils.get_notifications_thread(api_key, all_count = all_notifications_count)

first_time = True
first_search_parsing = True
last_clipboard = ''

def check_clipboard():
  '''
  Checks clipboard for an exact string containing asset ID.
  The string is generated on www.blenderkit.com as for example here:
  https://www.blenderkit.com/get-blenderkit/54ff5c85-2c73-49e9-ba80-aec18616a408/
  '''

  # clipboard monitoring to search assets from web
  if platform.system() != 'Linux':
    global last_clipboard
    if bpy.context.window_manager.clipboard != last_clipboard:
      last_clipboard = bpy.context.window_manager.clipboard
      instr = 'asset_base_id:'
      # first check if contains asset id, then asset type
      if last_clipboard[:len(instr)] == instr:
        atstr = 'asset_type:'
        ati = last_clipboard.find(atstr)
        # this only checks if the asset_type keyword is there but let's the keywords update function do the parsing.
        if ati > -1:
          search_props = utils.get_search_props()
          search_props.search_keywords = last_clipboard
          # don't run search after this - assigning to keywords runs the search_update function.
        # bpy.context.window_manager.clipboard = ''

def parse_result(r):
  '''
  needed to generate some extra data in the result(by now)
  Parameters
  ----------
  r - search result, also called asset_data
  '''
  scene = bpy.context.scene

  # TODO remove this fix when filesSize is fixed.
  # this is a temporary fix for too big numbers from the server.
  # try:
  #     r['filesSize'] = int(r['filesSize'] / 1024)
  # except:
  #     utils.p('asset with no files-size')
  asset_type = r['assetType']
  if len(r['files']) > 0:  # TODO remove this condition so all assets are parsed.
    get_author(r)

    r['available_resolutions'] = []
    allthumbs = []
    durl, tname, small_tname = '', '', ''

    if r['assetType'] == 'hdr':
      tname = paths.extract_filename_from_url(r['thumbnailLargeUrlNonsquared'])
    else:
      tname = paths.extract_filename_from_url(r['thumbnailMiddleUrl'])
    small_tname = paths.extract_filename_from_url(r['thumbnailSmallUrl'])
    allthumbs.append(tname)  # TODO just first thumb is used now.
    # if r['fileType'] == 'thumbnail':
    #     tname = paths.extract_filename_from_url(f['fileThumbnailLarge'])
    #     small_tname = paths.extract_filename_from_url(f['fileThumbnail'])
    #     allthumbs.append(tname)  # TODO just first thumb is used now.

    for f in r['files']:
      # if f['fileType'] == 'thumbnail':
      #     tname = paths.extract_filename_from_url(f['fileThumbnailLarge'])
      #     small_tname = paths.extract_filename_from_url(f['fileThumbnail'])
      #     allthumbs.append(tname)  # TODO just first thumb is used now.

      if f['fileType'] == 'blend':
        durl = f['downloadUrl'].split('?')[0]
        # fname = paths.extract_filename_from_url(f['filePath'])

      if f['fileType'].find('resolution') > -1:
        r['available_resolutions'].append(resolutions.resolutions[f['fileType']])

    # code for more thumbnails
    # tdict = {}
    # for i, t in enumerate(allthumbs):
    #     tdict['thumbnail_%i'] = t

    r['max_resolution'] = 0
    if r['available_resolutions']:  # should check only for non-empty sequences
      r['max_resolution'] = max(r['available_resolutions'])

    # tooltip = generate_tooltip(r)
    # for some reason, the id was still int on some occurances. investigate this.
    r['author']['id'] = str(r['author']['id'])

    # some helper props, but generally shouldn't be renaming/duplifiying original properties,
    # so blender's data is same as on server.
    asset_data = {'thumbnail': tname,
                  'thumbnail_small': small_tname,
                  # 'tooltip': tooltip,

                  }
    asset_data['downloaded'] = 0

    # parse extra params needed for blender here
    params = r['dictParameters']  # utils.params_to_dict(r['parameters'])

    if asset_type == 'model':
      if params.get('boundBoxMinX') != None:
        bbox = {
          'bbox_min': (
            float(params['boundBoxMinX']),
            float(params['boundBoxMinY']),
            float(params['boundBoxMinZ'])),
          'bbox_max': (
            float(params['boundBoxMaxX']),
            float(params['boundBoxMaxY']),
            float(params['boundBoxMaxZ']))
        }

      else:
        bbox = {
          'bbox_min': (-.5, -.5, 0),
          'bbox_max': (.5, .5, 1)
        }
      asset_data.update(bbox)
    if asset_type == 'material':
      asset_data['texture_size_meters'] = params.get('textureSizeMeters', 1.0)

    # asset_data.update(tdict)

    au = scene.get('assets used', {})
    if au == {}:
      scene['assets used'] = au
    if r['assetBaseId'] in au.keys():
      asset_data['downloaded'] = 100
      # transcribe all urls already fetched from the server
      r_previous = au[r['assetBaseId']]
      if r_previous.get('files'):
        for f in r_previous['files']:
          if f.get('url'):
            for f1 in r['files']:
              if f1['fileType'] == f['fileType']:
                f1['url'] = f['url']

    # attempt to switch to use original data gradually, since the parsing as itself should become obsolete.
    asset_data.update(r)
    return asset_data

def clear_searches():
  global search_tasks
  search_tasks.clear()

def cleanup_search_results():
  dicts = (
    'search results','bkit model search',
    'bkit scene search',
    'bkit hdr search',
    'bkit material search',
    'bkit texture search',
    'bkit brush search',
  )
  for sr in dicts:
    global_vars.DATA.pop(sr, None)
    global_vars.DATA.pop(f'{sr} orig', None)

def handle_search_task(task: tasks.Task) -> bool:
  '''parse search results, try to load all available previews.'''
  ##############original

  global search_tasks, first_search_parsing
  if len(search_tasks) == 0:
    # utils.p('end search timer')
    props = utils.get_search_props()
    props.is_searching = False
    return True

  # don't do anything while dragging - this could switch asset during drag, and make results list length different,
  # causing a lot of throuble.
  if bpy.context.window_manager.blenderkitUI.dragging:
    # utils.p('end search timer')
    return False
  # if original task was already removed (because user initiated another search), results are dropped- Returns True
  # because that's OK.
  orig_task = search_tasks.get(task.task_id)
  if orig_task is None:
    print('search task result not from active search', task.task_id, len(search_tasks), list(search_tasks.keys()))
    return True

  search_tasks.pop(task.task_id)

  # this fixes black thumbnails in asset bar, test if this bug still persist in blender and remove if it's fixed
  if bpy.app.version < (3, 3, 0):
    sys_prefs = bpy.context.preferences.system
    sys_prefs.gl_texture_limit = 'CLAMP_OFF'

  global first_search_parsing
  if first_search_parsing:
    comments_utils.check_notifications()
    first_search_parsing = False

  ###################

  asset_type = task.data['asset_type']
  props = utils.get_search_props()
  search_name = f'bkit {asset_type} search'

  if not task.data.get('get_next'):
    result_field = []
  else:
    result_field = []
    for r in global_vars.DATA[search_name]:
      result_field.append(r)

  ok, error = check_errors(task.result)
  if ok:
    ui_props = bpy.context.window_manager.blenderkitUI
    orig_len = len(result_field)

    for ri, r in enumerate(task.result['results']):
      asset_data = parse_result(r)
      if asset_data != None:
        result_field.append(asset_data)

    # Get ratings from BlenderKit server
    user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
    api_key = user_preferences.api_key
    headers = utils.get_headers(api_key)
    if utils.profile_is_validator():
      for r in task.result['results']:
        if ratings_utils.get_rating_local(r['id']) is None:
          rating_thread = threading.Thread(target=ratings_utils.get_rating, args=([r['id'], headers]),
                                           daemon=True)
          rating_thread.start()

    global_vars.DATA[search_name] = result_field
    global_vars.DATA[search_name + ' orig'] = task.result

    if result_field and result_field[0]['assetType'] == ui_props.asset_type.lower():
      global_vars.DATA['search results'] = result_field
      global_vars.DATA['search results orig'] = task.result

    if len(result_field) < ui_props.scroll_offset or not (task.data.get('get_next')):
      # jump back
      ui_props.scroll_offset = 0
    props.report = f"Found {global_vars.DATA['search results orig']['count']} results."
    if len(global_vars.DATA['search results']) == 0:
      tasks_queue.add_task((reports.add_report, ('No matching results found.',)))
    else:
      tasks_queue.add_task((reports.add_report, (f"Found {global_vars.DATA['search results orig']['count']} results.",)))
    # show asset bar automatically, but only on first page - others are loaded also when asset bar is hidden.
    if not ui_props.assetbar_on and not task.data.get('get_next'):
      bpy.ops.view3d.run_assetbar_fix_context()

  else:
    props.report = error
    reports.add_report(error, 15, 'ERROR')

  if len(search_tasks) == 0:
    props.is_searching = False
  return True

def handle_preview_task(task: tasks.Task) -> bool:
  """Parse search results, try to load all available previews."""

  global_vars.DATA['images available'][task.data['image_path']] = True
  if asset_bar_op.asset_bar_operator is not None:
    if task.data['thumbnail_type'] =='small':
      asset_bar_op.asset_bar_operator.update_image(task.data['assetBaseId'])
    if task.data['thumbnail_type'] == 'full':
      asset_bar_op.asset_bar_operator.update_tooltip_image(task.data['assetBaseId'])
  return True

def load_preview(asset):
  # FIRST START SEARCH
  props = bpy.context.window_manager.blenderkitUI
  directory = paths.get_temp_dir('%s_search' % props.asset_type.lower())

  tpath = os.path.join(directory, asset['thumbnail_small'])
  tpath_exists = os.path.exists(tpath)
  if not asset['thumbnail_small'] or asset['thumbnail_small'] == '' or not tpath_exists:
    # tpath = paths.get_addon_thumbnail_path('thumbnail_notready.jpg')
    asset['thumb_small_loaded'] = False

  iname = f".{asset['thumbnail_small']}"
  # if os.path.exists(tpath):  # sometimes we are unlucky...
  img = bpy.data.images.get(iname)

  if img is None or len(img.pixels) == 0:
    if not tpath_exists:
      return False
    # wrap into try statement since sometimes
    try:
      img = bpy.data.images.load(tpath, check_existing=True)

      img.name = iname
      if len(img.pixels) > 0:
        return True
    except:
      pass
    return False
  elif img.filepath != tpath:
    if not tpath_exists:
      # unload loaded previews from previous results
      bpy.data.images.remove(img)
      return False
    # had to add this check for autopacking files...
    if bpy.data.use_autopack and img.packed_file is not None:
      img.unpack(method='USE_ORIGINAL')
    img.filepath = tpath
    try:
      img.reload()
    except:
      return False

  if asset['assetType'] == 'hdr':
    # to display hdr thumbnails correctly, we use non-color, otherwise looks shifted
    image_utils.set_colorspace(img, 'Non-Color')
  else:
    image_utils.set_colorspace(img, 'sRGB')
  asset['thumb_small_loaded'] = True
  return True

def load_previews():
  results = global_vars.DATA.get('search results')
  if results is not None:
    for i, result in enumerate(results):
      load_preview(result)

#  line splitting for longer texts...
def split_subs(text, threshold=40):
  if text == '':
    return []
  # temporarily disable this, to be able to do this in drawing code

  text = text.rstrip()
  text = text.replace('\r\n', '\n')

  lines = []

  while len(text) > threshold:
    # first handle if there's an \n line ending
    i_rn = text.find('\n')
    if 1 < i_rn < threshold:
      i = i_rn
      text = text.replace('\n', '', 1)
    else:
      i = text.rfind(' ', 0, threshold)
      i1 = text.rfind(',', 0, threshold)
      i2 = text.rfind('.', 0, threshold)
      i = max(i, i1, i2)
      if i <= 0:
        i = threshold
    lines.append(text[:i])
    text = text[i:]
  lines.append(text)
  return lines

def list_to_str(input):
  output = ''
  for i, text in enumerate(input):
    output += text
    if i < len(input) - 1:
      output += ', '
  return output

def writeblock(t, input, width=40):  # for longer texts
  dlines = split_subs(input, threshold=width)
  for i, l in enumerate(dlines):
    t += '%s\n' % l
  return t

def writeblockm(tooltip, mdata, key='', pretext=None, width=40):  # for longer texts
  if mdata.get(key) == None:
    return tooltip
  else:
    intext = mdata[key]
    if type(intext) == list:
      intext = list_to_str(intext)
    if type(intext) == float:
      intext = round(intext, 3)
    intext = str(intext)
    if intext.rstrip() == '':
      return tooltip
    if pretext == None:
      pretext = key
    if pretext != '':
      pretext = pretext + ': '
    text = pretext + intext
    dlines = split_subs(text, threshold=width)
    for i, l in enumerate(dlines):
      tooltip += '%s\n' % l

  return tooltip

def has(mdata, prop):
  if mdata.get(prop) is not None and mdata[prop] is not None and mdata[prop] is not False:
    return True
  else:
    return False

def generate_tooltip(mdata):
  col_w = 40
  if type(mdata['parameters']) == list:
    mparams = utils.params_to_dict(mdata['parameters'])
  else:
    mparams = mdata['parameters']
  t = ''
  t = writeblock(t, mdata['displayName'], width=int(col_w * .6))
  # t += '\n'

  # t = writeblockm(t, mdata, key='description', pretext='', width=col_w)
  return t

def generate_author_textblock(adata):
  t = ''

  if adata not in (None, ''):
    col_w = 2000
    if len(adata['firstName'] + adata['lastName']) > 0:
      t = '%s %s\n' % (adata['firstName'], adata['lastName'])
      t += '\n'
      if adata.get('aboutMe') is not None:
        t = writeblockm(t, adata, key='aboutMe', pretext='', width=col_w)
  return t

def write_gravatar(a_id, gravatar_path):
  '''
  Write down gravatar path, as a result of thread-based gravatar image download.
  This should happen on timer in queue.
  '''
  # print('write author', a_id, type(a_id))
  authors = global_vars.DATA['bkit authors']
  if authors.get(a_id) is not None:
    adata = authors.get(a_id)
    adata['gravatarImg'] = gravatar_path

def fetch_gravatar(adata=None):
  '''
  Gets avatars from blenderkit server
  Parameters
  ----------
  adata - author data from elastic search result

  '''

  # fetch new avatars if available already
  if adata.get('avatar128') is not None:
    avatar_path = paths.get_temp_dir(subdir='bkit_g/') + adata['id'] + '.jpg'
    if os.path.exists(avatar_path):
      tasks_queue.add_task((write_gravatar, (adata['id'], avatar_path)))
      return

    url = global_vars.SERVER + adata['avatar128']
    r = rerequests.get(url, stream=False, headers=utils.get_headers())
    # print(r.body)
    if r.status_code == 200:
      # print(url)
      # print(r.headers['content-disposition'])
      with open(avatar_path, 'wb') as f:
        f.write(r.content)
      tasks_queue.add_task((write_gravatar, (adata['id'], avatar_path)))
    elif r.status_code == '404':
      adata['avatar128'] = None
      utils.p('avatar for author not available.')
    return

  # older gravatar code
  if adata.get('gravatarHash') is not None:
    gravatar_path = paths.get_temp_dir(subdir='bkit_g/') + adata['gravatarHash'] + '.jpg'

    if os.path.exists(gravatar_path):
      tasks_queue.add_task((write_gravatar, (adata['id'], gravatar_path)))
      return;

    url = "https://www.gravatar.com/avatar/" + adata['gravatarHash'] + '?d=404'
    r = rerequests.get(url, stream=False, headers=utils.get_headers())
    if r.status_code == 200:
      with open(gravatar_path, 'wb') as f:
        f.write(r.content)
      tasks_queue.add_task((write_gravatar, (adata['id'], gravatar_path)))
    elif r.status_code == '404':
      adata['gravatarHash'] = None
      utils.p('gravatar for author not available.')

fetching_gravatars = {}

def get_author(r):
  ''' Writes author info (now from search results) and fetches gravatar if needed.
  this is now tweaked to be able to get authors from
  '''
  global fetching_gravatars

  a_id = str(r['author']['id'])
  preferences = bpy.context.preferences.addons['blenderkit'].preferences
  authors = global_vars.DATA.get('bkit authors', {})
  if authors == {}:
    global_vars.DATA['bkit authors'] = authors
  a = authors.get(a_id)
  if a is None:  # or a is '' or (a.get('gravatarHash') is not None and a.get('gravatarImg') is None):
    a = r['author']
    a['id'] = a_id
    a['tooltip'] = generate_author_textblock(a)

    authors[a_id] = a
    if fetching_gravatars.get(a['id']) is None:
      fetching_gravatars[a['id']] = True

    thread = threading.Thread(target=fetch_gravatar, args=(a.copy(),), daemon=True)
    thread.start()
  return a

def write_profile(adata):
  user = adata['user']
  # we have to convert to MiB here, numbers too big for python int type
  if user.get('sumAssetFilesSize') is not None:
    user['sumAssetFilesSize'] /= (1024 * 1024)
  if user.get('sumPrivateAssetFilesSize') is not None:
    user['sumPrivateAssetFilesSize'] /= (1024 * 1024)
  if user.get('remainingPrivateQuota') is not None:
    user['remainingPrivateQuota'] /= (1024 * 1024)

  if adata.get('canEditAllAssets') is True:
    user['exmenu'] = True
  else:
    user['exmenu'] = False

  global_vars.DATA['bkit profile'] = adata

def request_profile(api_key):
  a_url = f'{paths.BLENDERKIT_API}/me/'
  headers = utils.get_headers(api_key)
  r = rerequests.get(a_url, headers=headers)
  adata = r.json()
  if adata.get('user') is None:
    utils.p(adata)
    utils.p('getting profile failed')
    return None
  return adata

def fetch_profile(api_key):
  try:
    adata = request_profile(api_key)
    if adata is not None:
      tasks_queue.add_task((write_profile, (adata,)))
  except Exception as e:
    bk_logger.error(e)

def get_profile():
  preferences = bpy.context.preferences.addons['blenderkit'].preferences
  profile = global_vars.DATA.get('bkit profile')
  thread = threading.Thread(target=fetch_profile, args=(preferences.api_key,), daemon=True)
  thread.start()

  return profile

def query_to_url(query={}, params={}):
  # build a new request
  url = f'{paths.BLENDERKIT_API}/search/'

  # build request manually
  # TODO use real queries
  requeststring = '?query='
  #
  if query.get('query') not in ('', None):
    requeststring += query['query'].lower()
  for i, q in enumerate(query):
    if q != 'query':
      requeststring += '+'
      requeststring += q + ':' + str(query[q]).lower()

  # add dict_parameters to make results smaller
  # result ordering: _score - relevance, score - BlenderKit score
  order = []
  if params['free_first']:
    order = ['-is_free', ]
  if query.get('query') is None and query.get('category_subtree') == None:
    # assumes no keywords and no category, thus an empty search that is triggered on start.
    # orders by last core file upload
    if query.get('verification_status') == 'uploaded':
      # for validators, sort uploaded from oldest
      order.append('created')
    else:
      order.append('-last_upload')
  elif query.get('author_id') is not None and utils.profile_is_validator():

    order.append('-created')
  else:
    if query.get('category_subtree') is not None:
      order.append('-score,_score')
    else:
      order.append('_score')
  if requeststring.find('+order:') == -1:
    requeststring += '+order:' + ','.join(order)
  requeststring += '&dict_parameters=1'

  requeststring += '&page_size=' + str(params['page_size'])
  requeststring += '&addon_version=%s' % params['addon_version']
  if not (query.get('query') and query.get('query').find('asset_base_id')>-1):
    requeststring += '&blender_version=%s' % params['blender_version']
  if params.get('scene_uuid') is not None:
    requeststring += '&scene_uuid=%s' % params['scene_uuid']
  # print('params', params)
  urlquery = url + requeststring
  return urlquery

def parse_html_formated_error(text):
  report = text[text.find('<title>') + 7: text.find('</title>')]

  return report

def build_query_common(query, props):
  '''add shared parameters to query'''
  query_common = {}
  if props.search_keywords != '':
    # keywords = urllib.parse.urlencode(props.search_keywords)
    keywords = props.search_keywords.replace('&', '%26')
    query_common["query"] = keywords

  if props.search_verification_status != 'ALL' and utils.profile_is_validator():
    query_common['verification_status'] = props.search_verification_status.lower()

  if props.unrated_only and utils.profile_is_validator():
    query["quality_count"] = 0

  if props.search_file_size:
    query_common["files_size_gte"] = props.search_file_size_min * 1024 * 1024
    query_common["files_size_lte"] = props.search_file_size_max * 1024 * 1024

  if props.quality_limit > 0:
    query["quality_gte"] = props.quality_limit

  query.update(query_common)

def build_query_model():
  '''use all search input to request results from server'''

  props = bpy.context.window_manager.blenderkit_models
  query = {
    "asset_type": 'model',
    # "engine": props.search_engine,
    # "adult": props.search_adult,
  }
  if props.search_style != 'ANY':
    if props.search_style != 'OTHER':
      query["model_style"] = props.search_style
    else:
      query["model_style"] = props.search_style_other

  # the 'free_only' parametr gets moved to the search command and is used for ordering the assets as free first
  # if props.free_only:
  #     query["is_free"] = True

  if props.search_condition != 'UNSPECIFIED':
    query["condition"] = props.search_condition

  if props.search_design_year:
    query["designYear_gte"] = props.search_design_year_min
    query["designYear_lte"] = props.search_design_year_max
  if props.search_polycount:
    query["faceCount_gte"] = props.search_polycount_min
    query["faceCount_lte"] = props.search_polycount_max
  if props.search_texture_resolution:
    query["textureResolutionMax_gte"] = props.search_texture_resolution_min
    query["textureResolutionMax_lte"] = props.search_texture_resolution_max
  if props.search_animated:
    query["animated"] = True
  build_query_common(query, props)

  return query

def build_query_scene():
  '''use all search input to request results from server'''

  props = bpy.context.window_manager.blenderkit_scene
  query = {
    "asset_type": 'scene',
    # "engine": props.search_engine,
    # "adult": props.search_adult,
  }
  build_query_common(query, props)
  return query

def build_query_HDR():
  '''use all search input to request results from server'''

  props = bpy.context.window_manager.blenderkit_HDR
  query = {
    "asset_type": 'hdr',

    # "engine": props.search_engine,
    # "adult": props.search_adult,
  }
  if props.true_hdr:
    query["trueHDR"] = props.true_hdr
  build_query_common(query, props)
  return query

def build_query_material():
  props = bpy.context.window_manager.blenderkit_mat
  query = {
    "asset_type": 'material',

  }
  # if props.search_engine == 'NONE':
  #     query["engine"] = ''
  # if props.search_engine != 'OTHER':
  #     query["engine"] = props.search_engine
  # else:
  #     query["engine"] = props.search_engine_other
  if props.search_style != 'ANY':
    if props.search_style != 'OTHER':
      query["style"] = props.search_style
    else:
      query["style"] = props.search_style_other
  if props.search_procedural == 'TEXTURE_BASED':
    # todo this procedural hack should be replaced with the parameter
    query["textureResolutionMax_gte"] = 0
    # query["procedural"] = False
    if props.search_texture_resolution:
      query["textureResolutionMax_gte"] = props.search_texture_resolution_min
      query["textureResolutionMax_lte"] = props.search_texture_resolution_max

  elif props.search_procedural == "PROCEDURAL":
    # todo this procedural hack should be replaced with the parameter
    query["files_size_lte"] = 1024 * 1024
    # query["procedural"] = True

  build_query_common(query, props)

  return query

def build_query_texture():
  props = bpy.context.scene.blenderkit_tex
  query = {
    "asset_type": 'texture',

  }

  if props.search_style != 'ANY':
    if props.search_style != 'OTHER':
      query["search_style"] = props.search_style
    else:
      query["search_style"] = props.search_style_other

  build_query_common(query, props)

  return query

def build_query_brush():
  props = bpy.context.window_manager.blenderkit_brush

  brush_type = ''
  if bpy.context.sculpt_object is not None:
    brush_type = 'sculpt'

  elif bpy.context.image_paint_object:  # could be just else, but for future p
    brush_type = 'texture_paint'

  query = {
    "asset_type": 'brush',

    "mode": brush_type
  }

  build_query_common(query, props)

  return query

def add_search_process(query, params):
  global search_tasks

  if len(search_tasks) > 0:
    # just remove all running search tasks.
    # we can also kill them in daemon, but not so urgent now
    # TODO stop tasks in daemon?
    bk_logger.debug('Removing old search tasks')
    search_tasks = dict()

  tempdir = paths.get_temp_dir('%s_search' % query['asset_type'])
  headers = utils.get_headers(params['api_key'])

  if params.get('get_next'):
    urlquery = params['next']
  else:
    urlquery = query_to_url(query, params)

  data = {
    'PREFS': utils.get_prefs_dir(),
    'tempdir': tempdir,
    'urlquery': urlquery,
    'asset_type': query['asset_type'],
  }
  data.update(params)
  response = daemon_lib.search_asset(data)
  search_tasks[response['task_id']] = data

def get_search_simple(parameters, filepath=None, page_size=100, max_results=100000000, api_key=''):
  '''
  Searches and returns the

  Parameters
  ----------
  parameters - dict of blenderkit elastic parameters
  filepath - a file to save the results. If None, results are returned
  page_size - page size for retrieved results
  max_results - max results of the search
  api_key - BlenderKit api key

  Returns
  -------
  Returns search results as a list, and optionally saves to filepath

  '''
  headers = utils.get_headers(api_key)
  url = f'{paths.BLENDERKIT_API}/search/'
  requeststring = url + '?query='
  for p in parameters.keys():
    requeststring += f'+{p}:{parameters[p]}'

  requeststring += '&page_size=' + str(page_size)
  requeststring += '&dict_parameters=1'

  bk_logger.debug(requeststring)
  response = rerequests.get(requeststring, headers=headers)  # , params = rparameters)
  # print(response.json())
  search_results = response.json()

  results = []
  results.extend(search_results['results'])
  page_index = 2
  page_count = math.ceil(search_results['count'] / page_size)
  while search_results.get('next') and len(results) < max_results:
    bk_logger.info(f'getting page {page_index} , total pages {page_count}')
    response = rerequests.get(search_results['next'], headers=headers)  # , params = rparameters)
    search_results = response.json()
    # print(search_results)
    results.extend(search_results['results'])
    page_index += 1

  if not filepath:
    return results

  with open(filepath, 'w', encoding='utf-8') as s:
    json.dump(results, s, ensure_ascii=False, indent=4)
  bk_logger.info(f'retrieved {len(results)} assets from elastic search')
  return results

def get_single_asset(asset_base_id):
  preferences = bpy.context.preferences.addons['blenderkit'].preferences
  params = {
    'asset_base_id': asset_base_id
  }
  results = get_search_simple(params, api_key=preferences.api_key)
  if len(results) > 0:
    return results[0]
  return None

def search(category='', get_next=False, query = None, author_id=''):
  ''' initialize searching
  query : submit an already built query from search history
  '''

  if global_vars.DAEMON_ACCESSIBLE != True:
    reports.add_report('Cannot search, daemon is not accessible.', timeout = 2, type='ERROR')
    return

  # print(category,get_next,author_id)
  user_preferences = bpy.context.preferences.addons['blenderkit'].preferences

  scene = bpy.context.scene
  wm = bpy.context.window_manager
  ui_props = bpy.context.window_manager.blenderkitUI

  props = utils.get_search_props()
  # it's possible get_next was requested more than once.
  if props.is_searching and get_next == True:
    # print('return because of get next and searching is happening')
    return;

  if not query:
    if ui_props.asset_type == 'MODEL':
      if not hasattr(wm, 'blenderkit_models'):
        return;
      query = build_query_model()

    if ui_props.asset_type == 'SCENE':
      if not hasattr(wm, 'blenderkit_scene'):
        return;
      query = build_query_scene()

    if ui_props.asset_type == 'HDR':
      if not hasattr(wm, 'blenderkit_HDR'):
        return;
      query = build_query_HDR()

    if ui_props.asset_type == 'MATERIAL':
      if not hasattr(wm, 'blenderkit_mat'):
        return;

      query = build_query_material()

    if ui_props.asset_type == 'TEXTURE':
      if not hasattr(wm, 'blenderkit_tex'):
        return;
      # props = scene.blenderkit_tex
      # query = build_query_texture()

    if ui_props.asset_type == 'BRUSH':
      if not hasattr(wm, 'blenderkit_brush'):
        return;
      query = build_query_brush()

    # crop long searches
    if query.get('query'):
      if len(query['query']) > 50:
        query['query'] = strip_accents(query['query'])

      if len(query['query']) > 150:
        idx = query['query'].find(' ', 142)
        query['query'] = query['query'][:idx]

    if category != '':
      if utils.profile_is_validator() and user_preferences.categories_fix:
        query['category'] = category
      else:
        query['category_subtree'] = category

    if author_id != '':
      query['author_id'] = author_id

    elif props.own_only:
      # if user searches for [another] author, 'only my assets' is invalid. that's why in elif.
      profile = global_vars.DATA.get('bkit profile')
      if profile is not None:
        query['author_id'] = str(profile['user']['id'])

    if not get_next:
      #write to search history and check history length
      if len(global_vars.DATA['search history'])>0 and global_vars.DATA['search history'][-1] == query:
        # don't send same query again, when user clicks multiple times and waits e.t.c.
        return

      global_vars.DATA['search history'].append(query)
  # utils.p('searching')
  props.is_searching = True

  page_size = min(40, ui_props.wcount * user_preferences.max_assetbar_rows + 5)
  params = {
    'scene_uuid': bpy.context.scene.get('uuid', None),
    'addon_version': version_checker.get_addon_version(),
    'blender_version': version_checker.get_blender_version(),
    'api_key': user_preferences.api_key,
    'get_next': get_next,
    'free_first': props.free_only,
    'page_size': page_size,
  }

  orig_results = global_vars.DATA.get(f'bkit {ui_props.asset_type.lower()} search orig')
  if orig_results is not None and get_next:
    params['next'] = orig_results['next']
  add_search_process(query, params)

  props.report = 'BlenderKit searching....'

def clean_filters():
  '''cleanup filters in case search needs to be reset, typicaly when asset id is copy pasted'''
  sprops = utils.get_search_props()
  ui_props = bpy.context.window_manager.blenderkitUI
  sprops.property_unset('own_only')
  sprops.property_unset('search_texture_resolution')
  sprops.property_unset('search_file_size')
  sprops.property_unset('search_procedural')
  sprops.property_unset('free_only')
  sprops.property_unset('quality_limit')
  if ui_props.asset_type == 'MODEL':
    sprops.property_unset('search_style')
    sprops.property_unset('search_condition')
    sprops.property_unset('search_design_year')
    sprops.property_unset('search_polycount')
    sprops.property_unset('search_animated')
  if ui_props.asset_type == 'HDR':
    sprops.true_hdr = False

def update_filters():
  sprops = utils.get_search_props()
  ui_props = bpy.context.window_manager.blenderkitUI
  fcommon = sprops.own_only or \
            sprops.search_texture_resolution or \
            sprops.search_file_size or \
            sprops.search_procedural != 'BOTH' or \
            sprops.free_only or \
            sprops.quality_limit > 0

  if ui_props.asset_type == 'MODEL':
    sprops.use_filters = fcommon or \
                         sprops.search_style != 'ANY' or \
                         sprops.search_condition != 'UNSPECIFIED' or \
                         sprops.search_design_year or \
                         sprops.search_polycount or \
                         sprops.search_animated
  elif ui_props.asset_type == 'MATERIAL':
    sprops.use_filters = fcommon
  elif ui_props.asset_type == 'HDR':
    sprops.use_filters = sprops.true_hdr

def search_update(self, context):
  # if self.search_keywords != '':
  update_filters()
  ui_props = bpy.context.window_manager.blenderkitUI
  if ui_props.down_up != 'SEARCH':
    ui_props.down_up = 'SEARCH'

  # here we tweak the input if it comes form the clipboard. we need to get rid of asset type and set it in UI
  sprops = utils.get_search_props()
  instr = 'asset_base_id:'
  atstr = 'asset_type:'
  kwds = sprops.search_keywords
  id_index = kwds.find(instr)
  if id_index>-1:
    asset_type_index = kwds.find(atstr)
    # if the asset type already isn't there it means this update function
    # was triggered by it's last iteration and needs to cancel
    if asset_type_index > -1:
      asset_type_string = kwds[asset_type_index:].lower()
      # uncertain length of the remaining string -  find as better method to check the presence of asset type
      if asset_type_string.find('model') > -1:
        target_asset_type = 'MODEL'
      elif asset_type_string.find('material') > -1:
        target_asset_type = 'MATERIAL'
      elif asset_type_string.find('brush') > -1:
        target_asset_type = 'BRUSH'
      elif asset_type_string.find('scene') > -1:
        target_asset_type = 'SCENE'
      elif asset_type_string.find('hdr') > -1:
        target_asset_type = 'HDR'
      if ui_props.asset_type != target_asset_type:
        sprops.search_keywords = ''
        ui_props.asset_type = target_asset_type

      # now we trim the input copypaste by anything extra that is there,
      # this is also a way for this function to recognize that it already has parsed the clipboard
      # the search props can have changed and this needs to transfer the data to the other field
      # this complex behaviour is here for the case where the user needs to paste manually into blender,
      # Otherwise it could be processed directly in the clipboard check function.
      sprops = utils.get_search_props()
      clean_filters()
      sprops.search_keywords = kwds[:asset_type_index].rstrip()
      # return here since writing into search keywords triggers this update function once more.
      return

  if global_vars.DAEMON_ACCESSIBLE:
    reports.add_report(f"Searching for: '{kwds}'", 2)
  search()

# accented_string is of type 'unicode'
def strip_accents(s):
  return ''.join(c for c in unicodedata.normalize('NFD', s)
                 if unicodedata.category(c) != 'Mn')

class SearchOperator(Operator):
  """Tooltip"""
  bl_idname = "view3d.blenderkit_search"
  bl_label = "BlenderKit asset search"
  bl_description = "Search online for assets"
  bl_options = {'REGISTER', 'UNDO', 'INTERNAL'}

  esc: BoolProperty(name="Escape window",
                    description="Escape window right after start",
                    default=False,
                    options={'SKIP_SAVE'}
                    )

  own: BoolProperty(name="own assets only",
                    description="Find all own assets",
                    default=False,
                    options={'SKIP_SAVE'})

  category: StringProperty(
    name="category",
    description="search only subtree of this category",
    default="",
    options={'SKIP_SAVE'}
  )

  author_id: StringProperty(
    name="Author ID",
    description="Author ID - search only assets by this author",
    default="",
    options={'SKIP_SAVE'}
  )

  get_next: BoolProperty(name="next page",
                         description="get next page from previous search",
                         default=False,
                         options={'SKIP_SAVE'}
                         )

  keywords: StringProperty(
    name="Keywords",
    description="Keywords",
    default="",
    options={'SKIP_SAVE'}
  )

  # close_window: BoolProperty(name='Close window',
  #                            description='Try to close the window below mouse before download',
  #                            default=False)

  tooltip: bpy.props.StringProperty(default='Runs search and displays the asset bar at the same time')

  @classmethod
  def description(cls, context, properties):
    return properties.tooltip

  @classmethod
  def poll(cls, context):
    return True

  def execute(self, context):
    # TODO ; this should all get transferred to properties of the search operator, so sprops don't have to be fetched here at all.
    if self.esc:
      bpy.ops.view3d.close_popup_button('INVOKE_DEFAULT')
    sprops = utils.get_search_props()
    if self.author_id != '':
      sprops.search_keywords = ''
    if self.keywords != '':
      sprops.search_keywords = self.keywords

    search(category=self.category, get_next=self.get_next, author_id=self.author_id)
    # bpy.ops.view3d.blenderkit_asset_bar_widget()

    return {'FINISHED'}

  # def invoke(self, context, event):
  #     if self.close_window:
  #         context.window.cursor_warp(event.mouse_x, event.mouse_y - 100);
  #         context.area.tag_redraw()
  #
  #         context.window.cursor_warp(event.mouse_x, event.mouse_y);
  #     return self. execute(context)

class UrlOperator(Operator):
  """"""
  bl_idname = "wm.blenderkit_url"
  bl_label = ""
  bl_description = "Search online for assets"
  bl_options = {'REGISTER', 'UNDO', 'INTERNAL'}

  tooltip: bpy.props.StringProperty(default='Open a web page')
  url: bpy.props.StringProperty(default='Runs search and displays the asset bar at the same time')

  @classmethod
  def description(cls, context, properties):
    return properties.tooltip

  def execute(self, context):
    bpy.ops.wm.url_open(url=self.url)
    return {'FINISHED'}

class TooltipLabelOperator(Operator):
  """"""
  bl_idname = "wm.blenderkit_tooltip"
  bl_label = ""
  bl_description = "Empty operator to be able to create tooltips on labels in UI"
  bl_options = {'REGISTER', 'UNDO', 'INTERNAL'}

  tooltip: bpy.props.StringProperty(default='Open a web page')

  @classmethod
  def description(cls, context, properties):
    return properties.tooltip

  def execute(self, context):
    return {'FINISHED'}

classes = [
  SearchOperator,
  UrlOperator,
  TooltipLabelOperator
]

def register_search():
  bpy.app.handlers.load_post.append(scene_load)
  bpy.app.handlers.load_post.append(undo_post_reload_previews)
  bpy.app.handlers.undo_post.append(undo_post_reload_previews)
  bpy.app.handlers.undo_pre.append(undo_pre_end_assetbar)

  for c in classes:
    bpy.utils.register_class(c)

def unregister_search():
  bpy.app.handlers.load_post.remove(scene_load)

  for c in classes:
    bpy.utils.unregister_class(c)


from flask import Flask, render_template, url_for, redirect, request
import os
from dotenv import load_dotenv
from api import MatchHistory as MH
from api import LiveGame as LG
from helpers import GenericImages as GI
from helpers import RegionDetails as RD
from helpers import UserID as UID
from helpers import cache

#------------------------------------------------------------------------------------------------------
load_dotenv()
api = os.getenv('RIOT_API_KEY')
#------------------------------------------------------------------------------------------------------

app = Flask(__name__)

@app.route('/', methods=['POST','GET'])
def Home():
    Page_Info = {'Title': 'LolStat - Home'}
    return render_template('home.html', PAGE_INFO=Page_Info)

#------------------------------------------------------------------------------------------------------

@app.route('/lol', methods=['POST'])
def Lol():
    username = str(request.form['username'])
    region = str(request.form['region'])
    if username and region:
        return redirect(f"/lol/{region}/{username}")
    return "Please enter both username & password"

@app.route('/lol/<region>/<username>', methods=['GET'])
def MatchHistory(region, username):
    shard = RD.getShard(region)
    version = RD.getVersion(api, region)

    user_info = UID.getUser(api, region, username)
    profile = UID.getRankInfo(api, region, user_info['summonerID'])

    match_ids = MH.getMatchIDs(api, shard, user_info['puuID'],)
    match_details = MH.getAllMatchDetails(api, shard, match_ids)
    match_info = MH.getMatchInfo(match_details)
    match_player_info = MH.getMatchPlayersDetails(version, match_details)

    # live_player_info = LG.getLiveGameInfo(api, region, version, user_info['summonerID'])
    PageInfo = {
        'Title': f"{user_info['name']} - Match History",
        'Username': user_info['name'],
        'len': len(match_player_info),
        'region': region
    }
    return render_template('MatchHistory.html', PAGE_INFO=PageInfo, Player_Details=match_player_info, Match_Details=match_info, Profile=profile )

# TEST ------------------------------------------------------------------------------------------------

@app.route('/test', methods=['POST'])
def Test():
    username = str(request.form['username'])
    region = str(request.form['region'])
    if username and region:
        return redirect(f"/test/{region}/{username}")
    return "Please enter both username & password"

@app.route('/test/<region>/<username>', methods=['GET'])
def TestJSON(region, username):
    shard = RD.getShard(region)
    version = RD.getVersion(api, region)

    user_info = UID.getUser(api, region, username)
    profile = UID.getRankInfo(api, region, user_info['summonerID'])

    match_ids = MH.getMatchIDs(api, shard, user_info['puuID'],)
    match_details = MH.getAllMatchDetails(api, shard, match_ids)
    match_info = MH.getMatchInfo(match_details)
    match_player_info = MH.getMatchPlayersDetails(version, match_details)

    live_player_info = LG.getLiveGameInfo(api, region, version, user_info['summonerID'])
    return profile

#------------------------------------------------------------------------------------------------------

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


# -*- coding: utf-8 -*-
import scrapy
from seleniumtest.items import ProductItem

class JingdonglistSpider(scrapy.Spider):
    name = 'jingdonglist'
    allowed_domains = ['list.jd.com']
    url = 'https://list.jd.com/list.html?cat=670,671,672'

    def start_requests(self):
        for page in range(1, self.settings.get('MAX_PAGE') + 1):
            yield scrapy.Request(url=self.url, callback=self.parse, meta={'page': page}, dont_filter=True)

    def parse(self, response):
        print('=============================================')
        productlist = response.xpath("//li[@class='gl-item']")
        for item in productlist:
            product = ProductItem()
            product['name'] = item.xpath('.//div[@class="p-name"]//em//text()').extract_first()
            product['comment'] = item.xpath('.//div[@class="p-commit-n"]//strong//a[@class="comment"]//text()').extract_first()
            product['price'] = item.xpath('.//div[@class="p-price"]//strong[@class="J_price"]//i//text()').extract_first()
            print(product)
            yield product


from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from googlecloudsdk.core import log as core_log
from googlecloudsdk.core.console.style import text
import six

def _PrintResourceChange(operation,
                         resource,
                         kind,
                         is_async,
                         details,
                         failed,
                         operation_past_tense=None):
  """Prints a status message for operation on resource.

  The non-failure status messages are disabled when user output is disabled.

  Args:
    operation: str, The completed operation name.
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message. For commands that operate on multiple
      resources and report all successes and failures before exiting. Failure
      messages use core_log.error. This will display the message on the standard
      error even when user output is disabled.
    operation_past_tense: str, The past tense version of the operation verb.
      If None assumes operation + 'd'
  """

  msg = []
  if failed:
    msg.append('Failed to ')
    msg.append(operation)
  elif is_async:
    msg.append(operation.capitalize())
    msg.append(' in progress for')
  else:
    verb = operation_past_tense or '{0}d'.format(operation)
    msg.append('{0}'.format(verb.capitalize()))

  if kind:
    msg.append(' {}'.format(kind))
  if resource:
    msg.append(' ')
    msg.append(text.TextTypes.RESOURCE_NAME(six.text_type(resource)))
  if details:
    msg.append(' ')
    msg.append(details)

  if failed:
    msg[-1] = '{0}: '.format(msg[-1])
    msg.append(failed)
  msg.append('.')
  msg = text.TypedText(msg)
  writer = core_log.error if failed else core_log.status.Print
  writer(msg)

def CreatedResource(resource, kind=None, is_async=False, details=None,
                    failed=None):
  """Prints a status message indicating that a resource was created.

  Args:
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message.
  """
  _PrintResourceChange('create', resource, kind, is_async, details, failed)

def DeletedResource(resource, kind=None, is_async=False, details=None,
                    failed=None):
  """Prints a status message indicating that a resource was deleted.

  Args:
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message.
  """
  _PrintResourceChange('delete', resource, kind, is_async, details, failed)

def RestoredResource(resource, kind=None, is_async=False, details=None,
                     failed=None):
  """Prints a status message indicating that a resource was restored.

  Args:
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message.
  """
  _PrintResourceChange('restore', resource, kind, is_async, details, failed)

def UpdatedResource(resource, kind=None, is_async=False, details=None,
                    failed=None):
  """Prints a status message indicating that a resource was updated.

  Args:
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message.
  """
  _PrintResourceChange('update', resource, kind, is_async, details, failed)

def ResetResource(resource, kind=None, is_async=False, details=None,
                  failed=None):
  """Prints a status message indicating that a resource was reset.

  Args:
    resource: str, The resource name.
    kind: str, The resource kind (instance, cluster, project, etc.).
    is_async: bool, True if the operation is in progress.
    details: str, Extra details appended to the message. Keep it succinct.
    failed: str, Failure message.
  """
  _PrintResourceChange('reset', resource, kind, is_async, details, failed,
                       operation_past_tense='reset')

# pylint: disable=invalid-name,protected-access
# Redirects to log copy of common logging functions. This way core.log and
# this module don't need to both be imported.
getLogger = core_log.getLogger
log = core_log.log
debug = core_log.debug
info = core_log.info
warning = core_log.warning
error = core_log.error
critical = core_log.critical
fatal = core_log.fatal
exception = core_log.exception
out = core_log._log_manager.stdout_writer
err = core_log._log_manager.stderr_writer
status = core_log.err
file_only_logger = core_log._log_manager.file_only_logger


# %%
#! pip install kaggle --upgrade
#! kaggle --version
#! kaggle competitions download -c ieee-fraud-detection --force
# %%

if __name__ == "__main__":

    import pandas as pd
    import numpy as np
    import os

    import sklearn.preprocessing as sk_prep
    from sklearn.model_selection import train_test_split
    import xgboost as xgb
    import joblib

    from sklearn.metrics import accuracy_score, recall_score, roc_auc_score, precision_score
    ######

    prod = True # boolean for production vs testing differences
    laptop = False #boolean for running on my laptop lol

    train = pd.DataFrame()

    if laptop:
        import zipfile
        zf = zipfile.ZipFile("C:\\Users\\Ben\\Documents\\YC\\mygit\\BenZuckier\\DSA\\06_Fraud_Model\\ieee-fraud-detection.zip")
        train = pd.read_csv(zf.open('train_transaction.csv'))
        del zf
    else:
        train = pd.read_csv('/content/train_transaction.csv.zip', compression='zip')
    # valid = pd.read_csv('/content/test_transaction.csv.zip', compression='zip')

    train = train.drop("TransactionID", axis=1)

    # lets remove V cols except v307 and v310 which perfromed well in testing
    v310 = train[["V310"]].copy()
    v307 = train[["V307"]].copy()
    train = train.loc[:,~train.columns.str.startswith('V')]
    train["V310"] = v310[["V310"]]
    train["V307"] = v307[["V307"]]

    # engineer two new features
    train['TimeInDay'] = train.TransactionDT % 86400
    train['Cents'] = train.TransactionAmt % 1

    # rename the cols we're gonna onehot
    # train = train.rename(columns={'card4':'card4d', 'card6':'card6f', 'P_emaildomain': 'pEmail','R_emaildomain':'rEmail', 'M1':'M1a', 'M2':'M2b', 'M3':'M3c', 'M4':'M4d', 'M5':'M5e', 'M6':'M6f', 'M7':'M7g', 'M8':'M8h', 'M9':'M9i'})

    # shplits
    train_txn, test_txn = train_test_split(train, test_size=0.25, stratify=train[['isFraud']])
    del train

    y_train = train_txn["isFraud"].copy()
    X_train = train_txn.drop("isFraud", axis=1)
    del train_txn

    y_test = test_txn['isFraud'].copy()
    X_test = test_txn.drop('isFraud',axis=1)
    del test_txn

    # y_valid = valid["isFraud"].copy()
    # X_valid = valid.drop("isFraud",axis=1).copy()
    # del valid

    # Label Encode any non numeric col
    toLabel = [f for f in X_train.columns if X_train[f].dtype=='object']

    onehot = sk_prep.OneHotEncoder(handle_unknown='ignore',sparse=False)
    onehot.fit(X_train[toLabel])
    onehotted = onehot.transform(X_train[toLabel])
    # pd.DataFrame(onehotted)
    col_names = onehot.get_feature_names(toLabel)
    onehotDat =  pd.DataFrame(onehotted, columns= col_names, index=X_train.index)
    X_train = pd.concat([X_train.drop(toLabel,axis=1), onehotDat], axis=1 )

    # # robust scaler our numerical data cols
    # robust = sk_prep.RobustScaler().fit(X_train[["TransactionAmt"]])
    # pd.options.mode.chained_assignment = None
    # train[['cdev','trainhr']] = robust.transform(train[['cdev','trainhr']])
    # pd.options.mode.chained_assignment = 'warn'

    clf = xgb.XGBClassifier(
        n_estimators=500,
        max_depth=9,
        learning_rate=0.05,
        subsample=0.9,
        colsample_bytree=0.9,
        random_state=69,
        tree_method='gpu_hist' # set runtime to gpu to take advantage
    )
    model = clf.fit(X_train, y_train, eval_metric="auc")

    folder = './models'
    try:
        os.makedirs(folder)
    except FileExistsError:
        pass

    for x in [(onehot,"onehot"),(toLabel,"labelCols"),(col_names,"onehottedLabels"),(model,"xgb")]:
        (mod,nom) = x
        with open(os.path.join(folder,nom), "wb") as f:
                joblib.dump(mod, f)

    if not prod:
        X_test_enc = X_test.copy()
        onehotted_test = onehot.transform(X_test_enc[toLabel])
        # pd.DataFrame(onehotted)
        # col_names = onehot.get_feature_names(toLabel)
        onehotDat_test =  pd.DataFrame(onehotted_test, columns=col_names, index=X_test_enc.index)
        X_test_enc = pd.concat([X_test_enc.drop(toLabel, axis=1), onehotDat_test], axis=1)

        pred = model.predict(X_test_enc)
        pred_proba = model.predict_proba(X_test_enc)
        xgb_df = pd.DataFrame(data=[accuracy_score(y_test, pred), recall_score(y_test, pred),
                        precision_score(y_test, pred), roc_auc_score(y_test, pred_proba[:,1])],
                    columns=['XGB Score'],
                    index=["Accuracy", "Recall", "Precision", "ROC AUC Score"])
        display(xgb_df)

    print("done")

# %%


import pygame
from pygame import mixer
import random

pygame.init()
pygame.font.init()
mixer.init()

WIDTH = 640
HEIGHT = 480
SIZE = (WIDTH, HEIGHT)

screen = pygame.display.set_mode(SIZE)
clock = pygame.time.Clock()

mixer.music.load("Snowball Park - Super Mario 3D World.mp3")
mixer.music.set_volume(0.5)
mixer.music.play(-1)
winter_sky = pygame.image.load("winter_sky.png")
presentbox = pygame.image.load("presentbox.png")
presentbox = pygame.transform.scale(presentbox, (50, 50))
santabag = pygame.image.load("santabag.png")
santabag = pygame.transform.scale(santabag, (90, 90))
pagging = pygame.image.load("pagging.gif")
pagging = pygame.transform.scale(pagging, (100, 100))
sadge = pygame.image.load("sadge.png")
sadge = pygame.transform.scale(sadge, (100, 100))
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)

FONT = pygame.font.SysFont(None, 70)
TITLE_FONT = pygame.font.SysFont(None, 30)

def moving_objs(boxes):
    global back_snow, middle_snow, front_snow, present_boxes, presents_num
    back_snow = []
    for i in range (150):
        bx = random.randrange(-10, 650)
        brlimit = bx + 4
        bllimit = bx - 4
        by = random.randrange(-480, 0)
        bz = 1
        brlimit, bllimit = snowspeed(bx, bz)[ :2]
        b_mvx, b_mvy = snowspeed(bx, bz)[2: ]
        back_snow.append([bx, by, bz, brlimit, bllimit, b_mvx, b_mvy])
    middle_snow = []
    for i in range (150):
        mdx = random.randrange(-10, 650)
        mdrlimit = mdx + 7
        mdllimit = mdx - 7
        mdy = random.randrange(-480, 0)
        mdz = 2
        mdrlimit, mdllimit = snowspeed(mdx, mdz)[ :2]
        md_mvx, md_mvy = snowspeed(mdx, mdz)[2: ]
        middle_snow.append([mdx, mdy, mdz, mdrlimit, mdllimit, md_mvx, md_mvy])
    front_snow = []
    for i in range (150):
        fx = random.randrange(-10, 650)
        frlimit = fx + 10
        fllimit = fx - 10
        fy = random.randrange(-480, 0)
        fz = 3
        frlimit, fllimit = snowspeed(fx, fz)[ :2]
        f_mvx, f_mvy = snowspeed(fx, fz)[2:]
        front_snow.append([fx, fy, fz, frlimit, fllimit, f_mvx, f_mvy])
    present_boxes = []
    for i in range(boxes):
        presentx = random.randrange(0, 590)
        presenty = random.randrange(-960, 0)
        present_boxes.append([None, presentx, presenty])
    presents_num = len(present_boxes)

def snowspeed(x, z):
    if z == 1:
        movex = round(random.uniform(0.2, 0.5), 1)
        movey = round(random.uniform(0.7, 1.3), 1)
        right_limit = x + 4
        left_limit = x - 4
    elif z == 2:
        movex = round(random.uniform(0.5, 0.8), 1)
        movey = round(random.uniform(1.3, 1.9), 1)
        right_limit = x + 7
        left_limit = x - 7
    else:
        movex = round(random.uniform(0.8, 1.1), 1)
        movey = round(random.uniform(1.9, 2.5), 1)
        right_limit = x + 10
        left_limit = x - 10
    return right_limit, left_limit, movex, movey

def back_snowfall():
    global back_snow, middle_snow
    for item in back_snow:
        if item[0] > item[3]:
            item[5] = -abs(item[5])
        elif item[0] < item[4]:
            item[5] = abs(item[5])
        item[0] += item[5]
        item[1] += item[6]
        pygame.draw.circle(screen, (105, 105, 105), item[0:2], item[2])
        if item[1] > 235:
            item[0] = random.randrange(-10, 650)
            item[1] = random.randrange(-480, 0)
    for item in middle_snow:
        if item[0] > item[3]:
            item[5] = -abs(item[5])
        elif item[0] < item[4]:
            item[5] = abs(item[5])
        item[0] += item[5]
        item[1] += item[6]
        pygame.draw.circle(screen, (170, 170, 170), item[0:2], item[2])
        if item[1] > 480:
            item[0] = random.randrange(-10, 650)
            item[1] = random.randrange(-480, 0)

def front_snowfall():
    global front_snow
    for item in front_snow:
        if item[0] > item[3]:
            item[5] = -abs(item[5])
        elif item[0] < item[4]:
            item[5] = abs(item[5])
        item[0] += item[5]
        item[1] += item[6]
        pygame.draw.circle(screen, WHITE, item[0:2], item[2])
        if item[1] > 480:
            item[0] = random.randrange(-10, 650)
            item[1] = random.randrange(-480, 0)

def intro_screen():
    screen.blit(winter_sky, (0, 0))
    description = TITLE_FONT.render("Santa keeps dropping his presents! Catch them!", True, WHITE)
    instructions = TITLE_FONT.render("Click the left and right arrow keys to catch Santa's droppings.", True, WHITE)
    start_title = TITLE_FONT.render("Click the left or right arrow key to start.", True, WHITE)
    screen.blit(description, (description.get_rect(center = screen.get_rect().center)[0], description.get_rect(center = screen.get_rect().center)[1]-30))
    screen.blit(instructions, (instructions.get_rect(center = screen.get_rect().center)[0], instructions.get_rect(center = screen.get_rect().center)[1]+30))
    screen.blit(start_title, (start_title.get_rect(center = screen.get_rect().center)[0], start_title.get_rect(center = screen.get_rect().center)[1]))

def game():
    lvl_state = [True, False, False, False, False]
    lvl1, lvl2, lvl3, lvl4, lvl5 = 0, 1, 2, 3, 4
    moving_objs(0)
    present_fall = 1
    bagx = 0
    bagy = 390
    caught = False
    points = 0
    lvl_finish = False
    win = False
    lose = False
    lvl = 0
    click = False
    intro = True
    running = True
    while running:
        mx, my = pygame.mouse.get_pos()
        if intro:
            intro_screen()
        else:
            if len(present_boxes) == 0:
                lvl_finish = True
            if lvl_finish:
                if points >= presents_num*5:
                    lvl_finish = False
                    pygame.time.wait(1000)
                    points = 0
                    lvl += 1
                    if lvl_state[lvl1]:
                        moving_objs(20)
                        lvl_state[lvl1] = False
                        lvl_state[lvl2] = True
                    elif lvl_state[lvl2]:
                        moving_objs(22)
                        lvl_state[lvl2] = False
                        lvl_state[lvl3] = True
                    elif lvl_state[lvl3]:
                        moving_objs(24)
                        present_fall += 0.5
                        lvl_state[lvl3] = False
                        lvl_state[lvl4] = True
                    elif lvl_state[lvl4]:
                        moving_objs(26)
                        lvl_state[lvl4] = False
                        lvl_state[lvl5] = True
                    elif lvl_state[lvl5]:
                        moving_objs(30)
                        present_fall += 0.5
                        lvl_state[lvl5] = False
                    else:
                        win = True
                else:
                    lose = True

            screen.blit(winter_sky, (0, 0))
            points_text = FONT.render(str(points), True, WHITE)
            screen.blit(points_text, (5, 5))
            lvl_text = FONT.render(str(lvl), True, WHITE)
            lvltxt_rect = lvl_text.get_rect()
            lvltxt_rect.topright = 635, 5
            screen.blit(lvl_text, lvltxt_rect)

            back_snowfall()

            for present in present_boxes:
                present[2] += present_fall
                present[0] = screen.blit(presentbox, (present[1], present[2]))

            bag = pygame.Rect([bagx+15, bagy+3, 60, 10])
            screen.blit(santabag, (bagx, bagy))

            for present in present_boxes:
                caught = False
                if present[0].colliderect((bag)):
                    if present[2] <= bagy - 45:
                        caught = True
                        if present[-1] != True:
                            present.append(True)
                    if caught or present[-1] == True:
                        present[1] = bagx + 20
                if present[2] > 400 and present[-1] == True:
                    present_boxes.remove(present)
                    points += 10
                elif present[2] > 480:
                    present_boxes.remove(present)

            front_snowfall()

            if win:
                screen.blit(winter_sky, (0, 0))
                win_text = FONT.render("You Win! I'm Pagging!", True, WHITE)
                screen.blit(win_text, (win_text.get_rect(center = screen.get_rect().center)))
                screen.blit(pagging, (10, 10))
                menu_button = pygame.draw.rect(screen, BLACK, [110, 300, 200, 80])
                menu_title = TITLE_FONT.render("BACK TO MENU", True, WHITE)
                menu_rect = menu_title.get_rect
                screen.blit(menu_title, (menu_rect(center = screen.get_rect().center)[0]-110, menu_rect(center = screen.get_rect().center)[1]+100))
                if menu_button.collidepoint((mx, my)):
                    if click:
                        pass
                        # main.main_menu()
                restart_button = pygame.draw.rect(screen, BLACK, [330, 300, 200, 80])
                restart_title = TITLE_FONT.render("RESTART", True, WHITE)
                restart_rect = restart_title.get_rect
                screen.blit(restart_title, (restart_rect(center = screen.get_rect().center)[0]+110, restart_rect(center = screen.get_rect().center)[1]+100))
                if restart_button.collidepoint((mx, my)):
                    if click:
                        game()
            elif lose:
                screen.blit(winter_sky, (0, 0))
                win_text = FONT.render("You Lose... Sadge...", True, WHITE)
                screen.blit(win_text, (win_text.get_rect(center = screen.get_rect().center)))
                screen.blit(sadge, (10, 10))
                menu_button = pygame.draw.rect(screen, BLACK, [110, 300, 200, 80])
                menu_title = TITLE_FONT.render("BACK TO MENU", True, WHITE)
                menu_rect = menu_title.get_rect
                screen.blit(menu_title, (menu_rect(center = screen.get_rect().center)[0]-110, menu_rect(center = screen.get_rect().center)[1]+100))
                if menu_button.collidepoint((mx, my)):
                    if click:
                        pass
                        # main.main_menu()
                restart_button = pygame.draw.rect(screen, BLACK, [330, 300, 200, 80])
                restart_title = TITLE_FONT.render("RESTART", True, WHITE)
                restart_rect = restart_title.get_rect
                screen.blit(restart_title, (restart_rect(center = screen.get_rect().center)[0]+110, restart_rect(center = screen.get_rect().center)[1]+100))
                if restart_button.collidepoint((mx, my)):
                    if click:
                        game()

        keys = pygame.key.get_pressed()
        if keys[pygame.K_LEFT]:
            if bagx >= 4:
                bagx -= 4
            if intro:
                intro = False
        if keys[pygame.K_RIGHT]:
            if bagx <= 546:
                bagx += 4
            if intro:
                intro = False

        click = False
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                running = False
                break
            elif event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    running = False
                    break
            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 1:
                    click = True

        pygame.display.update()
        clock.tick(30)

    pygame.quit()

if __name__ == "__main__":
    game()


"""
https://leetcode.com/problems/delete-nodes-and-return-forest/
"""

from test_helper import TreeNode

class Solution:
    def delNodes(self, root: TreeNode, to_delete: list[int]) -> list[TreeNode]:
        def check(currNode: TreeNode) -> TreeNode or None:
            if currNode:
                currNode.left = check(currNode.left)
                currNode.right = check(currNode.right)
                if currNode.val in deletedNodes:
                    if currNode.left:
                        remainNodes.append(currNode.left)

                    if currNode.right:
                        remainNodes.append(currNode.right)

                    return None

                return currNode

        deletedNodes = set(to_delete)
        remainNodes: list[TreeNode] = []
        if root.val not in deletedNodes:
            remainNodes.append(root)

        check(root)
        return remainNodes


from tika import parser
import shutil
import os
import subprocess
from PIL import Image
import pytesseract
import sys
from pdf2image import convert_from_path
from pdf2image.exceptions import (
    PDFInfoNotInstalledError,
    PDFPageCountError,
    PDFSyntaxError
)
from cleanup import cleanUp
import random

headers = {
    'X-Tika-PDFextractInlineImages': 'true',
    "X-Tika-OCRLanguage": "eng+nor"
}

path = './data/Docs_txt/'

# def makeFolder():
#     os.mkdir(path)

def readData():
    data = parser.from_file(
        './data/Templates/f1099msc.pdf')
    text = data['content']
    if text != None:
        return text
    else:
        return ''

def createSamples(text, n=1):
    for i in range(0, n):
        file = open(path+'sample'+str(i)+'.txt', 'w+')
        amt = random.randint(1, 5000)
        text = text.replace('$', '$'+str(amt))
        text = cleanUp(text)
        file.write(text)

# makeFolder()
text = readData()
createSamples(text, n=5)


__all__ = ["MLflowModelsClient"]

from dbacademy.clients.rest.common import ApiClient, ApiContainer

class MLflowModelsClient(ApiContainer):

    def __init__(self, client: ApiClient):
        self.client = client
        self.base_uri = f"{self.client.endpoint}/api/2.0/preview/mlflow/registered-models"

    def list(self):
        results = []
        max_results = 1000

        url = f"{self.base_uri}/search?max_results={max_results}"

        response = self.client.api("GET", url)
        results.extend(response.get("registered_models", []))

        while "next_page_token" in response:
            next_page_token = response["next_page_token"]
            response = self.client.api("GET", f"{url}&page_token={next_page_token}")
            results.extend(response.get("registered_models", []))

        return results

    def delete(self, name: str):
        url = f"{self.base_uri}/delete"
        payload = {
            "name": name
        }
        self.client.api("DELETE", url, payload)


"""
Django settings for statistical_computing_interface project.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.7/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))
PROJECT_ROOT = os.path.realpath(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '091vz8ah@o#@qwl%sq9&livg^ecvsn*5bh#^)@88^-idorquva'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = (
	'django.contrib.admin',
	'django.contrib.auth',
	'django.contrib.contenttypes',
	'django.contrib.sessions',
	'django.contrib.messages',
	'django.contrib.staticfiles',
	'statistical_computing_interface.sciApp',
	'highcharts',
)

MIDDLEWARE_CLASSES = (
	'django.contrib.sessions.middleware.SessionMiddleware',
	'django.middleware.common.CommonMiddleware',
	'django.middleware.csrf.CsrfViewMiddleware',
	'django.contrib.auth.middleware.AuthenticationMiddleware',
	'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
	'django.contrib.messages.middleware.MessageMiddleware',
	'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'statistical_computing_interface.urls'

WSGI_APPLICATION = 'statistical_computing_interface.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.7/ref/settings/#databases

DATABASES = {
	'default': {
		'ENGINE': 'django.db.backends.sqlite3',
		'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
	}
}

# Internationalization
# https://docs.djangoproject.com/en/1.7/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.7/howto/static-files/

MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media')

# URL that handles the media served from MEDIA_ROOT.
MEDIA_URL = '/media/'

STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')

STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        }
    },
    'loggers': {
        'django.request': {
            'handlers': ['mail_admins'],
            'level': 'ERROR',
            'propagate': True,
        },
    }
}


from featuretools.variable_types import Categorical, Numeric, Boolean, Discrete, Index, Id, Datetime, Timedelta, Text

DEFAULT_VARIABLE_TYPES = {
    "categorical": Categorical,
    "numerical": Numeric,
    "boolean": Boolean,
    "discrete": Discrete,
    "index": Index,
    "id": Id,
    "datetime": Datetime,
    "timedelta": Timedelta,
    "text": Text
}


# Steps to follow to get Selenium working in your project:
# run `pip install selenium`
# Add your we driver to the local directory
# `geckodriver.exe` or `chromedriver.exe`
# e.g.: In our case, the driver is in `qa_auto_testing`
# folder as the `functional_test.py` script that uses
# the driver. Once that is complete, you can start writing
# the automated test.

from selenium import webdriver

browser = webdriver.Chrome()
browser.get('http://127.0.0.1:8000/')
print(browser.title)
assert 'CPH Indicator Visualization Tool' in browser.title


from itertools import chain
from pathlib import Path

import pandas as pd
from allennlp.common import Params

from src.utils.helpers import filter_config, create_parent_dir_if_not_exists

def create_vocab(
        train_qa_result_file, val_qa_result_file,
        vocab_result_file):
    qa_train = pd.read_pickle(train_qa_result_file)
    qa_val = pd.read_pickle(val_qa_result_file)
    combined = pd.concat([qa_train, qa_val])
    question_tokens = set(" ".join(combined.preprocessed_question.values).split())
    train_ans_tokens = set((" ".join(qa_train.answer)).split())
    # val_ans_tokens = set(" ".join(chain(*qa_val.answer.values)).split())

    vocab = ["[UNK]", "[EOS]"] + sorted(question_tokens | train_ans_tokens)
    vocab_result_file = Path(vocab_result_file)
    create_parent_dir_if_not_exists(vocab_result_file)
    with vocab_result_file.open("w") as f:
        print("\n".join(vocab), file=f)
    return

if __name__ == "__main__":
    data_params = Params.from_file("config.jsonnet").pop("data").as_dict()
    data_params = filter_config(data_params, create_vocab)
    create_vocab(**data_params)


"""Atmospheric stability indices.

This module's methods fetch or calculate predictors related to
the stability of the atmosphere against air parcel convection.

"""

import logging
import os
import sys
import re
import pdb
import copy
import metpy.calc as calc
from metpy.units import units
import math
import numpy as np
import operator

from ...mospred import create as create
from ...mospred import parse_pred as parse_pred
from ...core.reader import read_var
from ...core.Time import epoch_to_datetime
from ...core import Time as Time
from ...core import Camps_data
from ...registry import constants as const
from . import moisture

def KIndex_setup(filepaths, time, predictor):
    r"""The K stability index is a linear function of several weather
        parameters: air temperatures at pressure levels of 850 hPa,
        700 hPa, and 500 hPa, and dewpoint temperatures at pressure
        levels of 850 hPa and 700 hPa.  It normally has units of
        degrees Centigrade.

    Args:
        control(instance): contains mospred_control.yaml file variables.
        time (int): The time in seconds since January 1, 1970 00Z
        predictor (Predictor): a generic predictor object carrying
            information necessary to fetch Camps data objects from
            netCDF4 files.

    Returns:
        kindex (Camps_data): a camps data object containing metadata
            and data of the K index.

    """

#   Instantiate the camps data object for the K index.
#   Certain attributes are set in this instantiation.
    kindex = Camps_data("k_index_instant")
    international_units = { 'Temperature' : 'kelvin' } #Future global dictionary

#   The K index has units of temperature, particularly degrees Celsius.
#   Get the international standard unit for temperature
#   and construct a MetPy object containing that unit.  It will be used
#   below to test the dimensionality of the units in objects of temperature.
    iu_unit = international_units.get('Temperature')
    iu_pint = units.Quantity(1., iu_unit)

#   The unit for K-index must be degrees Celsius.
    try:
        unit = kindex.metadata['units']
        if unit != 'degC':
            logging.info("Reset units of stability K index from %s to degC" % unit)
    except KeyError:
        logging.info("metadata key \'units\' does not exist or has no value.")
        logging.info("Inserted into metadata, set to \'degC\'.")
    unit = 'degC'
    kindex.metadata.update({'units' : unit})

#   Make a hard copy of the Predictor object predictor to
#   use in fetching the components of the K index.  It is
#   necessary to do this in order to prevent inadvertently
#   changing the key/value pairs for
    pred = copy.deepcopy(predictor)

#   Fetch the weather parameters that make up K index.
#   Check that they are camps data objects
#   and that they have units of temperature.
    pred['search_metadata'].update({'property' : parse_pred.observedProperty('Temp')})

#   Fetch temperature at isobar 850 mbar
    pred['search_metadata'].update({'vert_coord1' : 850})
    pred['search_metadata'].update({'vert_units' : 'hPa'})
    t850 = read_var(filepath=filepaths, forecast_time=time, **pred['search_metadata'])
    assert(isinstance(t850,Camps_data)),"t850 expected to be camps data object"
    mask = np.ma.getmaskarray(t850.data)
    try:
        t850_unit = t850.metadata['units']
        t_pint = units.Quantity(1., t850_unit)
        assert(t_pint.dimensionality == iu_pint.dimensionality),"t850 does not have dimensions of temperature"
    except KeyError:
        logging.info("Fetched temperature at 850hPa has no units!")
        raise
#   Construct MetPy object to be used in calculating the K index.
##   And add the camps data object of the 850 mb air temperature to
##   the list of components of the K index.
    q_t850 = units.Quantity(t850.data, t850_unit)
    kindex.add_component(t850)
    kindex.preprocesses = t850.preprocesses

#   Fetch temperature at isobar 700 mbar
    pred['search_metadata'].update({'vert_coord1' : 700})
    t700 = read_var(filepath=filepaths, forecast_time=time, **pred['search_metadata'])
    assert(isinstance(t700,Camps_data)),"t700 expected to be camps data object"
    mask += np.ma.getmaskarray(t700.data)
    try:
        t_unit = t700.metadata['units']
        t_pint = units.Quantity(1., t_unit)
        assert(t_pint.dimensionality == iu_pint.dimensionality),"t700 does not have dimensions of temperature"
    except KeyError:
        logging.info("Fetched temperature at 700hPa has no units")
        raise
#   Construct MetPy object to be used in calculating the K index.
##   And add the camps data object of the 700 mb air temperature to
##   the list of components of the K index.
    q_t700 = units.Quantity(t700.data, t_unit)
    q_t700.ito(t850_unit)
    kindex.add_component(t700)
    for proc in t700.preprocesses:
        kindex.add_preprocess(proc)

#   Fetch temperature at isobar 500 mbar
    pred['search_metadata'].update({'vert_coord1' : 500})
    t500 = read_var(filepath=filepaths, forecast_time=time, **pred['search_metadata'])
    assert(isinstance(t500,Camps_data)),"t500 expected to be camps data object"
    mask += np.ma.getmaskarray(t500.data)
    try:
        t_unit = t500.metadata['units']
        t_pint = units.Quantity(1., t_unit)
        assert(t_pint.dimensionality == iu_pint.dimensionality),"t500 does not have dimensions of temperature"
    except KeyError:
        logging.info("Fetched temperature at 500hPa has no units")
        raise
#   Construct MetPy object to be used in calculating the K index.
##   And add the camps data object of the 500 mb air temperature to
##   the list of components of the K index.
    q_t500 = units.Quantity(t500.data, t_unit)
    q_t500.ito(t850_unit)
    kindex.add_component(t500)
    for proc in t500.preprocesses:
        kindex.add_preprocess(proc)

#   Dewpoint temperature components of K index.
#   These parameters may not be available from netCDF files.
#   But if relative humidity is, then we calculate it from
#   the dewpoint temperature function within the module moisture.
    pred['search_metadata'].update({'property' : parse_pred.observedProperty('DewPt')})

#   Fetch dewpoint temperature at isobar 850 mbar.
#   If fetch fails, try the dewpoint temperature function.
    pred['search_metadata'].update({'vert_coord1' : 850})
    td850 = read_var(filepath=filepaths, forecast_time=time, **pred['search_metadata'])
    if td850 is None:
        td850 = create.calculate(filepaths, time, pred)
    assert(isinstance(td850,Camps_data)),"td850 expected to be camps data object"
    mask += np.ma.getmaskarray(td850.data)
    try:
        t_unit = td850.metadata['units']
        t_pint = units.Quantity(1., t_unit)
        assert(t_pint.dimensionality == iu_pint.dimensionality),"td850 does not have dimensions of temperature"
    except KeyError:
        logging.info("Fetched dewpoint temperature at 850hPa has no units")
        raise
#   Construct MetPy object to be used in calculating the K index.
##   And add the camps data object of the 850 mb dewpoint temperature to
##   the list of components of the K index.
    q_td850 = units.Quantity(td850.data, t_unit)
    q_td850.ito(t850_unit)
    kindex.add_component(td850)
    for proc in td850.preprocesses:
        kindex.add_preprocess(proc)

#   Fetch dewpoint temperature at isobar 700 mbar.
#   If fetch fails, try the dewpoint temperature function.
    pred['search_metadata'].update({'vert_coord1' : 700})
    td700 = read_var(filepath=filepaths, forecast_time=time, **pred['search_metadata'])
    if td700 is None:
        td700 = create.calculate(filepaths, time, pred)
    assert(isinstance(td700,Camps_data)),"td700 expected to be camps data object"
    mask += np.ma.getmaskarray(td700.data)
    try:
        t_unit = td700.metadata['units']
        t_pint = units.Quantity(1., t_unit)
        assert(t_pint.dimensionality == iu_pint.dimensionality),"td700 does not have dimensions of temperature"
    except KeyError:
        logging.info("Fetched dewpoint temperature at 700hPa has no units")
        raise
#   Construct MetPy object to be used in calculating the K index.
##   And add the camps data object of the 700 mb dewpoint temperature to
##   the list of components of the K index.
    q_td700 = units.Quantity(td700.data, t_unit)
    q_td700.ito(t850_unit)
    kindex.add_component(td700)
    for proc in td700.preprocesses:
        kindex.add_preprocess(proc)

#   Call the function that constructs the K index as a pint.Quantity object.
#   The key advantage of using pint.Quantity objects is that their
#   mathematical operators account for differences in units of the operands.
#   Add the dimensions to the camps data object before inserting the data
#   into it.
    q_kindex = KIndex(q_t850, q_t700, q_t500, q_td850, q_td700).to(unit)
    kindex.add_dimensions('phenomenonTime')
    kindex.add_dimensions('y')
    kindex.add_dimensions('x')
    kindex.add_data(np.ma.array(np.array(q_kindex), mask=mask))
    kindex.add_process('KIndexCalc')

#   Create the rest of the camps data object for K index
    kindex.add_vert_coord(0,vert_type='elev')
    kindex.time = copy.deepcopy(t850.time)
    kindex.location = t850.location
    kindex.metadata.update({'FcstTime_hour' : t850.metadata.get('FcstTime_hour')})
    kindex.metadata.update({'PROV__hadPrimarySource' : t850.metadata.get('PROV__hadPrimarySource')})

    return kindex

def KIndex(t850, t700, t500, td850, td700):
    r"""The K index formula.

    Args:
        t850 (pint.Quantity): Atmospheric temperature (K) at isobar 850mb.
        t700 (pint.Quantity): Atmospheric temperature (K) at isobar 700mb.
        t500 (pint.Quantity): Atmospheric temperature (K) at isobar 500mb.
        td850 (pint.Quantity): Atmospheric dewpoint temperature (K) at isobar 850mb.
        td700 (pint.Quantity): Atmospheric dewpoint temperature (K) at isobar 700mb.

    Returns:
        kindex (pint.Quantity): A stability index of the atmosphere (deg C)

    K-index = (t850 - t500) + td850 - (t700 - td700).
    It is useful for predicting non-severe warm season
    convective activity.  Its unit is commonly degrees
    Centigrade.

    """

    kindex = (t850 - t500) + td850 - (t700 - td700)

    return kindex


from django.db import models
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType

class SourceCopyist(models.Model):
    class Meta:
        app_label = "diamm_data"

    MUSIC_COPYIST = 1
    TEXT_COPYIST = 2
    INDEXER = 3
    LIMINARY_TEXT = 4
    ILLUMINATOR = 5
    TEXT_AND_MUSIC = 6
    UNKNOWN = 7

    SOURCE_COPYIST_TYPES = (
        (MUSIC_COPYIST, "Music"),
        (TEXT_COPYIST, "Text"),
        (INDEXER, "Indexer"),
        (LIMINARY_TEXT, "Liminary Text"),
        (ILLUMINATOR, "Illuminator"),
        (TEXT_AND_MUSIC, "Text and Music"),
        (UNKNOWN, "Unknown")
    )

    source = models.ForeignKey("diamm_data.Source",
                               on_delete=models.CASCADE,
                               related_name="copyists")

    uncertain = models.BooleanField(default=False)
    type = models.IntegerField(choices=SOURCE_COPYIST_TYPES, blank=True, null=True)

    # Copyists can be either People or Organizations.
    limit = models.Q(app_label='diamm_data', model="person") | models.Q(app_label='diamm_data', model='organization')
    content_type = models.ForeignKey(ContentType,
                                     on_delete=models.CASCADE,
                                     limit_choices_to=limit)
    object_id = models.PositiveIntegerField()
    copyist = GenericForeignKey()

    def __str__(self):
        return f"{self.copyist} ({self.copyist_type})"

    @property
    def copyist_type(self):
        if not self.type:
            return None

        d = dict(self.SOURCE_COPYIST_TYPES)
        return d[self.type]


from PyQt6.QtWidgets import QApplication, QPushButton, QMainWindow, QVBoxLayout, QWidget, QLabel, QColorDialog, QHBoxLayout
from PyQt6.QtCore import QTimer
from PyQt6.QtGui import *
import sys
import glob
from time import sleep
import serial
import threading

def setInterval(func,time):
    e = threading.Event()
    while not e.wait(time):
        func()

def serial_ports():
    if sys.platform.startswith('win'):
        ports = ['COM%s' % (i + 1) for i in range(256)]
    elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
        ports = glob.glob('/dev/tty[A-Za-z]*')
    elif sys.platform.startswith('darwin'):
        ports = glob.glob('/dev/tty.*')
    else:
        raise EnvironmentError('Unsupported platform')

    result = []
    for port in ports:
        try:
            s = serial.Serial(port)
            s.close()
            result.append(port)
        except (OSError, serial.SerialException):
            pass
    return result

ser = serial.Serial(serial_ports()[0], 115200)

class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.currentData = []
        ser.write(bytearray([0xFF]))
        self.get_data()

        self.setStyleSheet("background-color: #12860a;" "color: #ffffff;")
        self.temp = QLabel("Температура воздуха: " + str(self.currentData[3]) + "°C")
        self.humidity = QLabel("Влажность воздуха: " + str(self.currentData[4]) + "%")

        self.airLayout = QHBoxLayout()
        self.airLayout.addWidget(self.temp)
        self.airLayout.addWidget(self.humidity)

        self.soilhumidity = QLabel("Влажость почвы: " + str(self.currentData[8]) + "%")
        self.soiltemp = QLabel("Температура почвы: " + str(self.currentData[7]) + "°C")

        self.soilLayout = QHBoxLayout()
        self.soilLayout.addWidget(self.soilhumidity)
        self.soilLayout.addWidget(self.soiltemp)

        self.pressure = QLabel("Давление воздуха: " + str(self.currentData[5]) + "hPa")
        self.light = QLabel("Интенсивность света: " + str(self.currentData[6]) + "Люкс")

        self.otherLayout = QHBoxLayout()
        self.otherLayout.addWidget(self.pressure)
        self.otherLayout.addWidget(self.light)

        self.color = QColorDialog()

        self.submitColor = QPushButton("Задать цвет")
        self.submitColor.clicked.connect(self.onSubmitColor)

        self.pumpButton = QPushButton("Включить помпу" if self.currentData[-6] == 0 else "Выключить помпу")
        self.pumpButton.clicked.connect(self.onPumpButtonClick)

        self.buttonsLayout = QHBoxLayout()
        self.buttonsLayout.addWidget(self.submitColor)
        self.buttonsLayout.addWidget(self.pumpButton)

        self.windButton = QPushButton("Включить вентилятор" if self.currentData[-5] == 0 else "Выключить вентилятор")
        self.windButton.clicked.connect(self.onWindButtonClick)

        self.windowButton = QPushButton("Открыть окно" if self.currentData[-4] == 15 else "Закрыть окно")
        self.windowButton.clicked.connect(self.onWindowButtonClick)

        self.buttonsLayout2 = QHBoxLayout()
        self.buttonsLayout2.addWidget(self.windButton)
        self.buttonsLayout2.addWidget(self.windowButton)

        layout = QVBoxLayout()
        layout.addLayout(self.airLayout)
        layout.addLayout(self.soilLayout)
        layout.addLayout(self.otherLayout)
        layout.addWidget(self.color)
        layout.addLayout(self.buttonsLayout)
        layout.addLayout(self.buttonsLayout2)

        container = QWidget()
        container.setLayout(layout)

        self.setWindowTitle("Умная теплица")

        self.timer = QTimer()
        self.timer.setInterval(1000)
        self.timer.timeout.connect(self.updateData)
        self.timer.start()

        self.setCentralWidget(container)

    def updateData(self):
        ser.write(bytearray([0xFF]))
        self.get_data()
        self.reloadSensors()

    def reloadSensors(self):
        self.temp.setText("Температура воздуха: " + str(self.currentData[3]) + "°C")
        self.humidity.setText("Влажность воздуха: " + str(self.currentData[4]) + "%")
        self.pressure.setText("Давление воздуха: " + str(self.currentData[5]) + "hPa")
        self.soiltemp.setText("Температура почвы: " + str(self.currentData[7]) + "°C")
        self.soilhumidity.setText("Влажость почвы: " + str(self.currentData[8]) + "%")
        self.light.setText("Интенсивность света: " + str(self.currentData[6]) + "Люкс")

    def onSubmitColor(self):
        _currentColor = self.color.currentColor()
        _writeRGB = bytearray([0xA3])
        self.reloadSensors()

        _writeRGB.append(_currentColor.red())
        _writeRGB.append(_currentColor.green())
        _writeRGB.append(_currentColor.blue())

        ser.write(_writeRGB)
        self.get_data()
        self.color.show()

    def onPumpButtonClick(self):
        ser.write(bytearray([0xA0, 0x1]))
        self.reloadSensors()
        self.get_data()
        self.pumpButton.setText("Включить помпу" if self.currentData[-6] == 0 else "Выключить помпу")

    def onWindButtonClick(self):
        ser.write(bytearray([0xA1, 0x1]))
        self.get_data()
        self.reloadSensors()
        self.windButton.setText("Включить вентилятор" if self.currentData[-5] == 0 else "Выключить вентилятор")

    def onWindowButtonClick(self):
        if (self.currentData[-4] == 15):
            ser.write(bytearray([0xA2, 0xAA]))
            self.windowButton.setText("Закрыть окно")
        else:
            ser.write(bytearray([0xA2, 0xF]))
            self.windowButton.setText("Открыть окно")

        self.get_data()

    def get_data (self):
        sleep(0.1)
        _currentData = str(ser.read_all())
        _currentData = _currentData[2:_currentData.find('\\r\\n')]
        if (_currentData!=''):
            currentDataList = list(map(float, _currentData.split('|')))
            self.currentData = currentDataList
            print(currentDataList)

app = QApplication(sys.argv)
window = MainWindow()

window.show()
sys.exit(app.exec())


'''
   Author: Rohith Kumar Punithavel
   Email: rohithkumar@asu.edu
   Problem Statement: https://leetcode.com/problems/running-sum-of-1d-array/
'''
class Solution:
    def runningSum(self, nums: List[int]) -> List[int]:
        final_result=[]
        sum1=0
        for i in range(len(nums)):
            sum1=sum1+nums[i]
            final_result.append(sum1)
        return final_result


import processing_data
import pickle

def pre(text):
    text = processing_data.text_preprocess(text)
    with open('tfidf.pickle', 'rb') as f:
        tfidf = pickle.load(f)
    with open('model.pickle', 'rb') as f:
        model = pickle.load(f)

    category_codes = {0: 'đời sống',
                      1: 'giải trí',
                      2: 'giáo dục',
                      3: 'kinh doanh',
                      4: 'pháp luật',
                      5: 'sức khỏe',
                      6: 'others'}

    features = tfidf.transform([text]).toarray()

    predict = model.predict(features)[0]
    proba = int(model.predict_proba(features)[0][predict] * 100)

    if proba <= 50:
        predict = 6
    return category_codes[predict], proba

if __name__ == '__main__':
    pass