import argparse
import glob
import json
import os
import sys

MIN_PYTHON = (3, 0)
if sys.version_info < MIN_PYTHON:
    sys.exit("Python {}.{} or later is required.\n".format(*MIN_PYTHON))

parser = argparse.ArgumentParser()
parser.add_argument("input", help="input directory with files to annotate")
parser.add_argument("output", help="output directory to put packed files")
parser.add_argument("min_tokens", help="minimum number of tokens per file", type=int)
args = parser.parse_args()

if not os.path.isdir(args.input):
    sys.exit("Error: {} does not exist".format(args.input))

if not os.path.exists(args.output):
    os.makedirs(args.output)

basename = None
lines = []
files = []
count = 1
metadata = {}
for filename in sorted(glob.glob(os.path.join(args.input, "*.txt"))):
    # grab first 6 characters: IL6_NW
    if basename is None:
        basename = os.path.basename(filename)[0:6]

    with open(filename, 'r') as ifp:
        new_lines = [x for x in ifp]
        # must have an empty line at end of file
        if len(new_lines[-1].split('\t')) > 2:
            new_lines.append('\n')
        # remove tsv header after first file
        if len(lines) > 0 and new_lines[0][0:5] == 'TOKEN':
            del new_lines[0]
        num_lines = len(new_lines)
        # we don't write out header to annotation file
        if len(lines) == 0:
            num_lines -= 1

        files.append({'filename': os.path.basename(filename), 'num_lines': num_lines})
        lines.extend(new_lines)

    if len(lines) > args.min_tokens:
        out_filename = os.path.join(args.output, "{}_{:03d}.txt".format(basename, count))
        with open(out_filename, 'w') as ofp:
            ofp.writelines(lines)
        metadata[os.path.basename(out_filename)] = files
        lines = []
        files = []
        count += 1

with open(os.path.join(args.output, '.metadata'), 'w') as mfp:
    json.dump(metadata, mfp)


# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt
import numpy as np

"""
This script is for plotting G1/G2/G4 data of calculated Symm-Func for
train and predict of Lammps-MD 1000K LC7
"""

def plotG(symdt, plttitle, plotfile, xlb, clr):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    plt.title(plttitle)
    ax.set_ylabel("Value of G")
    ax.grid(True)

    num=0
    for symdata in symdt:
        for eachsample in symdata:
            print(f'{num}: {plttitle}')
            for gdata in eachsample:
                ax.scatter(xlb, gdata, c=clr, marker='.')
                num=num+1

    labels = ax.get_xticklabels()
    plt.setp(labels, rotation=90, fontsize=8);
    plt.savefig(plotfile)
    plt.close()

def gatherG(root, outfolder, grps, md):
    xlb=["G1"]
    clr=["b"]
    for i in range(1, 25):
        xlb.append("G2-"+str(i))
        clr.append("g")
    for i in range(1, 17):
        xlb.append("G4-"+str(i))
        clr.append("c")

    for grp in grps:
        symdtt, symdtp = [],[]
        plotfileT=outfolder+grp+"-Gdata-T.png"
        plotfileP=outfolder+grp+"-Gdata-P.png"
        for i in range(1, 11):
            if grp=='mix':
                for j in range(1, 11):
                    datadir=root+grp+"/"+str(i)+"/"+str(j)
                    symfft=datadir+"/data/CrystalSi64/symmetry_function.npz"
                    symt= np.load(symfft)
                    symdata= symt['sym_func']
                    symdtt.append(symdata)
                    symffp=datadir+"/predict-phono3py-2/output-phono3py/symmetry_function-pred.npz"
                    symp= np.load(symffp)
                    symdata= symp['sym_func']
                    symdtp.append(symdata)
            else:
                datadir=root+grp+"/"+str(i)
                symfft=datadir+"/data/CrystalSi64/symmetry_function.npz"
                symt= np.load(symfft)
                symdata= symt['sym_func']
                symdtt.append(symdata)
                symffp=datadir+"/predict-phono3py-2/output-phono3py/symmetry_function-pred.npz"
                symp= np.load(symffp)
                symdata= symp['sym_func']
                symdtp.append(symdata)
        print(f'Symm_func of {grp} is gathered')

        stnparr=np.array(symdtt)
        plttitle=f'[{md}/{grp}] Train({stnparr.shape}) Symm_Func G value'
        plotG(symdtt, plttitle, plotfileT, xlb, clr)
        spnparr=np.array(symdtp)
        plttitle=f'[{md}/{grp}] Predict({spnparr.shape}) Symm_Func G value'
        plotG(symdtp, plttitle, plotfileP, xlb, clr)
        print(f'Symm_func of {grp} is plotted')

if __name__ == '__main__':
    #Plot Lammps-MD LC7 Symm_Func of Train & Predict
    root="/home/okugawa/HDNNP/Si-190808/1000K-LC7/"
    outfolder="/home/okugawa/HDNNP/Si-190808/result-LC7/symf/"
    grps=['0.95','0.97','0.99','1.00','1.01','1.03','1.05','mix']
    gatherG(root, outfolder, grps, "Lammps-MD")

    #Plot AIMD LC7 Symm_Func of Train & Predict
    root="/home/okugawa/HDNNP/Si-190808-md/1000K-LC7n/"
    outfolder="/home/okugawa/HDNNP/Si-190808-md/result-LC7n/symf/"
    grps=['0.95','0.97','0.99','1.0','1.01','1.03','1.05','mix']
    gatherG(root, outfolder, grps, "AIMD")


#!/usr/bin/python3

import datetime, glob, os, subprocess, unittest

from slippi import Game, parse
from slippi.id import CSSCharacter, InGameCharacter, Item, Stage
from slippi.log import log
from slippi.metadata import Metadata
from slippi.event import Buttons, Direction, End, Frame, Position, Start, Triggers, Velocity
from slippi.parse import ParseEvent

BPhys = Buttons.Physical
BLog = Buttons.Logical

def norm(f):
    return 1 if f > 0.01 else -1 if f < -0.01 else 0

def path(name):
    return os.path.join(os.path.dirname(__file__), 'replays', name + '.slp')

class TestGame(unittest.TestCase):
    def __init__(self, *args, **kwargs) -> None:
        self.pkgname: str = "slippi"
        super().__init__(*args, **kwargs)
        my_env = os.environ.copy()
        self.pypath: str = my_env.get("PYTHONPATH", os.getcwd())
        self.mypy_opts: List[str] = ['--ignore-missing-imports']

    def _game(self, name):
        return Game(path(name))

    def _stick_seq(self, game):
        pass

    # not used yet because the recorder currently puts bogus values in triggers.physical
    def _trigger_seq(self, game):
        last_triggers = None
        trigger_seq = []
        for frame in game.frames:
            t = frame.ports[0].leader.pre.triggers
            t = Triggers(norm(t.logical), norm(t.physical.l), norm(t.physical.r))
            if (t.logical or t.physical.l or t.physical.r) and t != last_triggers:
                trigger_seq.append(t)
            last_triggers = t
        return trigger_seq

    def _button_seq(self, game):
        last_buttons = None
        button_seq = []
        for frame in game.frames:
            b = frame.ports[0].leader.pre.buttons
            if (b.logical or b.physical) and b != last_buttons:
                button_seq.append(b)
            last_buttons = b
        return button_seq

    def test_run_mypy_module(self):
        """Run mypy on all module sources"""
        mypy_call: List[str] = ["mypy"] + self.mypy_opts + ["-p", self.pkgname]
        browse_result: int = subprocess.call(mypy_call, env=os.environ, cwd=self.pypath)
        self.assertEqual(browse_result, 0, 'mypy on slippi')

    def test_run_mypy_tests(self):
        """Run mypy on all tests in module under the tests directory"""
        for test_file in glob.iglob(f'{os.getcwd()}/tests/*.py'):
            mypy_call: List[str] = ["mypy"] + self.mypy_opts + [test_file]
            test_result: int = subprocess.call(mypy_call, env=os.environ, cwd=self.pypath)
            self.assertEqual(test_result, 0, f'mypy on test {test_file}')

    def test_slippi_old_version(self):
        game = self._game('v0.1')
        self.assertEqual(game.start.slippi.version, Start.Slippi.Version(0,1,0,0))
        self.assertEqual(game.metadata.duration, None)
        self.assertEqual(game.metadata.players, (None, None, None, None))
        self.assertEqual(game.start.players[0].character, CSSCharacter.FOX)
        self.assertEqual(game.start.players[1].character, CSSCharacter.GANONDORF)

    def test_game(self):
        game = self._game('game')

        self.assertEqual(game.metadata, Metadata._parse({
            'startAt': '2018-06-22T07:52:59Z',
            'lastFrame': 5085,
            'playedOn': 'dolphin',
            'players': {
                '0': {'characters': {InGameCharacter.MARTH: 5209}},
                '1': {'characters': {InGameCharacter.FOX: 5209}}}}))
        self.assertEqual(game.metadata, Metadata(
            date=datetime.datetime(2018, 6, 22, 7, 52, 59, 0, datetime.timezone.utc),
            duration=5209,
            platform=Metadata.Platform.DOLPHIN,
            players=(
                Metadata.Player({InGameCharacter.MARTH: 5209}),
                Metadata.Player({InGameCharacter.FOX: 5209}),
                None, None)))

        self.assertEqual(game.start, Start(
            is_teams=False,
            random_seed=3803194226,
            slippi=Start.Slippi(Start.Slippi.Version(1,0,0,0)),
            stage=Stage.YOSHIS_STORY,
            players=(
                Start.Player(character=CSSCharacter.MARTH, type=Start.Player.Type.HUMAN, stocks=4, costume=3, team=None, ucf=Start.Player.UCF(False, False)),
                Start.Player(character=CSSCharacter.FOX, type=Start.Player.Type.CPU, stocks=4, costume=0, team=None, ucf=Start.Player.UCF(False, False)),
                None, None)))

        self.assertEqual(game.end, End(End.Method.CONCLUSIVE))

        self.assertEqual(game.metadata.duration, len(game.frames))

    def test_game_skip_frames(self):
        game = Game(path('game'), skip_frames=True)

        self.assertEqual(game.metadata, Metadata._parse({
            'startAt': '2018-06-22T07:52:59Z',
            'lastFrame': 5085,
            'playedOn': 'dolphin',
            'players': {
                '0': {'characters': {InGameCharacter.MARTH: 5209}},
                '1': {'characters': {InGameCharacter.FOX: 5209}}}}))
        self.assertEqual(game.metadata, Metadata(
            date=datetime.datetime(2018, 6, 22, 7, 52, 59, 0, datetime.timezone.utc),
            duration=5209,
            platform=Metadata.Platform.DOLPHIN,
            players=(
                Metadata.Player({InGameCharacter.MARTH: 5209}),
                Metadata.Player({InGameCharacter.FOX: 5209}),
                None, None)))

        self.assertEqual(game.start, Start(
            is_teams=False,
            random_seed=3803194226,
            slippi=Start.Slippi(Start.Slippi.Version(1,0,0,0)),
            stage=Stage.YOSHIS_STORY,
            players=(
                Start.Player(character=CSSCharacter.MARTH, type=Start.Player.Type.HUMAN, stocks=4, costume=3, team=None, ucf=Start.Player.UCF(False, False)),
                Start.Player(character=CSSCharacter.FOX, type=Start.Player.Type.CPU, stocks=4, costume=0, team=None, ucf=Start.Player.UCF(False, False)),
                None, None)))

        self.assertEqual(game.end, End(End.Method.CONCLUSIVE))

        self.assertFalse(game.frames)

    def test_ics(self):
        game = self._game('ics')
        self.assertEqual(game.metadata.players[0].characters, {
            InGameCharacter.NANA: 344,
            InGameCharacter.POPO: 344})
        self.assertEqual(game.start.players[0].character, CSSCharacter.ICE_CLIMBERS)
        self.assertIsNotNone(game.frames[0].ports[0].follower)

    def test_ucf(self):
        self.assertEqual(self._game('shield_drop').start.players[0].ucf, Start.Player.UCF(dash_back=False, shield_drop=True))
        self.assertEqual(self._game('dash_back').start.players[0].ucf, Start.Player.UCF(dash_back=True, shield_drop=False))

    def test_buttons_lrzs(self):
        game = self._game('buttons_lrzs')
        self.assertEqual(self._button_seq(game), [
            Buttons(BLog.TRIGGER_ANALOG, BPhys.NONE),
            Buttons(BLog.TRIGGER_ANALOG|BLog.L, BPhys.L),
            Buttons(BLog.TRIGGER_ANALOG, BPhys.NONE),
            Buttons(BLog.TRIGGER_ANALOG|BLog.R, BPhys.R),
            Buttons(BLog.TRIGGER_ANALOG|BLog.A|BLog.Z, BPhys.Z),
            Buttons(BLog.START, BPhys.START)])

    def test_buttons_abxy(self):
        game = self._game('buttons_abxy')
        self.assertEqual(self._button_seq(game), [
            Buttons(BLog.A, BPhys.A),
            Buttons(BLog.B, BPhys.B),
            Buttons(BLog.X, BPhys.X),
            Buttons(BLog.Y, BPhys.Y)])

    def test_dpad_udlr(self):
        game = self._game('dpad_udlr')
        self.assertEqual(self._button_seq(game), [
            Buttons(BLog.DPAD_UP, BPhys.DPAD_UP),
            Buttons(BLog.DPAD_DOWN, BPhys.DPAD_DOWN),
            Buttons(BLog.DPAD_LEFT, BPhys.DPAD_LEFT),
            Buttons(BLog.DPAD_RIGHT, BPhys.DPAD_RIGHT)])

    def test_cstick_udlr(self):
        game = self._game('cstick_udlr')
        self.assertEqual(self._button_seq(game), [
            Buttons(BLog.CSTICK_UP, BPhys.NONE),
            Buttons(BLog.CSTICK_DOWN, BPhys.NONE),
            Buttons(BLog.CSTICK_LEFT, BPhys.NONE),
            Buttons(BLog.CSTICK_RIGHT, BPhys.NONE)])

    def test_joystick_udlr(self):
        game = self._game('joystick_udlr')
        self.assertEqual(self._button_seq(game), [
            Buttons(BLog.JOYSTICK_UP, BPhys.NONE),
            Buttons(BLog.JOYSTICK_DOWN, BPhys.NONE),
            Buttons(BLog.JOYSTICK_LEFT, BPhys.NONE),
            Buttons(BLog.JOYSTICK_RIGHT, BPhys.NONE)])

    def test_nintendont(self):
        game = self._game('nintendont')
        self.assertEqual(game.metadata.platform, Metadata.Platform.NINTENDONT)

    def test_netplay_name(self):
        game = self._game('netplay')
        players = game.metadata.players
        self.assertEqual(players[0].netplay, Metadata.Player.Netplay(code='ABCD#123', name='abcdefghijk'))
        self.assertEqual(players[1].netplay, Metadata.Player.Netplay(code='XX#000', name='nobody'))

    def test_console_name(self):
        game = self._game('console_name')
        self.assertEqual(game.metadata.console_name, 'Station 1')

    def test_metadata_json(self):
        game = self._game('game')
        self.assertEqual(game.metadata_raw, {
            'lastFrame': 5085,
            'playedOn': 'dolphin',
            'players': {
                '0': {'characters': {'18': 5209}},
                '1': {'characters': {'1': 5209}}},
            'startAt': '2018-06-22T07:52:59Z'})

    def test_v2(self):
        game = self._game('v2.0')
        self.assertEqual(game.start.slippi.version, Start.Slippi.Version(2,0,1))

    def test_unknown_event(self):
        with self.assertLogs(log, 'INFO') as log_context:
            game = self._game('unknown_event')
        self.assertEqual(log_context.output, ['INFO:root:ignoring unknown event type: 0xff'])

    def test_items(self):
        game = self._game('items')
        items = {}
        for f in game.frames:
            for i in f.items:
                if not i.spawn_id in items:
                    items[i.spawn_id] = i
        self.assertEqual(items, {
            0: Frame.Item(
                damage=0,
                direction=Direction.RIGHT,
                position=Position(-62.7096061706543, -1.4932749271392822),
                spawn_id=0,
                state=0,
                timer=140.0,
                type=Item.PEACH_TURNIP,
                velocity=Velocity(0.0, 0.0)),
            1: Frame.Item(
                damage=0,
                direction=Direction.LEFT,
                position=Position(20.395559310913086, -1.4932749271392822),
                spawn_id=1,
                state=0,
                timer=140.0,
                type=Item.PEACH_TURNIP,
                velocity=Velocity(0.0, 0.0)),
            2: Frame.Item(
                damage=0,
                direction=Direction.RIGHT,
                position=Position(-3.982539176940918, -1.4932749271392822),
                spawn_id=2,
                state=0,
                timer=140.0,
                type=Item.PEACH_TURNIP,
                velocity=Velocity(0.0, 0.0))})

class TestParse(unittest.TestCase):
    def test_parse(self):
        metadata = None
        def set_metadata(x):
            nonlocal metadata
            metadata = x
        parse(path('game'), {ParseEvent.METADATA: set_metadata})
        self.assertEqual(metadata.duration, 5209)

if __name__ == '__main__':
    unittest.main()


class GroupRule:
  def __init__(self, group, tag_requires, tag_conflicts, group_requires, group_conflicts):
    self.group = group
    self.tag_requires = set(tag_requires)
    self.tag_conflicts = tag_conflicts
    self.group_requires = group_requires
    self.group_conflicts = group_conflicts

  def meets_requirements(self, tagset):
    if self.group.tags.intersection(tagset):
      if not tagset.issuperset(self.tag_requires):
        return False
      for group in self.group_requires:
        # tagset must share at least one tag that's a member of the current group
        if tagset.isdisjoint(group.tags):
          return False
    return True

  def no_conflicts(self, tagset):
    if self.group.tags.intersection(tagset):
      if tagset.intersection(self.tag_conflicts):
        return False
      for group in self.group_conflicts:
        # tagset must share no tag that is a member of the current group
        if tagset.intersection(group.tags):
          return False
    return True

  def check(self, tagset):
    return (
      self.meets_requirements(tagset) and
      self.no_conflicts(tagset)
    )


import unittest
from delivery import *
from pedidos import *
from loja import *
from motoboy import *

moto1 = Motoboy('João', 2, None, None)
moto2 = Motoboy('Carlos', 2, None, None)
moto3 = Motoboy('Alberto', 2, None, None)
moto4 = Motoboy('Bruna', 2, 'Magazine', None)
moto5 = Motoboy('Luiza', 3, None, None)

pedido1_loja1 = Pedido(50, 'pedido1')
pedido2_loja1 = Pedido(50, 'pedido2')
pedido3_loja1 = Pedido(50, 'pedido3')

pedido1_loja2 = Pedido(50, 'pedido1')
pedido2_loja2 = Pedido(50, 'pedido2')
pedido3_loja2 = Pedido(50, 'pedido3')
pedido4_loja2 = Pedido(50, 'pedido4')

pedido1_loja3 = Pedido(50, 'pedido1')
pedido1_loja3 = Pedido(50, 'pedido2')
pedido1_loja3 = Pedido(100, 'pedido3')

pedidos_loja1 = Pedidos([pedido1_loja1, pedido2_loja1, pedido3_loja1])
pedidos_loja2 = Pedidos([pedido1_loja2, pedido2_loja2,
                        pedido3_loja2, pedido4_loja2])
pedidos_loja3 = Pedidos([pedido1_loja3, pedido1_loja3, pedido1_loja3])

loja1 = Loja('Magazine', pedidos_loja1, 5)
loja2 = Loja('Americanas', pedidos_loja2, 5)
loja3 = Loja('Casas Bahia', pedidos_loja3, 15)

mock_teste1 = {
    'nome_loja': 'Magazine',
    'nome_motoboy': 'Bruna',
    'quantidade_pedidos': 3,
    'valor_rececbido_pelo_motoboy': 4.5
}

mock_teste2 = {'nome_loja': 'Magazine', 'nome_motoboy': 'João',
               'quantidade_pedidos': 2, 'valor_rececbido_pelo_motoboy': 4.5}

mock_teste3 = {'nome_loja': 'Americanas', 'nome_motoboy': 'Carlos',
               'quantidade_pedidos': 4, 'valor_rececbido_pelo_motoboy': 4.5}

mock_teste4 = {'nome_loja': 'Magazine', 'nome_motoboy': 'João',
               'quantidade_pedidos': 1, 'valor_rececbido_pelo_motoboy': 9.0}

mock_teste5 = {'nome_loja': 'Casas Bahia', 'nome_motoboy': 'Alberto',
               'quantidade_pedidos': 3, 'valor_rececbido_pelo_motoboy': 47.0}

class MyTest(unittest.TestCase):
    def test_exclusive_motoboy(self):
        self.assertEqual(executa_entrega(loja1, moto4), mock_teste1)

    def test_non_exclusive_motoboy(self):
        self.assertEqual(executa_entrega(loja1, moto1), mock_teste2)

    def test_non_exclusive_motoboy_another_store(self):
        self.assertEqual(executa_entrega(loja2, moto2), mock_teste3)

    def test_non_exclusive_motoboy_another_get_all_items(self):
        self.assertEqual(executa_entrega(loja1, moto1, True), mock_teste4)

    def test_non_exclusive_motoboy_another_store_all_items(self):
        self.assertEqual(executa_entrega(loja3, moto3, True), mock_teste5)

if __name__ == '__main__':
    unittest.main()


class Solution:
    def reverseWords(self, s: str) -> str:
        s =s.strip() #删除首尾空格
        #双指针，从右向左寻找空格
        i = j =len(s) -1
        new =[]
        while i>=0:
            while i>=0 and s[i] != ' ': i-=1 #寻找第一个空格
            new.append(s[i+1: j+1]) #添加进数组
            while i>=0 and s[i] == ' ': i-=1 #跳过空格
            j = i #j指向下个单词的尾字符
        return ' '.join(new)


import gtk
import thermo

class SteamSolver:
    def __init__(self):
        '''Initial window which has a combo box from which the user can choose either of     temperature and pressure'''

        self.window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        self.window.set_size_request(500 , 500)
        self.window.set_title('Steam Solver')
        self.window.connect('delete-event' , gtk.main_quit)
        self.window.modify_bg(gtk.STATE_NORMAL , gtk.gdk.Color(red = 20000, blue = 20000 , green = 20000))
        self.TableCreator()

    def TableCreator(self):
        self.PropertyList =['Pressure' , 'Temperature' , 'Volume(v)' , 'Energy(u)' ,'Enthalpy(h)' , 'Entropy(s)' , 'Quality(x)']
        self.FinalList = []
        self.table = gtk.Table(500 ,500 , True)
        self.label = gtk.Label('Choose any one')
        self.table.attach(self.label , 25 , 225 , 25 , 60 )
        self.box = gtk.combo_box_new_text()
        self.box.append_text('Pressure')
        self.box.append_text('Temperature')
        self.box.connect('changed' , self.PresTemp)
        self.table.attach(self.box , 275 , 450 , 25 , 60)
        self.window.add(self.table)
        self.window.show_all()

    def _helper(self , FinalList):
        try:

            Property1 = self.PresTempEnt.get_text()
            Property2 = self.PropertyEntry.get_text()
            if FinalList[0] == 'P':
                if FinalList[1] == 'Temperature':
                    a = thermo.State(P = float(Property1), T  = float(Property2))
                    return a

                elif FinalList[1] == 'Volume(v)':
                    a = thermo.State(P = float(Property1), v  = float(Property2))
                    return a

                elif FinalList[1] == 'Energy(u)':
                    a = thermo.State(P = float(Property1), u  = float(Property2))
                    return a

                elif FinalList[1] == 'Enthalpy(h)':
                    a = thermo.State(P = float(Property1), h = float(Property2))
                    return a

                elif FinalList[1] == 'Entropy(s)':
                    a = thermo.State(P = float(Property1), s  = float(Property2))
                    return a

                elif FinalList[1] == 'Quality(x)':
                    a = thermo.State(P = float(Property1), x  = float(Property2))
                    return a

            if FinalList[0] == 'T':
                if FinalList[1] == 'Volume(v)':
                    a = thermo.State(P = float(Property1), v  = float(Property2))
                    return a

                elif FinalList[1] == 'Energy(u)':
                    a = thermo.State(P = float(Property1), u  = float(Property2))
                    return a

                elif FinalList[1] == 'Enthalpy(h)':
                    a = thermo.State(P = float(Property1), h = float(Property2))
                    return a

                elif FinalList[1] == 'Entropy(s)':
                    a = thermo.State(P = float(Property1), s  = float(Property2))
                    return a

                elif FinalList[1] == 'Quality(x)':
                    a = thermo.State(P = float(Property1), x  = float(Property2))
                    return a

        except ValueError:
            a = 'Error'
            return a

    def PresTemp(self , widget):
        '''Function called when the combo box is changed , the window gets another label and an
        entry into which the user can tpe the required value of temperature and pressure'''
        index = widget.get_active()
        widget.destroy()
        self.Entry1 = gtk.Entry()
        self.table.attach(self.Entry1 , 275 , 450 , 25 , 60)
        self.PresTempLab = gtk.Label()
        self.PresTempEnt = gtk.Entry()
        self.table.attach(self.PresTempLab , 25 , 225 , 120 , 155)
        self.table.attach(self.PresTempEnt , 275 , 450 , 120 , 155)
        if index == 0:
            self.FinalList.append('P')
            self.PropertyList.remove('Pressure')
            self.PresTempLab.set_text('Enter Pressure')
            self.Entry1.set_text('Pressure')
            self.Entry1.set_editable(False)
        else:
            self.FinalList.append('T')
            self.PropertyList.remove('Temperature')
            self.PresTempLab.set_text('Enter Temperature')
            self.Entry1.set_text('Temperature')
            self.Entry1.set_editable(False)
        self.button = gtk.Button('Process')
        self.button.connect('clicked' , self.Process , index)
        self.table.attach(self.button , 200 , 300 , 420 , 480)
        self.window.show_all()

    def Process(self , widget , index):
        '''Function called by the button Process , Checks whether the input is valid or not
        If the input is valid , then another label and a combo box are added. The user can
        choose any other property other then temperature or pressure which was entered'''
        UserEntry = self.PresTempEnt.get_text()
        self.PresTempEnt.set_editable(False)
        widget.destroy()
        self.SecondStateLab = gtk.Label('Enter Second Property')
        self.Combo = gtk.combo_box_new_text()
        for Property in self.PropertyList:
            self.Combo.append_text(Property)
        self.Combo.connect('changed' , self.SecState)
        self.table.attach(self.SecondStateLab , 25 , 225 , 215 , 250)
        self.table.attach(self.Combo , 275 , 450 , 215 , 250)
        self.window.show_all()

    def SecState(self , widget):
        self.index = widget.get_active()
        self.PropertyEntry = gtk.Entry()
        self.PropertyLabel = gtk.Label('Enter ' + self.PropertyList[self.index])
        self.FinalList.append(self.PropertyList[self.index])
        widget.destroy()
        self.OptionsLabel = gtk.Entry()
        self.OptionsLabel.set_text(self.PropertyList[self.index])
        self.OptionsLabel.set_editable(False)
        self.table.attach(self.OptionsLabel , 275 , 450 , 215 , 250)
        self.table.attach(self.PropertyLabel , 25 , 225 , 310 , 345)
        self.table.attach(self.PropertyEntry , 275 , 450 , 310 , 345)
        self.button2= gtk.Button('Process')
        self.button2.connect('clicked' , self.FinalState)
        self.table.attach(self.button2 , 200 , 300 , 420 , 480)
        self.window.show_all()

    def FinalState(self , widget):
        widget.set_label('Reset')
        a = self._helper(self.FinalList)
        self.label.destroy()
        self.PresTempLab.destroy()
        self.PresTempEnt.destroy()
        self.Entry1.destroy()
        self.PropertyEntry.destroy()
        self.PropertyLabel.destroy()
        self.OptionsLabel.destroy()
        self.SecondStateLab.destroy()
        if a == 'Error':
            self.warning = gtk.Label('P L E A S E  E N T E R  V A L I D  I N P U T')
            self.table.attach(self.warning , 25 , 475 , 25 , 375)
            self.window.show_all()

        else:

            self.TemperatureEntry = gtk.Entry()
            self.TemperatureLabel = gtk.Label('Temperature')
            self.PressureEntry = gtk.Entry()
            self.PressureLabel = gtk.Label('Pressure')
            self.table.attach(self.TemperatureLabel, 25 , 225 , 25 , 55)
            self.table.attach(self.TemperatureEntry , 275 , 450 , 25 , 55)
            self.TemperatureEntry.set_text(str(a.GetTemp()))
            self.TemperatureEntry.set_editable(False)
            self.table.attach(self.PressureLabel, 25 , 225 , 75 , 105)
            self.table.attach(self.PressureEntry , 275 , 450 , 75 , 105)
            self.PressureEntry.set_text(str(a.GetPressure()))
            self.PressureEntry.set_editable(False)
            self.VolumeEntry = gtk.Entry()
            self.VolumeLabel = gtk.Label('Volume(v):')
            self.table.attach(self.VolumeLabel, 25 , 225 , 125 , 155)
            self.table.attach(self.VolumeEntry , 275 , 450 , 125 , 155)
            self.VolumeEntry.set_text(str(a.GetVolume()))
            self.VolumeEntry.set_editable(False)
            self.EnergyEntry = gtk.Entry()
            self.EnergyLabel = gtk.Label('Internal Energy(U):')
            self.table.attach(self.EnergyLabel, 25 , 225 , 175 , 205)
            self.table.attach(self.EnergyEntry , 275 , 450 , 175 , 205)
            self.EnergyEntry.set_text(str(a.GetEnergy()))
            self.EnergyEntry.set_editable(False)
            self.EnthalpyEntry = gtk.Entry()
            self.EnthalpyLabel = gtk.Label('Enthalpy(H):')
            self.table.attach(self.EnthalpyLabel, 25 , 225, 225 , 255)
            self.table.attach(self.EnthalpyEntry , 275 , 450 , 225 , 255)
            self.EnthalpyEntry.set_text(str(a.GetEnthalpy()))
            self.EnthalpyEntry.set_editable(False)
            self.EntropyEntry = gtk.Entry()
            self.EntropyLabel = gtk.Label('Entropy(s):')
            self.table.attach(self.EntropyLabel, 25 , 225 , 275 , 305)
            self.table.attach(self.EntropyEntry , 275 , 450 , 275 , 305)
            self.EntropyEntry.set_text(str(a.GetEntropy()))
            self.EntropyEntry.set_editable(False)
            self.QualityEntry = gtk.Entry()
            self.QualityLabel = gtk.Label('Quality(x):')
            self.table.attach(self.QualityLabel, 25 , 225 , 325 , 355)
            self.table.attach(self.QualityEntry , 275 , 450 ,325 , 355)
            self.QualityEntry.set_text(str(a.GetQuality()))
            self.QualityEntry.set_editable(False)
            self.window.show_all()

        widget.connect('clicked' , self.destroy)

    def destroy(self , widget):
        self.table.destroy()
        self.TableCreator()

class main:
    SteamSolver()
    gtk.main()

if __name__ == '__main__':
    main()


from tvm.relay.expr_functor import ExprMutator
#from collage.utils import *
import tvm.relay as relay

"""
This class annotates relay expression with compiler_begin and compiler_end
for each operator from target backend.
This will be merged by merge and partition pass following this.
"""
class ExtCompilerOpAnnotator(ExprMutator):
    def __init__(self, opt_match):
        super().__init__()
        self.opt_match = opt_match

    # read csv file and translate it into opt_match dictionary
    # (Key: expression / Value: annotation (group_id + op_name))
    def annotate(self, expr, target_str):
        self.target_str = target_str
        expr = self.visit(expr)
        return expr

    def is_target_op(self, expr):
        assert expr in self.opt_match

        is_target_op = False
        annotation = self.opt_match[expr]
        if get_backend_from_backend_pattern_annotation(annotation) == self.target_str:
            is_target_op = True

        return is_target_op

    # Visit Relay expressions in post-order
    def visit(self, expr):
        return super().visit(expr)

    def visit_call(self, call):
        new_fn = self.visit(call.op)
        new_args = [self.visit(arg) for arg in call.args]

        # Wanring(@soo): Is it okay?
        if self.is_target_op(call):
            new_args = [relay.annotation.compiler_begin(new_arg, self.target_str) for new_arg in new_args]

        new_call = relay.Call(new_fn, new_args, call.attrs, call.type_args, call.span)

        if self.is_target_op(call):
            new_call = relay.annotation.compiler_end(new_call, self.target_str)

        return new_call

    # def visit_function(self, fn):
    #     new_params = [self.visit(x) for x in fn.params]
    #     new_body = self.visit(fn.body)
    #     return Function(list(new_params), new_body, fn.ret_type, fn.type_params, fn.attrs)
    #
    # def visit_var(self, var):
    #     return var
    #
    # def visit_tuple(self, tup):
    #     # raise NotImplementedError
    #     return Tuple([self.visit(field) for field in tup.fields], tup.span)
    #
    # def visit_tuple_getitem(self, op):
    #     # raise NotImplementedError
    #     tuple_value = self.visit(op.tuple_value)
    #     if not tuple_value.same_as(op.tuple_value):
    #         return TupleGetItem(tuple_value, op.index)
    #     return op
    #
    # def visit_constant(self, const):
    #     return const


import gym

# Set learning parameters
LEARNING_RATE = .8
DISCOUNT_FACTOR = .95
NUM_EPISODES = 2000

def state_to_scalar(state):
    state_scalar = state.reshape(state.shape[0] * state.shape[1] * state.shape[2], 1)
    return state_scalar

env = gym.make('SpaceInvaders-v0')
env.reset()
print('env.observation_space.n', env.observation_space.n)

Q = np.zeros([env.observation_space.n,env.action_space.n])

for i_episode in range(NUM_EPISODES):
    state = observation = env.reset()
    state = state_to_scalar(state)
    for t in range(1000):
        print('state', state.shape)
        env.render()
        #print('env.action_space', env.action_space)
        #action = env.action_space.sample()
        action= np.argmax(Q[state,:] + np.random.randn(1,env.action_space.n)*(1./(i_episode+1)))
        state_updated, reward, done, info = env.step(action)
        state_updated = state_to_scalar(state_updated)
        print('reward', reward)

        state = state_updated
        if done:
            print("Episode finished after {} timesteps".format(t+1))
            break


# 複雑なソート
# 数値や文字列のようにそのものでソートできないものをソートする
class Tool:
  def __init__(self, name, weight):
    self.name = name
    self.weight = weight

  def __repr__(self):
    return f'Tool({self.name!r}, {self.weight})'

tools = [
  Tool('level', 3.5),
  Tool('hammer', 1.25),
  Tool('screwdriver', 0.5),
  Tool('chiel', 0.25),
]

print(tools)

# 単にソートはできない
try:
  tools.sort()
except Exception as e:
  print(e)

# keyを指定できる
print('Unsorted:', repr(tools))
tools.sort(key=lambda x: x.name)
print('Sorted:  ', tools)
tools.sort(key=lambda x: x.weight)
print('Sorted:  ', tools)

# lowerで大文字小文字を無視などもできる
places = ['home', 'work', 'New York', 'Paris']
places.sort()
print(places)
places.sort(key=lambda x: x.lower())
print(places)

# 複数基準のソートtupleをつかう
power_tools = [
  Tool('drill', 4),
  Tool('circular saw', 5),
  Tool('jackhammer', 40),
  Tool('sander', 4),
]
saw = (5, 'circular saw')
jackhammer = (40, 'jackhammer')
assert not (jackhammer < saw)
drill = (4, 'drill')
sander = (4, 'sander')
assert drill[0] == sander[0]
assert drill[1] < sander[1]
assert drill < sander
# keyをtupleにする
power_tools.sort(key=lambda x: (x.weight, x.name))
print(power_tools)
power_tools.sort(key=lambda x: (x.weight, x.name), reverse=True)
print(power_tools)
# 部分的にreverseにできる
power_tools.sort(key=lambda x: (-x.weight, x.name))
print(power_tools)
power_tools.sort(key=lambda x: (x.name, x.weight))
print(power_tools)


'''
283. Move Zeroes

Given an array nums, write a function to move all 0's to the end of it while maintaining the relative order of the non-zero elements.

Example:

    Input: [0,1,0,3,12]
    Output: [1,3,12,0,0]

Note:

    1.You must do this in-place without making a copy of the array.
    2.Minimize the total number of operations.
'''
class Solution(object):

    def move_zeroes(self, nums):
        # make non zeroes elements in [0, k)
        # k is slow runner point to the place index could swap in non zero element
        # i is fast runner iterate each element and swap non zeroes ones to k if i != k
        k = 0
        for i in range(len(nums)):
            if nums[i]:
                if(i != k):
                    nums[i], nums[k] = nums[k], nums[i]
                    k += 1
                else:
                    k += 1

if __name__ == '__main__':
    nums = [1, 2, 3, 0, 4, 5, 0, 3, 11]
    Solution().move_zeroes(nums)
    print(nums)


import pyaudio
import math
import struct

# Wut is diz??
Threshold = 10
SHORT_NORMALIZE = (1.0/32768.0)
swidth = 2

chunk = 1024
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 16000

f_name_directory = r'audiofiles'

class Listener:

    @staticmethod
    def rms(frame):
        count = len(frame) / swidth
        format = "%dh" % (count)
        shorts = struct.unpack(format, frame)

        sum_squares = 0.0
        for sample in shorts:
            n = sample * SHORT_NORMALIZE
            sum_squares += n * n
        rms = math.pow(sum_squares / count, 0.5)

        return rms * 1000

    def __init__(self):
        self.p = pyaudio.PyAudio()
        self.stream = self.p.open(format=FORMAT,
                                  channels=CHANNELS,
                                  rate=RATE,
                                  input=True,
                                  output=True,
                                  frames_per_buffer=chunk)

        self.sound_count = 0

    def react(self):
        self.sound_count += 1
        print(f'Noise detected. Count: {self.sound_count}')
        print("Listening for sound")

    def listen(self):
        print('Listening beginning')
        while True:
            input = self.stream.read(chunk)
            rms_val = self.rms(input)
            if rms_val > Threshold:
                self.react()

a = Listener()

a.listen()


import cv2
import numpy as np
from keras.models import load_model

model = load_model('CNNmodel.h5')

def prediction(pred):
    return chr(pred + 65)

def keras_predict(model1, image):
    data = np.asarray(image, dtype="int32")
    pred_probab = model1.predict(data)[0]
    pred_class = list(pred_probab).index(max(pred_probab))
    return max(pred_probab), pred_class

def keras_process_image(img):
    img = cv2.resize(img, (1, 28, 28), interpolation=cv2.INTER_AREA)

    return img

def crop_image(image, x, y, width, height):
    return image[y:y + height, x:x + width]

def main():
    while True:

        webcam = cv2.VideoCapture(0)
        rval, frame = webcam.read()
        frame = cv2.flip(frame, 1)

        im2 = crop_image(frame, 0, 300, 300, 300)
        image_grayscale = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY)

        image_grayscale_blurred = cv2.GaussianBlur(image_grayscale, (15, 15), 0)
        im3 = cv2.resize(image_grayscale_blurred, (28, 28), interpolation=cv2.INTER_AREA)

        im4 = np.resize(im3, (28, 28, 1))
        im5 = np.expand_dims(im4, axis=0)

        pred_probab, pred_class = keras_predict(model, im5)

        curr = prediction(pred_class)

        cv2.putText(frame, curr, (10, 300), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 255, 255), 2)

        cv2.rectangle(frame, (0, 300), (300, 600), (255, 255, 00), 3)
        cv2.imshow("frame", frame)

        cv2.imshow("Image3", image_grayscale_blurred)

        if cv2.waitKey(25) & 0xFF == ord('q'):
            cv2.destroyAllWindows()
            break

if __name__ == '__main__':
    main()

# cam_capture.release()
cv2.destroyAllWindows()


import shell_content
import os
import time
import signal
import subprocess
import numpy as np
import pydoop.hdfs as hdfs
import json

spark_conf_names = ['spark.default.parallelism', 'spark.driver.cores', 'spark.driver.memory',
                    'spark.driver.maxResultSize',
                    'spark.executor.instances', 'spark.executor.cores', 'spark.executor.memory',
                    'spark.executor.memoryOverhead',
                    'spark.files.maxPartitionBytes', 'spark.memory.fraction', 'spark.memory.storageFraction',
                    'spark.reducer.maxSizeInFlight',
                    'spark.shuffle.compress', 'spark.shuffle.file.buffer', 'spark.shuffle.spill.compress']

AppName = ['ConnectedComponent', 'DecisionTree', 'KMeans', 'LabelPropagation', 'LinearRegression',
           'LogisticRegression', 'PageRank',
           'PCA', 'PregelOperation', 'ShortestPaths', 'StronglyConnectedComponent', 'SVDPlusPlus', 'SVM', 'Terasort',
           'TriangleCount']

result_sizes = ['200m', '500m', '1g', '2g', '4g']

last_log = ""

def get_conf_str(params):
    executor_cores, executor_num, mem_fraction, executor_mem, parallelism_vals, driver_cores,\
    driver_mem, driver_maxResultSize, executor_memoryOverhead, files_maxPartitionBytes, mem_storageFraction,\
        reducer_maxSizeInFlight, shuffle_compress, shuffle_file_buffer, shuffle_spill_compress = params

    # executor_memoryOverhead = 16 - executor_mem
    return_str = ""
    return_str += ''' --conf "spark.executor.cores=''' + str(executor_cores) + '''" '''
    return_str += ''' --conf "spark.executor.memory=''' + str(executor_mem) + '''g" '''
    return_str += ''' --conf "spark.executor.instances=''' + str(executor_num) + '''" '''
    return_str += ''' --conf "spark.memory.fraction=''' + str(float(mem_fraction)/10) + '''" '''
    return_str += ('''--conf "spark.default.parallelism=''' + str(parallelism_vals) + '''" ''')
    return_str += ('''--conf "spark.driver.cores=''' + str(driver_cores) + '''" ''')
    return_str += ('''--conf "spark.driver.memory=''' + str(driver_mem) + '''g" ''')
    return_str += ('''--conf "spark.driver.maxResultSize=''' + str(result_sizes[int(driver_maxResultSize)]) + '''" ''')
    return_str += ('''--conf "spark.executor.memoryOverhead=''' + str(executor_memoryOverhead * 512) + '''m" ''')
    return_str += ('''--conf "spark.files.maxPartitionBytes=''' + str(files_maxPartitionBytes * 64) + '''m" ''')
    return_str += ('''--conf "spark.memory.storageFraction=''' + str(float(mem_storageFraction)/10) + '''" ''')
    return_str += ('''--conf "spark.reducer.maxSizeInFlight=''' + str(reducer_maxSizeInFlight*32) + '''m" ''')
    bool_vals = ['true', 'false']
    return_str += ('''--conf "spark.shuffle.compress=''' + str(bool_vals[int(shuffle_compress)]) + '''" ''')
    return_str += ('''--conf "spark.shuffle.file.buffer=''' + str(shuffle_file_buffer*32) + '''k" ''')
    return_str += ('''--conf "spark.shuffle.spill.compress=''' + str(bool_vals[int(shuffle_spill_compress)]) + '''" ''')
    return_str += ('''--conf "spark.network.timeout=''' + str(300) + '''" ''')
    return return_str

def get_shell_file(shell_file_path, params, workload_num):
    shell_file = open(shell_file_path, 'w', encoding='utf-8')
    shell_file.write(shell_content.front[workload_num])
    shell_file.write(
        '''    echo_and_run sh -c " ${SPARK_HOME}/bin/spark-submit --class $CLASS \
        --master ${APP_MASTER} ${SPARK_RUN_OPT} ''' +
        get_conf_str(params) + ''' $JAR ${OPTION} 2>&1|tee ${BENCH_NUM}/${APP}_run_${START_TS}.dat"''')
    shell_file.write(shell_content.rear)

def run_command(cmd_string, timeout=100):
    p = subprocess.Popen(cmd_string, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=True, close_fds=True, start_new_session=True)
    try:
        (msg, errs) = p.communicate(timeout=timeout)
        ret_code = p.poll()
        if ret_code:
            code = 1
            msg = "[Error]Called Error ： " + str(msg.decode('utf-8'))
        else:
            code = 0
            # msg = str(msg.decode('utf-8'))
            msg = "finished fucking successfully"
    except subprocess.TimeoutExpired:
        os.system("for i in  `yarn application  -list  | awk '{print $1}' | grep application_`; do yarn  application -kill $i; done ")
        p.kill()
        p.terminate()
        os.killpg(p.pid, signal.SIGTERM)
        code = 1
        msg = "[ERROR]Timeout Error : Command '" + cmd_string + "' timed out after " + str(timeout) + " seconds"
    except Exception as e:
        code = 1
        msg = "[ERROR]Unknown Error : " + str(e)
    return code, msg

def read_log(workload):
    path = "log/" + workload
    log = open(path,'r',encoding='utf-8')
    all_stage_info = {}
    for line in log:
        try:
            line_json = json.loads(line)
        except:
            print('json错误')
            continue
        # shuffle read/write、input/output
        if line_json['Event'] == 'SparkListenerTaskEnd':
            cur_stage = line_json['Stage ID']
            # new stage
            if line_json['Stage ID'] not in all_stage_info:
                all_stage_info[cur_stage] = [0, 0, 0, 0]
            # if line_json['Stage ID'] != cur_stage:
            #     cur_metrics, cur_stage = {'input': 0, 'output': 0, 'read': 0, 'write': 0}, line_json['Stage ID']
            try:
                all_stage_info[cur_stage][0] += line_json['Task Metrics']['Input Metrics']['Bytes Read']
                all_stage_info[cur_stage][1] += line_json['Task Metrics']['Output Metrics']['Bytes Written']
                all_stage_info[cur_stage][2] += (line_json['Task Metrics']['Shuffle Read Metrics']['Remote Bytes Read'] +
                                        line_json['Task Metrics']['Shuffle Read Metrics']['Local Bytes Read'])
                all_stage_info[cur_stage][3] += line_json['Task Metrics']['Shuffle Write Metrics']['Shuffle Bytes Written']
            except:
                print('metrics key error')
                break
    return len(all_stage_info.values()), list(all_stage_info.values())

def get_rs(action):

    # find log
    his_file_list = hdfs.ls("/history/")
    log_path = his_file_list[-1]
    global last_log

    if last_log == log_path:
        return 10000

    last_log = log_path
    print("Application:" + last_log)
    log_file = hdfs.open(log_path, 'rt', encoding='utf-8')

    start_timestamp = None
    end_timestamp = None
    for line in log_file:
        try:
            line_json = json.loads(line)
        except:
            print('json错误')
            continue

        if line_json['Event'] == 'SparkListenerEnvironmentUpdate':
            spark_props = line_json['Spark Properties']
            s = ''
            for conf_name in spark_conf_names:
                s = s + spark_props[conf_name] + ", "

            print()
            print(s)
            print(action)
            print()

        if line_json['Event'] == 'SparkListenerApplicationStart':
            start_timestamp = line_json['Timestamp']
        if line_json['Event'] == 'SparkListenerApplicationEnd':
            end_timestamp = line_json['Timestamp']

        if line_json['Event'] == 'SparkListenerJobEnd':
            if line_json['Job Result']['Result'] != 'JobSucceeded':
                break

    if start_timestamp and end_timestamp:
        duration = float(int(end_timestamp) - int(start_timestamp))/1000
    else:
        return 10000

    return duration

def run_bench(workload, params):
    workload_num = AppName.index(workload)
    # spark_bench_path = "C:/Users/86159/PycharmProjects/Reinforcement_Learning_in_Action-master/"
    spark_bench_path = "/home/spark_user/lib/spark-bench-legacy/"
    shell_file_path = spark_bench_path + workload + "/bin/my-run.sh"
    get_shell_file(shell_file_path, params, workload_num)
    # os.system("bash " + shell_file_path)
    code, msg = run_command("bash " + shell_file_path, timeout=600)
    time.sleep(10)
    # return code, msg

if __name__ == "__main__":
    run_bench("LinearRegression",[1, 1, 1, 1, 1, 1, 1, 0, 4, 1, 1, 1, 0, 1, 0])


import random
import time

def RandomSearch(A,k):
    n = len(A)
    checked = [0 for i in range(n)]
    while sum(checked)!=n:
        i = random.randint(0,n-1)
        if (A[i]==k):
             return i
        checked[i] = 1
    return -1

def LinearSearch(A,k):
    n = len(A)
    for i in range(n):
        if (A[i]==k):
            return i
    return -1

def ScrambleSearch(A,k):
    random.shuffle(A)
    n = len(A)
    for i in range(n):
        if (A[i]==k):
            return i
    return -1

def main():
    A = [random.randint(1,100000000)for i in range(100000000)]
    k = random.randint(1,100000000)
    start = time.time()
    print(LinearSearch(A,k), "time:",time.time()-start)
    start = time.time()
    print(RandomSearch(A,k), "time:",time.time()-start)
    start = time.time()
    print(ScrambleSearch(A,k),"time:",time.time()-start)

if __name__ == '__main__':
    main()


import sys
import atexit
import subprocess
import typing

class AutoPager:
    def __init__(self, no_pager: bool = False, always: bool = False):
        self.original_stdout = sys.stdout
        self.always = always
        self.no_pager = no_pager

        if self.original_stdout.isatty():
            self.process: typing.Optional[
                subprocess.Popen[str]] = subprocess.Popen(
                    ['/usr/bin/less', '-F'],
                    stdin=subprocess.PIPE,
                    encoding='utf-8')
            sys.stdout = self.process.stdin  # type: ignore
        else:
            self.process = None
        atexit.register(self.output)

    def output(self):
        if self.process:
            self.process.communicate()


# system imports
import logging
import sys
import os
import io
import concurrent.futures
import shutil
from time import sleep
#import keyring

# google and http imports
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaIoBaseDownload
from googleapiclient.http import MediaFileUpload
import googleapiclient
import google_auth_httplib2
import httplib2
from googleapiclient import discovery

# application imports
current = os.path.dirname(os.path.realpath(__file__))
parent = os.path.dirname(current)
sys.path.append(parent)
from libdata.data_types import *
from libdata.sqlite_store import *
from lib import mods
from lib import keyring
from lib import filewatcher
from config import config as cfg
#from lib.mods import *

# data structure for queueing changes
"""
class Change:
    def __init__(self, change: str = "", src_object=None, dst_object=None, type="", retry=0):
        if change not in ['modified', 'created', 'deleted', 'moved', 'closed']:
            raise "Invalid change type '%s'" % change
        if type not in ['file', 'directory']:
            raise "Invalid change type '%s'" % type
        self.change_type = change
        self.object_type=type
        self.change_object = src_object
        self.dst_object = dst_object
        self.retry = retry
"""

def test_func():
    print("test function called")

def login_to_drive():
    logging.info("initializing application credentials")
    creds = None
    # The file token.json stores the user's access and refresh tokens, and is
    # created automatically when the authorization flow completes for the first
    # time.
    if cfg.USE_KEYRING == True:
        kr = keyring.Keyring()
    else:
        kr = None

    if cfg.USE_KEYRING == True:
        logging.debug("looking for and existing token in the OS keyring")
        try:
            tokenStr = kr.get_data("gdrive", "token")
            if tokenStr is not None and tokenStr != "":
                tokenStr = json.loads(tokenStr)
                if tokenStr['scopes'] != cfg.TARGET_SCOPES:
                    creds = None
                    kr.delete_data("gdrive", "token")
                else:
                    creds = Credentials.from_authorized_user_info(tokenStr, cfg.TARGET_SCOPES)
        except Exception as err:
            logging.error("Unable to fetch the oauth token from the OS keyring. %s" % str(err))
    else:
        logging.debug("looking for an existing token in" + cfg.TOKEN_CACHE)
        if os.path.exists(cfg.TOKEN_CACHE):
            creds = Credentials.from_authorized_user_file(cfg.TOKEN_CACHE, cfg.TARGET_SCOPES)
            with open(cfg.TOKEN_CACHE, 'r') as tokenFile:
                token = json.loads(tokenFile.read())
                if token['scopes'] != cfg.TARGET_SCOPES:
                    logging.warning("token cache scopes are not valid, removing token")
                    creds = None
                    os.remove(cfg.TOKEN_CACHE)

    # If there are no (valid) credentials available, let the user log in.
    if not creds or not creds.valid:
        logging.warning("valid credentials weren't found, initializing oauth consent from in default browser.")
        try:
            if creds and creds.expired and creds.refresh_token:
                try:
                    creds.refresh(Request())
                except HttpError as err:
                    logging.error("error logging in to google drive. %s" % str(err))
                except Exception as err:
                    # if refersh token expired, remove the token cache and rerun self
                    if 'invalid_grant: Token has been expired or revoked.' in err.args[0]:
                        logging.warning("oauth refresh token expired, clearing token cache.")
                        if cfg.USE_KEYRING == True:
                            kr.delete_data("gdrive", "token")
                        else:
                            os.remove(cfg.TOKEN_CACHE)
                        login_to_drive()
                        return
                    logging.error("error logging in to Google Drive. %s" % str(err))
            else:
                flow = InstalledAppFlow.from_client_secrets_file(cfg.APP_CREDS, cfg.TARGET_SCOPES)
                creds = flow.run_local_server(port=0)
            # Save the credentials for the next run
            if cfg.USE_KEYRING == True:
                kr.store_data("gdrive", "token", creds.to_json())
            else:
                with open(cfg.TOKEN_CACHE, 'w+') as token:
                    logging.debug("saving credentials to " + cfg.TOKEN_CACHE)
                    token.write(creds.to_json())
        except HttpError as err:
            print(err)

    return creds

# Create a new Http() object for every request
# https://googleapis.github.io/google-api-python-client/docs/thread_safety.html
# overrides the constructor of the http2 object
def build_request(http, *args, **kwargs):
    new_http = google_auth_httplib2.AuthorizedHttp(cfg.CREDENTIALS, http=httplib2.Http())
    return googleapiclient.http.HttpRequest(new_http, *args, **kwargs)

# get the root folder
def get_root_folder(service) -> gFolder:
    logging.debug("fetching the root folder")
    rootFolder = None
    try:
        gServiceFiles = service.files()
        params = { "fileId": 'root'
        }
        request = gServiceFiles.get(**params)
        rootFolderResult =request.execute()
        rootFolder = gFolder(rootFolderResult)
    except HttpError as err:
        logging.error("error fetching the root folder." + str(err))
        print(err)
    return rootFolder

# print out the google drive folder tree (won't be used in production)
def print_folder_tree(folders = None):
    # grab the root folder
    rootFolder = list(filter(lambda rf: rf.id == cfg.ROOT_FOLDER_ID, folders))
    #print(rootFolder[0]['name'])

    def printTree(parent, level=0):
        print("-" * level + parent.name)
        for child in parent.children:
            printTree(child, level+1)

    #printTree(folders, rootFolder[0], 0)
    printTree(rootFolder[0], 0)
    return

def get_full_folder_path(service, folder: gFolder)-> str:
    full_path = str(folder.name)
    try:
        if 'parents' in folder.properties.keys():
            gServiceFiles = service.files()
            params = { "fileId": folder.properties['parents'][0], "fields": "parents, mimeType, id, name, ownedByMe"}
            request = gServiceFiles.get(**params)
            parent = request.execute()
            full_path = parent['name'] + "/" + full_path
            while 'parents' in parent.keys():
                params = { "fileId": parent['parents'][0], "fields": "parents, mimeType, id, name, ownedByMe"}
                request = gServiceFiles.get(**params)
                parent = request.execute()
                full_path = parent['name'] + "/" + full_path
            if parent['ownedByMe'] == False:
                # a folder shared outside of the current owner for the drive object.
                # stick in the root folder
                full_path = "_shared_withme/" + full_path

        else:
            if folder.properties['ownedByMe'] == False:
                full_path = "_shared_withme/" + full_path

    except Exception as err:
        logging.error("Error getting full local path for folder id %s. %s" % (folder.id, str(err)))
        print(str(err))

    return full_path

# download a single file (will be called multi-threaded)
def download_file(service, file: gFile, targetPath:str, threadSafeDB:sqlite_store = None):
    logging.debug("beginning to download file %s", file.name)
    sReturn = ""
    try:
        gServiceFiles = service.files()
        params = { "fileId": file.id,
                    "acknowledgeAbuse": True
        }
        request = gServiceFiles.get_media(**params)
        fileData = io.BytesIO()
        downloader = MediaIoBaseDownload(fileData, request)
        done = False
        logging.info("downloading file %s." % targetPath)
        #print("downloading file %s." % targetPath)

        fileDir = os.path.dirname(targetPath)
        if not os.path.exists(fileDir):
            logging.debug("file's parent directory '%s' doesn't exist, creating." % fileDir)
            os.makedirs(os.path.expanduser(fileDir))

        while done is False:
            status, done = downloader.next_chunk()
            #print(F'Download {int(status.progress() * 100)}.')

        with open(targetPath, "wb+") as f:
            f.write(fileData.getbuffer())

        file.localPath = targetPath
        file.md5 = mods.hash_file(targetPath)

        # update the file timestamp to match what's in Drive
        mod_time = int(datetime.datetime.strptime(file.properties['modifiedTime'][:-5], '%Y-%m-%dT%H:%M:%S').strftime("%s"))
        os.utime(targetPath, (mod_time, mod_time))

        if threadSafeDB is not None:
            threadSafeDB.insert_gObject(file=file)
        else:
            cfg.DATABASE.insert_gObject(file=file)

        fileSize = os.path.getsize(targetPath)
        sReturn = "file %s written %d byes." % (targetPath, fileSize)

    except HttpError as err:
        logging.error("error downloading file. %s" % str(err))
        print(err)
        sReturn = "file %s download failed with %s" % (targetPath, str(err))
    except Exception as err:
        logging.error("error downloading file. %s" % str(err))
        print(err)
        sReturn = "file %s download failed with %s" % (targetPath, str(err))
    return sReturn

# export a native google document format (can't be downloaded)
def export_native_file(service, file: gFile, targetPath: str)-> bool:
    logging.debug("exporting the native google application file %s.", file.name)
    bSuccess = False
    try:
        gServiceFiles = service.files()
        # get type of application
        targetMimeType = None
        if file.properties['mimeType'] in cfg.MEDIA_EXPORT_MATRIX.keys():
            targetMimeType = cfg.MEDIA_EXPORT_MATRIX[file.properties['mimeType']]["targetMimeType"]
            targetExtension = cfg.MEDIA_EXPORT_MATRIX[file.properties['mimeType']]["extension"]
            targetPath = targetPath + targetExtension
        if targetMimeType is None:
            return False
        params = { "fileId": file.id,
                    "mimeType": targetMimeType
        }
        request = gServiceFiles.export_media(**params)
        fileData = io.BytesIO()
        downloader = MediaIoBaseDownload(fileData, request)
        done = False
        while done is False:
            status, done = downloader.next_chunk()
            print(F'Download {int(status.progress() * 100)}.')

        with open(targetPath, "wb+") as f:
            f.write(fileData.getbuffer())

    except HttpError as err:
        logging.error("error exporting google application file. %s", str(err))
        print(err)
        bSuccess = False
    return bSuccess

# return a listing of files in a directory (non-recursive)
def list_files_in_dir(service, folder:gFolder, maxFiles = 1000) -> List[gFile]:
    logging.debug("listing files in %s directory", folder.name)
    files = []
    try:
        gServiceFiles = service.files()
        params = { "q": "mimeType!='application/vnd.google-apps.folder' and '" +
                    folder.id + "' in parents",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + cfg.FILE_FIELDS
        }
        request = gServiceFiles.list(**params)

        while (request is not None) and len(files) <= maxFiles:
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                objFile = gFile(f)
                objFile.md5 = None
                files.append(objFile)

            request = gServiceFiles.list_next(request, files_page)
    except HttpError as err:
        logging.error("error listing files." + str(err))
        print(err)
    return files

# download all files in a folder (non-recursive)
def download_files_from_folder(service, folder: gFolder, targetDir: str) -> bool:
    logging.debug("starting to download files from %s to %s" % (folder.name, targetDir))
    bResult = False
    try:
        files = list_files_in_dir(service, folder)

        # the google api module isn't thread safe, since it's based on http2 which also isn't thread safe
        # https://googleapis.github.io/google-api-python-client/docs/thread_safety.html
        with concurrent.futures.ThreadPoolExecutor(max_workers=cfg.MAX_THREADS) as executor:
            futures = []
            for f in files:
                if not "application/vnd.google-apps" in f.properties['mimeType']:
                    filePath = os.path.join(targetDir, folder.name, f.name)

                    # build a new http2 object to enable thread safety.  gets passed to each thread
                    credentials = Credentials.from_authorized_user_file(cfg.TOKEN_CACHE, cfg.TARGET_SCOPES)
                    authorized_http = google_auth_httplib2.AuthorizedHttp(credentials, http=httplib2.Http())
                    service = discovery.build('drive', 'v3', requestBuilder=build_request, http=authorized_http)

                    # build new database object for multi-threading too
                    threadSafeDB = sqlite_store()
                    threadSafeDB.open(cfg.DATABASE_PATH)

                    futures.append(executor.submit(
                            download_file, service, f, filePath, threadSafeDB
                        ))
            for future in concurrent.futures.as_completed(futures):
                result = future.result()
                logging.debug("Download result: %s" % str(result))
            #wait(futures) # want to make sure we don't start too many threads
    except Exception as err:
        logging.error("error downloading directory %s. %s." % (folder.name, str(err)))
        bResult = False
    return bResult

def write_folder_cache(service, localCachePath:str = cfg.FOLDERS_CACHE_PATH):
    logging.debug("writing local folder cache to %s." % str(localCachePath))
    try:
        # get the root folder
        gServiceFiles = service.files()
        if not cfg.ROOT_FOLDER_OBJECT:
            request = gServiceFiles.get(fileId = 'root')
            rootFolder = request.execute()

        else:
            rootFolder = cfg.ROOT_FOLDER_OBJECT.properties

        fRootFolder = open(cfg.FOLDERS_CACHE_PATH + "_root", "w+")
        fRootFolder.write(json.dumps(rootFolder, indent = 4))
        fRootFolder.close()

        #global ROOT_FOLDER_ID
        if cfg.ROOT_FOLDER_ID == '':
            cfg.ROOT_FOLDER_ID = rootFolder['id']

        #print('List files')

        pageToken = None
        params = { "q": "mimeType='application/vnd.google-apps.folder'",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + cfg.FOLDER_FIELDS
        }
        request = gServiceFiles.list(**params)

        while request is not None:
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                #print(f)
                with open(cfg.FOLDERS_CACHE_PATH + f['id'], 'w+') as folder_data:
                    folderObj = gFolder(f)
                    folderObj.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folderObj))
                    cfg.DATABASE.insert_gObject(folder=folderObj)
                    if 'parents' in folderObj.properties.keys():
                        cfg.DATABASE.insert_parents(folderObj.id, folderObj.properties['parents'])
                    folder_data.write(json.dumps(f, indent=5))
                    folder_data.close()

            request = gServiceFiles.list_next(request, files_page)

    except HttpError as err:
        logging.error("error writing local folder cache. %s", str(err))
        print(err)

# full sync down
def do_full_download(service, folder: gFolder, targetPath:str):
    logging.debug("starting full download from google drive to %s" % targetPath)
    try:
        download_files_from_folder(service, folder, os.path.join(targetPath))
        if folder.children is not None:
            for child in folder.children:
                do_full_download(service, child, os.path.join(targetPath, folder.name))

    except Exception as err:
        logging.error("error writing local folder cache. %s" % str(err))
        print(str(err))

# retrieve the metadata for Google object (file or folder)
def get_drive_object(service, id:str):
    return_object = None
    try:
        gServiceFiles = service.files()
        params = { "fileId": id,
                    "fields": "*"
        }
        request = gServiceFiles.get(**params)
        object = request.execute()
        if object is not None:
            if object['mimeType'] == 'application/vnd.google-apps.folder':
                return_object = gFolder(object)
            else:
                return_object = gFile(object)

    except HttpError as err:
        logging.error("Unable to fetch metadata from google drive for object id %s. %s" % (id, str(err)))
    except Exception as err:
        logging.error("Unable to fetch metadata from google drive for object id %s. %s" % (id, str(err)))

    return return_object

# create folder in Google Drive
def create_drive_folder(service, folderName:str, localPath:str, parentId:str=None) -> gFolder:
    folder = None
    try:

        if parentId is None or parentId == "":
            parentId = cfg.ROOT_FOLDER_ID
        file_metadata = {
            'name': folderName,
            'mimeType': 'application/vnd.google-apps.folder',
            'parents': parentId
        }

        logging.info("creating folder %s in Google Drive" % folderName)
        f = service.files().create(body=file_metadata, fields='*').execute()
        folder = gFolder(f)
        folder.localPath = localPath
        cfg.DATABASE.insert_gObject(folder=folder)

    except HttpError as err:
        logging.error("error creating Google Drive folder. %s" % str(err))
    except Exception as err:
        logging.error("error creating Google Drive folder. %s" % str(err))

    return folder

# create the entire folder tree, if any part doesn't exist
def create_drive_folder_tree(service, folderPath:str) -> gFolder:
    parentFolder = None
    try:
        if cfg.DRIVE_CACHE_PATH not in cfg.ROOT_FOLDER_OBJECT.localPath:
            raise "folder path isn't the defined drive cache path."
            return
        folderPath = folderPath.replace(cfg.ROOT_FOLDER_OBJECT.localPath + "/", "")
        folders = folderPath.split(os.sep)
        parent = cfg.ROOT_FOLDER_OBJECT
        currentFolder = cfg.ROOT_FOLDER_OBJECT.localPath
        for folder in folders:
            currentFolder = os.path.join(currentFolder, folder)
            c, dbFolders = cfg.DATABASE.fetch_gObjectSet(searchField="local_path", \
                            searchCriteria=currentFolder)
            if len(dbFolders) == 0:
                parent = create_drive_folder(service, folder, currentFolder, parent.id)
            else:
                parent = dbFolders[0]

        parentFolder = parent

    except HttpError as err:
        logging.error("error creating Google Drive folder tree. %s" % str(err))
    except Exception as err:
        logging.error("error creating Google Drive folder tree. %s" % str(err))
    return parentFolder

def upload_drive_file(service, filePath:str, parentId:str = None)-> gFile:
    file = None
    attempt = 1
    try:
        while attempt <= cfg.UPLOAD_RETRIES_MAX:
            # if file is under 5 mb perform a simple upload
            fileSize = os.path.getsize(filePath)
            fileHash = mods.hash_file(filePath)
            if fileSize <= (5 * 1024 * 1024):
                f = upload_drive_file_simple(service, filePath, parentId)
            else:
                # we need to figure out how to resumable uploads at some point later
                f = upload_drive_file_simple(service, filePath, parentId)
            file = gFile(f)
            if fileHash != file.properties['md5Checksum']:
                logging.warning("File upload resulted in a hash mismatch.")
                # remove the file
                file = delete_drive_file(service, file)
                attempt += 1
            else:
                file.localPath = filePath
                file.md5 = fileHash
                cfg.DATABASE.insert_gObject(file=file)
                break
        if attempt == cfg.UPLOAD_RETRIES_MAX:
            logging.error("Exceeded max retries to upload file '%s'" % filePath)
    except HttpError as err:
        logging.error("error uploading file to Google Drive. %s" % str(err))
    except Exception as err:
        logging.error("error uploading file to Google Drive. %s" % str(err))
    return file

def upload_drive_file_simple(service, filePath:str, parentId:str=None)->gFile:
    file = None
    try:
        if parentId is None or parentId == "":
            parentId = cfg.ROOT_FOLDER_ID
        fileDir = os.path.dirname(filePath)
        fileName = os.path.basename(filePath)
        logging.info("performing simple upload of file '%s'" % filePath)
        file_metadata = {'name': fileName, 'parents': parentId}
        media = MediaFileUpload(filePath, resumable=True)
        file = service.files().create(body=file_metadata, media_body=media,
                                      fields='*').execute()
    except HttpError as err:
        logging.error("error downing a simple file upload to Google Drive. %s" % str(err))
    except Exception as err:
        logging.error("error downing a simple file upload to Google Drive. %s" % str(err))
    return file

# uploads new files that have been identified as missing from the cloud post reconciliation
def upload_new_local_files(service):
    logging.debug("starting to upload new local files to the cloud.")
    try:
        records, new_local_files = cfg.DATABASE.fetch_newLocalFiles()
        recordsParsed = 0
        while records > 0:
            for f in new_local_files:
                if cfg.ROOT_FOLDER_OBJECT.localPath in f.localPath:
                    if f.mimeType == cfg.TYPE_GOOGLE_FOLDER:
                        return
                    else:
                        parentFolder = os.path.dirname(f.localPath)
                        c, db_parentFolders = cfg.DATABASE.fetch_gObjectSet(searchField = "local_path", \
                                                    searchCriteria=parentFolder)
                        db_parentFolder = db_parentFolders[0]
                        if db_parentFolder is not None:
                            f.properties['parents'] = [db_parentFolder.id]
                        else:
                            parent = create_drive_folder_tree(service, parentFolder)
                            f.properties['parents'] = parent.id
                        #file = upload_drive_file(service, f.localPath, f.properties['parents'][0])
                        change = filewatcher.Change('created', f.localPath, None, 'file')
                        cfg.LOCAL_QUEUE.put(change)
                else:
                    logging.warning("skipping file '%s'. path not in local cache directory." % f.localPath)
                recordsParsed += 1
            records, new_local_files = cfg.DATABASE.fetch_newLocalFiles(offset=recordsParsed)
    except Exception as err:
        logging.error("error uploading new files to the cloud. %s" % str(err))

def update_drive_file(service, file:gFile, localPath:str):
    logging.info("updating Google drive file %s." % file.name)
    updated_file = None
    try:
        media_body = MediaFileUpload(localPath, resumable=True)

        # Send the request to the API.
        updated_file = service.files().update(
            fileId=file.id,
            #body=file.properties,
            media_body=media_body).execute()
        # get the encriched full metadata
        updated_file = get_drive_object(service, updated_file['id'])
        updated_file.localPath = file.localPath
        updated_file.md5 = file.md5

        cfg.DATABASE.update_gObject(file=updated_file)

    except HttpError as err:
        logging.error("error updating Google drive file '%s'. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error updating Google drive file '%s'. %s" % (file.name, str(err)))
    return updated_file

def update_drive_files(service):
    logging.debug("starting to update changed local files to the cloud.")
    try:
        c, changed_local_files = cfg.DATABASE.fetch_changedLocalFiles()
        recordsParsed = 0
        while c > 0:
            for f in changed_local_files:
                if cfg.ROOT_FOLDER_OBJECT.localPath in f.localPath:
                    if f.mimeType == cfg.TYPE_GOOGLE_FOLDER:
                        return
                    else:
                        #parentFolder = os.path.dirname(f.localPath)
                        #db_parentFolders, c = cfg.DATABASE.fetch_gObjectSet(searchField = "local_path", \
                        #                            searchCriteria=parentFolder)
                        #db_parentFolder = db_parentFolders[0]
                        #if db_parentFolder is not None:
                        #    f.properties['parents'] = [db_parentFolder.id]
                        #else:
                        #    parent = create_drive_folder_tree(service, parentFolder)
                        #    f.properties['parents'] = parent.id
                        #file = update_drive_file(service, f, f.localPath)
                        change = filewatcher.Change('modified', f.localPath, None, 'file')
                        cfg.LOCAL_QUEUE.put(change)
                else:
                    logging.warning("skipping file '%s'. path not in local cache directory." % f.localPath)
                recordsParsed += 1
            c, changed_local_files = cfg.DATABASE.fetch_newLocalFiles(offset=recordsParsed)
    except Exception as err:
        logging.error("error updating cloudfile '%s'. %s" % (f.name, str(err)))

def move_drive_file(service, file:gFile, newParent_id: str=None, newName:str = None) -> gFile:
    try:
        if file.properties['parents'][0] != newParent_id and newParent_id is not None:
            prev_parents = ','.join(file.properties['parents'])
            file = service.files().update(fileId=file.id, addParents=newParent_id,
                                        removeParents=prev_parents,
                                        fields='id, parents').execute()
            file = get_drive_object(service, file['id'])
            logging.info("moved file ID '%s' to new parent ID '%s'" % (file.id, newParent_id))
        else:
            if file.name != newName:
                file = service.files().update(fileId=file.id, body={'name': newName}).execute()
                file = get_drive_object(service, file['id'])
            else:
                logging.warning("Unable to process file '%s' move.  Can't parse the change." % file.id)
    except HttpError as err:
        logging.error("error moving file '%s' in Google Drive. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error moving file '%s' in Google Drive. %s" % (file.name, str(err)))
    return file

def delete_drive_file(service, file:gFile):
    #file = None
    try:
        file.properties['trashed'] = True
        gSerivceFiles = service.files()
        gSerivceFiles.delete(fileId = file.id).execute()
        logging.info("deleted Google Drive file with id %s" % file.id)
        #cfg.DATABASE.update_gObject(file=file)
        cfg.DATABASE.delete_gObject(id=file.id) # do we want to delete the file?  or just mark it as trashed?

    except HttpError as err:
        # 404, delete from db since it's gone from Drive
        if err.resp.status == 404:
            logging.info("File not found in Drive, removing from db.")
            cfg.DATABASE.delete_gObject(id=file.id) # do we want to delete the file?  or just mark it as trashed?
        else:
            logging.error("error deleteing file '%s' from Google Drive. %s" % (file.name, str(err)))
    except Exception as err:
        logging.error("error deleteing file '%s' from Google Drive. %s" % (file.name, str(err)))
    return file

# region : Change tracking in Google drive

# gets the change token for changes in the drive since last sync
# https://developers.google.com/drive/api/guides/manage-changes
def get_drive_changes_token(service):
    logging.info("fetching the start changes token from Google Drive.")
    startToken = None
    try:
        response = service.changes().getStartPageToken().execute()
        startToken = response.get("startPageToken")
    except HttpError as err:
        logging.error("error getting changes start token. %s", str(err))
        print(err)
    except Exception as err:
        logging.error("error getting changes start token. %s", str(err))
        print(str(err))

    return startToken

# get changes since the last change token fetch
# https://developers.google.com/drive/api/guides/manage-changes
def get_drive_changes(service, changeToken):
    changes = []
    try:
        while changeToken is not None:
            response = service.changes().list(pageToken=changeToken,
                                              spaces='drive').execute()
            for change in response.get('changes'):
                # Process change
                changes.append(change)
            if 'newStartPageToken' in response:
                # Last page, save this token for the next polling interval
                #global CHANGES_TOKEN
                cfg.CHANGES_TOKEN = response.get('newStartPageToken')
            changeToken = response.get('nextPageToken')
    except HttpError as err:
        logging.error("error getting changes from Drive. %s", str(err))
        print(err)
    except Exception as err:
        logging.error("error getting changes from Drive. %s", str(err))
        print(str(err))

    return changes

# handles any sort of change in a file in google drive (create, update, delete)
def handle_changed_file(service, file:gFile = None):
    try:
        parents = []
        if file is not None:
            # ******************************************
            #      create or update an existing file
            # ******************************************
            dbFiles = cfg.DATABASE.fetch_gObject(file.id)
            # **** handle file creation ****
            if len(dbFiles) > 1:
                logging.warn("file id %s has multiple entries in the database. skipping." % file.id)
            elif len(dbFiles) == 0:
                # **** handle new files from Google Drive ****
                logging.debug("file id %s isn't in the database, assuming a new object." % file.id)
                if 'parents' in file.properties.keys():
                    for parent_id in file.properties['parents']:
                        parent_folder = get_drive_object(service, parent_id)
                        full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                            get_full_folder_path(service, parent_folder), \
                            file.name)
                        full_path = os.path.expanduser(full_path)
                        file.localPath = full_path
                        cfg.DATABASE.insert_gObject(file=file)

                        if file.properties['trashed'] == False:
                            cfg.LQUEUE_IGNORE.append(full_path)
                            download_file(service, file, full_path)

            else:
                # **** handle file updates ****
                dbFile  = dbFiles[0]
                if file.properties != dbFile.properties and int(file.properties['version']) > int(dbFile.properties['version']):
                    # if the md5 is different for the file, then we are going to remove the local version and re-download
                    logging.info("file id %s is newer in the cloud and has changes, processing." % file.id)
                    if (file.properties['trashed'] == False):
                        file.localPath = dbFile.localPath
                        if (file.properties['md5Checksum'] != dbFile.md5 or file.name != dbFile.name):
                            file.md5 = dbFile.md5 # we'll download it later if we need to
                            try:
                                # delete the existing files and redownload for each instance of the file
                                if 'parents' in file.properties.keys():
                                    for parent_id in file.properties['parents']:
                                        for db_parent_id in dbFile.properties['parents']:
                                            parent_folder = get_drive_object(service, parent_id)
                                            root_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                                get_full_folder_path(service, parent_folder))
                                            full_path = os.path.join(root_path, file.name)
                                            full_path = os.path.expanduser(full_path)

                                            parent_folder = get_drive_object(service, db_parent_id)
                                            root_path_old = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                                get_full_folder_path(service, parent_folder))
                                            full_path_old = os.path.join(root_path_old, dbFile.name)
                                            full_path_old = os.path.expanduser(full_path_old)

                                            # do the the redownload if the md5 doesn't match
                                            if file.properties['md5Checksum'] != dbFile.md5:
                                                logging.info("file id %s checksum is different and cloud version is newer, redownloading." % file.id)

                                                if file.properties['trashed'] == False:
                                                    cfg.LQUEUE_IGNORE.append(full_path)
                                                    if os.path.exists(full_path):
                                                        logging.info("removing outdated file '%s'." % full_path)
                                                        os.remove(full_path)
                                                    download_file(service, file, full_path)

                                            # do the rename
                                            if file.name != dbFile.name:
                                                if root_path_old == root_path:
                                                    if file.properties['trashed'] == False:
                                                        cfg.LQUEUE_IGNORE.append(full_path_old)
                                                        cfg.LQUEUE_IGNORE.append(full_path)
                                                        os.rename(full_path_old, full_path)
                                                        cfg.DATABASE.update_gObject(file=file)
                                                        #sleep(0.2) # give the Watchdog service time to catch up
                                                        #cfg.LQUEUE_IGNORE.remove(full_path_old)
                                                        #cfg.LQUEUE_IGNORE.remove(full_path)

                            except Exception as err:
                                logging.error("unable to update file id %s. %s" % (file.id, str(err)))
                        else:
                            file.md5 = dbFile.md5
                            cfg.DATABASE.update_gObject(file=file)

                    # ***** delete a local file ******
                    elif file.properties['trashed'] == True:
                        file.md5 = dbFile.md5
                        file.localPath = dbFile.localPath
                        cfg.DATABASE.update_gObject(file=file)
                        if 'parents' in file.properties.keys():
                            for parent_id in file.properties['parents']:
                                try:
                                    parent_folder = get_drive_object(service, parent_id)
                                    full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                        get_full_folder_path(service, parent_folder), \
                                        file.name)
                                    full_path = os.path.expanduser(full_path)
                                    if os.path.exists(full_path):
                                        logging.info("removing trashed file '%s'" % full_path)
                                        cfg.LQUEUE_IGNORE.append(full_path)
                                        os.remove(full_path)
                                        #sleep(0.2) # give the Watchdog service time to catch up
                                        #cfg.LQUEUE_IGNORE.remove(full_path)
                                except Exception as err:
                                    logging.error("unable to remove local file %s. %s" % (full_path, str(err)))

    except Exception as err:
        logging.error("error processing Google object change. %s" % str(err))
    except HttpError as err:
        logging.error("error processing Google object change. %s" % str(err))
    return

# handles any sort of folder change in google drive (create, update, delete)
def handle_changed_folder(service, folder: gFolder = None):
    try:

        parents = []

        if folder is not None:
            # *************************************************************************
            #    create or update an existing folder
            # *************************************************************************
            dbFolders = cfg.DATABASE.fetch_gObject(folder.id)
            if len(dbFolders) > 1:
                logging.warn("folder id %s has multiple entries in the database. skipping." % folder.id)
            elif len(dbFolders) == 0:
                logging.debug("folder %s isn't in the database, assuming a new object." % folder.id)
                folder.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folder))
                cfg.DATABASE.insert_gObject(folder=folder)
                if 'parents' in folder.properties.keys():
                    for parent_id in folder.properties['parents']:
                        parent_folder = get_drive_object(service, parent_id)
                        full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                            get_full_folder_path(service, parent_folder), \
                            folder.name)
                        full_path = os.path.expanduser(full_path)
                        if not os.path.exists(full_path):
                            cfg.LQUEUE_IGNORE.append(full_path)
                            logging.info("creating new local folder '%s'" % full_path)
                            os.mkdir(os.path.expanduser(full_path))

            else:
                # if folder name is different, rename it.  if it's trashed, remove it.  only changes possible for folders
                dbFolder = dbFolders[0]
                if folder.properties != dbFolder.properties and int(folder.properties['version']) > int(dbFolder.properties['version']):
                    logging.info("folder id %s has a later version and different properties in Google Drive, applying changes" % folder.id)
                    # update the folder properties in the db
                    folder.localPath = os.path.join(cfg.DRIVE_CACHE_PATH, get_full_folder_path(service, folder))
                    cfg.DATABASE.update_gObject(folder=folder)
                    # **** rename the local folder(s) ****
                    if folder.name != dbFolder.name and folder.properties['trashed'] == False:
                        for parent_id in folder.properties['parents']:
                            for db_parent_id in dbFolder.properties['parents']:
                                parent_folder = get_drive_object(service, parent_id)
                                root_path_new = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder))
                                full_path_new = os.path.join(root_path_new, folder.name)
                                full_path_new = os.path.expanduser(full_path_new)

                                parent_folder = get_drive_object(service, db_parent_id)
                                root_path_old = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder))
                                full_path_old = os.path.join(root_path_old, dbFolder.name)
                                full_path_old = os.path.expanduser(full_path_old)

                                if root_path_old == root_path_new:
                                    cfg.LQUEUE_IGNORE.append(full_path_new)
                                    cfg.LQUEUE_IGNORE.append(full_path_old)
                                    os.rename(full_path_old, full_path_new)

                    # ***** delete a local folder ******
                    if folder.properties['trashed'] == True:
                        if 'parents' in folder.properties.keys():
                            for parent_id in folder.properties['parents']:
                                parent_folder = get_drive_object(service, parent_id)
                                full_path = os.path.join(cfg.DRIVE_CACHE_PATH, \
                                    get_full_folder_path(service, parent_folder), \
                                    folder.name)
                                full_path = os.path.expanduser(full_path)
                                if os.path.exists(full_path):
                                    cfg.LQUEUE_IGNORE.append(full_path)
                                    logging.info("removing trashed directory '%s'" % full_path)
                                    shutil.rmtree(full_path)

    except Exception as err:
        logging.error("error processing Google object change. %s" % str(err))
    except HttpError as err:
        logging.error("error processing Google object change. %s" % str(err))
    return

# scans all files in Google drive that aren't in the db.  that's our change set.
# this only needs to happen during startup.  otherwise, change notifications will do the job
def get_gdrive_changes(service) -> List:
    # loop through the pages of files from google drive
    # return the md5Checksum property, along with name, id, mimeType, version, parents
    # compare files by id with the db.  look where the md5Checksum != md5 stored in the db
    # also look for files not in the db
    # important: if the file in google drive is a later version, it's authoritative
    # this will be the changes from the side of google drive

    logging.info("scanning google drive files, looking for files and folders that have changed.")
    differences = []
    try:
        gServiceFiles = service.files()
        params = { "q": "'me' in owners",
                    "pageSize": cfg.PAGE_SIZE,
                    "fields": "nextPageToken," + "files(id, name, mimeType, version, md5Checksum, parents, ownedByMe)"
        }
        request = gServiceFiles.list(**params)

        while (request is not None):
            files_page = request.execute()
            fs = files_page.get('files', [])
            for f in fs:
                dbFile = None
                rows = cfg.DATABASE.fetch_gObject(f['id'])
                if len(rows) > 0:
                    dbFile = rows[0]
                if f['mimeType'] == cfg.TYPE_GOOGLE_FOLDER:
                    googleFolder = gFolder(f)
                    if dbFile is not None and 'version' in dbFile.properties.keys():
                        # if (dbFile.id != googleFolder.id or \
                        #            dbFile.name != googleFolder.name) and \
                        #            dbFile.properties['version'] < googleFolder.properties['version']:
                        if (int(dbFile.properties['version']) < int(googleFolder.properties['version'])):
                            # fetch full metadata of the file
                            get_params = {"fileId": googleFolder.id, "fields": "*"}
                            get_req = gServiceFiles.get(**get_params)
                            full_folder = gFolder(get_req.execute())
                            full_folder.localPath = get_full_folder_path(service, full_folder)
                            #differences.append(full_folder)
                            cfg.REMOTE_QUEUE.put(full_folder)
                    else:
                        get_params = {"fileId": googleFolder.id, "fields": "*"}
                        get_req = gServiceFiles.get(**get_params)
                        full_folder = gFolder(get_req.execute())
                        #differences.append(full_folder)
                        cfg.REMOTE_QUEUE.put(full_folder)

                else:
                    googleFile = gFile(f)
                    if dbFile is not None and 'version' in dbFile.properties.keys():
                        #if (dbFile.md5 != googleFile.properties['md5Checksum'] or \
                        #    dbFile.mimeType != googleFile.mimeType) and \
                        #    dbFile.properties['version'] < googleFile.properties['version']:
                        if (int(dbFile.properties['version']) < int(googleFile.properties['version'])):
                                # fetch full metadata of the file
                                get_params = {"fileId": googleFile.id, "fields": "*"}
                                get_req = gServiceFiles.get(**get_params)
                                full_file = gFile(get_req.execute())
                                #differences.append(full_file)
                                cfg.REMOTE_QUEUE.put(full_file)
                    else:
                        if cfg.TYPE_GOOGLE_APPS not in googleFile.mimeType:
                            get_params = {"fileId": googleFile.id, "fields": "*"}
                            get_req = gServiceFiles.get(**get_params)
                            full_file = gFile(get_req.execute())
                            #differences.append(full_file)
                            cfg.REMOTE_QUEUE.put(full_file)
            request = gServiceFiles.list_next(request, files_page)
    except HttpError as err:
        #exc_type, exc_obj, exc_tb = sys.exc_info()
        #fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        #logging.error(exc_type, fname, exc_tb.tb_lineno)
        logging.error("error scanning google drive files. %s" % str(err))
        print(err)
    except Exception as err:
        #exc_type, exc_obj, exc_tb = sys.exc_info()
        #fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
        #logging.error(exc_type, fname, exc_tb.tb_lineno)
        logging.error("error scanning google drive files. %s" % str(err))
        print(err)
    return differences

# endregion


from base64 import b64encode

from django import template

CSS_CLASS = 'dj-special-block'
KEY_PREFIX = 'dj-special-{}'
block_count = 0
register = template.Library()

class ShideNode(template.Node):
    template = "

"\
               "" % CSS_CLASS

    def __init__(self, nodelist):
        self.nodelist = nodelist

    def render(self, context):
        output = self.nodelist.render(context)
        prepared_output = self.template.format(
            key_name=KEY_PREFIX.format(block_count), data=b64encode(output))
        block_count += 1
        return prepared_output

def do_shide(parser, token):
    nodelist = parser.parse(('endshide',))
    parser.delete_first_token()
    return ShideNode(nodelist)

@register.simple_tag
def post_shide():
    """ This is just minified version of repo root file unhide.js
    """
    return """""".format(css_name=CSS_CLASS)