from odoo import models, api, fields,_

class SaleOrder(models.Model):
    _inherit = "sale.order"

    responsible_id = fields.Many2one( "res.users",string='Resposible')

    # ('warehouse_user', 'in', self.env.user.ids)

class StockPicking(models.Model):
    _inherit = "stock.picking"

    user_id = fields.Many2one('res.users', 'Responsible', tracking=True,
        domain=lambda self: [('groups_id', 'in', self.env.ref('stock.group_stock_user').id),
        ('groups_id', 'in', self.env.ref('odoov13_7.group_stock_user_reposible').id),
        ('warehouse_user' ,'in', self.domain_user_id.ids)],

        states={'done': [('readonly', True)], 'cancel': [('readonly', True)], 'draft': [('readonly', False)],'draft': [('readonly', False)],
        		'assigned': [('readonly', False)],'confirmed': [('readonly', False)],'waiting': [('readonly', False)]},
        store=True,compute='_compute_user')
    domain_user_id = fields.Many2one( "stock.warehouse",string='Domain',compute='get_warehouse_id',store=True)

    @api.depends('picking_type_id')
    def get_warehouse_id(self):
        for l in self:
            if l.picking_type_id.warehouse_id:
                l.domain_user_id = l.picking_type_id.warehouse_id.id

    @api.depends('sale_id')
    def _compute_user(self):
        for l in self:
            ordr_id = self.env['sale.order'].search([('id','=',l.sale_id.id)])
            for line in ordr_id:
                l.user_id = line.responsible_id.id

class WarehouseUser(models.Model):
    _inherit='res.users'
    warehouse_user = fields.Many2one("stock.warehouse", string="Warehouse")


import wandb
import pandas as pd

api = wandb.Api()
run = api.run("/surcyf2/openai_qa/runs/wev5by5x")
history = run.history()

# Write the history to a file in JSON format using pandas
history.to_json('run_history.json', orient='records', lines=True, indent=4)

print("History has been saved to run_history.json")


import os
import json
import collections
import buildbot
import buildbot.process.factory
from buildbot.steps.source import SVN
from buildbot.steps.shell import Configure, SetProperty
from buildbot.steps.shell import ShellCommand, WarningCountingShellCommand
from buildbot.steps.slave import RemoveDirectory
from buildbot.process.properties import WithProperties, Property
from buildbot.steps import trigger
import zorg.buildbot.commands.BatchFileDownload as batch_file_download
from zorg.buildbot.commands.LitTestCommand import LitTestCommand
from zorg.buildbot.builders.Util import getVisualStudioEnvironment
from zorg.buildbot.builders.Util import extractSlaveEnvironment
from zorg.buildbot.process.factory import LLVMBuildFactory

# We *must* checkout at least Clang, LLVM, and LLDB.  Also check out LLD since
# it is needed to run the LLDB test suite.
def getLLDBSource(f,llvmTopDir='llvm'):
    f.addStep(SVN(name='svn-llvm',
                  mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
                  defaultBranch='trunk',
                  workdir=llvmTopDir))
    f.addStep(SVN(name='svn-clang',
                  mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
                  defaultBranch='trunk',
                  workdir='%s/tools/clang' % llvmTopDir))
    f.addStep(SVN(name='svn-lld',
                  mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/',
                  defaultBranch='trunk',
                  workdir='%s/tools/lld' % llvmTopDir))
    f.addStep(SVN(name='svn-lldb',
                  mode='update', baseURL='http://llvm.org/svn/llvm-project/lldb/',
                  defaultBranch='trunk',
                  workdir='%s/tools/lldb' % llvmTopDir))
    return f

# Clean SVN source tree
# SVN doesn't provide build-in commands to remove all untracked files
# List all untracked files, and remove one by one
def cleanSVNSourceTree(f, srcdir='llvm'):
    f.addStep(ShellCommand(name='clean svn source %s' % srcdir,
                           command="svn status --no-ignore | grep '^[I?]' | cut -c 9- | while IFS= read -r f; do echo \"$f\"; rm -rf \"$f\"; done",
                           description="clean SVN source tree",
                           workdir='%s' % srcdir))
    return f

# CMake builds
def getLLDBCMakeBuildFactory(
            clean=False,
            cmake='cmake',
            jobs="%(jobs)s",

            # Source directory containing a built python
            python_source_dir=None,

            # Default values for VS devenv and build configuration
            vs=None,
            config='Release',
            target_arch='x86',

            extra_cmake_args=None,
            test=False,
            testTimeout=2400,
            install=False):

    ############# PREPARING
    f = buildbot.process.factory.BuildFactory()

    # Determine Slave Environment and Set MSVC environment.
    if vs:
        f.addStep(SetProperty(
            command=getVisualStudioEnvironment(vs, target_arch),
            extract_fn=extractSlaveEnvironment))

    f = getLLDBSource(f,'llvm')

    build_cmd=['ninja']
    install_cmd = ['ninja','install']
    test_cmd = ['ninja','check-lldb']

    if jobs:
        build_cmd.append(WithProperties("-j%s" % jobs))
        install_cmd.append(WithProperties("-j%s" % jobs))
        test_cmd.append(WithProperties("-j%s" % jobs))

    # Global configurations
    build_dir='build'

    # get full path to build directory
    f.addStep(SetProperty(name="get_builddir",
                          command=["pwd"],
                          property="builddir",
                          description="set build dir",
                          workdir=build_dir))

    ############# CLEANING
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(RemoveDirectory(name='clean '+build_dir,
                dir=build_dir,
                haltOnFailure=False,
                flunkOnFailure=False,
                doStepIf=cleanBuildRequested
                ))

    cmake_cmd = [
        "cmake", "-G", "Ninja", "../llvm",
        "-DCMAKE_BUILD_TYPE=" + config,
        "-DCMAKE_INSTALL_PREFIX=../install"
        ]
    if python_source_dir:
        cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir)
    if extra_cmake_args:
        cmake_cmd += extra_cmake_args
    # Note: ShellCommand does not pass the params with special symbols right.
    # The " ".join is a workaround for this bug.
    f.addStep(ShellCommand(name="cmake-configure",
                           description=["cmake configure"],
                           command=WithProperties(" ".join(cmake_cmd)),
                           haltOnFailure=True,
                           warnOnWarnings=True,
                           workdir=build_dir,
                           env=Property('slave_env')))

    f.addStep(WarningCountingShellCommand(name='build',
                          command=build_cmd,
                          haltOnFailure=True,
                          description='ninja build',
                          workdir=build_dir,
                          env=Property('slave_env')))

    ignoreInstallFail = bool(install != 'ignoreFail')
    f.addStep(ShellCommand(name='install',
                          command=install_cmd,
                          flunkOnFailure=ignoreInstallFail,
                          description='ninja install',
                          workdir=build_dir,
                          doStepIf=bool(install),
                          env=Property('slave_env')))

    ignoreTestFail = bool(test != 'ignoreFail')
    f.addStep(ShellCommand(name='test',
                          command=test_cmd,
                          flunkOnFailure=ignoreTestFail,
                          timeout=testTimeout,
                          description='ninja test',
                          workdir=build_dir,
                          doStepIf=bool(test),
                          env=Property('slave_env')))

    return f

def getLLDBBuildFactory(
            triple,
            useTwoStage=False,
            make='make',
            jobs='%(jobs)s',
            extra_configure_args=[],
            env={},
            *args,
            **kwargs):

    llvm_srcdir = "llvm.src"
    llvm_objdir = "llvm.obj"

    f = buildbot.process.factory.BuildFactory()

    # Determine the build directory.
    f.addStep(SetProperty(name="get_builddir",
              command=["pwd"],
              property="builddir",
              description="set build dir",
              workdir="."))

    # Find out what version of llvm and clang are needed to build this version
    # of lldb. Right now we will assume they use the same version.
    # XXX - could this be done directly on the master instead of the slave?
    f.addStep(SetProperty(command='svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
                          property='llvmrev'))

    # The SVN build step provides no mechanism to check out a specific revision
    # based on a property, so just run the commands directly here.

    svn_co = ['svn', 'checkout', '--force']
    svn_co += ['--revision', WithProperties('%(llvmrev)s')]

    # build llvm svn checkout command
    svn_co_llvm = svn_co + \
     [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
                     llvm_srcdir]
    # build clang svn checkout command
    svn_co_clang = svn_co + \
     [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
                     '%s/tools/clang' % llvm_srcdir]

    f.addStep(ShellCommand(name='svn-llvm',
                           command=svn_co_llvm,
                           haltOnFailure=True,
                           workdir='.'))
    f.addStep(ShellCommand(name='svn-clang',
                           command=svn_co_clang,
                           haltOnFailure=True,
                           workdir='.'))

    f.addStep(SVN(name='svn-lldb',
                  mode='update',
                  baseURL='http://llvm.org/svn/llvm-project/lldb/',
                  defaultBranch='trunk',
                  always_purge=True,
                  workdir='%s/tools/lldb' % llvm_srcdir))

    # Run configure
    config_args = [WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
                   "--disable-bindings",
                   "--without-llvmgcc",
                   "--without-llvmgxx",
                  ]
    if triple:
        config_args += ['--build=%s' % triple]
    config_args += extra_configure_args

    f.addStep(Configure(name='configure',
        command=config_args,
        env=env,
        workdir=llvm_objdir))

    f.addStep(WarningCountingShellCommand(name="compile",
                                          command=['nice', '-n', '10',
                                                   make, WithProperties("-j%s" % jobs)],
                                          env=env,
                                          haltOnFailure=True,
                                          workdir=llvm_objdir))

    # Test.
    f.addStep(LitTestCommand(name="test lldb",
                             command=['nice', '-n', '10',
                                      make],
                             description="test lldb",
                             env=env,
                             workdir='%s/tools/lldb/test' % llvm_objdir))

    return f
# Add test steps from list of compilers and archs
def getLLDBTestSteps(f,
                     bindir,
                     test_archs,
                     test_compilers,
                     remote_platform=None,
                     remote_host=None,
                     remote_port=None,
                     remote_dir=None,
                     env=None):
    # Skip test steps if no test compiler or arch is specified
    if None in (test_archs, test_compilers):
        return f
    llvm_srcdir = "llvm"
    llvm_builddir = "build"
    if env is None:
        env = {}
    flunkTestFailure = True
    extraTestFlag = ''
    # TODO: for now, run tests with 8 threads and without mi tests on android
    # come back when those issues are addressed
    testenv=dict(env)
    for compiler in test_compilers:
        # find full path for top of tree clang
        if compiler=='totclang':
            compilerPath=bindir + '/clang'
        elif remote_platform is 'android':
            compilerPath = os.path.join('%(toolchain_test)s', 'bin', compiler)
        else:
            compilerPath = compiler
        for arch in test_archs:
            DOTEST_OPTS=''.join(['--executable ' + bindir + '/lldb ',
                                 '--filecheck ' + bindir + '/FileCheck ',
                                 '-A %s ' % arch,
                                 '-C %s ' % compilerPath,
                                 '-s lldb-test-traces-%s-%s ' % (compiler, arch),
                                 '-u CXXFLAGS ',
                                 '-u CFLAGS ',
                                 '--channel ',
                                 '"gdb-remote packets" ',
                                 '--channel ',
                                 '"lldb all"'])
            testname = "local"
            if remote_platform is not None:
                urlStr='connect://%(remote_host)s:%(remote_port)s'
                if remote_platform is 'android':
                    #i386/x86_64 are the only android archs that are expected to pass at this time
                    flunkTestFailure = arch in ('i386', 'x86_64')
                    testenv['LLDB_TEST_THREADS'] = '8'
                    extraTestFlag = ' -m'
                    urlStr = 'adb://%(deviceid)s:%(remote_port)s'
                    # for Android, remove all forwarded ports before running test
                    # it is noticed that forwarded socket connections were not cleaned for certain crashed tests
                    # clean it here to avoid too many "LISTEN" ports left on slave
                    f.addStep(ShellCommand(name="remove port forwarding %s" % arch,
                                           command=['adb',
                                                    'forward',
                                                    '--remove-all'],
                                           description="Remove port forwarding",
                                           env=env,
                                           haltOnFailure=False,
                                           workdir='%s' % llvm_builddir))
                DOTEST_OPTS += ''.join([' --platform-name remote-' + remote_platform,
                                        ' --platform-url ' + urlStr,
                                        ' --platform-working-dir %(remote_dir)s',
                                        ' --env OS=' + remote_platform.title()])
                testname = "remote-" + remote_platform
            DOTEST_OPTS += extraTestFlag
            f.addStep(LitTestCommand(name="test lldb %s (%s-%s)" % (testname, compiler, arch),
                                     command=['../%s/tools/lldb/test/dosep.py' % llvm_srcdir,
                                              '--options',
                                              WithProperties(DOTEST_OPTS)],
                                     description="test lldb",
                                     parseSummaryOnly=True,
                                     flunkOnFailure=flunkTestFailure,
                                     warnOnFailure=flunkTestFailure,
                                     workdir='%s' % llvm_builddir,
                                     timeout=1800,
                                     env=testenv))
            f=cleanSVNSourceTree(f, '%s/tools/lldb/test' % llvm_srcdir)
    return f

# Define a structure to describe remote target
# For example, RemoteConfig('linux','x86_64',['gcc4.8.2','clang'],['i386'])
RemoteConfig = collections.namedtuple("RemoteConfig", ["platform", "host_arch", "test_compilers", "test_archs"])

# Add steps to run lldb test on remote target
def getLLDBRemoteTestSteps(f,
                           bindir,
                           build_type,
                           remote_config,
                           env):
    if None in (remote_config.test_archs, remote_config.test_compilers):
        return f
    # only supports linux and android as remote target at this time
    if remote_config.platform not in ('linux', 'android'):
        return f
    llvm_srcdir = "llvm"
    llvm_builddir = "build"
    stepDesc = remote_config.platform + "-" + remote_config.host_arch
    # get hostname
    slave_hostname = None
    f.addStep(SetProperty(name="get hostname",
                          command=["hostname"],
                          property="slave_hostname",
                          description="set slave hostname",
                          workdir="."))
    # get configuration of remote target
    # config file should be placed under builddir on builder machine
    # file name: remote_cfg.json
    # content: json format with keys [remote_platform]-[remote_arch]
    # the value for each key defines "remote_host", "remote_port", "remote_dir", "toolchain", "deviceId"
    # example: {"android-i386": {"remote_host":"localhost",
    #                            "remote_port":"5430",
    #                            "remote_dir":"/data/local/tmp/lldb",
    #                            "toolchain_build":"/home/lldb_build/Toolchains/i386-android-toolchain-21",
    #                            "toolchain_test":"/home/lldb_build/Toolchains/i386-android-toolchain-16",
    #                            "deviceid":"XXXXXXX"},

    def getRemoteCfg(rc, stdout, stderr):
        return json.loads(stdout)[stepDesc]
    f.addStep(SetProperty(name="get remote target " + stepDesc,
                          command="cat remote_cfg.json",
                          extract_fn=getRemoteCfg,
                          description="get remote target",
                          workdir="."))
    # rsync
    if remote_config.platform is 'linux':
        shellcmd = ['ssh',
                    WithProperties('%(remote_host)s')]
        hostname = '%(slave_hostname)s'
        launchcmd = shellcmd + ['screen', '-d', '-m']
        terminatecmd = shellcmd + ['pkill', 'lldb-server']
        cleandircmd = WithProperties('ssh %(remote_host)s rm -r %(remote_dir)s/*')
        f.addStep(ShellCommand(name="rsync lldb-server",
                               command=['rsync',
                                        '-havL',
                                        'bin/lldb-server',
                                        WithProperties('%(remote_host)s:%(remote_dir)s')],
                               description="rsync lldb-server " + stepDesc,
                               haltOnFailure=True,
                               env=env,
                               workdir='%s' % llvm_builddir))
        f.addStep(ShellCommand(name="rsync python2.7",
                               command=['rsync',
                                        '-havL',
                                        'lib/python2.7',
                                        WithProperties('%(remote_host)s:%(remote_dir)s')],
                               description="rsync python2.7 " + stepDesc,
                               haltOnFailure=True,
                               env=env,
                               workdir='%s' % llvm_builddir))
    elif remote_config.platform is 'android':
        shellcmd = ['adb',
                    '-s',
                    WithProperties('%(deviceid)s'),
                    'shell']
        hostname = '127.0.0.1'
        launchcmd = ['screen', '-d', '-m'] + shellcmd + [WithProperties("TMPDIR=%(remote_dir)s/tmp")]
        terminatecmd = 'ps | grep lldb-server | awk \'{print $2}\' | xargs'
        terminatecmd = WithProperties('adb -s %(deviceid)s shell ' + terminatecmd + ' adb -s %(deviceid)s shell kill')
        cleandircmd = WithProperties('adb -s %(deviceid)s shell rm -rf %(remote_dir)s/*')
        # compile lldb-server for target platform
        f = getLLDBCmakeAndCompileSteps(f,
                                        'gcc',
                                        build_type,
                                        ['lldb-server'],
                                        bindir,
                                        remote_config.platform,
                                        remote_config.host_arch,
                                        env)

        f.addStep(ShellCommand(name="adb push lldb-server " + stepDesc,
                               command=['adb',
                                        '-s',
                                        WithProperties('%(deviceid)s'),
                                        'push',
                                        remote_config.platform+'-' + remote_config.host_arch + '/bin/lldb-server',
                                        WithProperties('%(remote_dir)s/')],
                               description="lldb-server",
                               env=env,
                               haltOnFailure=True,
                               workdir='%s' % llvm_builddir))
        f.addStep(ShellCommand(name="Build fingerprint " + stepDesc,
                               command=['adb',
                                        '-s',
                                        WithProperties('%(deviceid)s'),
                                        'shell',
                                        'getprop',
                                        'ro.build.fingerprint'],
                               description="get build fingerprint",
                               env=env,
                               haltOnFailure=False,
                               workdir='%s' % llvm_builddir))
    # launch lldb-server
    f.addStep(ShellCommand(name="launch lldb-server " + stepDesc,
                           command=launchcmd +
                                   [WithProperties('%(remote_dir)s/lldb-server'),
                                    'platform',
                                    '--listen',
                                    WithProperties(hostname + ':%(remote_port)s'),
                                    '--server'],
                           description="launch lldb-server on remote host",
                           env=env,
                           haltOnFailure=True,
                           workdir='%s' % llvm_builddir))
    # test steps
    f = getLLDBTestSteps(f,
                         bindir,
                         remote_config.test_archs,
                         remote_config.test_compilers,
                         remote_config.platform,
                         '%(remote_host)s',
                         '%(remote_port)s',
                         '%(remote_dir)s',
                         env)
    # terminate lldb-server on remote host
    f.addStep(ShellCommand(name="terminate lldb-server " + stepDesc,
                           command=terminatecmd,
                           description="terminate lldb-server",
                           env=env,
                           workdir='%s' % llvm_builddir))
    # clean remote test directory
    f.addStep(ShellCommand(name="clean remote dir " + stepDesc,
                           command=cleandircmd,
                           description="clean remote dir",
                           env=env))
    return f

# Cmake bulid on Ubuntu
# Build command sequence - cmake, ninja, ./dosep
# Note: If test_archs or test_compilers is not specified, lldb-test will not be added to build factory
def getLLDBUbuntuCMakeBuildFactory(build_compiler,
                                   build_type,
                                   local_test_archs=None,
                                   local_test_compilers=None,
                                   remote_configs=None,
                                   jobs='%(jobs)s',
                                   env=None):
    """Generate factory steps for ubuntu cmake builder

       Arguments:
       build_compiler       -- string of compile name, example 'clang',
                               the compiler will be used to build binaries for host platform
       build_type           -- 'Debug' or 'Release',
                               used to define build type for host platform as well as remote platform if any
       local_test_archs     -- list of architectures, example ['i386','x86_64'],
                               defines architectures to run local tests against, if None, local tests won't be executed
       local_test_compiler  -- list of compilers, example ['clang','gcc4.8.2'],
                               definds compilers to run local tests with, if None, local tests won't be executed
       remote_configs       -- list of RemoteConfig objects, example [RemoteConfig(...)], if None, remote tests won't be executed
       jobs                 -- number of threads for compilation step, example 40
                               default value is jobs number defined during slave creation
       env                  -- environment variables passed to shell commands

    """
    if env is None:
        env = {}

    llvm_srcdir = "llvm"
    llvm_builddir = "build"
    bindir='%(builddir)s/' + llvm_builddir + '/bin'

    f = buildbot.process.factory.BuildFactory()

    # Determine the build directory.
    f.addStep(SetProperty(name="get_builddir",
                          command=["pwd"],
                          property="builddir",
                          description="set build dir",
                          workdir="."))
    # Determine the binary directory of *-tblgen.
    f.addStep(SetProperty(name="get tblgen dir",
                          command=["echo", WithProperties(bindir)],
                          property="tblgen_bindir",
                          description="set tblgen dir",
                          workdir="."))
    # Get source code
    f = getLLDBSource(f,llvm_srcdir)

    # Clean Build Folder
    f.addStep(ShellCommand(name="clean",
                           command="rm -rf *",
                           description="clear build folder",
                           env=env,
                           workdir='%s' % llvm_builddir))

    f = getLLDBCmakeAndCompileSteps(f,
                                    build_compiler,
                                    build_type,
                                    [],
                                    bindir,
                                    'linux',
                                    'x86_64',
                                    env)

    # TODO: it will be good to check that architectures listed in test_archs are compatible with host architecture
    # For now, the caller of this function should make sure that each target architecture is supported by builder machine

    # Add local test steps
    f = getLLDBTestSteps(f,
                         bindir,
                         local_test_archs,
                         local_test_compilers)
    # Remote test steps
    if remote_configs is not None:
        for config in remote_configs:
            f = getLLDBRemoteTestSteps(f,
                                       bindir,
                                       build_type,
                                       config,
                                       env)
    # archive test traces
    f = archiveLLDBTestTraces(f, "build/lldb-test-traces-*")
    return f

# zip and upload test traces to google storage
def archiveLLDBTestTraces(f, test_trace):
    f.addStep(ShellCommand(name="compress test traces",
                           command=WithProperties("zip -r build-%(buildnumber)s " + test_trace),
                           description="zip",
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           workdir='.'))
    f.addStep(ShellCommand(name="upload test traces",
                           command=['gsutil',
                                    'mv',
                                    WithProperties('build-%(buildnumber)s.zip'),
                                    WithProperties('gs://lldb_test_traces/%(buildername)s/')],
                           description="upload to Google Storage",
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           workdir='.'))
    return f

# for cmake and compile
def getLLDBCmakeAndCompileSteps(f,
                                build_compiler,
                                build_type,
                                ninja_target,
                                bindir,
                                target_platform,
                                target_arch,
                                env=None):

    if env is None:
        env={}
    llvm_builddir = 'build'
    if target_platform is 'android':
        llvm_builddir = 'build/android-' + target_arch
    # Configure
    f = getLLDBCMakeStep(f,
                         build_compiler,
                         build_type,
                         bindir,
                         target_platform,
                         target_arch,
                         env)
    # Compile
    f.addStep(WarningCountingShellCommand(name='ninja-%s-%s'%(target_platform, target_arch),
                                          command=['nice','-n', '10',
                                                   'ninja',
                                                   WithProperties('-j%(jobs)s')] + ninja_target,
                                          env=env,
                                          haltOnFailure=True,
                                          workdir=llvm_builddir))
    return f

def getLLDBCMakeStep(f,
                     build_compiler,
                     build_type,
                     bindir,
                     target_platform,
                     target_arch,
                     env=None):
    if target_platform is 'linux':
        return getLLDBLinuxCMakeStep(f,
                                     build_compiler,
                                     build_type,
                                     target_arch,
                                     env)
    elif target_platform is 'android':
        return getLLDBAndroidCMakeStep(f,
                                       build_compiler,
                                       build_type,
                                       bindir,
                                       target_arch,
                                       env)

def getCCompilerCmd(compiler):
  if compiler == "clang":
    return "clang"
  elif compiler == "gcc":
    return "gcc"

def getCxxCompilerCmd(compiler):
  if compiler == "clang":
    return "clang++"
  elif compiler == "gcc":
    return "g++"

def getLLDBLinuxCMakeStep(f,
                          build_compiler,
                          build_type,
                          target_arch,
                          env=None):
    if env is None:
        env = {}
    llvm_srcdir = 'llvm'
    llvm_builddir = 'build'
    # Construct cmake
    cmake_args = ["cmake", "-GNinja"]
    cmake_args.append(WithProperties("-DCMAKE_BUILD_TYPE=%s" % build_type))
    cmake_args.append(WithProperties('%(builddir)s/' + llvm_srcdir))
    cmake_args.append("-DCMAKE_C_COMPILER=%s" % getCCompilerCmd(build_compiler))
    cmake_args.append("-DCMAKE_CXX_COMPILER=%s" % getCxxCompilerCmd(build_compiler))

    f.addStep(Configure(name='cmake-linux-%s' % (target_arch),
                        command=cmake_args,
                        env=env,
                        haltOnFailure=True,
                        workdir=llvm_builddir))
    return f

def getLLDBAndroidCMakeStep(f,
                            build_compiler,
                            build_type,
                            bindir,
                            target_arch,
                            env):
    if env is None:
        env = {}
    llvm_srcdir = 'llvm'
    llvm_builddir = 'build/android-' + target_arch
    abiMap={
            'i386':'x86',
            'arm':'armeabi',
            'aarch64':'aarch64'
           }
    # Construct cmake
    cmake_args = ["cmake", "-GNinja"]
    cmake_args.append(WithProperties("-DCMAKE_BUILD_TYPE=%s" % build_type))
    cmake_args.append(WithProperties('%(builddir)s/' + llvm_srcdir))
    cmake_args.append(WithProperties('-DCMAKE_TOOLCHAIN_FILE=' + '%(builddir)s/' + llvm_srcdir + '/tools/lldb/cmake/platforms/Android.cmake'))
    cmake_args.append(WithProperties('-DANDROID_TOOLCHAIN_DIR=' + '%(toolchain_build)s'))
    cmake_args.append('-DANDROID_ABI=' + abiMap[target_arch])
    cmake_args.append('-DCMAKE_CXX_COMPILER_VERSION=4.9')
    cmake_args.append('-DLLVM_TARGET_ARCH=' + target_arch)
    cmake_args.append('-DLLVM_HOST_TRIPLE=' + target_arch + '-unknown-linux-android')
    cmake_args.append(WithProperties('-DLLVM_TABLEGEN=%(tblgen_bindir)s/llvm-tblgen'))
    cmake_args.append(WithProperties('-DCLANG_TABLEGEN=%(tblgen_bindir)s/clang-tblgen'))

    f.addStep(Configure(name='cmake-android-%s' % target_arch,
                        command=cmake_args,
                        env=env,
                        haltOnFailure=True,
                        workdir=llvm_builddir))
    return f
# Set symbolic links, so the folder structure will be llvm, llvm/tools/clang, llvm/tools/lldb
def getSymbLinkSteps(f, lldb_srcdir):
    f.addStep(ShellCommand(name='set symbolic link clang',
                           command=['ln', '-nfs',
                                    WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm/tools/clang'),
                                    'clang'],
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(ShellCommand(name='set symbolic link lldb',
                           command=['ln', '-nfs',
                                    WithProperties('%(builddir)s/' + lldb_srcdir),
                                    lldb_srcdir + '/llvm/tools/lldb'],
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(ShellCommand(name='set symbolic link llvm',
                           command=['ln', '-nfs',
                                    WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm'),
                                    'llvm'],
                           workdir=WithProperties('%(builddir)s')))
    return f

def getLLDBxcodebuildFactory(use_cc=None,
                             build_type='Debug',
                             remote_configs=None,
                             env=None):
    if env is None:
        env = {}
    f = buildbot.process.factory.BuildFactory()
    f.addStep(SetProperty(name='get_builddir',
                          command=['pwd'],
                          property='builddir',
                          description='set build dir',
                          workdir='.'))
    lldb_srcdir = 'lldb'
    OBJROOT='%(builddir)s/' + lldb_srcdir + '/build'
    f.addStep(SetProperty(name='get_bindir',
                          command=['echo',
                                   WithProperties('%(builddir)s/' + lldb_srcdir + '/build/' + build_type)],
                          property='lldb_bindir',
                          description='set bin dir',
                          workdir='.'))
    # cleaning out the build directory is vital for codesigning.
    f.addStep(ShellCommand(name='clean.lldb-buid',
                           command=['rm', '-rf', WithProperties(OBJROOT)],
                           haltOnFailure=True,
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(ShellCommand(name='clean.llvm-buid',
                           command=['rm', '-rf', '%s/llvm-build' % lldb_srcdir ],
                           haltOnFailure=True,
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(ShellCommand(name='clean.test trace',
                           command='rm -rf build/*',
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           workdir='.'))
    # Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency
    f.addStep(ShellCommand(name='remove symbolic link lldb',
                           command=['rm',
                                    lldb_srcdir + '/llvm/tools/lldb'],
                           haltOnFailure=False,
                           flunkOnFailure=False,
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(SVN(name='svn-lldb',
                  mode='update',
                  baseURL='http://llvm.org/svn/llvm-project/lldb/',
                  defaultBranch='trunk',
                  workdir=lldb_srcdir))
    f.addStep(SVN(name='svn-llvm',
                  mode='update',
                  baseURL='http://llvm.org/svn/llvm-project/llvm/',
                  defaultBranch='trunk',
                  workdir='%s/llvm' % lldb_srcdir))
    f.addStep(SVN(name='svn-clang',
                  mode='update',
                  baseURL='http://llvm.org/svn/llvm-project/cfe/',
                  defaultBranch='trunk',
                  workdir='%s/llvm/tools/clang' % lldb_srcdir))
# setup keychain for codesign
# In order for the codesigning to work inside of buildbot, security must be
# called to unlock the keychain, which requires a password.
# I've set up a special keychain for this purpose, so as to not compromise
# the login password of the buildslave.
# This means I have to set the special keychain as the default and unlock it
# prior to building the sources.
    f.addStep(ShellCommand(name='check.keychain',
                           command=['security', 'default-keychain'],
                           haltOnFailure=True,
                           workdir=WithProperties('%(builddir)s')))
    f.addStep(ShellCommand(name='find.certificate',
                           command=['security', 'find-certificate', '-c',
                                    'lldb_codesign'],
                           haltOnFailure=True,
                           workdir=WithProperties('%(builddir)s')))
# Building the sources
#
    buildcmd = ' '.join(['xcrun',
                         'xcodebuild',
                         '-target',
                         'desktop',
                         '-configuration',
                         build_type,
                         'SYMROOT=' + OBJROOT,
                         'OBJROOT=' + OBJROOT])
    f.addStep(ShellCommand(name='lldb-build',
                           command=WithProperties(buildcmd + " || " + buildcmd),
                           haltOnFailure=True,
                           workdir=lldb_srcdir))

# Testing
#
    if not use_cc:
        use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/'
        use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang'
        f.addStep(SetProperty(name='set.cc',
                  command=['xcrun', '-find', 'clang'],
                  property='use_cc',
                  description='set cc',
                  workdir=lldb_srcdir))
    else:
        f.addStep(SetProperty(name='set.cc',
                  command=['echo', use_cc],
                  property='use_cc',
                  description='set cc',
                  workdir=lldb_srcdir))
    DOTEST_OPTS = ' '.join(['--executable',
                            '%(lldb_bindir)s/lldb',
                            '--filecheck',
                            '%(lldb_bindir)s/FileCheck',
                            '--framework', '%(lldb_bindir)s/LLDB.framework',
                            '-A', 'x86_64',
                            '-C', 'clang',
                            '-s', '../../build/lldb-test-traces'])
    f.addStep(LitTestCommand(name='lldb-test',
                             command=['./dosep.py',
                                      '--options',
                                      WithProperties(DOTEST_OPTS)],
                             haltOnFailure=False,
                             workdir='%s/test' % lldb_srcdir,
                             env={'DYLD_FRAMEWORK_PATH' : WithProperties('%(lldb_bindir)s')}))
# Remote test steps
    if remote_configs is not None:
        # Source structure to use cmake command
        f.addStep(SetProperty(name='get tblgen bindir',
                              command=['echo',
                                       WithProperties('%(builddir)s/' + lldb_srcdir + '/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin')],
                              property='tblgen_bindir',
                              description='set tblgen binaries dir',
                              workdir='.'))
        f = getSymbLinkSteps(f, lldb_srcdir)
        for config in remote_configs:
            f = getLLDBRemoteTestSteps(f,
                                       '%(lldb_bindir)s',
                                       build_type,
                                       config,
                                       env={'DYLD_FRAMEWORK_PATH' : WithProperties('%(lldb_bindir)s')})
# Compress and upload test log
    f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")

# Results go in a directory coded named according to the date and time of the test run, e.g.:
#
# 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
#
# Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error.    return f
    return f

def getShellCommandStep(f,
                        name,
                        command,
                        description="",
                        flunkOnFailure=True,
                        haltOnFailure=True,
                        alwaysRun=False,
                        workdir='scripts',
                        env=None):
    if env is None:
        env = {}
    f.addStep(ShellCommand(name=name,
                           command=command,
                           description=description,
                           env=env,
                           flunkOnFailure=flunkOnFailure,
                           haltOnFailure=haltOnFailure,
                           alwaysRun=alwaysRun,
                           workdir=workdir))

# get configuration of tests
# config file should be placed under builddir on builder machine
# file name: test_cfg.json
# content: json format with keys test[num]:target-compiler-architecture
# example: {"test1":"android-gcc4.9-i386",
#           "test2":"local-clang-x86",
#           "test3":"local-clang-i386"}
def getTestConfig(f):
    def getRemoteCfg(rc, stdout, stderr):
        return json.loads(stdout)
    f.addStep(SetProperty(name="get test config",
                          command="cat test_cfg.json",
                          extract_fn=getRemoteCfg,
                          description="get remote target",
                          workdir="scripts"))
    return f
def getTestSteps(f, scriptExt, pathSep):
    # buildbot doesn't support dynamic step creation, so create 9 test steps as place holder
    # then each builder will define available tests in test_cfg.json
    # if there're less than 9 tests defined on certain builder, extra steps will be skipped and hidden from test details view
    # **hide step is not supported by buildbot 0.8.5
    # flunkOnFailure only takes boolean value, and cannot take configurable property.
    # workaround: don't flunk the last three tests
    # put non flunkable tests at the last three, test7, test8, test9
    getTestConfig(f)
    for x in range(1, 10):
        test='test'+str(x)
        f.addStep(LitTestCommand(name=test,
                                 command=[pathSep + 'test' + scriptExt,
                                          Property(test)],
                                 description=["testing"],
                                 descriptionDone=[WithProperties('%('+test+':-)s')],
                                 doStepIf=lambda step: step.build.hasProperty(step.name),
                                 flunkOnFailure=(x<7),
                                 warnOnFailure=(x<7),
                                 workdir='scripts'))

def getLLDBScriptCommandsFactory(
                       downloadBinary=True,
                       buildAndroid=False,
                       runTest=True,
                       scriptExt='.sh',
                       extra_cmake_args=None,
                       depends_on_projects=None,
                       ):
    if scriptExt is '.bat':
        pathSep = '.\\'
    else:
        pathSep = './'

    if extra_cmake_args is None:
        extra_cmake_args = []

    if depends_on_projects is None:
        f = buildbot.process.factory.BuildFactory()
    else:
        f = LLVMBuildFactory(
                depends_on_projects=depends_on_projects)

    # Update scripts
    getShellCommandStep(f, name='update scripts',
                        command=['updateScripts' + scriptExt])

    # Acquire lock
    if downloadBinary:
        getShellCommandStep(f, name='acquire lock',
                            command=[pathSep + 'acquireLock' + scriptExt,
                                     'totBuild'],
                            description='get')

    # Checkout source code
    getShellCommandStep(f, name='checkout source code',
                        command=[pathSep + 'checkoutSource' + scriptExt,
                                 WithProperties('%(revision)s')])

    # Set source revision
    f.addStep(SetProperty(name="set revision",
              command=[pathSep + 'getRevision' + scriptExt],
              property="got_revision",
              workdir="scripts"))

    # Configure
    getShellCommandStep(f, name='cmake local',
                        command=[pathSep + 'cmake' + scriptExt] + extra_cmake_args)

    # Build
    getShellCommandStep(f, name='ninja build local',
                        command=[pathSep + 'buildLocal' + scriptExt])
    if buildAndroid:
        getShellCommandStep(f, name='build android',
                            command=[pathSep + 'buildAndroid' + scriptExt])

    # Get lldb-server binaries
    if downloadBinary:
        getShellCommandStep(f, name='get lldb-server binaries',
                            command=[pathSep + 'downloadBinaries' + scriptExt,
                                      WithProperties('%(got_revision)s')])

    # Test
    if runTest:
        f.addStep(LitTestCommand(name="run unit tests",
                                 command=[pathSep + 'testUnit' + scriptExt],
                                 description=["testing"],
                                 descriptionDone=["unit test"],
                                 workdir='scripts'))
        getTestSteps(f, scriptExt, pathSep)
        # upload test traces
        getShellCommandStep(f, name='upload test traces',
                            command=[pathSep + 'uploadTestTrace' + scriptExt,
                                     WithProperties('%(buildnumber)s'),
                                     WithProperties('%(buildername)s')],
                            flunkOnFailure=False)

    # Upload lldb-server binaries and trigger android builders
    if buildAndroid:
        getShellCommandStep(f, name='upload lldb-server binaries',
                            command=[pathSep + 'uploadBinaries' + scriptExt])
        f.addStep(trigger.Trigger(schedulerNames=['lldb_android_scheduler'],
                                  updateSourceStamp=False,
                                  waitForFinish=False))
    # Release lock
    if downloadBinary:
        getShellCommandStep(f, name='release lock',
                            command=[pathSep + 'releaseLock' + scriptExt,
                                     'totBuild'],
                            description='release',
                            alwaysRun=True)
    return f


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 16 00:11:32 2020

@author: lukepinkel
"""

import re
import patsy
import pandas as pd
import numpy as np # analysis:ignore
import scipy as sp # analysis:ignore
import scipy.sparse as sps # analysis:ignore
from ..utilities.linalg_operations import (dummy, vech, invech, _check_np,
                                           khatri_rao, sparse_woodbury_inversion,
                                           _check_shape, woodbury_inversion)
from ..utilities.special_mats import lmat, nmat
from ..utilities.numerical_derivs import so_gc_cd, so_fc_cd
from .families import (Binomial, ExponentialFamily, Gamma, Gaussian,  # analysis:ignore
                            InverseGaussian, Poisson, NegativeBinomial)

from ..utilities.output import get_param_table
from sksparse.cholmod import cholesky

def replace_duplicate_operators(match):
    return match.group()[-1:]

def parse_random_effects(formula):
    matches = re.findall("\([^)]+[|][^)]+\)", formula)
    groups = [re.search("\(([^)]+)\|([^)]+)\)", x).groups() for x in matches]
    frm = formula
    for x in matches:
        frm = frm.replace(x, "")
    fe_form = re.sub("(\+|\-)(\+|\-)+", replace_duplicate_operators, frm)
    return fe_form, groups

def construct_random_effects(groups, data, n_vars):
    re_vars, re_groupings = list(zip(*groups))
    re_vars, re_groupings = set(re_vars), set(re_groupings)
    Zdict = dict(zip(re_vars, [_check_np(patsy.dmatrix(x, data=data, return_type='dataframe')) for x in re_vars]))
    Jdict = dict(zip(re_groupings, [dummy(data[x]) for x in re_groupings]))
    dim_dict = {}
    Z = []
    for x, y in groups:
        Ji, Xi = Jdict[y], Zdict[x]
        dim_dict[y] = {'n_groups':Ji.shape[1], 'n_vars':Xi.shape[1]}
        Zi = khatri_rao(Ji.T, Xi.T).T
        Z.append(Zi)
    Z = np.concatenate(Z, axis=1)
    return Z, dim_dict

def construct_model_matrices(formula, data, return_fe=False):
    fe_form, groups = parse_random_effects(formula)
    yvars, fe_form = re.split("[~]", fe_form)
    fe_form = re.sub("\+$", "", fe_form)
    yvars = re.split(",", re.sub("\(|\)", "", yvars))
    yvars = [x.strip() for x in yvars]
    n_vars = len(yvars)
    Z, dim_dict = construct_random_effects(groups, data, n_vars)
    X = patsy.dmatrix(fe_form, data=data, return_type='dataframe')
    fe_vars = X.columns
    y = data[yvars]
    X, y = _check_np(X), _check_np(y)
    if return_fe:
        return X, Z, y, dim_dict, list(dim_dict.keys()), fe_vars
    else:
        return X, Z, y, dim_dict, list(dim_dict.keys())

def make_theta(dims):
    theta, indices, index_start = [], {}, 0
    dims = dims.copy()
    dims['error'] = dict(n_groups=0, n_vars=1)
    for key, value in dims.items():
        n_vars = value['n_vars']
        n_params = int(n_vars * (n_vars+1) //2)
        indices[key] = np.arange(index_start, index_start+n_params)
        theta.append(vech(np.eye(n_vars)))
        index_start += n_params
    theta = np.concatenate(theta)
    return theta, indices

def make_gcov(theta, indices, dims, inverse=False):
    Gmats, g_indices, start = {}, {}, 0
    for key, value in dims.items():
        dims_i = dims[key]
        ng, nv = dims_i['n_groups'],  dims_i['n_vars']
        nv2, nvng = nv*nv, nv*ng
        theta_i = theta[indices['theta'][key]]
        if inverse:
            theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
        else:
            theta_i = invech(theta_i).reshape(-1, order='F')
        row = np.repeat(np.arange(nvng), nv)
        col = np.repeat(np.arange(ng)*nv, nv2)
        col = col + np.tile(np.arange(nv), nvng)
        data = np.tile(theta_i, ng)
        Gmats[key] = sps.csc_matrix((data, (row, col)))
        g_indices[key] = np.arange(start, start+len(data))
        start += len(data)
    G = sps.block_diag(list(Gmats.values())).tocsc()
    return G, g_indices

def lndet_gmat(theta, dims, indices):
    lnd = 0.0
    for key, value in dims.items():
        dims_i = dims[key]
        ng = dims_i['n_groups']
        Sigma_i = invech(theta[indices['theta'][key]])
        lnd += ng*np.linalg.slogdet(Sigma_i)[1]
    return lnd

def lndet_gmat_chol(theta, dims, indices):
    lnd = 0.0
    for key, value in dims.items():
        dims_i = dims[key]
        ng = dims_i['n_groups']
        theta_i = theta[indices['theta'][key]]
        L_i = invech_chol(theta_i)
        Sigma_i = L_i.dot(L_i.T)
        lnd += ng*np.linalg.slogdet(Sigma_i)[1]
    return lnd

def invech_chol(lvec):
    p = int(0.5 * ((8*len(lvec) + 1)**0.5 - 1))
    L = np.zeros((p, p))
    a, b = np.triu_indices(p)
    L[(b, a)] = lvec
    return L

def transform_theta(theta, dims, indices):
    for key in dims.keys():
        G = invech(theta[indices['theta'][key]])
        L = np.linalg.cholesky(G)
        theta[indices['theta'][key]] = vech(L)
    return theta

def inverse_transform_theta(theta, dims, indices):
    for key in dims.keys():
        L = invech_chol(theta[indices['theta'][key]])
        G = L.dot(L.T)
        theta[indices['theta'][key]] = vech(G)
    return theta

def get_d2_chol(dim_i):
    p = dim_i['n_vars']
    Lp = lmat(p).A
    T = np.zeros((p, p))
    H = []
    Ip = np.eye(p)
    for j, i in list(zip(*np.triu_indices(p))):
        T[i, j] = 1
        Hij = (Lp.dot(np.kron(Ip, T+T.T)).dot(Lp.T))[np.newaxis]
        H.append(Hij)
        T[i, j] = 0
    H = np.concatenate(H, axis=0)
    return H

def get_jacmats2(Zs, dims, indices, g_indices, theta):
    start = 0
    jac_mats = {}
    for key, value in dims.items():
        nv, ng =  value['n_vars'], value['n_groups']
        jac_mats[key] = []
        Zi = Zs[:, start:start+ng*nv]
        theta_i = theta[indices[key]]
        nv2, nvng = nv*nv, nv*ng
        row = np.repeat(np.arange(nvng), nv)
        col = np.repeat(np.arange(ng)*nv, nv2)
        col = col + np.tile(np.arange(nv), nvng)
        for i in range(len(theta_i)):
            dtheta_i = np.zeros_like(theta_i)
            dtheta_i[i] = 1.0
            dtheta_i = invech(dtheta_i).reshape(-1, order='F')
            data = np.tile(dtheta_i, ng)
            dGi = sps.csc_matrix((data, (row, col)))
            dVi = Zi.dot(dGi).dot(Zi.T)
            jac_mats[key].append(dVi)
        start+=ng*nv
    jac_mats['error'] = [sps.eye(Zs.shape[0])]
    return jac_mats

class LMEC:

    def __init__(self, formula, data, weights=None):

        indices = {}
        X, Z, y, dims, levels, fe_vars = construct_model_matrices(formula, data, return_fe=True)
        theta, theta_indices = make_theta(dims)

        indices['theta'] = theta_indices

        G, g_indices = make_gcov(theta, indices, dims)

        indices['g'] = g_indices

        XZ, Xty, Zty, yty = np.hstack([X, Z]), X.T.dot(y), Z.T.dot(y), y.T.dot(y)

        C, m = sps.csc_matrix(XZ.T.dot(XZ)), sps.csc_matrix(np.vstack([Xty, Zty]))
        M = sps.bmat([[C, m], [m.T, yty]])
        M = M.tocsc()
        self.fe_vars = fe_vars
        self.X, self.Z, self.y, self.dims, self.levels = X, Z, y, dims, levels
        self.XZ, self.Xty, self.Zty, self.yty = XZ, Xty, Zty, yty
        self.C, self.m, self.M = C, m, M
        self.theta, self.theta_chol = theta, transform_theta(theta, dims, indices)
        self.G = G
        self.indices = indices
        self.R = sps.eye(Z.shape[0])
        self.Zs = sps.csc_matrix(Z)
        self.jac_mats = get_jacmats2(self.Zs, self.dims, self.indices['theta'],
                                     self.indices['g'], self.theta)
        self.t_indices = list(zip(*np.triu_indices(len(theta))))
        self.elim_mats, self.symm_mats, self.iden_mats = {}, {}, {}
        self.d2g_dchol = {}
        for key in self.levels:
            p = self.dims[key]['n_vars']
            self.elim_mats[key] = lmat(p).A
            self.symm_mats[key] = nmat(p).A
            self.iden_mats[key] = np.eye(p)
            self.d2g_dchol[key] = get_d2_chol(self.dims[key])
        self.bounds = [(0, None) if x==1 else (None, None) for x in self.theta]
        self.bounds_2 = [(1e-6, None) if x==1 else (None, None) for x in self.theta]

    def update_mme(self, Ginv, s):
        """
        Parameters
        ----------
        Ginv: sparse matrix
             scipy sparse matrix with inverse covariance block diagonal

        s: scalar
            error covariance

        Returns
        -------
        M: sparse matrix
            updated mixed model matrix

        """
        M = self.M.copy()/s
        M[-Ginv.shape[0]-1:-1, -Ginv.shape[0]-1:-1] += Ginv
        return M

    def update_gmat(self, theta, inverse=False):
        """
        Parameters
        ----------
        theta: ndarray
             covariance parameters on the original scale

        inverse: bool
            whether or not to inverse G

        Returns
        -------
        G: sparse matrix
            updated random effects covariance

        """
        G = self.G
        for key in self.levels:
            ng = self.dims[key]['n_groups']
            theta_i = theta[self.indices['theta'][key]]
            if inverse:
                theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
            else:
                theta_i = invech(theta_i).reshape(-1, order='F')
            G.data[self.indices['g'][key]] = np.tile(theta_i, ng)
        return G

    def loglike(self, theta, use_sw=False):
        """
        Parameters
        ----------

        theta: array_like
            The original parameterization of the model parameters

        Returns
        -------
        loglike: scalar
            Log likelihood of the model
        """
        Ginv = self.update_gmat(theta, inverse=True)
        M = self.update_mme(Ginv, theta[-1])
        logdetG = lndet_gmat(theta, self.dims, self.indices)
        L = np.linalg.cholesky(M.A)
        ytPy = np.diag(L)[-1]**2
        logdetC = np.sum(2*np.log(np.diag(L))[:-1])
        logdetR = np.log(theta[-1]) * self.Z.shape[0]
        ll = logdetR + logdetC + logdetG + ytPy
        return ll

    def gradient2(self, theta, use_sw=False):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function has a memory requirement proportional to O(n^2), as
        a dense (n x n) matrix (P) needs to be formed.  For models with
        n>10000, it is generally both faster and more feasible to use
        gradient_me

        """
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        if use_sw:
            Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        else:
            Vinv = woodbury_inversion(self.Z, Cinv=Ginv.A, Ainv=Rinv.A)
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                gi = np.einsum("ij,ji->", dVdi.A, P) - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def gradient(self, theta, use_sw=False):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function has a memory requirement proportional to O(n^2), as
        a dense (n x n) matrix (P) needs to be formed.  For models with
        n>10000, it is generally both faster and more feasible to use
        gradient_me

        """
        G = self.update_gmat(theta, inverse=False)
        R = self.R * theta[-1]
        V = self.Zs.dot(G).dot(self.Zs.T) + R
        chol_fac = cholesky(V)
        if use_sw:
            Rinv = self.R / theta[-1]
            Ginv = self.update_gmat(theta, inverse=True)
            RZ = Rinv.dot(self.Zs)
            Q = Ginv + self.Zs.T.dot(RZ)
            Vinv = Rinv - RZ.dot(cholesky(Q).inv()).dot(RZ.T)
        else:
            Vinv = chol_fac.solve_A(sp.sparse.eye(V.shape[0], format='csc'))

        if Vinv.nnz / np.product(Vinv.shape) > 0.1:
            Vinv = Vinv.A
        W = chol_fac.solve_A(self.X)
        XtW = W.T.dot(self.X)
        U = np.linalg.solve(XtW, W.T)
        Py = chol_fac.solve_A(self.y) - W.dot(U.dot(self.y))

        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                VdVdi = dVdi.dot(Vinv).diagonal().sum()
                trPdV = VdVdi - np.einsum('ij,ji->', W,
                                          sp.sparse.csc_matrix.dot(U, dVdi))
                gi = trPdV - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def gradient_me(self, theta):
        """
        Parameters
        ----------
        theta: array_like
            The original parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        Notes
        -----
        This function avoids forming the (n x n) matrix P, and instead takes
        advantage of the fact that yP(dV)Py can be computed using mostly matrix
        vector products, while tr(P(dV)) can be computed by accumulating
        n vector-vector products where each component of P, P_i, can be formed
        only when needed.
        """
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        X, Z, y = self.X, self.Zs, self.y
        W = Rinv.dot(Z)
        Omega = cholesky((Z.T.dot(W) + Ginv).tocsc()).inv()
#        U = Rinv - W.dot(Omega).dot(W.T)
        UX = Rinv.dot(X) - W.dot(Omega).dot(W.T.dot(X))
        Uy = Rinv.dot(y) - W.dot(Omega).dot(W.T.dot(y))
        self.jac_mats['error'] = [self.jac_mats['error'][0].tocsc()]
        S = X.T.dot(UX)
        Sinv = np.linalg.inv(S)
        Py = Uy - UX.dot(np.linalg.inv(S).dot(UX.T.dot(y)))
        UXS = UX.dot(Sinv)
        grad = np.zeros_like(theta)
        k=0
        for key in (self.levels+['error']):
             for dVdi in self.jac_mats[key]:
                 grad[k] += -Py.T.dot(dVdi.dot(Py))[0][0]
                 k+=1
        for i in range(y.shape[0]):
#            P_i = np.asarray(U[i] - UXS[i].dot(UX.T))
            P_i = np.asarray((Rinv.tocsc()[i].T - W.dot(Omega.dot(W[i].T))).A.T[0] - UXS[i].dot(UX.T))
            k=0
            for key in (self.levels+['error']):
                for dVdi in self.jac_mats[key]:
#                    grad[k] = grad[k] + dVdi[:, i].T.dot(P_i[0])[0]
                    grad[k] = grad[k] + dVdi[:, i].T.dot(P_i)[0]
                    k=k+1
        return grad

    def hessian(self, theta):
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.R / theta[-1]
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        H = []
        PJ, yPJ = [], []
        for key in (self.levels+['error']):
            J_list = self.jac_mats[key]
            for i in range(len(J_list)):
                Ji = J_list[i].T
                PJ.append((Ji.dot(P)).T)
                yPJ.append((Ji.dot(Py)).T)
        t_indices = self.t_indices
        for i, j in t_indices:
            PJi, PJj = PJ[i], PJ[j]
            yPJi, JjPy = yPJ[i], yPJ[j].T
            Hij = -(PJi.dot(PJj)).diagonal().sum()\
                        + (2 * (yPJi.dot(P)).dot(JjPy))[0]
            H.append(np.array(Hij[0]))
        H = invech(np.concatenate(H)[:, 0])
        return H

    def update_chol(self, theta, inverse=False):
        """
        Parameters
        ----------
        theta: array_like
            array containing the lower triangular components of the cholesky
            for each random effect covariance

        inverse: bool

        Returns
        -------
        L_dict: dict of array_like
            Dictionary whose keys and values correspond to level names
            and the corresponding cholesky of the level's random effects
            covariance

        """
        L_dict = {}
        for key in self.levels:
            theta_i = theta[self.indices['theta'][key]]
            L_i = invech_chol(theta_i)
            L_dict[key] = L_i
        return L_dict

    def dg_dchol(self, L_dict):
        """

        Parameters
        ----------

        L_dict: dict of array_like
            Dictionary whose keys and values correspond to level names
            and the corresponding cholesky of the level's random effects
            covariance

        Returns
        -------

        Jf: dict of array_like
            For each level contains the derivative of the cholesky parameters
            with respect to the covariance

        Notes
        -----

        Function evaluates the derivative of the cholesky parameterization
        with respect to the lower triangular components of the covariance

        """

        Jf = {}
        for key in self.levels:
            L = L_dict[key]
            E = self.elim_mats[key]
            N = self.symm_mats[key]
            I = self.iden_mats[key]
            Jf[key] = E.dot(N.dot(np.kron(L, I))).dot(E.T)
        return Jf

    def loglike_c(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        loglike: scalar
            Log likelihood of the model
        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.loglike(theta, use_sw)

    def gradient_c(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient(theta, use_sw)

    def gradient_me_c(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient_me(theta)

    def hessian_c(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        hessian: array_like
            The hessian of the log likelihood with respect to the covariance
            parameterization

        """
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.hessian(theta)

    def gradient_chol(self, theta_chol, use_sw=False):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_c(theta_chol, use_sw)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def gradient_me_chol(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        gradient: array_like
            The gradient of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_me_c(theta_chol)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def hessian_chol(self, theta_chol):
        """
        Parameters
        ----------

        theta_chol: array_like
            The cholesky parameterization of the components

        Returns
        -------
        hessian: array_like
            The hessian of the log likelihood with respect to the cholesky
            parameterization

        """
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Hq = self.hessian_c(theta_chol)
        Jg = self.gradient_c(theta_chol)
        Hf = self.d2g_dchol
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        A = Jf.T.dot(Hq).dot(Jf)
        B = np.zeros_like(Hq)

        for key in self.levels:
            ix = self.indices['theta'][key]
            Jg_i = Jg[ix]
            Hf_i = Hf[key]
            C = np.einsum('i,ijk->jk', Jg_i, Hf_i)
            B[ix, ix[:, None]] += C
        H = A + B
        return H

    def _compute_effects(self, theta=None):
        G = self.update_gmat(theta, inverse=False)
        R = self.R * theta[-1]
        V = self.Zs.dot(G).dot(self.Zs.T) + R
        chol_fac = cholesky(V)
        XtVi = (chol_fac.solve_A(self.X)).T
        XtViX = XtVi.dot(self.X)
        XtViX_inv = np.linalg.inv(XtViX)
        beta = _check_shape(XtViX_inv.dot(XtVi.dot(self.y)))
        fixed_resids = _check_shape(self.y) - _check_shape(self.X.dot(beta))
        #Should be G.dot(Z).T.dot(solve(fixed_resids))
        Vinvr = chol_fac.solve_A(fixed_resids)
        u = G.dot(self.Zs.T).dot(Vinvr)

        return beta, XtViX_inv, u, G, R, V

    def _fit(self, use_grad=True, use_hess=False, opt_kws={}):

        if use_grad:
            default_opt_kws = dict(verbose=0, gtol=1e-6, xtol=1e-6)
            if use_hess:
               hess = self.hessian_chol
            else:
                hess = None
            for key, value in default_opt_kws.items():
                if key not in opt_kws.keys():
                    opt_kws[key] = value
            optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                             jac=self.gradient_chol, hess=hess,
                                             options=opt_kws, bounds=self.bounds,
                                             method='trust-constr')
        else:
            default_opt_kws = dict(disp=True, gtol=1e-14, ftol=1e-14,
                                   finite_diff_rel_step='3-point', eps=1e-7,
                                   iprint=99)
            for key, value in default_opt_kws.items():
                if key not in opt_kws.keys():
                    opt_kws[key] = value
            optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                             bounds=self.bounds_2,
                                             method='L-BFGS-B',
                                             options=opt_kws)
        theta_chol = optimizer.x
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)

        beta, XtWX_inv, u, G, R,  V = self._compute_effects(theta)
        params = np.concatenate([beta, theta])
        self.theta, self.beta, self.u, self.params = theta, beta, u, params
        self.Hinv_beta = XtWX_inv
        self.se_beta = np.sqrt(np.diag(XtWX_inv))
        self._G, self._R, self._V = G, R, V
        self.optimizer = optimizer
        self.theta_chol = theta_chol
        self.llconst = (self.X.shape[0] - self.X.shape[1])*np.log(2*np.pi)
        self.lltheta = self.optimizer.fun
        self.ll = (self.llconst + self.lltheta)
        self.llf = self.ll / -2.0

    def _post_fit(self, use_grad=True, analytic_se=False):
        if analytic_se:
            Htheta = self.hessian(self.theta)
        elif use_grad:
            Htheta = so_gc_cd(self.gradient, self.theta)
        else:
            Htheta = so_fc_cd(self.loglike, self.theta)
        self.Hinv_theta = np.linalg.pinv(Htheta/2.0)
        self.se_theta = np.sqrt(np.diag(self.Hinv_theta))
        self.se_params = np.concatenate([self.se_beta, self.se_theta])

    def predict(self, X=None, Z=None):
        if X is None:
            X = self.X
        if Z is None:
            Z = self.Z
        return X.dot(self.beta)+Z.dot(self.u)

    def fit(self, use_grad=True, use_hess=False, analytic_se=False, opt_kws={}):
        self._fit(use_grad, use_hess, opt_kws)
        self._post_fit(use_grad, analytic_se)
        param_names = list(self.fe_vars)
        for level in self.levels:
            for i, j in list(zip(*np.triu_indices(self.dims[level]['n_vars']))):
                param_names.append(f"{level}:G[{i}][{j}]")
        param_names.append("error_cov")
        self.param_names = param_names
        res = np.vstack((self.params, self.se_params)).T
        res = pd.DataFrame(res, index=param_names, columns=['estimate', 'SE'])
        res['t'] = res['estimate'] / res['SE']
        res['p'] = sp.stats.t(self.X.shape[0]-self.X.shape[1]).sf(np.abs(res['t']))
        self.res = res

class WLMEC:

    def __init__(self, formula, data, weights=None, fix_error=False):
        if weights is None:
            weights = np.eye(len(data))
        self.weights = sps.csc_matrix(weights)
        self.weights_inv = sps.csc_matrix(np.linalg.inv(weights))

        indices = {}
        X, Z, y, dims, levels = construct_model_matrices(formula, data)
        theta, theta_indices = make_theta(dims)

        indices['theta'] = theta_indices

        G, g_indices = make_gcov(theta, indices, dims)

        indices['g'] = g_indices

        XZ, Xty, Zty, yty = np.hstack([X, Z]), X.T.dot(y), Z.T.dot(y), y.T.dot(y)

        C, m = sps.csc_matrix(XZ.T.dot(XZ)), sps.csc_matrix(np.vstack([Xty, Zty]))
        M = sps.bmat([[C, m], [m.T, yty]])
        M = M.tocsc()
        self.F = sps.csc_matrix(XZ)
        self.X, self.Z, self.y, self.dims, self.levels = X, Z, y, dims, levels
        self.XZ, self.Xty, self.Zty, self.yty = XZ, Xty, Zty, yty
        self.C, self.m, self.M = C, m, M
        self.theta, self.theta_chol = theta, transform_theta(theta, dims, indices)
        self.G = G
        self.indices = indices
        self.R = sps.eye(Z.shape[0])
        self.Zs = sps.csc_matrix(Z)
        self.jac_mats = get_jacmats2(self.Zs, self.dims, self.indices['theta'],
                                     self.indices['g'], self.theta)
        self.t_indices = list(zip(*np.triu_indices(len(theta))))
        self.elim_mats, self.symm_mats, self.iden_mats = {}, {}, {}
        self.d2g_dchol = {}
        for key in self.levels:
            p = self.dims[key]['n_vars']
            self.elim_mats[key] = lmat(p).A
            self.symm_mats[key] = nmat(p).A
            self.iden_mats[key] = np.eye(p)
            self.d2g_dchol[key] = get_d2_chol(self.dims[key])
        self.bounds = [(0, None) if x==1 else (None, None) for x in self.theta]
        self.fix_error = fix_error

    def update_mme(self, Ginv, Rinv, s):
        C = sps.csc_matrix(self.F.T.dot(Rinv).dot(self.F))
        m = self.F.T.dot(Rinv).dot(self.y)
        yty = np.array(np.atleast_2d(self.y.T.dot(Rinv.A).dot(self.y)))
        M = sps.bmat([[C, m], [m.T, yty]]).tocsc()
        M[-Ginv.shape[0]-1:-1, -Ginv.shape[0]-1:-1] += Ginv
        return M

    def update_gmat(self, theta, inverse=False):
        G = self.G
        for key in self.levels:
            ng = self.dims[key]['n_groups']
            theta_i = theta[self.indices['theta'][key]]
            if inverse:
                theta_i = np.linalg.inv(invech(theta_i)).reshape(-1, order='F')
            else:
                theta_i = invech(theta_i).reshape(-1, order='F')
            G.data[self.indices['g'][key]] = np.tile(theta_i, ng)
        return G

    def loglike(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R /s).dot(self.weights_inv)
        M = self.update_mme(Ginv, Rinv, s)
        logdetG = lndet_gmat(theta, self.dims, self.indices)
        L = np.linalg.cholesky(M.A)
        ytPy = np.diag(L)[-1]**2
        logdetC = np.sum(2*np.log(np.diag(L))[:-1])
        R = (self.weights.dot(self.R * theta[-1]).dot(self.weights))
        logdetR = np.sum(np.log(R.diagonal()))
        ll = logdetR + logdetC + logdetG + ytPy
        return ll

    def gradient(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R / s).dot(self.weights_inv)
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        grad = []
        for key in (self.levels+['error']):
            for dVdi in self.jac_mats[key]:
                gi = np.einsum("ij,ji->", dVdi.A, P) - Py.T.dot(dVdi.dot(Py))
                grad.append(gi)
        grad = np.concatenate(grad)
        grad = _check_shape(np.array(grad))
        return grad

    def hessian(self, theta):
        if self.fix_error:
            s = 1
        else:
            s = theta[-1]
        Ginv = self.update_gmat(theta, inverse=True)
        Rinv = self.weights_inv.dot(self.R / s).dot(self.weights_inv)
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        W = (Vinv.dot(self.X))
        XtW = W.T.dot(self.X)
        XtW_inv = np.linalg.inv(XtW)
        P = Vinv - np.linalg.multi_dot([W, XtW_inv, W.T])
        Py = P.dot(self.y)
        H = []
        PJ, yPJ = [], []
        for key in (self.levels+['error']):
            J_list = self.jac_mats[key]
            for i in range(len(J_list)):
                Ji = J_list[i].T
                PJ.append((Ji.dot(P)).T)
                yPJ.append((Ji.dot(Py)).T)
        t_indices = self.t_indices
        for i, j in t_indices:
            PJi, PJj = PJ[i], PJ[j]
            yPJi, JjPy = yPJ[i], yPJ[j].T
            Hij = -(PJi.dot(PJj)).diagonal().sum()\
                        + (2 * (yPJi.dot(P)).dot(JjPy))[0]
            H.append(np.array(Hij[0]))
        H = invech(np.concatenate(H)[:, 0])
        return H

    def update_chol(self, theta, inverse=False):
        L_dict = {}
        for key in self.levels:
            theta_i = theta[self.indices['theta'][key]]
            L_i = invech_chol(theta_i)
            L_dict[key] = L_i
        return L_dict

    def dg_dchol(self, L_dict):
        Jf = {}
        for key in self.levels:
            L = L_dict[key]
            E = self.elim_mats[key]
            N = self.symm_mats[key]
            I = self.iden_mats[key]
            Jf[key] = E.dot(N.dot(np.kron(L, I))).dot(E.T)
        return Jf

    def loglike_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.loglike(theta)

    def gradient_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.gradient(theta)

    def hessian_c(self, theta_chol):
        theta = inverse_transform_theta(theta_chol.copy(), self.dims, self.indices)
        theta[-1] = theta_chol[-1]
        return self.hessian(theta)

    def gradient_chol(self, theta_chol):
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Jg = self.gradient_c(theta_chol)
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        return Jg.dot(Jf)

    def hessian_chol(self, theta_chol):
        L_dict = self.update_chol(theta_chol)
        Jf_dict = self.dg_dchol(L_dict)
        Hq = self.hessian_c(theta_chol)
        Jg = self.gradient_c(theta_chol)
        Hf = self.d2g_dchol
        Jf = sp.linalg.block_diag(*Jf_dict.values())
        Jf = np.pad(Jf, [[0, 1]])
        Jf[-1, -1] = 1
        A = Jf.T.dot(Hq).dot(Jf)
        B = np.zeros_like(Hq)

        for key in self.levels:
            ix = self.indices['theta'][key]
            Jg_i = Jg[ix]
            Hf_i = Hf[key]
            C = np.einsum('i,ijk->jk', Jg_i, Hf_i)
            B[ix, ix[:, None]] += C
        H = A + B
        return H

    def _compute_effects(self, theta=None):
        G = self.update_gmat(theta, inverse=False)
        Ginv = self.update_gmat(theta, inverse=True)
        R = (self.weights.dot(self.R * theta[-1]).dot(self.weights))
        Rinv = self.weights_inv.dot(self.R / theta[-1]).dot(self.weights_inv)
        V = self.Zs.dot(G).dot(self.Zs.T)+R
        Vinv = sparse_woodbury_inversion(self.Zs, Cinv=Ginv, Ainv=Rinv.tocsc())
        XtVi = (Vinv.dot(self.X)).T
        XtViX = XtVi.dot(self.X)
        XtViX_inv = np.linalg.inv(XtViX)
        beta = _check_shape(XtViX_inv.dot(XtVi.dot(self.y)))
        fixed_resids = _check_shape(self.y) - _check_shape(self.X.dot(beta))

        Zt = self.Zs.T
        u = G.dot(Zt.dot(Vinv)).dot(fixed_resids)

        return beta, XtViX_inv, u, G, R, Rinv, V, Vinv

    def _fit(self, use_hess=False, opt_kws={}):
        default_opt_kws = dict(verbose=3, gtol=1e-6, xtol=1e-6)
        if use_hess:
            hess = self.hessian_chol
        else:
            hess = None
        for key, value in default_opt_kws.items():
            if key not in opt_kws.keys():
                opt_kws[key] = value

        optimizer = sp.optimize.minimize(self.loglike_c, self.theta,
                                              jac=self.gradient_chol, hess=hess,
                                              options=opt_kws, bounds=self.bounds,
                                              method='trust-constr')
        theta_chol = optimizer.x
        theta = inverse_transform_theta(theta_chol, self.dims, self.indices)

        beta, XtWX_inv, u, G, R, Rinv, V, Vinv = self._compute_effects(theta)
        params = np.concatenate([beta, theta])
        self.theta, self.beta, self.u, self.params = theta, beta, u, params
        self.Hinv_beta = XtWX_inv
        self.se_beta = np.sqrt(np.diag(XtWX_inv))
        self._G, self._R, self._Rinv, self._V, self._Vinv = G, R, Rinv, V, Vinv

    def _post_fit(self):
        Htheta = self.hessian(self.theta)
        self.Hinv_theta = np.linalg.pinv(Htheta)
        self.se_theta = np.sqrt(np.diag(self.Hinv_theta))
        self.se_params = np.concatenate([self.se_beta, self.se_theta])

    def predict(self, X=None, Z=None):
        if X is None:
            X = self.X
        if Z is None:
            Z = self.Z
        return X.dot(self.beta)+Z.dot(self.u)

class GLMMC(WLMEC):
    '''
    Currently an ineffecient implementation of a GLMM, mostly done
    for fun.  A variety of implementations for GLMMs have been proposed in the
    literature, and a variety of names have been used to refer to each model;
    the implementation here is based of off linearization using a taylor
    approximation of the error (assumed to be gaussian) around the current
    estimates of fixed and random effects.  This type of approach may be
    referred to as penalized quasi-likelihood, or pseudo-likelihood, and
    may be abbreviated PQL, REPL, RPL, or RQL.

    '''
    def __init__(self, formula, data, weights=None, fam=None):
        if isinstance(fam, ExponentialFamily) is False:
            fam = fam()
        self.f = fam
        self.mod = WLMEC(formula, data, weights=None)
        self.theta_init = self.mod.theta.copy()
        self.mod._fit()
        self.y = self.mod.y
        self.non_continuous = [isinstance(self.f, Binomial),
                               isinstance(self.f, NegativeBinomial),
                               isinstance(self.f, Poisson)]
        if np.any(self.non_continuous):
            self.mod.bounds = self.mod.bounds[:-1]+[(1, 1)]
            self.mod.fix_error=True
        self._nfixed_params = self.mod.X.shape[1]
        self._n_obs = self.mod.X.shape[0]
        self._n_cov_params = len(self.mod.bounds)
        self._df1 = self._n_obs - self._nfixed_params
        self._df2 = self._n_obs - self._nfixed_params - self._n_cov_params - 1
        self._ll_const = self._df1 / 2 * np.log(2*np.pi)

    def _update_model(self, W, nu):
        nu = _check_shape(nu, 2)
        self.mod.weights = sps.csc_matrix(W)
        self.mod.weights_inv = sps.csc_matrix(np.diag(1.0/np.diag((W))))
        self.mod.y = nu
        self.mod.Xty = self.mod.X.T.dot(nu)
        self.mod.Zty = self.mod.Z.T.dot(nu)
        self.mod.theta = self.theta_init
        self.mod.yty = nu.T.dot(nu)

    def _get_pseudovar(self):
        eta = self.mod.predict()
        mu = self.f.inv_link(eta)
        var_mu = _check_shape(self.f.var_func(mu=mu), 1)
        gp = self.f.dlink(mu)
        nu = eta + gp * (_check_shape(self.y, 1) - mu)
        W = np.diag(np.sqrt(var_mu * (self.f.dlink(mu)**2)))
        return W, nu

    def _sandwich_cov(self, r):
        M = self.mod.Hinv_beta
        X = self.mod.X
        Vinv = self.mod.Vinv
        B = (Vinv.dot(X))
        d = _check_shape(r, 2)**2
        B = B * d
        C = B.T.dot(B)
        Cov = M.dot(C).dot(M)
        return Cov

    def predict(self, fixed=True, random=True):
        yhat = 0.0
        if fixed:
            yhat += self.mod.X.dot(self.mod.beta)
        if random:
            yhat += self.mod.Z.dot(self.mod.u)
        return yhat

    def fit(self, n_iters=200, tol=1e-3, optimizer_kwargs={},
            verbose_outer=True, hess=False):
        if 'options' in optimizer_kwargs.keys():
            if 'verbose' not in optimizer_kwargs['options'].keys():
                optimizer_kwargs['options']['verbose'] = 0
        else:
            optimizer_kwargs['options'] = dict(verbose=0)

        if hess:
            hessian = self.mod.hessian
        else:
            hessian = None

        theta = self.mod.theta.copy()
        fit_hist = {}
        for i in range(n_iters):
            W, nu = self._get_pseudovar()
            self._update_model(W, nu)
            self.mod._fit(opt_kws={}, use_hess=hessian)
            tvar = (np.linalg.norm(theta)+np.linalg.norm(self.mod.theta))
            eps = np.linalg.norm(theta - self.mod.theta) / tvar
            fit_hist[i] = dict(param_change=eps, theta=self.mod.theta,
                               nu=nu)
            if verbose_outer:
                print(eps)
            if eps < tol:
                break
            theta = self.mod.theta.copy()
        self.mod._postfit()
        self.res = get_param_table(self.mod.params, self.mod.se_params,
                                   self.mod.X.shape[0]-len(self.mod.params))

        eta_fe = self.predict(fixed=True, random=False)
        eta = self.predict(fixed=True, random=True)
        mu = self.f.inv_link(eta)
        gp = self.f.dlink(mu)
        var_mu  =  _check_shape(self.f.var_func(mu=mu), 1)
        r_eta_fe = _check_shape(self.mod.y, 1) - eta_fe

        generalized_chi2 = r_eta_fe.T.dot(self.mod.Vinv.dot(r_eta_fe))
        resids_raw_linear = _check_shape(self.mod.y, 1) - eta
        resids_raw_mean = _check_shape(self.y, 1) - mu

        var_pearson_linear = self.mod.R.diagonal() / gp**2
        var_pearson_mean = var_mu

        resids_pearson_linear = resids_raw_linear / np.sqrt(var_pearson_linear)
        resids_pearson_mean = resids_raw_mean / np.sqrt(var_pearson_mean)

        pll = self.mod.loglike(self.mod.theta) / -2.0 - self._ll_const
        aicc = -2 * pll + 2 * self._n_cov_params  * self._df1 / self._df2
        bic = -2 * pll + self._n_cov_params * np.log(self._df1)
        self.sumstats = dict(generalized_chi2=generalized_chi2,
                             pseudo_loglike=pll,
                             AICC=aicc,
                             BIC=bic)
        self.resids = dict(resids_raw_linear=resids_raw_linear,
                           resids_raw_mean=resids_raw_mean,
                           resids_pearson_linear=resids_pearson_linear,
                           resids_pearson_mean=resids_pearson_mean)

"""
from pystats.utilities.random_corr import vine_corr
from pystats.tests.test_data import generate_data
from pylmm.pylmm.lmm import LME
from pylmm.pylmm.glmm import WLME, GLMM

from pystats.utilities import numerical_derivs

np.set_printoptions(precision=3, suppress=True, linewidth=200)
formula = "y~1+x1+x2+(1+x3|id1)+(1+x4|id2)"
model_dict = {}
model_dict['gcov'] = {'id1':invech(np.array([2., 0.4, 2.])),
                      'id2':invech(np.array([2.,-0.4, 2.]))}

model_dict['ginfo'] = {'id1':dict(n_grp=200, n_per=10),
                       'id2':dict(n_grp=400, n_per=5)}

model_dict['mu'] = np.zeros(4)
model_dict['vcov'] = vine_corr(4, 20)
model_dict['beta'] = np.array([1, -1, 1])
model_dict['n_obs'] = 2000
data, formula = generate_data(formula, model_dict, r=0.6**0.5)

model_original = LME(formula, data)
model_cholesky = LME3(formula, data)
model_original._fit()
model_cholesky._fit(opt_kws=dict(verbose=3))
model_cholesky._post_fit()

model_original.se_params
model_cholesky.se_params

"""


import os
from fastapi import FastAPI
from dotenv import load_dotenv
import requests

load_dotenv()

p52_active = False
clockify_active = False

token = os.getenv("HUBITAT_API_TOKEN")
hubitatApi = os.getenv("HUBITAT_API_BASE")

app = FastAPI()
print("FastAPI started")

@app.get("/habitat/")
async def root():
    return {"message": "alive"}

# any timer started
@app.post("/habitat/clockify/start")
async def timeIn():
    global clockify_active
    clockify_active = True
    requests.get(os.getenv("HUBITAT_CLOCK_IN"))
    return {"message": "started"}

# any timer stopped
@app.post("/habitat/clockify/stop")
async def timeOut():
    global clockify_active
    clockify_active = False
    requests.get(os.getenv("HUBITAT_CLOCK_OUT"))
    return {"message": "stopped"}

@app.get("/habitat/location/{device}/{loc}")
async def location(device, loc):
    msg = "located {} {}".format(device, loc)
    print(msg)
    return {"message": msg}

@app.get("/habitat/phone-ring/{device}")
async def ring(device):
    msg = "ringing {}".format(device)
    print(msg)
    return {"message": msg}

@app.get("/habitat/battery-low/{device}")
async def lowBattery(device):
    msg = "low battery {}".format(device)
    print(msg)
    return {"message": msg}

async def getAllDevices():
    requests.get("{}/all?access_token={}".format(hubitatApi, token))

@app.get("/habitat/p52/active")
async def p52Active():
    global p52_active
    p52_active = True
    return {"p52_active": p52_active}

@app.get("/habitat/p52/inactive")
async def p52Inactive():
    global p52_active
    p52_active = False
    return {"p52_active": p52_active}

@app.get("/habitat/office")
async def officeStatus():
    return {"p52_active": p52_active, "clockify_active": clockify_active}


from django.shortcuts import render,HttpResponse,redirect
from django.views.decorators.csrf import csrf_exempt

import glob

import os
import cv2
import numpy as np
import tensorflow as tf
import sys

# This is needed since the notebook is stored in the object_detection folder.
sys.path.append("..")

# Import utilites
from .utils import label_map_util
from .utils import visualization_utils as vis_util
from .forms import *

# Create your views here.
@csrf_exempt
def index(request):
    context = {
        'variable':"ALEE"}

    if request.method == 'POST':
        form = Image(request.POST, request.FILES)

        if form.is_valid():
            form.save()
            detection(request,"./media/media/{}".format(request.FILES['Img']))
            #file(request)
            for name in glob.glob('./media/*.*'):
                head_tail = os.path.split(name)
                if head_tail[1] == str(request.FILES['Img']):
                    return render(request,'show.html',{"names" : os.path.join("./media/",head_tail[1])})

    else:
        form = Image()

    return render(request, 'index.html', {'form' : form})

def about(request):
    #return HttpResponse("this is about")
    return render(request,'about.html')
def file(request):
    print("ok")
    for name in glob.glob('./media/*.*'):
        head_tail = os.path.split(name)
        if head_tail[1] == str(request.FILES['Img']):
            print("ook")
            print()
            return render(request,'show.html',{"names" : os.path.join("./media/media/",head_tail[1])})
            #return success(request)

def success(request):
    return render(request, 'show.html')

def show(request):
        print("kk")
        return render(request, 'show.html')

def services(request):
    #return HttpResponse("this is services")
    return render(request,'services.html')

def contact(request):
    #return HttpResponse("this is contact")
    return render(request,'contact.html')

def detection(request,image):
    MODEL_NAME = 'home/inference_graph'
    IMAGE_NAME = image

    # Grab path to current working directory
    CWD_PATH = os.getcwd()

    # Path to frozen detection graph .pb file, which contains the model that is used
    # for object detection.
    PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,'frozen_inference_graph.pb')

    # Path to label map file
    PATH_TO_LABELS = os.path.join(CWD_PATH,'home/training','labelmap.pbtxt')

    # Path to image
    PATH_TO_IMAGE = os.path.join(CWD_PATH,IMAGE_NAME)

    # Number of classes the object detector can identify
    NUM_CLASSES = 10

    # Load the label map.
    # Label maps map indices to category names, so that when our convolution
    # network predicts `5`, we know that this corresponds to `king`.
    # Here we use internal utility functions, but anything that returns a
    # dictionary mapping integers to appropriate string labels would be fine
    label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
    categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
    category_index = label_map_util.create_category_index(categories)

    # Load the Tensorflow model into memory.
    detection_graph = tf.Graph()
    with detection_graph.as_default():
        od_graph_def = tf.GraphDef()
        with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf.import_graph_def(od_graph_def, name='')

        sess = tf.Session(graph=detection_graph)

    # Define input and output tensors (i.e. data) for the object detection classifier

    # Input tensor is the image
    image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')

    # Output tensors are the detection boxes, scores, and classes
    # Each box represents a part of the image where a particular object was detected
    detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

    # Each score represents level of confidence for each of the objects.
    # The score is shown on the result image, together with the class label.
    detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
    detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')

    # Number of objects detected
    num_detections = detection_graph.get_tensor_by_name('num_detections:0')

    # Load image using OpenCV and
    # expand image dimensions to have shape: [1, None, None, 3]
    # i.e. a single-column array, where each item in the column has the pixel RGB value
    image = cv2.imread(PATH_TO_IMAGE)
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    image_expanded = np.expand_dims(image_rgb, axis=0)

    # Perform the actual detection by running the model with the image as input
    (boxes, scores, classes, num) = sess.run(
        [detection_boxes, detection_scores, detection_classes, num_detections],
        feed_dict={image_tensor: image_expanded})

    # Draw the results of the detection (aka 'visulaize the results')

    vis_util.visualize_boxes_and_labels_on_image_array(
        image,
        np.squeeze(boxes),
        np.squeeze(classes).astype(np.int32),
        np.squeeze(scores),
        category_index,
        use_normalized_coordinates=True,
        line_thickness=8,
        min_score_thresh=0.60)

    # All the results have been drawn on image. Now display the image.
    #cv2.imshow('Object detector', image)
    cv2.imwrite("./media/{}".format(request.FILES['Img']),image)
    # Press any key to close the image
    #cv2.waitKey(0)

    # Clean up
    #cv2.destroyAllWindows()