import sys
import platform
import re
import os
import subprocess
import shutil

# The set of platforms for which this branch supports automated builds
BUILDBOT_PLATFORM_TRIPLES = (
    'x86-linux-debian8',
    'x86_64-linux-debian8',
    'armv7-android-ndk16r15',
    'arm64-android-ndk16r15',
    'x86-android-ndk16r15',
    'x86_64-android-ndk16r15',
    'universal-mac-macosx10.9', # Minimum deployment target
    'universal-ios-iphoneos14.5',
    'universal-ios-iphoneos14.4',
    'universal-ios-iphoneos13.2',
    'universal-ios-iphoneos12.1',
    'universal-ios-iphoneos11.2',
    'universal-ios-iphonesimulator14.5',
    'universal-ios-iphonesimulator14.4',
    'universal-ios-iphonesimulator13.2',
    'universal-ios-iphonesimulator12.1',
    'universal-ios-iphonesimulator11.2',
    'x86-win32', # TODO[2017-03-23] More specific ABI
    'x86_64-win32',
    'js-emscripten-sdk1.35',
)

KNOWN_PLATFORMS = (
    'linux-x86', 'linux-x86_64', 'linux-armv6hf', 'linux-armv7',
    'android-armv6', 'android-armv7', 'android-arm64', 'android-x86', 'android-x86_64',
    'mac', 'ios',
    'win-x86', 'win-x86_64',
    'emscripten'
)

def usage(exit_status):
    print(
"""Use gyp to generate project files when compiling LiveCode.

Usage:
  config.py [--platform PLATFORM] [OPTION...] [GYP_OPTION ...]

Options:
  -p, --platform PLATFORM
                    Choose which platform to build for
  -h, --help        Print this message
  --sysroot SYSROOT
                    Use the given folder for system headers and libraries (only
                    useful when cross-compiling).
  --aux-sysroot SYSROOT
                    Like --sysroot but --sysroot's headers and libraries
                    come first. Passed to the compiler as -I/-L options
                    instead of a --sysroot option.
  --triple TRIPLE
                    Required if --aux-sysroot has a `multilib' setup where
                    some headers and libraries are under a subdirectory named
                    with the target triple. This is often the case for Debian/
                    Ubuntu derived distributions.
  --cc-prefix PREFIX
                    Compiler prefix path e.g.
                    ${HOME}/toolchain/bin/arm-linux-gnueabi-
  --cross
                    Indicates cross-compilation (you probably want to specify
                    --sysroot/--aux-sysroot and --cc-prefix too).
  --use-lto
                    [EXPERIMENTAL] Use link-time optimisation when building.

gyp options:
  --generator-output DIR
                    Put generated build files under DIR
  --depth PATH      Set DEPTH gyp variable to a relative path to PATH
  -f, --format FORMATS
                    Output formats to generate
  -Gmsvs_version=WIN_MSVS_VERSION
                    Version of Microsoft Visual Studio to use
  -Gandroid_ndk_version=ANDROID_NDK_VERSION
                    Version of Android Native Development Kit to use
  -DOS=OS           Target operating system
  -Dtarget_arch=TARGET_ARCH
                    Target LiveCode to run on ARCH processors
  -Dtarget_sdk=XCODE_TARGET_SDK
                    Compile LiveCode using the specified SDK in Xcode
  -Dhost_sdk=XCODE_HOST_SDK
                    Compile build tools using the specified SDK in Xcode

All unrecognised options get passed directly to gyp.  If you don't specify
a required option, config.py will try to guess a suitable value.

The currently-supported PLATFORMs are:
""")
    for p in KNOWN_PLATFORMS:
        print("  " + p)
    sys.exit(exit_status)

def error(message):
    print('ERROR: ' + message)
    sys.exit(1)

def guess_platform():
    system = platform.system()
    arch = platform.machine()
    if system == 'Darwin':
        return 'mac'
    if system == 'Linux':
        if re.match('^(x|i.?)86$', arch) is not None:
            return 'linux-x86'
        else:
            return 'linux-' + arch
    if system == 'Windows':
        if arch == 'AMD64':
            return 'win-x86_64'
        else:
            return 'win-x86'

def exec_gyp(args):
    gyp_lib = os.path.join(os.path.dirname(sys.argv[0]), 'gyp', 'pylib')
    sys.path.insert(0, gyp_lib)
    import gyp
    print('gyp ' + ' '.join(args))
    sys.exit(gyp.main(args))

################################################################
# Parse command-line options
################################################################

def process_env_options(opts):
    vars = ('OS', 'PLATFORM', 'GENERATOR_OUTPUT', 'FORMATS', 'DEPTH',
        'WIN_MSVS_VERSION', 'XCODE_TARGET_SDK', 'XCODE_HOST_SDK',
        'TARGET_ARCH', 'PERL', 'ANDROID_NDK_VERSION', 'ANDROID_LIB_PATH',
        'ANDROID_NDK_PLATFORM_VERSION', 'ANDROID_PLATFORM',
        'ANDROID_SDK', 'ANDROID_NDK', 'ANDROID_BUILD_TOOLS', 'LTO',
        'ANDROID_TOOLCHAIN_DIR', 'ANDROID_TOOLCHAIN', 'ANDROID_API_VERSION',
        'AR', 'CC', 'CXX', 'LINK', 'OBJCOPY', 'OBJDUMP',
        'STRIP', 'JAVA_SDK', 'NODE_JS', 'BUILD_EDITION', 'CC_PREFIX', 'CROSS',
        'SYSROOT', 'AUX_SYSROOT', 'TRIPLE', 'MS_SPEECH_SDK5', 'QUICKTIME_SDK',
        )
    for v in vars:
        opts[v] = os.getenv(v)

    if opts['FORMATS'] is not None:
        opts['FORMATS'] = opts['FORMATS'].split()
    else:
        opts['FORMATS'] = []

def process_arg_options(opts, args):
    gyp_options = []
    offset = 0
    while offset < len(args):
        key = args[offset]
        if offset + 1 < len(args):
            value = args[offset + 1]
        else:
            value = None

        if key in ('-h', '--help'):
            usage(0)
        if key in ('-p', '--platform'):
            opts['PLATFORM'] = value
            offset += 2
            continue
        if key in ('--use-lto'):
             opts['LTO'] = True
             offset += 1
             continue
        if key in ('--cross'):
             opts['CROSS'] = True
             offset += 1
             continue
        if key in ('--sysroot'):
            opts['SYSROOT'] = value
            offset += 2
            continue
        if key in ('--aux-sysroot'):
            opts['AUX_SYSROOT'] = value
            offset += 2
            continue
        if key in ('--triple'):
            opts['TRIPLE'] = value
            offset += 2
            continue
        if key in ('--cc-prefix'):
            opts['CC_PREFIX'] = value
            offset += 2
            continue
        if key in ('--generator-output'):
            opts['GENERATOR_OUTPUT'] = value
            offset += 2
            continue
        if key in ('--depth'):
            opts['DEPTH'] = value
            offset += 2
            continue
        if key in ('-f', '--format'):
            opts['FORMATS'].insert(0, value)
            offset += 2
            continue

        # Intercept -D & -G options that config.py tries to generate
        intercepted_options = {
            '-Dhost_sdk': 'XCODE_HOST_SDK',
            '-Dtarget_sdk': 'XCODE_TARGET_SDK',
            '-Dtarget_arch': 'TARGET_ARCH',
            '-DOS': 'OS',
            '-Dperl': 'PERL',
            '-Dms_speech_sdk5': 'MS_SPEECH_SDK5',
            '-Dquicktime_sdk': 'QUICKTIME_SDK',
            '-Gmsvs_version': 'WIN_MSVS_VERSION',
            '-Gandroid_ndk_version': 'ANDROID_NDK_VERSION',
        }
        if key.startswith('-D') or key.startswith('-G'):
            prefix, suffix = key.split('=',2)
            if prefix in intercepted_options:
                opts[intercepted_options[prefix]] = suffix
            else:
                gyp_options.append(key)
            offset += 1
            continue

        # Unrecognised option
        error("Unrecognised option '{}'".format(key))

    opts['GYP_OPTIONS'] = gyp_options

################################################################
# Guess and validate platform and OS
################################################################

def validate_platform(opts):
    platform = opts['PLATFORM']
    if platform is None:
        platform = guess_platform()
    if platform is None:
        error("Cannot guess platform; specify '--platform <name>-'")

    if not platform in KNOWN_PLATFORMS:
        error("Unrecognised platform: '{}'".format(platform))

    opts['PLATFORM'] = platform

def validate_os(opts):
    validate_platform(opts)

    # Windows systems may have $OS set automatically in the
    # environment
    if opts['OS'] == 'Windows_NT':
        opts['OS'] = 'win'

    if opts['OS'] is None:
        opts['OS'] = opts['PLATFORM'].split('-')[0]

def host_platform(opts):
    opts['HOST_PLATFORM'] = guess_platform()

def guess_xcode_arch(target_sdk):
    sdk, ver = re.match('^([^\d]*)(\d*)', target_sdk).groups()
    if sdk == 'macosx':
        return 'x86_64'
    if sdk == 'iphoneos':
        if int(ver) < 8:
            return 'armv7'
        else:
            return 'armv7 arm64'
    if sdk == 'iphonesimulator':
        return 'x86_64'

def validate_target_arch(opts):
    if opts['TARGET_ARCH'] is None:
        validate_platform(opts)

        platform = opts['PLATFORM']
        if platform == 'emscripten':
            opts['TARGET_ARCH'] = 'js'
            opts['UNIFORM_ARCH'] = opts['TARGET_ARCH']
            return

        platform_arch = re.search('-(x86|x86_64|arm(64|v(6(hf)?|7)))$', platform)
        if platform_arch is not None:
            opts['TARGET_ARCH'] = platform_arch.group(1)
            opts['UNIFORM_ARCH'] = opts['TARGET_ARCH']
            return

        if re.match('^(ios|mac)', platform) is not None:
            validate_xcode_sdks(opts)
            arch = guess_xcode_arch(opts['XCODE_TARGET_SDK'])
            if arch is not None:
                opts['TARGET_ARCH'] = arch
                opts['UNIFORM_ARCH'] = opts['TARGET_ARCH']
                return

        error("Couldn't guess target architecture for '{}'".format(platform))
    else:
        opts['UNIFORM_ARCH'] = opts['TARGET_ARCH']

################################################################
# Guess other general options
################################################################

def validate_gyp_settings(opts):
    if opts['GENERATOR_OUTPUT'] is None:
        validate_platform(opts)

        opts['GENERATOR_OUTPUT'] = \
            os.path.join('build-' + opts['PLATFORM'], 'livecode')

    if len(opts['FORMATS']) < 1:
        validate_os(opts)

        build_os = opts['OS']
        if build_os in ('linux', 'android', 'emscripten'):
            format = 'make-linux'
        elif build_os in ('mac', 'ios'):
            format = 'xcode'
        elif build_os in ('win'):
            format = 'msvs'
        opts['FORMATS'] = [format,]

    if opts['DEPTH'] is None:
        opts['DEPTH'] = '.'

    if opts['BUILD_EDITION'] is None:
        opts['BUILD_EDITION'] = 'community'

def guess_java_home(platform):
    if platform.startswith('linux'):
        try:
            javac_str = '/bin/javac'
            javac_path = subprocess.check_output(['/usr/bin/env',
                         'readlink', '-f', '/usr' + javac_str]).strip()
            if (os.path.isfile(javac_path) and
                javac_path.endswith(javac_str)):
                return javac_path[:-len(javac_str)]
        except subprocess.CalledProcessError as e:
            print(e)
            pass # Fall through to other ways of guessing

    # More guesses
    try:
        if os.path.isfile('/usr/libexec/java_home'):
            return subprocess.check_output('/usr/libexec/java_home').strip()
    except subprocess.CalledProcessError as e:
        print(e)
        pass
    for d in ('/usr/lib/jvm/default', '/usr/lib/jvm/default-java'):
        if os.path.isdir(d):
            return d

def validate_java_tools(opts):
    if opts['JAVA_SDK'] is None:
        validate_platform(opts)
        sdk = guess_java_home(opts['HOST_PLATFORM'])
        if sdk is None:
            error('Java SDK not found; set $JAVA_SDK')
        opts['JAVA_SDK'] = sdk

def configure_toolchain(opts):
    ccprefix = ''
    if opts['CC_PREFIX'] is not None:
        ccprefix = opts['CC_PREFIX']
    if opts['CC'] is None:
        opts['CC'] = ccprefix + 'cc'
    if opts['CXX'] is None:
        opts['CXX'] = ccprefix + 'c++'
    if opts['AR'] is None:
        opts['AR'] = ccprefix + 'ar'
    if opts['LINK'] is None:
        opts['LINK'] = opts['CXX']
    if opts['STRIP'] is None:
        opts['STRIP'] = ccprefix + 'strip'
    if opts['OBJCOPY'] is None:
        opts['OBJCOPY'] = ccprefix + 'objcopy'
    if opts['OBJDUMP'] is None:
        opts['OBJDUMP'] = ccprefix + 'objdump'

    # Enable LTO if requested
    if opts['LTO'] is True:
        for key in ('CC', 'CXX', 'LINK'):
            opts[key] += ' -flto -ffunction-sections -fdata-sections -fuse-linker-plugin -fuse-ld=gold'

    # If cross-compiling, link libgcc and libstdc++ statically.
    # This is done because the cross-compilers are likely to be different
    # versions to those available on the target system.
    if opts['CROSS'] is True:
        for key in ('CC', 'CXX', 'LINK'):
            opts[key] += ' -static-libgcc -static-libstdc++'

    # Append a --sysroot option for the compilers and linker
    if opts['SYSROOT'] is not None:
        for key in ('CC', 'CXX', 'LINK'):
            opts[key] += ' --sysroot="' + opts['SYSROOT'] + '"'
            opts[key] += ' -Wl,--sysroot,"' + opts['SYSROOT'] + '"'

    # Configure an auxiliary sysroot, if one is specified.
    #
    # The -L options beginning '-L=/' are required to ensure the original
    # sysroot gets searched before the auxiliary one. Similarly with
    # -idirafter; these come after the sysroot include directories.
    if opts['AUX_SYSROOT'] is not None:
        for key in ('CC', 'CXX', 'LINK'):
            opts[key] += ' -idirafter "' + opts['AUX_SYSROOT'] + '/usr/include"'
            opts[key] += ' -L=/lib'
            opts[key] += ' -L=/usr/lib'
            if opts['TRIPLE'] is not None:
                opts[key] += ' -L"=/lib/' + opts['TRIPLE'] + '"'
                opts[key] += ' -L"=/usr/lib/' + opts['TRIPLE'] + '"'
            opts[key] += ' -L"' + opts['AUX_SYSROOT'] + '/lib"'
            opts[key] += ' -L"' + opts['AUX_SYSROOT'] + '/usr/lib"'
            opts[key] += ' -Wl,-rpath-link,"' + opts['AUX_SYSROOT'] + '/lib"'
            opts[key] += ' -Wl,-rpath-link,"' + opts['AUX_SYSROOT'] + '/usr/lib"'
            if opts['TRIPLE'] is not None:
                opts[key] += ' -idirafter "' + opts['AUX_SYSROOT'] + '/usr/include/' + opts['TRIPLE'] + '"'
                opts[key] += ' -L"' + opts['AUX_SYSROOT'] + '/lib/' + opts['TRIPLE'] + '"'
                opts[key] += ' -L"' + opts['AUX_SYSROOT'] + '/usr/lib/' + opts['TRIPLE'] + '"'
                opts[key] += ' -Wl,-rpath-link,"' + opts['AUX_SYSROOT'] + '/lib/' + opts['TRIPLE'] + '"'
                opts[key] += ' -Wl,-rpath-link,"' + opts['AUX_SYSROOT'] + '/usr/lib/' + opts['TRIPLE'] + '"'

    # Add LD as an alias for LINK
    opts['LD'] = opts['LINK']

################################################################
# Windows-specific options
################################################################

def get_program_files_x86():
    return os.environ.get('ProgramFiles(x86)',
                          os.environ.get('ProgramFiles',
                                         'C:\\Program Files\\'))

def guess_windows_perl():
    # Check the PATH first
    if (any(os.access(os.path.join(p, 'perl.exe'), os.X_OK)
            for p in os.environ['PATH'].split(os.pathsep))):
        return 'perl.exe'

    for p in ('C:\\perl64\\bin', 'C:\\perl\\bin'):
        perl = os.path.join(p, 'perl.exe')
        if os.access(perl, os.X_OK):
            return perl

    # If this is running on a non-Windows platform, default to "perl"
    if platform.system() != 'windows':
        return 'perl'

    error('Perl not found; set $PERL')

def guess_ms_speech_sdk5():
    d = os.path.join(get_program_files_x86(), 'Microsoft Speech SDK 5.1')
    if not os.path.isdir(d):
        return None
    return d

def guess_quicktime_sdk():
    d = os.path.join(get_program_files_x86(), 'QuickTime SDK')
    if not os.path.isdir(d):
        return None
    return d

def validate_windows_tools(opts):
    if opts['PERL'] is None:
        opts['PERL'] = guess_windows_perl()

    if opts['MS_SPEECH_SDK5'] is None:
        opts['MS_SPEECH_SDK5'] = guess_ms_speech_sdk5()

    if opts['QUICKTIME_SDK'] is None:
        opts['QUICKTIME_SDK'] = guess_quicktime_sdk()

    if opts['WIN_MSVS_VERSION'] is None:
        # TODO [2017-04-11]: This should be 2017, but it is not
        # compatible with our gyp as is.
        opts['WIN_MSVS_VERSION'] = '2015'

################################################################
# Mac & iOS-specific options
################################################################

def validate_xcode_sdks(opts):
    if opts['XCODE_TARGET_SDK'] is None:
        validate_os(opts)
        if opts['OS'] == 'mac':
            opts['XCODE_TARGET_SDK'] = 'macosx10.9'
        elif opts['OS'] == 'ios':
            opts['XCODE_TARGET_SDK'] = 'iphoneos'

    if opts['XCODE_HOST_SDK'] is None:
        validate_os(opts)
        if opts['OS'] == 'mac':
            opts['XCODE_HOST_SDK'] = opts['XCODE_TARGET_SDK']
        elif opts['OS'] == 'ios':
            opts['XCODE_HOST_SDK'] = 'macosx'

################################################################
# Android-specific options
################################################################

# We suggest some symlinks for Android toolchain components in the
# INSTALL-android.md file.  This checks if a directory is present
def guess_android_tooldir(toolchain, name):
    if toolchain is None:
        dir = os.path.join(os.path.expanduser('~'), 'android', 'toolchain', name)
    else:
        dir = os.path.join(toolchain, name)
    if os.path.isdir(dir):
        return dir
    return None

# Attempt to guess the Android build tools version by looking for directories
# in the SDK's build-tools subdirectory.  This is pretty fragile if someone
# has (potentially?) multiple sets of build tools installed.
def guess_android_build_tools(sdkdir):
    dirs = os.listdir(os.path.join(sdkdir, 'build-tools'))
    if len(dirs) == 1:
        return dirs[0]
    return None

# Guess the standalone toolchain directory name.
def guess_standalone_toolchain_dir_name(target_arch):
    if target_arch == 'armv6' or target_arch == 'armv7':
        return 'standalone-arm'
    else:
        return 'standalone-' + target_arch

# Guess the triple to use for a given Android target architecture.
def guess_android_triple(target_arch):
    if target_arch == 'armv6':
        return 'arm-linux-androideabi'
    elif target_arch == 'armv7':
        return 'armv7-linux-androideabi'
    elif target_arch == 'arm64':
        return 'aarch64-linux-android'
    elif target_arch == 'x86':
        return 'i686-linux-android'
    elif target_arch == 'x86_64':
        return 'x86_64-linux-android'
    else:
        return target_arch

# Guess the value to pass with the -march flag for Android builds.
def guess_android_march(target_arch):
    if target_arch == 'armv7':
        return 'armv7-a'
    elif target_arch == 'arm64':
        # The -march flag is not used for baseline arm64.
        return ''
    elif target_arch == 'x86':
        return 'i686'
    elif target_arch == 'x86_64':
        # The -march flag is not used for baseline x86_64
        return ''
    return target_arch

# Guess the prefix used on the compiler's name.
def guess_compiler_prefix(target_arch):
    if target_arch == 'armv7':
        # The ARMv7 triple is different from its compiler's prefix.
        triple = guess_android_triple('armv6')
    else:
        triple = guess_android_triple(target_arch)
    if triple == target_arch:
        return ''
    return triple + '-'

# Returns any extra C/C++ compiler flags required when targeting Android on the
# given architecture.
def android_extra_cflags(target_arch):
    if target_arch == 'armv7':
        # The first three flags are required in order to guarantee ABI compatibility.
        # Additionally, Google recommends generating thumb instructions on Android.
        return '-mfloat-abi=softfp -mfpu=vfpv3-d16 -Wl,--fix-cortex-a8 -mthumb'
    elif target_arch == 'armv6':
        # Google recommends generating thumb instructions on Android.
        return '-mthumb'
    return ''

# Returns any extra linker flags required when targeting Android on the given
# architecture.
def android_extra_ldflags(target_arch):
    if target_arch == 'armv7':
        return '-Wl,--fix-cortex-a8'
    return ''

def validate_android_tools(opts):
    if opts['ANDROID_NDK_VERSION'] is None:
        opts['ANDROID_NDK_VERSION'] = 'r15'

    ndk_ver = opts['ANDROID_NDK_VERSION']

    toolchain_dir = opts['ANDROID_TOOLCHAIN_DIR']

    if opts['ANDROID_NDK'] is None:
        ndk = guess_android_tooldir(toolchain_dir, 'android-ndk')
        if ndk is None:
            error('Android NDK not found; set $ANDROID_NDK')
        opts['ANDROID_NDK'] = ndk

    if opts['ANDROID_NDK_PLATFORM_VERSION'] is None:
        opts['ANDROID_NDK_PLATFORM_VERSION'] = '16'

    if opts['ANDROID_API_VERSION'] is None:
        opts['ANDROID_API_VERSION'] = '29'

    api_ver = opts['ANDROID_API_VERSION']

    if opts['ANDROID_PLATFORM'] is None:
        opts['ANDROID_PLATFORM'] = 'android-' + api_ver

    if opts['ANDROID_SDK'] is None:
        sdk = guess_android_tooldir(toolchain_dir, 'android-sdk')
        if sdk is None:
            error('Android SDK not found; set $ANDROID_SDK')
        opts['ANDROID_SDK'] = sdk

    if opts['ANDROID_BUILD_TOOLS'] is None:
        tools = guess_android_build_tools(opts['ANDROID_SDK'])
        if tools is None:
            error('Android build tools not found; set $ANDROID_BUILD_TOOLS')
        opts['ANDROID_BUILD_TOOLS'] = tools

    if opts['ANDROID_TOOLCHAIN'] is None:
        dir = guess_android_tooldir(toolchain_dir, guess_standalone_toolchain_dir_name(opts['TARGET_ARCH']))
        if dir is None:
            error('Android toolchain not found for architecture {}; set $ANDROID_TOOLCHAIN'.format(opts['TARGET_ARCH']))
        prefix = guess_compiler_prefix(opts['TARGET_ARCH'])
        opts['ANDROID_TOOLCHAIN'] = os.path.join(dir,'bin',prefix)

    def android_tool(name, env, extra=""):
        if opts[env] is None:
            tool = opts['ANDROID_TOOLCHAIN'] + name
            if extra is not None:
                tool += ' ' + extra
            opts[env] = tool

    target_arch = opts['TARGET_ARCH']
    march = guess_android_march(target_arch)
    triple = guess_android_triple(target_arch)
    cflags = android_extra_cflags(target_arch)
    ldflags = android_extra_ldflags(target_arch)

    if opts['ANDROID_LIB_PATH'] is None:
        dir =guess_standalone_toolchain_dir_name(opts['TARGET_ARCH'])
        if dir is None:
            error('Android standalone toolchain not found for architecture {}'.format(opts['TARGET_ARCH']))

        opts['ANDROID_LIB_PATH'] = os.path.join(dir,triple,'lib')

    # All Android builds use Clang and make a lot of noise about unused
    # arguments (e.g. linker-specific arguments). Suppress them.
    cflags += ' -Qunused-arguments'

    if march != '':
        march = '-march=' + march
    android_tool('ar', 'AR')
    android_tool('clang', 'CC',
                 '-target {} {} -integrated-as {}'.format(triple,march,cflags))
    android_tool('clang++', 'CXX',
                 '-target {} {} -integrated-as {}'.format(triple,march,cflags))
    android_tool('clang++', 'LINK',
                 '-target {} {} -integrated-as -fuse-ld=bfd {}'.format(triple,march,ldflags))
    android_tool('objcopy', 'OBJCOPY')
    android_tool('objdump', 'OBJDUMP')
    android_tool('strip', 'STRIP')

################################################################
# Emscripten-specific options
################################################################

def validate_emscripten_tools(opts):
    if opts['NODE_JS'] is None:
        opts['NODE_JS'] = 'node'

################################################################
# Linux-specific options
################################################################

################################################################
# Gyp invocation
################################################################

def core_gyp_args(opts):
    validate_gyp_settings(opts)
    validate_os(opts)

    args = []
    for f in opts['FORMATS']:
        args += ['--format', f]

    args += ['--depth', opts['DEPTH'],
             '--generator-output', opts['GENERATOR_OUTPUT'],
             '-DOS=' + opts['OS']]

    if opts['PERL'] is not None:
        args.append('-Dperl=' + opts['PERL'])

    if opts['BUILD_EDITION'] == 'commercial':
        args.append(os.path.join('..', 'livecode-commercial.gyp'))

    if opts['CROSS'] is not None:
        args.append('-Dcross_compile=1')

    args.append('-Dbuild_edition=' + opts['BUILD_EDITION'])

    args.append('-Duniform_arch=' + opts['UNIFORM_ARCH'])

    return args

def export_opts(opts, names):
    for n in names:
        if opts[n] is not None:
            print (n + '=' + opts[n])
            os.environ[n] = opts[n]

def gyp_define_args(opts, names):
    return ['-D{}={}'.format(key, opts[value])
            for key, value in names.iteritems()
            if opts[value] is not None]

def configure_linux(opts):
    host_platform(opts)
    validate_target_arch(opts)
    validate_java_tools(opts)

    configure_toolchain(opts)
    export_opts(opts, ('CC', 'CXX', 'AR', 'LINK', 'OBJCOPY', 'OBJDUMP', 'STRIP', 'LD'))

    args = core_gyp_args(opts) + ['-Dtarget_arch=' + opts['TARGET_ARCH'],
                                  '-Djavahome=' + opts['JAVA_SDK']]
    exec_gyp(args + opts['GYP_OPTIONS'])

def configure_emscripten(opts):
    host_platform(opts)
    validate_target_arch(opts)
    validate_emscripten_tools(opts)

    export_opts(opts, ('NODE_JS',))
    args = core_gyp_args(opts) + ['-Dtarget_arch=' + opts['TARGET_ARCH']]
    exec_gyp(args + opts['GYP_OPTIONS'])

def configure_android(opts):
    host_platform(opts)
    validate_target_arch(opts)
    validate_android_tools(opts)
    validate_java_tools(opts)

    export_opts(opts, ('ANDROID_BUILD_TOOLS', 'ANDROID_NDK',
                       'ANDROID_PLATFORM', 'ANDROID_SDK',
                       'ANDROID_NDK_VERSION', 'ANDROID_NDK_PLATFORM_VERSION',
                       'ANDROID_API_VERSION', 'ANDROID_LIB_PATH',
                       'JAVA_SDK', 'AR', 'CC', 'CXX', 'LINK', 'OBJCOPY',
                       'OBJDUMP', 'STRIP'))
    args = core_gyp_args(opts) + ['-Dtarget_arch=' + opts['TARGET_ARCH'],
                                  '-Dcross_compile=1',
                                  '-Gandroid_ndk_version=' + opts['ANDROID_NDK_VERSION'],
                                  '-Djavahome=' + opts['JAVA_SDK']]
    exec_gyp(args + opts['GYP_OPTIONS'])

def configure_win(opts):
    host_platform(opts)
    validate_target_arch(opts)
    validate_windows_tools(opts)

    # Make sure we strictly enforce TARGET_ARCH being x86 or x86_64
    if opts['TARGET_ARCH'] != 'x86' and opts['TARGET_ARCH'] != 'x86_64':
        error("TARGET_ARCH must be x86 or x86_64")

    # Map target_arch for gyp - x86_64 -> x64
    if opts['TARGET_ARCH'] == 'x86_64':
        opts['TARGET_ARCH'] = 'x64'

    args = core_gyp_args(opts) + ['-Gmsvs_version=' + opts['WIN_MSVS_VERSION']]
    args += gyp_define_args(opts, {'target_arch':    'TARGET_ARCH',
                                   'ms_speech_sdk5': 'MS_SPEECH_SDK5',
                                   'quicktime_sdk':  'QUICKTIME_SDK', })

    if platform.system() != 'Windows':
        args.append('-Dunix_configure=1')

    exec_gyp(args + opts['GYP_OPTIONS'])

def configure_mac(opts):
    host_platform(opts)
    validate_target_arch(opts)
    validate_xcode_sdks(opts)
    validate_java_tools(opts)
    copy_workspace_settings(opts)

    args = core_gyp_args(opts) + ['-Dtarget_sdk=' + opts['XCODE_TARGET_SDK'],
                                  '-Dhost_sdk=' + opts['XCODE_HOST_SDK'],
                                  '-Dtarget_arch=' + opts['TARGET_ARCH'],
                                  '-Djavahome=' + opts['JAVA_SDK']]
    exec_gyp(args + opts['GYP_OPTIONS'])

def configure_ios(opts):
    configure_mac(opts)

def configure(args):
    opts = {}
    process_env_options(opts)
    process_arg_options(opts, args)

    validate_os(opts)
    configure_procs = {
        'linux': configure_linux,
        'emscripten': configure_emscripten,
        'android': configure_android,
        'win': configure_win,
        'mac': configure_mac,
        'ios': configure_ios,
    }
    configure_procs[opts['OS']](opts)

def copy_workspace_settings(opts):
    validate_gyp_settings(opts)
    if opts['BUILD_EDITION'] == 'commercial':
        project = os.path.join(opts['GENERATOR_OUTPUT'], '..', 'livecode-commercial.xcodeproj')
    else:
        project = os.path.join(opts['GENERATOR_OUTPUT'], 'livecode.xcodeproj')

    xcshareddata = os.path.join(project, 'project.xcworkspace', 'xcshareddata')

    if not os.path.exists(xcshareddata):
        os.makedirs(xcshareddata)

    workspacesettingsdest= os.path.join(xcshareddata, 'WorkspaceSettings.xcsettings')
    workspacesettingssource = os.path.join('config', 'WorkspaceSettings.xcsettings')

    if not os.path.exists(workspacesettingsdest):
        shutil.copyfile(workspacesettingssource, workspacesettingsdest)

if __name__ == '__main__':
    configure(sys.argv[1:])


"""
pynamodb attributes tests
"""
import json
import six

from base64 import b64encode
from datetime import datetime

from dateutil.parser import parse
from dateutil.tz import tzutc

from mock import patch, Mock, call
import pytest

from pynamodb.constants import UTC, DATETIME_FORMAT
from pynamodb.models import Model

from pynamodb.attributes import (
    BinarySetAttribute, BinaryAttribute, NumberSetAttribute, NumberAttribute,
    UnicodeAttribute, UnicodeSetAttribute, UTCDateTimeAttribute, BooleanAttribute, LegacyBooleanAttribute,
    MapAttribute, MapAttributeMeta, ListAttribute, Attribute,
    JSONAttribute, DEFAULT_ENCODING, NUMBER, STRING, STRING_SET, NUMBER_SET, BINARY_SET,
    BINARY, MAP, LIST, BOOLEAN, _get_value_for_deserialize)

UTC = tzutc()

class AttributeTestModel(Model):

    class Meta:
        host = 'http://localhost:8000'
        table_name = 'test'

    binary_attr = BinaryAttribute()
    binary_set_attr = BinarySetAttribute()
    number_attr = NumberAttribute()
    number_set_attr = NumberSetAttribute()
    unicode_attr = UnicodeAttribute()
    unicode_set_attr = UnicodeSetAttribute()
    datetime_attr = UTCDateTimeAttribute()
    bool_attr = BooleanAttribute()
    json_attr = JSONAttribute()
    map_attr = MapAttribute()

class CustomAttrMap(MapAttribute):
    overridden_number_attr = NumberAttribute(attr_name="number_attr")
    overridden_unicode_attr = UnicodeAttribute(attr_name="unicode_attr")

class DefaultsMap(MapAttribute):
    map_field = MapAttribute(default={})

class TestAttributeDescriptor:
    """
    Test Attribute Descriptors
    """
    def setup(self):
        self.instance = AttributeTestModel()

    def test_binary_attr(self):
        """
        Binary attribute descriptor
        """
        self.instance.binary_attr = b'test'
        assert self.instance.binary_attr == b'test'

    def test_binary_set_attr(self):
        """
        Binary set attribute descriptor
        """
        self.instance.binary_set_attr = set([b'test', b'test2'])
        assert self.instance.binary_set_attr == set([b'test', b'test2'])

    def test_number_attr(self):
        """
        Number attribute descriptor
        """
        self.instance.number_attr = 42
        assert self.instance.number_attr == 42

    def test_number_set_attr(self):
        """
        Number set attribute descriptor
        """
        self.instance.number_set_attr = set([1, 2])
        assert self.instance.number_set_attr == set([1, 2])

    def test_unicode_attr(self):
        """
        Unicode attribute descriptor
        """
        self.instance.unicode_attr = u"test"
        assert self.instance.unicode_attr == u"test"

    def test_unicode_set_attr(self):
        """
        Unicode set attribute descriptor
        """
        self.instance.unicode_set_attr = set([u"test", u"test2"])
        assert self.instance.unicode_set_attr == set([u"test", u"test2"])

    def test_datetime_attr(self):
        """
        Datetime attribute descriptor
        """
        now = datetime.now()
        self.instance.datetime_attr = now
        assert self.instance.datetime_attr == now

    def test_bool_attr(self):
        """
        Boolean attribute descriptor
        """
        self.instance.bool_attr = True
        assert self.instance.bool_attr is True

    def test_json_attr(self):
        """
        JSON attribute descriptor
        """
        self.instance.json_attr = {'foo': 'bar', 'bar': 42}
        assert self.instance.json_attr == {'foo': 'bar', 'bar': 42}

class TestUTCDateTimeAttribute:
    """
    Tests UTCDateTime attributes
    """
    def test_utc_datetime_attribute(self):
        """
        UTCDateTimeAttribute.default
        """
        attr = UTCDateTimeAttribute()
        assert attr is not None
        assert attr.attr_type == STRING
        tstamp = datetime.now()
        attr = UTCDateTimeAttribute(default=tstamp)
        assert attr.default == tstamp

    def test_utc_date_time_deserialize(self):
        """
        UTCDateTimeAttribute.deserialize
        """
        tstamp = datetime.now(UTC)
        attr = UTCDateTimeAttribute()
        assert attr.deserialize(tstamp.strftime(DATETIME_FORMAT)) == tstamp

    def test_dateutil_parser_fallback(self):
        """
        UTCDateTimeAttribute.deserialize
        """
        expected_value = datetime(2047, 1, 6, 8, 21, tzinfo=tzutc())
        attr = UTCDateTimeAttribute()
        assert attr.deserialize('January 6, 2047 at 8:21:00AM UTC') == expected_value

    @patch('pynamodb.attributes.datetime')
    @patch('pynamodb.attributes.parse')
    def test_utc_date_time_deserialize_parse_args(self, parse_mock, datetime_mock):
        """
        UTCDateTimeAttribute.deserialize
        """
        tstamp = datetime.now(UTC)
        attr = UTCDateTimeAttribute()

        tstamp_str = tstamp.strftime(DATETIME_FORMAT)
        attr.deserialize(tstamp_str)

        parse_mock.assert_not_called()
        datetime_mock.strptime.assert_called_once_with(tstamp_str, DATETIME_FORMAT)

    def test_utc_date_time_serialize(self):
        """
        UTCDateTimeAttribute.serialize
        """
        tstamp = datetime.now()
        attr = UTCDateTimeAttribute()
        assert attr.serialize(tstamp) == tstamp.replace(tzinfo=UTC).strftime(DATETIME_FORMAT)

class TestBinaryAttribute:
    """
    Tests binary attributes
    """
    def test_binary_attribute(self):
        """
        BinaryAttribute.default
        """
        attr = BinaryAttribute()
        assert attr is not None
        assert attr.attr_type == BINARY

        attr = BinaryAttribute(default=b'foo')
        assert attr.default == b'foo'

    def test_binary_round_trip(self):
        """
        BinaryAttribute round trip
        """
        attr = BinaryAttribute()
        value = b'foo'
        serial = attr.serialize(value)
        assert attr.deserialize(serial) == value

    def test_binary_serialize(self):
        """
        BinaryAttribute.serialize
        """
        attr = BinaryAttribute()
        serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
        assert attr.serialize(b'foo') == serial

    def test_binary_deserialize(self):
        """
        BinaryAttribute.deserialize
        """
        attr = BinaryAttribute()
        serial = b64encode(b'foo').decode(DEFAULT_ENCODING)
        assert attr.deserialize(serial) == b'foo'

    def test_binary_set_serialize(self):
        """
        BinarySetAttribute.serialize
        """
        attr = BinarySetAttribute()
        assert attr.attr_type == BINARY_SET
        assert attr.serialize(set([b'foo', b'bar'])) == [
            b64encode(val).decode(DEFAULT_ENCODING) for val in sorted(set([b'foo', b'bar']))
        ]
        assert attr.serialize(None) is None

    def test_binary_set_round_trip(self):
        """
        BinarySetAttribute round trip
        """
        attr = BinarySetAttribute()
        value = set([b'foo', b'bar'])
        serial = attr.serialize(value)
        assert attr.deserialize(serial) == value

    def test_binary_set_deserialize(self):
        """
        BinarySetAttribute.deserialize
        """
        attr = BinarySetAttribute()
        value = set([b'foo', b'bar'])
        assert attr.deserialize(
            [b64encode(val).decode(DEFAULT_ENCODING) for val in sorted(value)]
        ) == value

    def test_binary_set_attribute(self):
        """
        BinarySetAttribute.serialize
        """
        attr = BinarySetAttribute()
        assert attr is not None

        attr = BinarySetAttribute(default=set([b'foo', b'bar']))
        assert attr.default == set([b'foo', b'bar'])

class TestNumberAttribute:
    """
    Tests number attributes
    """
    def test_number_attribute(self):
        """
        NumberAttribute.default
        """
        attr = NumberAttribute()
        assert attr is not None
        assert attr.attr_type == NUMBER

        attr = NumberAttribute(default=1)
        assert attr.default == 1

    def test_number_serialize(self):
        """
        NumberAttribute.serialize
        """
        attr = NumberAttribute()
        assert attr.serialize(3.141) == '3.141'
        assert attr.serialize(1) == '1'
        assert attr.serialize(12345678909876543211234234324234) == '12345678909876543211234234324234'

    def test_number_deserialize(self):
        """
        NumberAttribute.deserialize
        """
        attr = NumberAttribute()
        assert attr.deserialize('1') == 1
        assert attr.deserialize('3.141') == 3.141
        assert attr.deserialize('12345678909876543211234234324234') == 12345678909876543211234234324234

    def test_number_set_deserialize(self):
        """
        NumberSetAttribute.deserialize
        """
        attr = NumberSetAttribute()
        assert attr.attr_type == NUMBER_SET
        assert attr.deserialize([json.dumps(val) for val in sorted(set([1, 2]))]) == set([1, 2])

    def test_number_set_serialize(self):
        """
        NumberSetAttribute.serialize
        """
        attr = NumberSetAttribute()
        assert attr.serialize(set([1, 2])) == [json.dumps(val) for val in sorted(set([1, 2]))]
        assert attr.serialize(None) is None

    def test_number_set_attribute(self):
        """
        NumberSetAttribute.default
        """
        attr = NumberSetAttribute()
        assert attr is not None

        attr = NumberSetAttribute(default=set([1, 2]))
        assert attr.default == set([1, 2])

class TestUnicodeAttribute:
    """
    Tests unicode attributes
    """
    def test_unicode_attribute(self):
        """
        UnicodeAttribute.default
        """
        attr = UnicodeAttribute()
        assert attr is not None
        assert attr.attr_type == STRING

        attr = UnicodeAttribute(default=six.u('foo'))
        assert attr.default == six.u('foo')

    def test_unicode_serialize(self):
        """
        UnicodeAttribute.serialize
        """
        attr = UnicodeAttribute()
        assert attr.serialize('foo') == six.u('foo')
        assert attr.serialize(u'foo') == six.u('foo')
        assert attr.serialize(u'') is None
        assert attr.serialize(None) is None

    def test_unicode_deserialize(self):
        """
        UnicodeAttribute.deserialize
        """
        attr = UnicodeAttribute()
        assert attr.deserialize('foo') == six.u('foo')
        assert attr.deserialize(u'foo') == six.u('foo')

    def test_unicode_set_serialize(self):
        """
        UnicodeSetAttribute.serialize
        """
        attr = UnicodeSetAttribute()
        assert attr.attr_type == STRING_SET
        assert attr.deserialize(None) is None

        expected = sorted([six.u('foo'), six.u('bar')])
        assert attr.serialize(set([six.u('foo'), six.u('bar')])) == expected

        expected = sorted([six.u('True'), six.u('False')])
        assert attr.serialize(set([six.u('True'), six.u('False')])) == expected

        expected = sorted([six.u('true'), six.u('false')])
        assert attr.serialize(set([six.u('true'), six.u('false')])) == expected

    def test_round_trip_unicode_set(self):
        """
        Round trip a unicode set
        """
        attr = UnicodeSetAttribute()
        orig = set([six.u('foo'), six.u('bar')])
        assert orig == attr.deserialize(attr.serialize(orig))

        orig = set([six.u('true'), six.u('false')])
        assert orig == attr.deserialize(attr.serialize(orig))

        orig = set([six.u('1'), six.u('2.8')])
        assert orig == attr.deserialize(attr.serialize(orig))

        orig = set([six.u('[1,2,3]'), six.u('2.8')])
        assert orig == attr.deserialize(attr.serialize(orig))

    def test_unicode_set_deserialize(self):
        """
        UnicodeSetAttribute.deserialize
        """
        attr = UnicodeSetAttribute()
        value = set([six.u('foo'), six.u('bar')])
        assert attr.deserialize(value) == value

        value = set([six.u('True'), six.u('False')])
        assert attr.deserialize(value) == value

        value = set([six.u('true'), six.u('false')])
        assert attr.deserialize(value) == value

        value = set([six.u('1'), six.u('2.8')])
        assert attr.deserialize(value) == value

    def test_unicode_set_attribute(self):
        """
        UnicodeSetAttribute.default
        """
        attr = UnicodeSetAttribute()
        assert attr is not None
        assert attr.attr_type == STRING_SET
        attr = UnicodeSetAttribute(default=set([six.u('foo'), six.u('bar')]))
        assert attr.default == set([six.u('foo'), six.u('bar')])

class TestLegacyBooleanAttribute:
    def test_legacy_boolean_attribute_can_read_future_boolean_attributes(self):
        """
        LegacyBooleanAttribute.deserialize
        :return:
        """
        attr = LegacyBooleanAttribute()
        assert attr.deserialize('1') is True
        assert attr.deserialize('0') is False
        assert attr.deserialize(json.dumps(True)) is True
        assert attr.deserialize(json.dumps(False)) is False

    def test_legacy_boolean_attribute_get_value_can_read_both(self):
        """
        LegacyBooleanAttribute.get_value
        :return:
        """
        attr = LegacyBooleanAttribute()
        assert attr.get_value({'N': '1'}) == '1'
        assert attr.get_value({'N': '0'}) == '0'
        assert attr.get_value({'BOOL': True}) == json.dumps(True)
        assert attr.get_value({'BOOL': False}) == json.dumps(False)

    def test_legacy_boolean_attribute_get_value_and_deserialize_work_together(self):
        attr = LegacyBooleanAttribute()
        assert attr.deserialize(attr.get_value({'N': '1'})) is True
        assert attr.deserialize(attr.get_value({'N': '0'})) is False
        assert attr.deserialize(attr.get_value({'BOOL': True})) is True
        assert attr.deserialize(attr.get_value({'BOOL': False})) is False

    def test_legacy_boolean_attribute_serialize(self):
        """
        LegacyBooleanAttribute.serialize
        """
        attr = LegacyBooleanAttribute()
        assert attr.serialize(True) == '1'
        assert attr.serialize(False) == '0'
        assert attr.serialize(None) is None

class TestBooleanAttribute:
    """
    Tests boolean attributes
    """
    def test_boolean_attribute(self):
        """
        BooleanAttribute.default
        """
        attr = BooleanAttribute()
        assert attr is not None

        assert attr.attr_type == BOOLEAN
        attr = BooleanAttribute(default=True)
        assert attr.default is True

    def test_boolean_serialize(self):
        """
        BooleanAttribute.serialize
        """
        attr = BooleanAttribute()
        assert attr.serialize(True) is True
        assert attr.serialize(False) is False
        assert attr.serialize(None) is None

    def test_boolean_deserialize(self):
        """
        BooleanAttribute.deserialize
        """
        attr = BooleanAttribute()
        assert attr.deserialize('1') is True
        assert attr.deserialize('0') is True
        assert attr.deserialize(True) is True
        assert attr.deserialize(False) is False

class TestJSONAttribute:
    """
    Tests json attributes
    """
    def test_quoted_json(self):
        attr = JSONAttribute()
        serialized = attr.serialize('\\t')
        assert attr.deserialize(serialized) == '\\t'

        serialized = attr.serialize('"')
        assert attr.deserialize(serialized) == '"'

    def test_json_attribute(self):
        """
        JSONAttribute.default
        """
        attr = JSONAttribute()
        assert attr is not None

        assert attr.attr_type == STRING
        attr = JSONAttribute(default={})
        assert attr.default == {}

    def test_json_serialize(self):
        """
        JSONAttribute.serialize
        """
        attr = JSONAttribute()
        item = {'foo': 'bar', 'bool': True, 'number': 3.141}
        assert attr.serialize(item) == six.u(json.dumps(item))
        assert attr.serialize({}) == six.u('{}')
        assert attr.serialize(None) is None

    def test_json_deserialize(self):
        """
        JSONAttribute.deserialize
        """
        attr = JSONAttribute()
        item = {'foo': 'bar', 'bool': True, 'number': 3.141}
        encoded = six.u(json.dumps(item))
        assert attr.deserialize(encoded) == item

    def test_control_chars(self):
        """
        JSONAttribute with control chars
        """
        attr = JSONAttribute()
        item = {'foo\t': 'bar\n', 'bool': True, 'number': 3.141}
        encoded = six.u(json.dumps(item))
        assert attr.deserialize(encoded) == item

class TestMapAttribute:
    """
    Tests map with str, int, float
    """
    def test_attribute_children(self):
        person_attribute = {
            'name': 'Justin',
            'age': 12345678909876543211234234324234,
            'height': 187.96
        }
        attr = MapAttribute()
        serialized = attr.serialize(person_attribute)
        assert attr.deserialize(serialized) == person_attribute

    def test_map_of_map(self):
        attribute = {
            'name': 'Justin',
            'metrics': {
                'age': 31,
                'height': 187.96
            }
        }
        attr = MapAttribute()
        serialized = attr.serialize(attribute)
        assert attr.deserialize(serialized) == attribute

    def test_map_overridden_attrs_accessors(self):
        attr = CustomAttrMap(**{
            'overridden_number_attr': 10,
            'overridden_unicode_attr': "Hello"
        })

        assert attr.overridden_number_attr == 10
        assert attr.overridden_unicode_attr == "Hello"

    def test_map_overridden_attrs_serialize(self):
        attribute = {
            'overridden_number_attr': 10,
            'overridden_unicode_attr': "Hello"
        }
        expected = {'number_attr': {'N': '10'}, 'unicode_attr': {'S': six.u('Hello')}}
        assert CustomAttrMap().serialize(attribute) == expected

    def test_defaults(self):
        item = DefaultsMap()
        assert item.validate()
        assert DefaultsMap().serialize(item) == {
            'map_field': {
                'M': {}
            }
        }

    def test_raw_set_attr(self):
        item = AttributeTestModel()
        item.map_attr = {}
        item.map_attr.foo = 'bar'
        item.map_attr.num = 3
        item.map_attr.nested = {'nestedfoo': 'nestedbar'}

        assert item.map_attr['foo'] == 'bar'
        assert item.map_attr['num'] == 3
        assert item.map_attr['nested']['nestedfoo'] == 'nestedbar'

    def test_raw_map_from_dict(self):
        item = AttributeTestModel(
            map_attr={
                "foo": "bar",
                "num": 3,
                "nested": {
                    "nestedfoo": "nestedbar"
                }
            }
        )

        assert item.map_attr['foo'] == 'bar'
        assert item.map_attr['num'] == 3

    def test_raw_map_access(self):
        raw = {
            "foo": "bar",
            "num": 3,
            "nested": {
                "nestedfoo": "nestedbar"
            }
        }
        attr = MapAttribute(**raw)

        for k, v in six.iteritems(raw):
            assert attr[k] == v

    def test_raw_map_json_serialize(self):
        raw = {
            "foo": "bar",
            "num": 3,
            "nested": {
                "nestedfoo": "nestedbar"
            }
        }

        serialized_raw = json.dumps(raw, sort_keys=True)
        serialized_attr_from_raw = json.dumps(
            AttributeTestModel(map_attr=raw).map_attr.as_dict(),
            sort_keys=True)
        serialized_attr_from_map = json.dumps(
            AttributeTestModel(map_attr=MapAttribute(**raw)).map_attr.as_dict(),
            sort_keys=True)

        assert serialized_attr_from_raw == serialized_raw
        assert serialized_attr_from_map == serialized_raw

    def test_typed_and_raw_map_json_serialize(self):
        class TypedMap(MapAttribute):
            map_attr = MapAttribute()

        class SomeModel(Model):
            typed_map = TypedMap()

        item = SomeModel(
            typed_map=TypedMap(map_attr={'foo': 'bar'})
        )

        assert json.dumps({'map_attr': {'foo': 'bar'}}) == json.dumps(item.typed_map.as_dict())

    def test_json_serialize(self):
        class JSONMapAttribute(MapAttribute):
            arbitrary_data = JSONAttribute()

            def __eq__(self, other):
                return self.arbitrary_data == other.arbitrary_data

        item = {'foo': 'bar', 'bool': True, 'number': 3.141}
        json_map = JSONMapAttribute(arbitrary_data=item)
        serialized = json_map.serialize(json_map)
        deserialized = json_map.deserialize(serialized)
        assert isinstance(deserialized, JSONMapAttribute)
        assert deserialized == json_map
        assert deserialized.arbitrary_data == item

    def test_serialize_datetime(self):
        class CustomMapAttribute(MapAttribute):
            date_attr = UTCDateTimeAttribute()

        cm = CustomMapAttribute(date_attr=datetime(2017, 1, 1))
        serialized_datetime = cm.serialize(cm)
        expected_serialized_value = {
            'date_attr': {
                'S': u'2017-01-01T00:00:00.000000+0000'
            }
        }
        assert serialized_datetime == expected_serialized_value

    def test_complex_map_accessors(self):
        class NestedThing(MapAttribute):
            double_nested = MapAttribute()
            double_nested_renamed = MapAttribute(attr_name='something_else')

        class ThingModel(Model):
            nested = NestedThing()

        t = ThingModel(nested=NestedThing(
            double_nested={'hello': 'world'},
            double_nested_renamed={'foo': 'bar'})
        )

        assert t.nested.double_nested.as_dict() == {'hello': 'world'}
        assert t.nested.double_nested_renamed.as_dict() == {'foo': 'bar'}
        assert t.nested.double_nested.hello == 'world'
        assert t.nested.double_nested_renamed.foo == 'bar'
        assert t.nested['double_nested'].as_dict() == {'hello': 'world'}
        assert t.nested['double_nested_renamed'].as_dict() == {'foo': 'bar'}
        assert t.nested['double_nested']['hello'] == 'world'
        assert t.nested['double_nested_renamed']['foo'] == 'bar'

        with pytest.raises(AttributeError):
            bad = t.nested.double_nested.bad
        with pytest.raises(AttributeError):
            bad = t.nested.bad
        with pytest.raises(AttributeError):
            bad = t.nested.something_else
        with pytest.raises(KeyError):
            bad = t.nested.double_nested['bad']
        with pytest.raises(KeyError):
            bad = t.nested['something_else']

    def test_metaclass(self):
        assert type(MapAttribute) == MapAttributeMeta

    def test_attribute_paths_subclassing(self):
        class SubMapAttribute(MapAttribute):
            foo = UnicodeAttribute(attr_name='dyn_foo')

        class SubSubMapAttribute(SubMapAttribute):
            bar = UnicodeAttribute(attr_name='dyn_bar')

        class SubModel(Model):
            sub_map = SubMapAttribute(attr_name='dyn_sub_map')

        class SubSubModel(SubModel):
            sub_sub_map = SubSubMapAttribute()

        assert SubModel.sub_map.foo.attr_name == 'dyn_foo'
        assert SubModel.sub_map.foo.attr_path == ['dyn_sub_map', 'dyn_foo']
        assert SubSubModel.sub_map.foo.attr_name == 'dyn_foo'
        assert SubSubModel.sub_map.foo.attr_path == ['dyn_sub_map', 'dyn_foo']
        assert SubSubModel.sub_sub_map.foo.attr_name == 'dyn_foo'
        assert SubSubModel.sub_sub_map.foo.attr_path == ['sub_sub_map', 'dyn_foo']
        assert SubSubModel.sub_sub_map.bar.attr_name == 'dyn_bar'
        assert SubSubModel.sub_sub_map.bar.attr_path == ['sub_sub_map', 'dyn_bar']

    def test_attribute_paths_wrapping(self):
        class InnerMapAttribute(MapAttribute):
            map_attr = MapAttribute(attr_name='dyn_map_attr')

        class MiddleMapAttributeA(MapAttribute):
            inner_map = InnerMapAttribute(attr_name='dyn_in_map_a')

        class MiddleMapAttributeB(MapAttribute):
            inner_map = InnerMapAttribute(attr_name='dyn_in_map_b')

        class OuterMapAttribute(MapAttribute):
            mid_map_a = MiddleMapAttributeA()
            mid_map_b = MiddleMapAttributeB()

        class MyModel(Model):
            outer_map = OuterMapAttribute(attr_name='dyn_out_map')

        mid_map_a_map_attr = MyModel.outer_map.mid_map_a.inner_map.map_attr
        mid_map_b_map_attr = MyModel.outer_map.mid_map_b.inner_map.map_attr

        assert mid_map_a_map_attr.attr_name == 'dyn_map_attr'
        assert mid_map_a_map_attr.attr_path == ['dyn_out_map', 'mid_map_a', 'dyn_in_map_a', 'dyn_map_attr']
        assert mid_map_b_map_attr.attr_name == 'dyn_map_attr'
        assert mid_map_b_map_attr.attr_path == ['dyn_out_map', 'mid_map_b', 'dyn_in_map_b', 'dyn_map_attr']

class TestValueDeserialize:
    def test__get_value_for_deserialize(self):
        expected = '3'
        data = {'N': '3'}
        actual = _get_value_for_deserialize(data)
        assert expected == actual

    def test__get_value_for_deserialize_null(self):
        data = {'NULL': 'True'}
        actual = _get_value_for_deserialize(data)
        assert actual is None

class TestMapAndListAttribute:

    def test_map_of_list(self):
        grocery_list = {
            'fruit': ['apple', 'pear', 32],
            'veggies': ['broccoli', 'potatoes', 5]
        }
        serialized = MapAttribute().serialize(grocery_list)
        assert MapAttribute().deserialize(serialized) == grocery_list

    def test_map_of_list_of_map(self):
        family_attributes = {
            'phillips': [
                {
                    'name': 'Justin',
                    'age': 31,
                    'height': 187.96
                },
                {
                    'name': 'Galen',
                    'age': 29,
                    'height': 193.04,
                    'male': True
                },
                {
                    'name': 'Colin',
                    'age': 32,
                    'height': 203.2,
                    'male': True,
                    'hasChild': True
                }
            ],
            'workman': [
                {
                    'name': 'Mandy',
                    'age': 29,
                    'height': 157.48,
                    'female': True
                },
                {
                    'name': 'Rodney',
                    'age': 31,
                    'height': 175.26,
                    'hasChild': False
                }
            ]
        }
        serialized = MapAttribute().serialize(family_attributes)
        assert MapAttribute().deserialize(serialized) == family_attributes

    def test_list_of_map_with_of(self):
        class Person(MapAttribute):
            name = UnicodeAttribute()
            age = NumberAttribute()

            def __lt__(self, other):
                return self.name < other.name

            def __eq__(self, other):
                return (self.name == other.name and
                        self.age == other.age)

        person1 = Person()
        person1.name = 'john'
        person1.age = 40

        person2 = Person()
        person2.name = 'Dana'
        person2.age = 41

        inp = [person1, person2]

        list_attribute = ListAttribute(default=[], of=Person)
        serialized = list_attribute.serialize(inp)
        deserialized = list_attribute.deserialize(serialized)
        assert sorted(deserialized) == sorted(inp)

    def test_list_of_map_with_of_and_custom_attribute(self, mocker):

        class CustomMapAttribute(MapAttribute):
            custom = NumberAttribute()

            def __eq__(self, other):
                return self.custom == other.custom

        serialize_mock = mocker.spy(CustomMapAttribute.custom, 'serialize',)
        deserialize_mock = mocker.spy(CustomMapAttribute.custom, 'deserialize')

        attribute1 = CustomMapAttribute()
        attribute1.custom = 1

        attribute2 = CustomMapAttribute()
        attribute2.custom = 2

        inp = [attribute1, attribute2]

        list_attribute = ListAttribute(default=[], of=CustomMapAttribute)
        serialized = list_attribute.serialize(inp)
        deserialized = list_attribute.deserialize(serialized)

        assert deserialized == inp
        assert serialize_mock.call_args_list == [call(1), call(2)]
        assert deserialize_mock.call_args_list == [call('1'), call('2')]


#! /usr/bin/python3
import sys
import random as rd
import multiprocessing as mp
from nestable_pool import NestablePool
from deap import algorithms
from exception import LogExceptions
import builtins

def print(*objects):
    string = ""
    for o in objects:
        string += str(o) + " "
    builtins.print(string,flush=True)

def generate(parents,toolbox,size):
        ''' generate takes a list already evaluated parents and the size of the offspring to generate'''

        #select the individuals from the offspring will be produced
        selected_parents = toolbox.parents_select(parents, size)

        # Clone the selected individuals
        offspring = list(map(toolbox.clone, selected_parents))

        # Apply crossover and mutation on the offspring
        offspring = algorithms.varAnd(offspring, toolbox, 0.1, 0.2)

        return offspring

class AsynchEA:
    def __init__(self,pop_size,nb_workers,sync=0):
        self.pop_size = pop_size
        self.nbr_ind_to_wait = int(pop_size)*sync
        if sync == 0:
            self.nbr_ind_to_wait = 1
        print("number ind to wait:", self.nbr_ind_to_wait)
        self.parents = []
        self.pop = []
        self.evaluated_ind = []
        self.iteration = 0
        self.max_workers = nb_workers
        self.pool = NestablePool(processes=nb_workers)#,maxtasksperchild=100)
        self.in_evaluation = []
        self.workers_failed = False

    def remove(self,select):
        dead = select(self.parents,len(self.parents)-self.pop_size)
        for ind in dead:
            self.parents.remove(ind)
            del ind

    def worker_callback(self,results):
        if results == None:
            self.workers_failed = True
        else:
            self.evaluated_ind.append(results)
            self.in_evaluation.remove(results)

    def asynch_map(self,eval):
        for ind in self.pop:
            is_new_ind = True
            for e_ind in self.in_evaluation + self.evaluated_ind:
                if e_ind == ind:
                    is_new_ind = False
                    break
            if is_new_ind and len(self.in_evaluation) < self.max_workers:
                #print("ind",ind.index,"send to evaluation")
                self.in_evaluation.append(ind)
                self.pool.apply_async(LogExceptions(eval),(ind,),callback=self.worker_callback)
            if len(self.in_evaluation) >= self.max_workers:
                break

    #sequential execution. Use only for debugging
    def seq_map(self,eval):
        for ind in self.pop:
            results = eval(ind)
            self.evaluated_ind.append(results)

    def update(self,eval):
        # Evaluate the individuals with asynch map. Evaluate as to return a ref to the ind at the end
        #self.seq_map(eval)
        self.asynch_map(eval)
        if self.workers_failed:
            self.terminate()
            sys.exit("Exiting because of workers crash")

        if len(self.evaluated_ind) >= self.nbr_ind_to_wait:
            print("number individual evaluated",len(self.evaluated_ind))
            for e_ind in self.evaluated_ind:
                for i in range(len(self.pop)):
                    if self.pop[i] == e_ind:
                        self.pop[i] = e_ind
                        break
            del self.evaluated_ind
            self.evaluated_ind = []

        new_parents = [ind for ind in self.pop if ind.fitness.valid]
        if(len(new_parents) > 0):
            print("new_parents",[ind.index for ind in new_parents])
        for ind in new_parents:
            self.pop.remove(ind)

        return new_parents

    def init(self,toolbox):
        #initialisation
        self.pop = toolbox.population(self.pop_size)
        while len(self.parents) < self.pop_size:
            new_par = self.update(toolbox.eval)
            self.parents = self.parents + new_par
            if(len(new_par) > 0):
                print("init progress:",float(len(self.parents))/float(self.pop_size)*100,"%")
        assert(len(self.pop) == 0)
        self.pop = toolbox.generate(self.parents,toolbox,self.pop_size)
        print(len(self.pop))
        toolbox.extra(toolbox,self.parents,self.iteration)
        return self.parents

    def step(self,toolbox):
        #update - evaluation
        new_par = self.update(toolbox.eval)
        if len(new_par) > 0:
            self.parents = self.parents + new_par

            #survival
            self.remove(toolbox.death_select)

            #selection - mutation - crossover
            offspring = toolbox.generate(self.parents,toolbox,self.pop_size - len(self.pop))
            self.pop = self.pop + offspring

            self.iteration += 1

            toolbox.extra(toolbox,self.parents,self.iteration)

        return self.parents, new_par

    def terminate(self):
        self.pool.terminate()
        self.pool.join()


# -*- coding: utf-8 -*-

# @Time : 2022/1/13 16:59

# @Author : xiaowangwang
# @Email : www@qq.com
# @File : crm_user_tag.py
import os, sys, json
father_dir = os.path.dirname(os.path.dirname(__file__)).split("/")[-1]
sys.path.append(sys.path[0].split(father_dir)[0])
from flask import *
from analyzeserver.common import *
from config import *
import traceback
from util.help_fun import *
import pandas as pd
from analyzeserver.user.sysuser import check_token

usertagbp = Blueprint('crm_user_tag', __name__, url_prefix='/user/tag/')

'''标签列表'''
@usertagbp.route("list",methods=["GET"])
def user_tag_list():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        logger.info(request.json)
        token = request.headers["Token"]
        user_id = request.args.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        sql = '''select id,tag_name from crm_tag'''
        tag_list = pd.read_sql(sql,conn_analyze)
        tag_list.columns=["id","tag_name"]
        tag_list = tag_list.to_dict("records")
        return {"code":"0000","msg":tag_list,"status":"success"}

    except Exception as e:
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

'''用户打标'''
@usertagbp.route("uinsert",methods=["POST"])
def user_tag_uinsert():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        cursor_analyze = conn_analyze.cursor()

        logger.info(request.json)
        token = request.headers["Token"]
        user_id = request.json.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        logger.info("dddddd")
        tag_list = request.json.get("tag_list")
        unionid_list = request.json.get("unionid_list")

        params = []
        if len(unionid_list) == 1:
            #先删除这个用户的所有标签 再插入
            delete_sql = '''delete from crm_user_tag where unionid = %s''' %unionid_list[0]
            cursor_analyze.execute(delete_sql)

        for tag_id in tag_list:
            for unionid in unionid_list:
                #判断标签是否存在 如果存在不在进行打标
                sql = '''select * from crm_user_tag where unionid = %s and tag_id = %s'''
                cursor_analyze.execute(sql,(unionid,tag_id))
                exist = cursor_analyze.fetchone()
                if not exist:
                    params.append((unionid,tag_id))

        logger.info(params)

        insert_sql = '''insert into crm_user_tag (unionid,tag_id) values (%s,%s)'''
        cursor_analyze.executemany(insert_sql,(params))

        #日志记录
        t_data = []
        if params:
            #查询标签列表
            tag_sql = '''select tag_name from crm_tag where id in (%s)''' %(json.dumps(tag_list)[1:-1])
            cursor_analyze.execute(tag_sql)
            tag_data = cursor_analyze.fetchall()
            for td in tag_data:
                t_data.append(td[0])
            logger.info(t_data)

            compare = []
            update_unionid = []
            for p in params:
                update_unionid.append(p[0])
            update_unionid = list(set(update_unionid))
            compare.append("符合修改的unionid:"+",".join(update_unionid))
            compare.append("打上标签:"+",".join(t_data))

            insert_sql = '''insert into sys_log (user_id,log_url,log_req,log_action,remark) values (%s,%s,%s,%s,%s)'''
            params = []
            params.append(user_id)
            params.append("/user/relate/update/user/ascription")
            params.append(json.dumps(request.json))
            params.append("用户打标")
            params.append("
".join(compare))
            logger.info(params)
            cursor_analyze.execute(insert_sql, params)

        conn_analyze.commit()
        return {"code":"0000","msg":"更新成功","status":"success"}
    except Exception as e:
        conn_analyze.rollback()
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

'''新增标签'''
@usertagbp.route("insert",methods=["POST"])
def user_tag_insert():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        cursor_analyze = conn_analyze.cursor()

        logger.info(request.json)
        token = request.headers["Token"]
        user_id = request.json.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        tag_name = request.json.get("tag_name")
        params = []
        params.append(tag_name)

        insert_sql = '''INSERT INTO `lh_analyze`.`crm_tag` (`tag_name`) VALUES (%s)'''
        cursor_analyze.execute(insert_sql,(params))

        #日志记录
        compare = []
        compare.append("新增标签:%s" %tag_name)
        insert_sql = '''insert into sys_log (user_id,log_url,log_req,log_action,remark) values (%s,%s,%s,%s,%s)'''
        params = []
        params.append(user_id)
        params.append("/user/relate/update/user/ascription")
        params.append(json.dumps(request.json))
        params.append("新增标签")
        params.append("
".join(compare))
        logger.info(params)
        cursor_analyze.execute(insert_sql, params)

        conn_analyze.commit()
        return {"code":"0000","msg":"标签新增成功","status":"success"}
    except Exception as e:
        conn_analyze.rollback()
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

'''标签的删除'''
@usertagbp.route("del",methods=["delete"])
def user_tag_delete():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        cursor_analyze = conn_analyze.cursor()

        logger.info(request.json)
        token = request.headers["Token"]
        user_id = request.args.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        tag_id = request.args.get("tag_id")
        params = []
        params.append(tag_id)

        tag_sql = '''select tag_name from crm_tag where id = %s'''
        cursor_analyze.execute(tag_sql, (tag_id))
        tag_name = cursor_analyze.fetchone()

        #删除标签列表
        delete_tag_sql = '''delete from crm_tag where id = %s'''
        cursor_analyze.execute(delete_tag_sql,(tag_id))

        delete_user_tag = '''delete from crm_user_tag where tag_id = %s'''
        cursor_analyze.execute(delete_user_tag,(tag_id))

        # 日志记录
        compare = []
        logger.info(tag_id)
        tag_name = tag_name[0]
        compare.append("删除标签:%s" % tag_name)
        insert_sql = '''insert into sys_log (user_id,log_url,log_req,log_action,remark) values (%s,%s,%s,%s,%s)'''
        params = []
        params.append(user_id)
        params.append("/user/relate/update/user/ascription")
        params.append(json.dumps(request.json))
        params.append("删除标签")
        params.append("
".join(compare))
        logger.info(params)
        cursor_analyze.execute(insert_sql, params)

        conn_analyze.commit()
        return {"code":"0000","msg":"删除标签成功","status":"success"}
    except Exception as e:
        conn_analyze.rollback()
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

'''标签的修改'''
@usertagbp.route("update",methods=["put"])
def user_tag_update():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        cursor_analyze = conn_analyze.cursor()

        logger.info(request.json)
        token = request.headers["Token"]
        user_id = request.json.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        tag_id = request.json.get("tag_id")
        tag_name = request.json.get("tag_name")

        tag_sql = '''select tag_name from crm_tag where id = %s'''
        cursor_analyze.execute(tag_sql, (tag_id))
        old_tag_name = cursor_analyze.fetchone()

        if not tag_id or not tag_name:
            return {"code":"10001","msg":message["10001"],"status":"failed"}

        params = []
        params.append(tag_name)
        params.append(tag_id)

        update_sql = '''update crm_tag set tag_name = %s where id = %s'''
        cursor_analyze.execute(update_sql,params)

        if str(old_tag_name[0]) != str(tag_name):
            compare = []
            logger.info(tag_id)
            compare.append("原标签:%s 修改为:%s" %(old_tag_name[0],tag_name))
            insert_sql = '''insert into sys_log (user_id,log_url,log_req,log_action,remark) values (%s,%s,%s,%s,%s)'''
            params = []
            params.append(user_id)
            params.append("/user/relate/update/user/ascription")
            params.append(json.dumps(request.json))
            params.append("编辑标签")
            params.append("
".join(compare))
            logger.info(params)
            cursor_analyze.execute(insert_sql, params)

        conn_analyze.commit()
        return {"code":"0000","msg":"修改标签成功","status":"success"}
    except Exception as e:
        conn_analyze.rollback()
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

@usertagbp.route("list/distribute",methods=["POST"])
def user_tag_distribute():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)
        token = request.headers["Token"]
        user_id = request.json.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        tag_name = request.json.get("tag_name")
        page = request.json.get("page")
        size = request.json.get("size")

        sql = '''select id tag_id,tag_name from crm_tag'''
        tag_data = pd.read_sql(sql, conn_analyze)

        sql = '''select tag_id,unionid from crm_user_tag'''
        user_tag_data = pd.read_sql(sql, conn_analyze)
        # tag_data["tag_id"] = tag_data["tag_id"].astype(str)

        data = pd.merge(tag_data, user_tag_data, how="left", on="tag_id")
        if tag_name:
            data = data[data["tag_name"].str.contains(tag_name)]

        data = data.groupby(["tag_id", "tag_name"]).agg({"unionid": "count"}).reset_index().rename(columns={"unionid": "count"})
        count = data.shape[0]

        if page and size:
            code_page = (page - 1) * size
            code_size = page * size
            data = data[code_page:code_size]
        logger.info(data)
        data = data.to_dict("records")

        return {"code":"0000","msg":data,"count":count,"status":"success"}

    except Exception as e:
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()

@usertagbp.route("mes",methods=["POST"])
def user_tag_mes():
    try:
        conn_analyze = direct_get_conn(analyze_mysql_conf)

        token = request.headers["Token"]
        user_id = request.json.get("user_id")
        if not user_id and not token:
            return {"code": "10001", "status": "failed", "msg": message["10001"]}

        check_token_result = check_token(token, user_id)
        if check_token_result["code"] != "0000":
            return check_token_result

        unionid = request.json.get("unionid")
        sql = '''select tag_id from crm_user_tag where unionid = %s''' %unionid
        tag_data = pd.read_sql(sql,conn_analyze)
        tag_id_list = tag_data["tag_id"].to_list()

        return {"code":"0000","msg":tag_id_list,"status":"success"}

    except Exception as e:
        logger.error(e)
        logger.exception(traceback.format_exc())
        # 参数名错误
        return {"code": "10000", "status": "failed", "msg": message["10000"]}
    finally:
        conn_analyze.close()


"""
Code for simulating, visualizing and manipulating additive and
multiplicative functionals.

@authors: Chase Coleman, Balint Skoze, Tom Sargent

"""

import numpy as np
import scipy as sp
import scipy.linalg as la
import quantecon as qe
import matplotlib.pyplot as plt
from scipy.stats import norm, lognorm

class AMF_LSS_VAR:
    """
    This class transforms an additive (multipilcative)
    functional into a QuantEcon linear state space system.
    """

    def __init__(self, A, B, D, F=None, nu=None):
        # Unpack required elements
        self.nx, self.nk = B.shape
        self.A, self.B = A, B

        # checking the dimension of D (extended from the scalar case)
        if len(D.shape) > 1 and D.shape[0] != 1:
            self.nm = D.shape[0]
            self.D = D
        elif len(D.shape) > 1 and D.shape[0] == 1:
            self.nm = 1
            self.D = D
        else:
            self.nm = 1
            self.D = np.expand_dims(D, 0)

        # Create space for additive decomposition
        self.add_decomp = None
        self.mult_decomp = None

        # Set F
        if not np.any(F):
            self.F = np.zeros((self.nk, 1))
        else:
            self.F = F

        # Set nu
        if not np.any(nu):
            self.nu = np.zeros((self.nm, 1))
        elif type(nu) == float:
            self.nu = np.asarray([[nu]])
        elif len(nu.shape) == 1:
            self.nu = np.expand_dims(nu, 1)
        else:
            self.nu = nu

        if self.nu.shape[0] != self.D.shape[0]:
            raise ValueError("The dimension of nu is inconsistent with D!")

        # Construct BIG state space representation
        self.lss = self.construct_ss()

    def construct_ss(self):
        """
        This creates the state space representation that can be passed
        into the quantecon LSS class.
        """
        # Pull out useful info
        nx, nk, nm = self.nx, self.nk, self.nm
        A, B, D, F, nu = self.A, self.B, self.D, self.F, self.nu
        if self.add_decomp:
            nu, H, g = self.add_decomp
        else:
            nu, H, g = self.additive_decomp()

        # Auxiliary blocks with 0's and 1's to fill out the lss matrices
        nx0c = np.zeros((nx, 1))
        nx0r = np.zeros(nx)
        nx1 = np.ones(nx)
        nk0 = np.zeros(nk)
        ny0c = np.zeros((nm, 1))
        ny0r = np.zeros(nm)
        ny1m = np.eye(nm)
        ny0m = np.zeros((nm, nm))
        nyx0m = np.zeros_like(D)

        # Build A matrix for LSS
        # Order of states is: [1, t, xt, yt, mt]
        A1 = np.hstack([1, 0, nx0r, ny0r, ny0r])            # Transition for 1
        A2 = np.hstack([1, 1, nx0r, ny0r, ny0r])            # Transition for t
        A3 = np.hstack([nx0c, nx0c, A, nyx0m.T, nyx0m.T])   # Transition for x_{t+1}
        A4 = np.hstack([nu, ny0c, D, ny1m, ny0m])           # Transition for y_{t+1}
        A5 = np.hstack([ny0c, ny0c, nyx0m, ny0m, ny1m])     # Transition for m_{t+1}
        Abar = np.vstack([A1, A2, A3, A4, A5])

        # Build B matrix for LSS
        Bbar = np.vstack([nk0, nk0, B, F, H])

        # Build G matrix for LSS
        # Order of observation is: [xt, yt, mt, st, tt]
        G1 = np.hstack([nx0c, nx0c, np.eye(nx), nyx0m.T, nyx0m.T])    # Selector for x_{t}
        G2 = np.hstack([ny0c, ny0c, nyx0m, ny1m, ny0m])               # Selector for y_{t}
        G3 = np.hstack([ny0c, ny0c, nyx0m, ny0m, ny1m])               # Selector for martingale
        G4 = np.hstack([ny0c, ny0c, -g, ny0m, ny0m])                  # Selector for stationary
        G5 = np.hstack([ny0c, nu, nyx0m, ny0m, ny0m])                 # Selector for trend
        Gbar = np.vstack([G1, G2, G3, G4, G5])

        # Build H matrix for LSS
        Hbar = np.zeros((Gbar.shape[0], nk))

        # Build LSS type
        x0 = np.hstack([1, 0, nx0r, ny0r, ny0r])
        S0 = np.zeros((len(x0), len(x0)))
        lss = qe.lss.LinearStateSpace(Abar, Bbar, Gbar, Hbar, mu_0=x0, Sigma_0=S0)

        return lss

    def additive_decomp(self):
        """
        Return values for the martingale decomposition
            - nu        : unconditional mean difference in Y
            - H         : coefficient for the (linear) martingale component (kappa_a)
            - g         : coefficient for the stationary component g(x)
            - Y_0       : it should be the function of X_0 (for now set it to 0.0)
        """
        I = np.identity(self.nx)
        A_res = la.solve(I - self.A, I)
        g = self.D @ A_res
        H = self.F + self.D @ A_res @ self.B

        return self.nu, H, g

    def multiplicative_decomp(self):
        """
        Return values for the multiplicative decomposition (Example 5.4.4.)
            - nu_tilde  : eigenvalue
            - H         : vector for the Jensen term
        """
        nu, H, g = self.additive_decomp()
        nu_tilde = nu + (.5)*np.expand_dims(np.diag(H @ H.T), 1)

        return nu_tilde, H, g

    def loglikelihood_path(self, x, y):
        A, B, D, F = self.A, self.B, self.D, self.F
        k, T = y.shape
        FF = F @ F.T
        FFinv = la.inv(FF)
        temp = y[:, 1:] - y[:, :-1] - D @ x[:, :-1]
        obs =  temp * FFinv * temp
        obssum = np.cumsum(obs)
        scalar = (np.log(la.det(FF)) + k*np.log(2*np.pi))*np.arange(1, T)

        return -(.5)*(obssum + scalar)

    def loglikelihood(self, x, y):
        llh = self.loglikelihood_path(x, y)

        return llh[-1]

    def plot_additive(self, T, npaths=25, show_trend=True):
        """
        Plots for the additive decomposition

        """
        # Pull out right sizes so we know how to increment
        nx, nk, nm = self.nx, self.nk, self.nm

        # Allocate space (nm is the number of additive functionals - we want npaths for each)
        mpath = np.empty((nm*npaths, T))
        mbounds = np.empty((nm*2, T))
        spath = np.empty((nm*npaths, T))
        sbounds = np.empty((nm*2, T))
        tpath = np.empty((nm*npaths, T))
        ypath = np.empty((nm*npaths, T))

        # Simulate for as long as we wanted
        moment_generator = self.lss.moment_sequence()
        # Pull out population moments
        for t in range (T):
            tmoms = next(moment_generator)
            ymeans = tmoms[1]
            yvar = tmoms[3]

            # Lower and upper bounds - for each additive functional
            for ii in range(nm):
                li, ui = ii*2, (ii+1)*2
                madd_dist = norm(ymeans[nx+nm+ii], np.sqrt(yvar[nx+nm+ii, nx+nm+ii]))
                mbounds[li:ui, t] = madd_dist.ppf([0.01, .99])

                sadd_dist = norm(ymeans[nx+2*nm+ii], np.sqrt(yvar[nx+2*nm+ii, nx+2*nm+ii]))
                sbounds[li:ui, t] = sadd_dist.ppf([0.01, .99])

        # Pull out paths
        for n in range(npaths):
            x, y = self.lss.simulate(T)
            for ii in range(nm):
                ypath[npaths*ii+n, :] = y[nx+ii, :]
                mpath[npaths*ii+n, :] = y[nx+nm + ii, :]
                spath[npaths*ii+n, :] = y[nx+2*nm + ii, :]
                tpath[npaths*ii+n, :] = y[nx+3*nm + ii, :]

        add_figs = []

        for ii in range(nm):
            li, ui = npaths*(ii), npaths*(ii+1)
            LI, UI = 2*(ii), 2*(ii+1)
            add_figs.append(self.plot_given_paths(T, ypath[li:ui,:], mpath[li:ui,:], spath[li:ui,:],
                                                  tpath[li:ui,:], mbounds[LI:UI,:], sbounds[LI:UI,:],
                                                  show_trend=show_trend))

            add_figs[ii].suptitle( r'Additive decomposition of $y_{%s}$' % str(ii+1), fontsize=14 )

        return add_figs

    def plot_multiplicative(self, T, npaths=25, show_trend=True):
        """
        Plots for the multiplicative decomposition

        """
        # Pull out right sizes so we know how to increment
        nx, nk, nm = self.nx, self.nk, self.nm
        # Matrices for the multiplicative decomposition
        nu_tilde, H, g = self.multiplicative_decomp()

        # Allocate space (nm is the number of functionals - we want npaths for each)
        mpath_mult = np.empty((nm*npaths, T))
        mbounds_mult = np.empty((nm*2, T))
        spath_mult = np.empty((nm*npaths, T))
        sbounds_mult = np.empty((nm*2, T))
        tpath_mult = np.empty((nm*npaths, T))
        ypath_mult = np.empty((nm*npaths, T))

        # Simulate for as long as we wanted
        moment_generator = self.lss.moment_sequence()
        # Pull out population moments
        for t in range (T):
            tmoms = next(moment_generator)
            ymeans = tmoms[1]
            yvar = tmoms[3]

            # Lower and upper bounds - for each multiplicative functional
            for ii in range(nm):
                li, ui = ii*2, (ii+1)*2
                Mdist = lognorm(np.asscalar(np.sqrt(yvar[nx+nm+ii, nx+nm+ii])),
                                scale=np.asscalar( np.exp( ymeans[nx+nm+ii]- \
                                                t*(.5)*np.expand_dims(np.diag(H @ H.T),1)[ii])))
                Sdist = lognorm(np.asscalar(np.sqrt(yvar[nx+2*nm+ii, nx+2*nm+ii])),
                                scale = np.asscalar( np.exp(-ymeans[nx+2*nm+ii])))
                mbounds_mult[li:ui, t] = Mdist.ppf([.01, .99])
                sbounds_mult[li:ui, t] = Sdist.ppf([.01, .99])

        # Pull out paths
        for n in range(npaths):
            x, y = self.lss.simulate(T)
            for ii in range(nm):
                ypath_mult[npaths*ii+n, :] = np.exp(y[nx+ii, :])
                mpath_mult[npaths*ii+n, :] = np.exp(y[nx+nm + ii, :] - np.arange(T)*(.5)*np.expand_dims(np.diag(H @ H.T),1)[ii])
                spath_mult[npaths*ii+n, :] = 1/np.exp(-y[nx+2*nm + ii, :])
                tpath_mult[npaths*ii+n, :] = np.exp(y[nx+3*nm + ii, :] + np.arange(T)*(.5)*np.expand_dims(np.diag(H @ H.T),1)[ii])

        mult_figs = []

        for ii in range(nm):
            li, ui = npaths*(ii), npaths*(ii+1)
            LI, UI = 2*(ii), 2*(ii+1)

            mult_figs.append(self.plot_given_paths(T, ypath_mult[li:ui,:], mpath_mult[li:ui,:],
                                                   spath_mult[li:ui,:], tpath_mult[li:ui,:],
                                                   mbounds_mult[LI:UI,:], sbounds_mult[LI:UI,:], 1,
                                                   show_trend=show_trend))
            mult_figs[ii].suptitle( r'Multiplicative decomposition of $y_{%s}$' % str(ii+1), fontsize=14)

        return mult_figs

    def plot_martingales(self, T, npaths=25):

        # Pull out right sizes so we know how to increment
        nx, nk, nm = self.nx, self.nk, self.nm
        # Matrices for the multiplicative decomposition
        nu_tilde, H, g = self.multiplicative_decomp()

        # Allocate space (nm is the number of functionals - we want npaths for each)
        mpath_mult = np.empty((nm*npaths, T))
        mbounds_mult = np.empty((nm*2, T))

        # Simulate for as long as we wanted
        moment_generator = self.lss.moment_sequence()
        # Pull out population moments
        for t in range (T):
            tmoms = next(moment_generator)
            ymeans = tmoms[1]
            yvar = tmoms[3]

            # Lower and upper bounds - for each functional
            for ii in range(nm):
                li, ui = ii*2, (ii+1)*2
                Mdist = lognorm(np.asscalar(np.sqrt(yvar[nx+nm+ii, nx+nm+ii])),
                                scale=np.asscalar( np.exp( ymeans[nx+nm+ii]- \
                                                t*(.5)*np.expand_dims(np.diag(H @ H.T),1)[ii])))
                mbounds_mult[li:ui, t] = Mdist.ppf([.01, .99])

        # Pull out paths
        for n in range(npaths):
            x, y = self.lss.simulate(T)
            for ii in range(nm):
                mpath_mult[npaths*ii+n, :] = np.exp(y[nx+nm + ii, :] - np.arange(T)*(.5)*np.expand_dims(np.diag(H @ H.T),1)[ii])

        mart_figs = []

        for ii in range(nm):
            li, ui = npaths*(ii), npaths*(ii+1)
            LI, UI = 2*(ii), 2*(ii+1)
            mart_figs.append(self.plot_martingale_paths(T, mpath_mult[li:ui,:],
                                                        mbounds_mult[LI:UI,:], horline=1))
            mart_figs[ii].suptitle(r'Martingale components for many paths of $y_{%s}$' % str(ii+1),
                                   fontsize=14)

        return mart_figs

    def plot_given_paths(self, T, ypath, mpath, spath, tpath, mbounds, sbounds, horline=0, show_trend = True):

        # Allocate space
        trange = np.arange(T)

        # Create figure
        fig, ax = plt.subplots(2, 2, sharey=True, figsize=(15, 8))

        # Plot all paths together
        ax[0, 0].plot(trange, ypath[0, :], label=r"$y_t$", color="k")
        ax[0, 0].plot(trange, mpath[0, :], label=r"$m_t$", color="m")
        ax[0, 0].plot(trange, spath[0, :], label=r"$s_t$", color="g")
        if show_trend:
            ax[0, 0].plot(trange, tpath[0, :], label=r"$t_t$", color="r")
        ax[0, 0].axhline(horline, color="k", linestyle = "-.")
        ax[0, 0].set_title("One Path of All Variables")
        ax[0, 0].legend(loc="top left")

        # Plot Martingale Component
        ax[0, 1].plot(trange, mpath[0, :], "m")
        ax[0, 1].plot(trange, mpath.T, alpha=0.45, color="m")
        ub = mbounds[1, :]
        lb = mbounds[0, :]
        ax[0, 1].fill_between(trange, lb, ub, alpha=0.25, color="m")
        ax[0, 1].set_title("Martingale Components for Many Paths")
        ax[0, 1].axhline(horline, color="k", linestyle = "-.")

        # Plot Stationary Component
        ax[1, 0].plot(spath[0, :], color="g")
        ax[1, 0].plot(spath.T, alpha=0.25, color="g")
        ub = sbounds[1, :]
        lb = sbounds[0, :]
        ax[1, 0].fill_between(trange, lb, ub, alpha=0.25, color="g")
        ax[1, 0].axhline(horline, color="k", linestyle = "-.")
        ax[1, 0].set_title("Stationary Components for Many Paths")

        # Plot Trend Component
        if show_trend:
            ax[1, 1].plot(tpath.T, color="r")
        ax[1, 1].set_title("Trend Components for Many Paths")
        ax[1, 1].axhline(horline, color="k", linestyle = "-.")

        return fig

    def plot_martingale_paths(self, T, mpath, mbounds, horline=1, show_trend = False):
        # Allocate space
        trange = np.arange(T)

        # Create figure
        fig, ax = plt.subplots(1,1, figsize=(10,6))

        # Plot Martingale Component
        ub = mbounds[1, :]
        lb = mbounds[0, :]
        ax.fill_between(trange, lb, ub, color="#ffccff")
        ax.axhline(horline, color="k", linestyle = "-.")
        ax.plot(trange, mpath.T, linewidth=0.25, color="#4c4c4c")

        return fig


# 1.找到这个文件，双击打开它
# open(文件路径,mode="",encoding="")
# 1,绝对路径  d:/test/xxxx.txt
# 2,相对路径 相对于当前你的程序所在的文件夹
#
# ../上一层文件夹

# mode:
# r:read读取
# w:write写
# a:append追加写入
# b:读写的是非文本文件
# with:上下文，不需要手动去关闭一个文件
import os  #导入操作系统的相关类库
import time #和时间相关的类库
# 想当于一条管子，需要就读取
# f = open("夜晚数据.txt", mode="r", encoding="utf-8")
# content = f.read() #全部读取
# print(content)
#
# line = f.readline().strip() #去掉字符串左右两端的空白，空格，换行，制表符
# print(line)
#
# line1 = f.readlines()
# print(line1)

#最重要的一种文本读取方式（必须掌握）     #从f中读取到每一行数据
# for line in f:
#     line4 =line.strip()
#
#     print(line4)

# 写入文件
#     在w模式下,如果文件不存在,自动的创建一个文件
    # w模式下，每一次open会清空文件中的内容
    # k = open("奇幻.txt", mode="w",encoding="utf-8")
    # k.write("流量王")
    # k.close() #每次操作过后记得关闭
    #准备一个列表，要求把列表中的每一项内容写入到文件中
# lst = ['赵子龙','黄忠','张飞','关羽','刘邦']
#
# v = open("三国志.txt", mode="w",encoding="utf-8")
# #大多数情况要吧open写在循环外面,  千万不要选错自动模板，不然更改后无效
# for item in lst:
#     v.write(item)
#     v.write("\n")
#
# v.close()

#追加数据
# s = open("三国志.txt", mode="a",encoding="utf-8")
# s.write("黎明")

#with 自动关闭，不用close
# with open("三国志.txt", mode="r", encoding="utf-8") as f:
#     for  line in f:
#         print(line.strip())

# 想要读取图片
#在读写非文本文件的时候要加上b
# with open("小狗狗.png", mode="rb") as f:
#     for line in f:
#         print(line) #读取出来为字节码

# 文件的复制
# 从源文件读取内容，写入到新路径去
# with open("小狗狗.png", mode="rb") as f1,\
#     open("../len/大狗狗.png", mode="wb") as f2:
#     for line in f1:
#         f2.write(line)

#把文件中的周->张
with open("人名单.txt", mode="r",encoding="utf-8") as f1, \
    open("人名单_副本.txt", mode="w",encoding="utf-8" ) as f2:
    for line in f1:
        line = line.strip() #去掉换行
        if line.startswith("张"):
            line = line.replace("张","肖")  #修改
            f2.write(line)
            f2.write("\n")

time.sleep(3) #让程序休眠3秒钟
#删除源文件
os.remove("人名单.txt")

time.sleep(3) #让程序休眠3秒钟
#把副本文件从命名成源文件
os.rename("人名单_副本.txt","人名单.txt")