import wradlib as wrl
import datetime as dt
import numpy as np
from osgeo import gdal
# flake8: noqa

def read_gpm(filename):

    pr_data = wrl.io.read_generic_hdf5(filename)

    lon = pr_data['NS/Longitude']['data']
    lat = pr_data['NS/Latitude']['data']

    year = pr_data['NS/ScanTime/Year']['data']
    month = pr_data['NS/ScanTime/Month']['data']
    dayofmonth = pr_data['NS/ScanTime/DayOfMonth']['data']
    dayofyear = pr_data['NS/ScanTime/DayOfYear']['data']
    hour = pr_data['NS/ScanTime/Hour']['data']
    minute = pr_data['NS/ScanTime/Minute']['data']
    second = pr_data['NS/ScanTime/Second']['data']
    secondofday = pr_data['NS/ScanTime/SecondOfDay']['data']
    millisecond = pr_data['NS/ScanTime/MilliSecond']['data']
    date_array = zip(year, month, dayofmonth,
                     hour, minute, second,
                     millisecond.astype(np.int32) * 1000)
    pr_time = np.array(
        [dt.datetime(d[0], d[1], d[2], d[3], d[4], d[5], d[6]) for d in
         date_array])

    sfc = pr_data['NS/PRE/landSurfaceType']['data']
    pflag = pr_data['NS/PRE/flagPrecip']['data']

    bbflag = pr_data['NS/CSF/flagBB']['data']
    zbb = pr_data['NS/CSF/heightBB']['data']
    print(zbb.dtype)
    bbwidth = pr_data['NS/CSF/widthBB']['data']
    qbb = pr_data['NS/CSF/qualityBB']['data']
    qtype = pr_data['NS/CSF/qualityTypePrecip']['data']
    ptype = pr_data['NS/CSF/typePrecip']['data']

    quality = pr_data['NS/scanStatus/dataQuality']['data']
    refl = pr_data['NS/SLV/zFactorCorrected']['data']

    # Check for bad data
    if max(quality) != 0:
        raise ValueError('GPM contains Bad Data')

    pflag = pflag.astype(np.int8)

    # Determine the dimensions
    ndim = refl.ndim
    if ndim != 3:
        raise ValueError('GPM Dimensions do not match! '
                         'Needed 3, given {0}'.format(ndim))

    tmp = refl.shape
    nscan = tmp[0]
    nray = tmp[1]
    nbin = tmp[2]

    # Reverse direction along the beam
    # TODO: Why is this reversed?
    refl = refl[::-1]

    # Change pflag=1 to pflag=2 to be consistent with 'Rain certain' in TRMM
    pflag[pflag == 1] = 2

    # Simplify the precipitation types
    ptype = (ptype/1e7).astype(np.int16)

    # Simplify the surface types
    imiss = (sfc == -9999)
    sfc = (sfc/1e2).astype(np.int16) + 1
    sfc[imiss] = 0

    # Set a quality indicator for the BB and precip type data
    # TODO: Why is the `quality` variable overwritten?

    quality = np.zeros((nscan, nray), dtype=np.uint8)

    i1 = ((qbb == 0) | (qbb == 1)) & (qtype == 1)
    quality[i1] = 1

    i2 = ((qbb > 1) | (qtype > 2))
    quality[i2] = 2

    gpm_data = {}
    gpm_data.update({'nscan': nscan, 'nray': nray, 'nbin': nbin,
                     'date': pr_time, 'lon': lon, 'lat': lat,
                     'pflag': pflag, 'ptype': ptype, 'zbb': zbb,
                     'bbwidth': bbwidth, 'sfc': sfc, 'quality': quality,
                     'refl': refl})

    return gpm_data

def read_trmm(filename1, filename2):

    # trmm 2A23 and 2A25 data is hdf4
    # it can be read with `read_generic_netcdf`
    pr_data1 = wrl.io.read_generic_netcdf(filename1)
    pr_data2 = wrl.io.read_generic_netcdf(filename2)

    lon = pr_data1['variables']['Longitude']['data']
    lat = pr_data1['variables']['Latitude']['data']

    year = pr_data1['variables']['Year']['data']
    month = pr_data1['variables']['Month']['data']
    dayofmonth = pr_data1['variables']['DayOfMonth']['data']
    dayofyear = pr_data1['variables']['DayOfYear']['data']
    hour = pr_data1['variables']['Hour']['data']
    minute = pr_data1['variables']['Minute']['data']
    second = pr_data1['variables']['Second']['data']
    secondofday = pr_data1['variables']['scanTime_sec']['data']
    millisecond = pr_data1['variables']['MilliSecond']['data']
    date_array = zip(year, month, dayofmonth,
                     hour, minute, second,
                     millisecond.astype(np.int32) * 1000)
    pr_time = np.array(
        [dt.datetime(d[0], d[1], d[2], d[3], d[4], d[5], d[6]) for d in
         date_array])

    pflag = pr_data1['variables']['rainFlag']['data']
    ptype = pr_data1['variables']['rainType']['data']

    status = pr_data1['variables']['status']['data']
    zbb = pr_data1['variables']['HBB']['data'].astype(np.float32)
    bbwidth = pr_data1['variables']['BBwidth']['data'].astype(np.float32)

    quality = pr_data2['variables']['dataQuality']['data']
    refl = pr_data2['variables']['correctZFactor']['data'] / 100.
    print(refl.dtype, refl)

    # Ground clutter
    refl[refl == -8888.] = np.nan
    # Misssing data
    refl[refl == -9999.] = np.nan
    # Scaling
    refl /= 100.

    # Check for bad data
    if max(quality) != 0:
        raise ValueError('TRMM contains Bad Data')

    # Determine the dimensions
    ndim = refl.ndim
    if ndim != 3:
        raise ValueError('TRMM Dimensions do not match!'
                         'Needed 3, given {0}'.format(ndim))

    tmp = refl.shape
    nscan = tmp[0]
    nray = tmp[1]
    nbin = tmp[2]

    # Reverse direction along the beam
    # TODO: Why is this reversed?
    refl = refl[::-1]

    # Simplify the precipitation flag
    ipos = (pflag >= 10) & (pflag <= 20)
    icer = (pflag >= 20)
    pflag[ipos] = 1
    pflag[icer] = 2

    # Simplify the precipitation types
    istr = (ptype >= 100) & (ptype <= 200)
    icon = (ptype >= 200) & (ptype <= 300)
    ioth = (ptype >= 300)
    inone = (ptype == -88)
    imiss = (ptype == -99)
    ptype[istr] = 1
    ptype[icon] = 2
    ptype[ioth] = 3
    ptype[inone] = 0
    ptype[imiss] = -1

    # Extract the surface type
    sfc = np.zeros((nscan, nray), dtype=np.uint8)
    i0 = (status == 168)
    sfc[i0] = 0
    i1 = (status % 10 == 0)
    sfc[i1] = 1
    i2 = ((status - 1) % 10 == 0)
    sfc[i2] = 2
    i3 = ((status - 3) % 10 == 0)
    sfc[i3] = 3
    i4 = ((status - 4) % 10 == 0)
    sfc[i4] = 4
    i5 = ((status - 5) % 10 == 0)
    sfc[i5] = 5
    i9 = ((status - 9) % 10 == 0)
    sfc[i9] = 9

    # Extract 2A23 quality
    # TODO: Why is the `quality` variable overwritten?
    quality = np.zeros((nscan, nray), dtype=np.uint8)
    i0 = (status == 168)
    quality[i0] = 0
    i1 = (status < 50)
    quality[i1] = 1
    i2 = ((status >= 50) & (status < 109))
    quality[i2] = 2

    trmm_data = {}
    trmm_data.update({'nscan': nscan, 'nray': nray, 'nbin': nbin,
                      'date': pr_time, 'lon': lon, 'lat': lat,
                      'pflag': pflag, 'ptype': ptype, 'zbb': zbb,
                      'bbwidth': bbwidth, 'sfc': sfc, 'quality': quality,
                      'refl': refl})

    return trmm_data

# Hidden subdatasets in TRMM HDF4 files
sds_hidden_2a23 = {
    "0": {"name": "Year",         "units": "years",   "dtype": np.integer},
    "1": {"name": "Month",        "units": "months",  "dtype": np.integer},
    "2": {"name": "DayOfMonth",   "units": "days",    "dtype": np.integer},
    "3": {"name": "Hour",         "units": "hours",   "dtype": np.integer},
    "4": {"name": "Minute",       "units": "minutes", "dtype": np.integer},
    "5": {"name": "Second",       "units": "s",       "dtype": np.integer},
    "6": {"name": "MilliSecond",  "units": "ms",      "dtype": np.integer}
}
# NOT USED
#    "7": {"name": "DayOfYear",    "units": "days",    "dtype": np.integer},
#    "8": {"name": "scanTime_sec", "units": "s",       "dtype": np.float},

sds_hidden_2a25 = {
    "8": {"name": "dataQuality", "units": None, "dtype": np.integer}
}
# NOT USED
# "0": {"name": "Year",         "units": "years",   "dtype": np.integer},
# "1": {"name": "Month", "units": "months", "dtype": np.integer},
# "2": {"name": "DayOfMonth", "units": "days", "dtype": np.integer},
# "3": {"name": "Hour", "units": "hours", "dtype": np.integer},
# "4": {"name": "Minute", "units": "minutes", "dtype": np.integer},
# "5": {"name": "Second", "units": "s", "dtype": np.integer},
# "6": {"name": "MilliSecond", "units": "ms", "dtype": np.integer}
# "7": {"name": "DayOfYear",    "units": "days",    "dtype": np.integer},
# "9": {"name": "scanTime_sec", "units": "s",       "dtype": np.float},

trmm_sd_prefix = "HDF4_SDS:UNKNOWN:"

def read_trmm_gdal(f2a23, f2a25):
    """
    TRMM 2A23 and 2A25 data comes in hdf4. It can be read with
    `read_generic_netcdf`, but only if netcDF4 was compiled with hdf4 support.
    If not, we need to use gdal.

    The use of this function is discouraged in case read_trmm works. This
    is because the mapping is fragile since GDAL is unable to access all
    subdatasets by name. Unfortunately, we have to assume that the order
    of subdatasets remains the same. However, this might not be true depending
    on how the TRMM file was generated from the STORM database!

    """
    vars2a23 = {"Latitude": "lat", "Longitude": "lon", "rainFlag": "pflag",
                "rainType": "ptype", "status": "status", "HBB": "zbb",
                "BBwidth": "bbwidth"}
    vars2a25 = {"correctZFactor": "refl"}

    # Extract "visible" 2A23 subdatasets into output dictionary
    trmm = {}
    hdf = gdal.Open(f2a23)
    sds = hdf.GetSubDatasets()
    for var in vars2a23.keys():
        for sd in sds:
            if (var in sd[1]):
                trmm[vars2a23[var]] = gdal.Open(sd[0]).ReadAsArray()
    # TODO: What about the flags (-1111: no bb, -8888: no rain, -9999: no data)
    trmm["zbb"] = trmm["zbb"].astype(np.float32)
    trmm["bbwidth"] = trmm["bbwidth"].astype(np.float32)

    # Extract "hidden" 2A23 subdatasets into output dictionary
    d = {}
    for sd in sds_hidden_2a23.keys():
        sdstr = "%s%s:%s" % (trmm_sd_prefix, hdf.GetDescription(), sd)
        tmp = gdal.Open(sdstr)
        d[sds_hidden_2a23[sd]["name"]] = tmp.ReadAsArray()
        # Check consistency
        #   of units
        if not sds_hidden_2a23[sd]["units"] == tmp.GetMetadata_Dict()["units"]:
            raise ValueError("Unexpected TRMM HDF4 subdataset units: %s"
                             "instead of %s" % (sds_hidden_2a23[sd]["units"],
                                                tmp.GetMetadata_Dict()["units"]))
        #   of data type
        if not np.issubdtype(d[sds_hidden_2a23[sd]["name"]].dtype,
                             sds_hidden_2a23[sd]["dtype"]):
            raise ValueError("Unexpected TRMM HDF4 subdataset dtype:"
                             "%s instead of %s" %
                             (d[sds_hidden_2a23[sd]["name"]].dtype,
                              sds_hidden_2a23[sd]["dtype"]))

    date_array = zip(d["Year"].astype(np.int32).ravel(),
                     d["Month"].astype(np.int32).ravel(),
                     d["DayOfMonth"].astype(np.int32).ravel(),
                     d["Hour"].astype(np.int32).ravel(),
                     d["Minute"].astype(np.int32).ravel(),
                     d["Second"].astype(np.int32).ravel(),
                     # Millisecond to Microsecond
                     d["MilliSecond"].astype(np.int32).ravel() * 1000)

    trmm["date"] = np.array(
        [dt.datetime(d[0], d[1], d[2], d[3], d[4], d[5], d[6]) for d in
         date_array]
    )

    # Extract "visible" 2A25 subdatasets into output dictionary
    hdf = gdal.Open(f2a25)
    sds = hdf.GetSubDatasets()
    for var in vars2a25.keys():
        for sd in sds:
            if (var in sd[1]):
                trmm[vars2a25[var]] = np.swapaxes(
                    gdal.Open(sd[0]).ReadAsArray(), 0, 1)
    trmm["refl"] = trmm["refl"].astype(np.float32)
    # Ground clutter
    trmm["refl"][trmm["refl"] == -8888.] = np.nan
    # Misssing data
    trmm["refl"][trmm["refl"] == -9999.] = np.nan
    # Scaling
    trmm["refl"] = trmm["refl"] / 100.

    # Extract "hidden" 2A25 subdatasets into output dictionary
    d2 = {}
    for sd in sds_hidden_2a25.keys():
        sdstr = "%s%s:%s" % (trmm_sd_prefix, hdf.GetDescription(), sd)
        tmp = gdal.Open(sdstr)
        d2[sds_hidden_2a25[sd]["name"]] = tmp.ReadAsArray()
        # Check consistency
        #   of units
        if "units" not in tmp.GetMetadata_Dict().keys():
            # Quality flag does not have a units keyword
            if not sds_hidden_2a25[sd]["units"] is None:
                raise ValueError("Unexpected TRMM HDF4 subdataset units: %s"
                                 "is not expected to have units." %
                                 sds_hidden_2a25[sd]["name"])
        elif not sds_hidden_2a25[sd]["units"] == tmp.GetMetadata_Dict()["units"]:
            raise ValueError("Unexpected TRMM HDF4 subdataset units: %s"
                             "instead of %s" % (sds_hidden_2a25[sd]["units"],
                                                tmp.GetMetadata_Dict()["units"]))
        #   of data type
        if not np.issubdtype(d2[sds_hidden_2a25[sd]["name"]].dtype,
                             sds_hidden_2a25[sd]["dtype"]):
            raise ValueError("Unexpected TRMM HDF4 subdataset dtype:"
                             "%s instead of %s" %
                             (d2[sds_hidden_2a25[sd]["name"]].dtype,
                              sds_hidden_2a25[sd]["dtype"]))

    # Check for bad data
    if d2["dataQuality"].max() != 0:
        raise ValueError('TRMM contains Bad Data.')

    # Check dimensions
    ndim = trmm["refl"].ndim
    if ndim != 3:
        raise ValueError("TRMM Dimensions do not match!"
                         "Needed 3, given {0}".format(ndim))

    tmp = trmm["refl"].shape
    nscan = tmp[0]
    nray = tmp[1]
    nbin = tmp[2]

    # Reverse direction along the beam
    # TODO: Why is this reversed?
    trmm["refl"] = trmm["refl"][::-1]

    # Simplify the precipitation flag
    ipos = (trmm["pflag"] >= 10) & (trmm["pflag"] <= 20)
    icer = (trmm["pflag"] >= 20)
    trmm["pflag"][ipos] = 1
    trmm["pflag"][icer] = 2

    # Simplify the precipitation types
    istr = (trmm["ptype"] >= 100) & (trmm["ptype"] <= 200)
    icon = (trmm["ptype"] >= 200) & (trmm["ptype"] <= 300)
    ioth = (trmm["ptype"] >= 300)
    inone = (trmm["ptype"] == -88)
    imiss = (trmm["ptype"] == -99)
    trmm["ptype"][istr] = 1
    trmm["ptype"][icon] = 2
    trmm["ptype"][ioth] = 3
    trmm["ptype"][inone] = 0
    trmm["ptype"][imiss] = -1

    # Extract the surface type
    sfc = np.zeros((nscan, nray), dtype=np.uint8)
    i0 = (trmm["status"] == 168)
    sfc[i0] = 0
    i1 = (trmm["status"] % 10 == 0)
    sfc[i1] = 1
    i2 = ((trmm["status"] - 1) % 10 == 0)
    sfc[i2] = 2
    i3 = ((trmm["status"] - 3) % 10 == 0)
    sfc[i3] = 3
    i4 = ((trmm["status"] - 4)  % 10 == 0)
    sfc[i4] = 4
    i5 = ((trmm["status"] - 5) % 10 == 0)
    sfc[i5] = 5
    i9 = ((trmm["status"] - 9) % 10 == 0)
    sfc[i9] = 9

    # Extract 2A23 quality
    # TODO: Why is the `quality` variable overwritten?
    trmm["quality"] = np.zeros((nscan, nray), dtype=np.uint8)
    i0 = (trmm["status"] == 168)
    sfc[i0] = 0
    i1 = (trmm["status"] < 50)
    sfc[i1] = 1
    i2 = ((trmm["status"] >= 50) & (trmm["status"] < 109))
    sfc[i2] = 2

    # Update output dictionary
    trmm["sfs"] = sfc
    del trmm["status"]
    trmm["nscan"] = nscan
    trmm["nray"] = nray
    trmm["nbin"] = nbin

    return trmm

def _get_tilts(dic):
    i = 0
    for k in dic.keys():
        if 'dataset' in k:
            i += 1
    return i

def read_gr(filename, loaddata=True):

    gr_data = wrl.io.read_generic_netcdf(filename)
    dat = gr_data['what']['date']
    tim = gr_data['what']['time']
    date = dt.datetime.strptime(dat + tim, "%Y%d%m%H%M%S")
    source = gr_data['what']['source']

    lon = gr_data['where']['lon']
    lat = gr_data['where']['lat']
    alt = gr_data['where']['height']

    if gr_data['what']['object'] == 'PVOL':
        ntilt = _get_tilts(gr_data)
    else:
        raise ValueError('GR file is no PPI/Volume File')

    ngate = np.zeros(ntilt, dtype=np.int16)
    nbeam = np.zeros(ntilt)
    elang = np.zeros(ntilt)
    r0 = np.zeros(ntilt)
    dr = np.zeros(ntilt)
    a0 = np.zeros(ntilt)

    for i in range(0, ntilt):
        dset = gr_data['dataset{0}'.format(i+1)]
        a0[i] = dset['how']['astart']
        elang[i] = dset['where']['elangle']
        ngate[i] = dset['where']['nbins']
        r0[i] = dset['where']['rstart']
        dr[i] = dset['where']['rscale']
        nbeam[i] = dset['where']['nrays']

    if ((len(np.unique(r0)) != 1) |
            (len(np.unique(dr)) != 1) |
            (len(np.unique(a0)) != 1) |
            (len(np.unique(nbeam)) != 1) |
            (nbeam[0] != 360)):
        raise ValueError('GroundRadar Data layout dos not match')

    gr_dict = {}
    gr_dict.update({'source': source, 'date': date, 'lon': lon, 'lat': lat,
                    'alt': alt, 'ngate': ngate, 'nbeam': nbeam, 'ntilt': ntilt,
                    'r0': r0, 'dr': dr, 'a0': a0, 'elang': elang})
    if not loaddata:
        return gr_dict

    sdate = []
    refl = []
    for i in range(0, ntilt):
        dset = gr_data['dataset{0}'.format(i+1)]
        dat = dset['what']['startdate']
        tim = dset['what']['starttime']
        date = dt.datetime.strptime(dat + tim, "%Y%d%m%H%M%S")
        sdate.append(date)
        data = dset['data1']
        quantity = data['what']['quantity']
        factor = data['what']['gain']
        offset = data['what']['offset']
        if quantity == 'DBZH':
            dat = data['variables']['data']['data'] * factor + offset
            refl.append(dat)

    sdate = np.array(sdate)
    refl = np.array(refl)

    gr_dict.update({'sdate': sdate, 'refl': refl})

    return gr_dict

def _get_tilts2(dic):
    i = 0
    for k in dic.keys():
        if 'dataset' in k:
            i += 1
    return i/5

def read_gr2(filename, loaddata=True):

    gr_data = wrl.io.read_generic_hdf5(filename)
    dat = gr_data['what']['attrs']['date'].decode()
    tim = gr_data['what']['attrs']['time'].decode()
    date = dt.datetime.strptime(dat + tim, "%Y%d%m%H%M%S")
    source = gr_data['what']['attrs']['source']

    lon = gr_data['where']['attrs']['lon']
    lat = gr_data['where']['attrs']['lat']
    alt = gr_data['where']['attrs']['height']

    if gr_data['what']['attrs']['object'].decode() == 'PVOL':
        ntilt = int(_get_tilts2(gr_data))
        print("ntilt:", ntilt)
    else:
        raise ValueError('GR file is no PPI/Volume File')

    ngate = np.zeros(ntilt, dtype=np.int16)
    nbeam = np.zeros(ntilt)
    elang = np.zeros(ntilt)
    r0 = np.zeros(ntilt)
    dr = np.zeros(ntilt)
    a0 = np.zeros(ntilt)

    for i in np.arange(0, ntilt, dtype=np.uint8):
        a0[i] = gr_data['dataset{0}/how'.format(i+1)]['attrs']['astart']
        elang[i] = gr_data['dataset{0}/where'.format(i+1)]['attrs']['elangle']
        ngate[i] = gr_data['dataset{0}/where'.format(i+1)]['attrs']['nbins']
        r0[i] = gr_data['dataset{0}/where'.format(i+1)]['attrs']['rstart']
        dr[i] = gr_data['dataset{0}/where'.format(i+1)]['attrs']['rscale']
        nbeam[i] = gr_data['dataset{0}/where'.format(i+1)]['attrs']['nrays']

    if ((len(np.unique(r0)) != 1) |
            (len(np.unique(dr)) != 1) |
            (len(np.unique(a0)) != 1) |
            (len(np.unique(nbeam)) != 1) |
            (nbeam[0] != 360)):
        raise ValueError('GroundRadar Data layout dos not match')

    gr_dict = {}
    gr_dict.update({'source': source, 'date': date, 'lon': lon, 'lat': lat,
                    'alt': alt, 'ngate': ngate, 'nbeam': nbeam, 'ntilt': ntilt,
                    'r0': r0, 'dr': dr, 'a0': a0, 'elang': elang})
    if not loaddata:
        return gr_dict

    sdate = []
    refl = []
    for i in np.arange(0, ntilt, dtype=np.uint8):
        dat = gr_data['dataset{0}/what'.format(i+1)]['attrs']['startdate'].decode()
        tim = gr_data['dataset{0}/what'.format(i+1)]['attrs']['starttime'].decode()
        date = dt.datetime.strptime(dat + tim, "%Y%d%m%H%M%S")
        sdate.append(date)
        quantity = gr_data['dataset{0}/data1/what'.format(i+1)]['attrs']['quantity']
        factor = gr_data['dataset{0}/data1/what'.format(i+1)]['attrs']['gain']
        offset = gr_data['dataset{0}/data1/what'.format(i+1)]['attrs']['offset']
        if quantity.decode() == 'DBZH':
            dat = gr_data['dataset{0}/data1/data'.format(i+1)]['data'] * factor + offset
            refl.append(dat)

    sdate = np.array(sdate)
    refl = np.array(refl)

    gr_dict.update({'sdate': sdate, 'refl': refl})

    return gr_dict

if __name__ == '__main__':

    out = read_trmm_gdal(r"E:/src/git/heistermann/wradlib-data/trmm/2A-RW-BRS.TRMM.PR.2A23.20100206-S111422-E111519.069662.7.HDF",
                         r"E:/src/git/heistermann/wradlib-data/trmm/2A-RW-BRS.TRMM.PR.2A25.20100206-S111422-E111519.069662.7.HDF")


#!/usr/bin/python3
''' Query reddit API subreddit to retrieve top 10 hot posts titles'''
import requests

def top_ten(subreddit):
    ''' Query the subreddit for top 10 host posts'''
    url = 'https://api.reddit.com/r/{}/hot'.format(subreddit)
    headers = {
        'User-Agent': 'My User Agent 1.0'
    }

    r = requests.get(url, headers=headers, allow_redirects=False)
    if r.status_code != 200:
        print('None')
        return

    content = r.json()
    data = content['data']

    children = data['children']

    for idx, item in enumerate(children):
        if idx < 10:
            title = item['data']['title']
            print(title)


import numpy as np
import pandas as pd

from ._utils import check_column_in_df, is_categorical

__all__ = [
    "Contrast",
    "CorrelationContrast",
    "GroupContrast",
    "GroupContrastWithInteraction",
]

class Contrast:
    def __init__(self, name: str, built_contrast: pd.Series):
        self.name = name
        self.built_contrast = built_contrast

class CorrelationContrast(Contrast):
    def __init__(
        self, name: str, absolute_name: str, built_contrast: pd.Series, sign: str
    ):
        super().__init__(name, built_contrast)
        self.sign = sign
        self.absolute_name = absolute_name

    @classmethod
    def from_string(cls, contrast: str, df: pd.DataFrame):
        absolute_contrast_name = contrast
        contrast_sign = "positive"
        if contrast.startswith("-"):
            absolute_contrast_name = contrast[1:].lstrip()
            contrast_sign = "negative"
        check_column_in_df(df, absolute_contrast_name)
        built_contrast = df[absolute_contrast_name]
        if contrast_sign == "negative":
            built_contrast *= -1
        return cls(contrast, absolute_contrast_name, built_contrast, contrast_sign)

class GroupContrastWithInteraction(Contrast):
    def __init__(self, name: str, built_contrast: pd.Series):
        super().__init__(name, built_contrast)

    @classmethod
    def from_string(cls, contrast: str, df: pd.DataFrame):
        contrast_elements = [_.strip() for _ in contrast.split("*")]
        for contrast_element in contrast_elements:
            check_column_in_df(df, contrast_element)
        categorical = [is_categorical(df, _) for _ in contrast_elements]
        if len(contrast_elements) != 2 or sum(categorical) != 1:
            raise ValueError(
                "The contrast must be an interaction between one continuous "
                "variable and one categorical variable. Your contrast contains "
                f"the following variables : {contrast_elements}"
            )
        idx = 0 if categorical[0] else 1
        categorical_contrast = contrast_elements[idx]
        continue_contrast = contrast_elements[(idx + 1) % 2]
        group_values = np.unique(df[categorical_contrast])
        built_contrast = df[continue_contrast].where(
            df[categorical_contrast] == group_values[0], 0
        ) - df[continue_contrast].where(df[categorical_contrast] == group_values[1], 0)
        return cls(contrast, built_contrast)

class GroupContrast(Contrast):
    def __int__(self, name: str, built_contrast: pd.Series):
        super().__init__(name, built_contrast)

    @classmethod
    def from_string(cls, contrast: str, df: pd.DataFrame, sign: str):
        check_column_in_df(df, contrast)
        if not is_categorical(df, contrast):
            raise ValueError(
                "Contrast should refer to a categorical variable for group comparison. "
                "Please select 'correlation' for 'glm_type' otherwise."
            )
        group_values = np.unique(df[contrast])
        i, j = (0, 1) if sign == "positive" else (1, 0)
        contrast_name = f"{group_values[j]}-lt-{group_values[i]}"
        built_contrast = (df[contrast] == group_values[i]).astype(int) - (
            df[contrast] == group_values[j]
        ).astype(int)

        return cls(contrast_name, built_contrast)


from flask import Flask, request, jsonify
from flask_cors import CORS
import numpy as np
import ast
app = Flask(__name__)
CORS(app)

@app.route('/check',methods=['GET', 'POST'])
def hello():
    data = request.json
    print(data)
    if data['polygon']:

        data=data['polygon']
        polyarr = data['polygonArray']
        # polyarr =np.array(polyarr)
        # print(type(polyarr))
        point = data['point']
        point = point.strip('][').split(',')
        x = int(point[0])
        y = int(point[1])
        # print(polyarr)
        isInside = point_inside_polygon(x,y,polyarr)
        if isInside==True:
            isInside = "True"
        else:
            isInside = "False"
    return jsonify(isInside)

    # print(isInside)

@app.route('/sun',methods=['GET', 'POST'])
def sun():
    data = request.json
    if data['sun']:
        data=data['sun']
        buildarr = data['Buildings']
        buildarr =eval(buildarr)
        point = data['point']
        point = point.strip('][').split(',')
        # print(buildarr)
        n = len(buildarr)
        if n ==1:
            output=exposedToSunSingle(buildarr,point)
        else:
            near=nearestBuilding(buildarr)
            output=exposedToSunMultiple(buildarr,point,near)
    return jsonify(output)

def point_inside_polygon(x,y,poly,include_edges=True):
    poly=eval(poly)
    n = len(poly)
    inside = False

    p1x, p1y = poly[0]
    for i in range(1, n + 1):
        p2x, p2y = poly[i % n]
        if p1y == p2y:
            if y == p1y:
                if min(p1x, p2x) <= x <= max(p1x, p2x):
                    # point is on horisontal edge
                    inside = include_edges
                    break
                elif x < min(p1x, p2x):  # point is to the left from current edge
                    inside = not inside
        else:  # p1y!= p2y
            if min(p1y, p2y) <= y <= max(p1y, p2y):
                xinters = (y - p1y) * (p2x - p1x) / float(p2y - p1y) + p1x

                if x == xinters:  # point is right on the edge
                    inside = include_edges
                    break

                if x < xinters:  # point is to the left from current edge
                    inside = not inside

        p1x, p1y = p2x, p2y

    return inside

def exposedToSunMultiple(arr,sun,near):
    # inde =arr.index(near)
    index_row = [arr.index(row) for row in arr if near in row][0]
    first_build = arr[index_row]
    if index_row == 0:
        sec_build = arr[1]
    sec_build = arr[0]
    exposed = abs(first_build[1][1] - first_build[0][1])
    exposed += abs(first_build[3][0]- first_build[0][0])
    tanTheta = findTheta(first_build,sun)
    diffBuild = sec_build[0][0] - first_build[3][0]
    extra_len = tanTheta*diffBuild
    exposed += extra_len + abs(first_build[0][1])
    exposed += sec_build[3][0] - sec_build[0][0]
    exposed = round(float(exposed),2)
    return exposed

def exposedToSunSingle(arr,sun):
    # print(len(arr))
    first_build = arr[0]

    exposed = abs(first_build[1][1] - first_build[0][1])
    exposed += abs(first_build[3][0]- first_build[0][0])
    exposed = round(float(exposed),2)

    return exposed

def findTheta(first_build,sun):
    side1 = float(sun[1]) + float(abs(first_build[0][1]))
    width1 = abs(float(sun[0])) + first_build[3][0]
    # print(width1)
    tanTheta = side1/width1
    # print(tanTheta)

    return tanTheta

def nearestBuilding(arr):
    near_list = []
    xy = [0,0]
    for i in arr:
        dist = lambda x, y: (x[0]-y[0])**2 + (x[1]-y[1])**2
        a = min(i, key=lambda co: dist(co, xy))
        # print(a)
        near_list.append(a)
    dist = lambda x, y: (x[0]-y[0])**2 + (x[1]-y[1])**2
    a = min(near_list, key=lambda co: dist(co, xy))
    return a

if __name__ == '__main__':
    app.run(debug=True, port=8000)


import matplotlib.pyplot as plt
import numpy as np
from scipy import stats
from typing import List, Tuple

from SW import signal
from SW.signal import BOUNDS, PATH, get_signal

def get_pressure_info(sample: str, path: str = PATH, bounds: Tuple[int, int] = BOUNDS) \
        -> Tuple[List[int], List[float], List[float]]:
    signals = get_signal(sample, 'pressure', path, bounds)
    name = ['idx400', 'idx1400', 'idx2000', 'idx200']
    length = []
    slope = []
    amplitude = []

    for sig in signals:
        step = 0
        idx_max = 0
        idx = {}

        for i, value in enumerate(sig):
            cond = [value >= 400, value >= 1400, value >= 2000, value <= 200, False]
            if value > sig[idx_max]:
                idx_max = i
            if cond[step]:
                idx[name[step]] = (i, value)
                step += 1

        if step < 4:
            raise ValueError

        length.append(idx[name[3]][0] - idx[name[0]][0])
        slope.append((idx[name[2]][1] - idx[name[1]][1]) / (idx[name[2]][0] - idx[name[1]][0]))
        amplitude.append(sig[idx_max])

    return length, slope, amplitude

if __name__ == '__main__':
    boundary = (-200, 1300)
    len1, slope1, amp1 = get_pressure_info(signal.name, bounds=boundary)
    len2, slope2, amp2 = get_pressure_info('copper_0919_2', bounds=boundary)

    t_len, p_len = stats.ttest_ind(len1, len2, equal_var=False)
    t_slope, p_slope = stats.ttest_ind(slope1, slope2, equal_var=False)
    t_amp, p_amp = stats.ttest_ind(amp1, amp2, equal_var=False)

    print('len1 mean: ' + str(np.mean(len1)))
    print('len1 std: ' + str(np.std(len1)))
    print('slope1 mean: ' + str(np.mean(slope1)))
    print('slope1 std: ' + str(np.std(slope1)))
    print('amp1 mean: ' + str(np.mean(amp1)))
    print('amp1 std: ' + str(np.std(amp1)))

    print('len2 mean: ' + str(np.mean(len2)))
    print('len2 std: ' + str(np.std(len2)))
    print('slope2 mean: ' + str(np.mean(slope2)))
    print('slope2 std: ' + str(np.std(slope2)))
    print('amp2 mean: ' + str(np.mean(amp2)))
    print('amp2 std: ' + str(np.std(amp2)))

    print(t_len, p_len)
    print(t_slope, p_slope)
    print(t_amp, p_amp)

    plt.figure('len')
    plt.title('len')
    plt.plot(len1, label=1)
    plt.plot(len2, label=2)
    plt.legend()
    plt.figure('slope')
    plt.title('slope')
    plt.plot(slope1, label=1)
    plt.plot(slope2, label=2)
    plt.legend()
    plt.figure('amp')
    plt.title('amp')
    plt.plot(amp1, label=1)
    plt.plot(amp2, label=2)
    plt.legend()

    plt.figure('len_distribution')
    plt.title('len')
    plt.hist([len1, len2])
    plt.figure('slope_distribution')
    plt.title('slope')
    plt.hist([slope1, slope2])
    plt.figure('amp_distribution')
    plt.title('amp')
    plt.hist([amp1, amp2])

    plt.show()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

import tensorflow
import tensorflow_hub as hub
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import LSTM, Bidirectional
from tensorflow.keras.layers import Dense, Flatten, MaxPooling1D, Input, Activation, concatenate
from tensorflow.keras.models import Sequential, Model
from sklearn.preprocessing import OneHotEncoder
from tensorflow.keras.layers import Dropout, BatchNormalization, Reshape
from sklearn.utils import resample
from sklearn.utils import shuffle
from tensorflow.keras.models import load_model
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.utils import to_categorical

print("Loading Features and Labels \n\n")

######################
## Train Set ##
######################

train = pd.read_csv("Merged_Train_OG.csv")
print(train.shape)
train = train.sample(frac=1)

feature1 = train.iloc[:, 0:512]
feature2 = train.iloc[:, 512:1024]

print(feature1.shape, feature2.shape)

train_x_f1 = np.array(feature1)
train_x_f2 = np.array(feature2)

labels = train.iloc[:, 1024:1025]

encoder = LabelEncoder()
encoder.fit(labels)
labels = encoder.transform(labels)
labels = to_categorical(labels)

train_y = labels

######################
## Validation ##
######################

val = pd.read_csv("Merged_Val_OG.csv")

print(val.shape)
val = val.sample(frac=1)

feature1_val = val.iloc[:, 0:512]
feature2_val = val.iloc[:, 512:1024]

print(feature1_val.shape, feature2_val.shape)

val_x_f1 = np.array(feature1_val)
val_x_f2 = np.array(feature2_val)

labels_val = val.iloc[:, 1024:1025]

encoder = LabelEncoder()
encoder.fit(labels_val)
labels_val = encoder.transform(labels_val)
labels_val = to_categorical(labels_val)

val_y = labels_val

print("Training Model TALOS Merge \n\n")

input1 = Input(shape=(None, 512))
inner = Conv1D(128, 1, padding='same', name='conv1',
               kernel_initializer='he_normal')(input1)
inner = BatchNormalization()(inner)
inner = Dropout(0.5)(inner)
inner = Activation('relu')(inner)
inner1 = MaxPooling1D(pool_size=1, name='max1')(inner)

inner = Conv1D(128, 1, padding='same', name='conv1',
               kernel_initializer='he_normal')(inner1)
inner = BatchNormalization()(inner)
inner = Dropout(0.5)(inner)
inner = Activation('relu')(inner)
inner1 = MaxPooling1D(pool_size=1, name='max1')(inner)

inner = Conv1D(512, 1, padding='same', name='conv1',
               kernel_initializer='he_normal')(inner1)
inner = BatchNormalization()(inner)
inner = Dropout(0.5)(inner)
inner = Activation('relu')(inner)
inner1 = MaxPooling1D(pool_size=1, name='max1')(inner)

inner = Conv1D(512, 1, padding='same', name='conv1',
               kernel_initializer='he_normal')(inner1)
inner = BatchNormalization()(inner)
inner = Dropout(0.5)(inner)
inner = Activation('relu')(inner)
inner1 = MaxPooling1D(pool_size=1, name='max1')(inner)

inner = Conv1D(768, 1, padding='same', name='conv1',
               kernel_initializer='he_normal')(inner1)
inner = BatchNormalization()(inner)
inner = Dropout(0.5)(inner)
inner = Activation('relu')(inner)
inner1 = MaxPooling1D(pool_size=1, name='max1')(inner)

input2 = Input(shape=(None, 512))
inner2 = Conv1D(128, 1, padding='same', name='conv1',
                kernel_initializer='he_normal')(input2)
inner2 = BatchNormalization()(inner2)
inner2 = Dropout(0.5)(inner2)
inner2 = Activation('relu')(inner2)
inner2 = MaxPooling1D(pool_size=1, name='max1')(inner2)

inner2 = Conv1D(128, 1, padding='same', name='conv1',
                kernel_initializer='he_normal')(inner2)
inner2 = BatchNormalization()(inner2)
inner2 = Dropout(0.5)(inner2)
inner2 = Activation('relu')(inner2)
inner2 = MaxPooling1D(pool_size=1, name='max1')(inner2)

inner2 = Conv1D(512, 1, padding='same', name='conv1',
                kernel_initializer='he_normal')(inner2)
inner2 = BatchNormalization()(inner2)
inner2 = Dropout(0.5)(inner2)
inner2 = Activation('relu')(inner2)
inner2 = MaxPooling1D(pool_size=1, name='max1')(inner2)

inner2 = Conv1D(512, 1, padding='same', name='conv1',
                kernel_initializer='he_normal')(inner2)
inner2 = BatchNormalization()(inner2)
inner2 = Dropout(0.5)(inner2)
inner2 = Activation('relu')(inner2)
inner2 = MaxPooling1D(pool_size=1, name='max1')(inner2)

inner2 = Conv1D(768, 1, padding='same', name='conv1',
                kernel_initializer='he_normal')(inner2)
inner2 = BatchNormalization()(inner2)
inner2 = Dropout(0.5)(inner2)
inner2 = Activation('relu')(inner2)
inner2 = MaxPooling1D(pool_size=1, name='max1')(inner2)

x = concatenate([input1, input2])

x = Dense(1024, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation='relu')(x)
x = BatchNormalization()(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation='relu')(x)

x = Dense(3, activation='softmax')(x)

model = Model(inputs=[input1, input2], outputs=x)
model.compile(loss='binary_crossentropy',
              optimizer='adam',  metrics=['accuracy'])

train_x_f1 = np.array(train_x_f1).reshape(
    (train_x_f1.shape[0], 1, train_x_f1.shape[1]))
train_x_f2 = np.array(train_x_f2).reshape(
    (train_x_f2.shape[0], 1, train_x_f2.shape[1]))
train_y = np.array(train_y).reshape((train_y.shape[0], 1, train_y.shape[1]))
val_x_f1 = np.array(val_x_f1).reshape(
    (val_x_f1.shape[0], 1, val_x_f1.shape[1]))
val_x_f2 = np.array(val_x_f2).reshape(
    (val_x_f2.shape[0], 1, val_x_f2.shape[1]))
val_y = np.array(val_y).reshape((val_y.shape[0], 1, val_y.shape[1]))

history = model.fit([train_x_f1, train_x_f2], train_y,
                    epochs=5, shuffle=True, validation_data=([val_x_f1, val_x_f2], val_y))

print("Model Trained \n\n")

model.save("TV_Model")

print("Model Saved \n\n")


# How many bag colors can eventually contain at least one shiny gold bag?
# https://adventofcode.com/2020/day/7

from src.aoc_core import *

c = Core(7)
c.tic()

data = c.get_str_input()

class Bag:
    # List of all bags
    all = []

    # Gets or makes new bag
    @staticmethod
    def get(color):
        for b in Bag.all:
            if b.color == color.strip():
                return b

        # Make new bag
        n = Bag(color)
        Bag.all.append(n)
        return n

    def __init__(self, color):
        self.color = color
        self.children = dict()
        self.parents = set()
        self.ancestors = set()

    def __str__(self):
        return self.color + " | " + str([child.color for child in self.children])

    def add_child(self, color, quantity):
        child = Bag.get(color)
        self.children[child] = quantity
        child.add_parent(self)

    def add_parent(self, parent):
        if type(parent) is str:
            parent = Bag.get(parent)
        self.parents.add(parent)

    def get_all_ancestors(self):
        for p in self.parents:
            self.ancestors = self.ancestors.union(p.get_all_ancestors())

        self.ancestors = self.ancestors.union(self.parents)
        return self.ancestors

    def get_size(self):
        count = 0
        for child in self.children.keys():
            quantity = int(self.children.get(child))
            count += quantity + (quantity * child.get_size())
        return count

# Make Bag objects
for line in data:
    split_line = line.split(" bags contain")
    if split_line[1].strip() != "no other bags.":
        for rule in split_line[1].split(","):
            split_rule = rule.replace("bags", "").replace(".", "").replace("bag", "").strip().split(" ", 1)

            # Make new bag
            Bag.get(split_line[0]).add_child(split_rule[1], split_rule[0])

print("1)", len(Bag.get("shiny gold").get_all_ancestors()))
print("2)", Bag.get("shiny gold").get_size())

c.print_toc()


from .base_page import BasePage
from .locators import ProductPageLocators

class ProductPage(BasePage):
    def should_be_added_to_cart(self):
        self.should_be_addtocart_button()
        self.should_be_successful_added()
        self.should_be_correct_name()
        self.should_be_correct_price()

    def should_be_clicked_addtocart_button(self):
        button = self.browser.find_element(
            *ProductPageLocators.ADDTOCART_BUTTON)
        button.click()

    def should_be_successful_added(self):
        self.should_be_clicked_addtocart_button()
        self.solve_quiz_and_get_code()
        assert self.is_element_present(
            *ProductPageLocators.SUCCESS_INFO
        ), "The product was not added"

    def should_be_addtocart_button(self):
        assert self.is_element_present(
            *ProductPageLocators.ADDTOCART_BUTTON
        ), "Add to cart button is not presented"

    def should_be_correct_name(self):
        name = self.browser.find_element(
            *ProductPageLocators.PRODUCT_NAME).text
        info = self.browser.find_element(
            *ProductPageLocators.INFO_NAME).text
        assert name == info, "Another product was added"

    def should_be_correct_price(self):
        price = self.browser.find_element(
            *ProductPageLocators.PRICE).text
        info = self.browser.find_element(
            *ProductPageLocators.INFO_PRICE).text
        assert price == info, "Another product was displayed"

    def cant_see_success_message(self):
        assert self.is_not_element_present(*ProductPageLocators.SUCCESS_INFO), \
            "Success info was presented"

    def should_disappear_success_message(self):
        assert self.is_disappeared(*ProductPageLocators.SUCCESS_INFO), \
            "The success message didn't disappear"


from flask import Flask, jsonify, request
from elasticsearch import Elasticsearch
import face_recognition

app = Flask(__name__)

es = Elasticsearch("http://elasticsearch:9200")

@app.route("/")
def welcome():
    return jsonify({
        'message': 'welcome1'
    })

@app.route("/search", methods=["POST"])
def search():
    image = face_recognition.load_image_file(request.files['file'])
    face_encoding = face_recognition.face_encodings(image)[0]
    query = {
            "script_score": {
                "query" : {
                    "match_all": {}
                },
                "script": {
                    "source": "cosineSimilarity(params.query_vector, 'face_encoding')",
                    "params": {
                        "query_vector":face_encoding.tolist()
                    }
                }
            }
        }
    response = es.search(index='faces', query=query, size=1, source='face_name', min_score=0.93)
    print(response)
    if len(response['hits']['hits']) == 1:
        return jsonify({
            'status': True,
            "face_name": response['hits']['hits'][0]['_source']['face_name'],
            "score": response['hits']['hits'][0]['_score']
        })
    else:
        return jsonify({
            'status': False,
        })

@app.route("/face_register", methods=["POST"])
def face_register():
    image = face_recognition.load_image_file(request.files['file'])
    face_encoding = face_recognition.face_encodings(image)[0]
    doc = {
        'face_name': request.form['name'],
        'face_encoding': face_encoding.tolist()
    }

    es.index(index='faces', document=doc)

    return jsonify({
        'message': 'success'
    })

if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=True)


import io
from collections import Counter

def load_dictionary(file_path):
    with io.open(file_path, 'r', encoding='latin-1') as file:
        dictionary = {word.strip().lower() for word in file}
    return dictionary

def get_letter_frequencies(data):
    total_letters = sum(1 for char in data if char.isalpha())
    letter_counts = Counter(char.lower() for char in data if char.isalpha())
    letter_frequencies = {char: count / total_letters for char, count in letter_counts.items()}
    return letter_frequencies

def decrypt(data, dictionary):
    letter_frequencies = get_letter_frequencies(data)
    decrypted_results = []
    for shift in range(1, 121):
        decrypted_data = ''
        for char in data:
            if char.isalpha():
                if char.islower():
                    decrypted_char = chr((ord(char) - shift - 97) % 26 + 97)
                else:
                    decrypted_char = chr((ord(char) - shift - 65) % 26 + 65)
                decrypted_data += decrypted_char
            else:
                decrypted_data += char

        words = decrypted_data.lower().split()
        word_count = sum(1 for word in words if word in dictionary)
        word_frequency = word_count / len(words) if len(words) > 0 else 0
        letter_frequency = sum(letter_frequencies.get(char.lower(), 0) for char in decrypted_data)
        score = word_frequency * 0.8 + letter_frequency * 0.2
        decrypted_results.append((decrypted_data, shift, score))

    decrypted_results.sort(key=lambda x: x[2], reverse=True)
    return decrypted_results

def main():
    file_path = 'Resources/engmix.txt'  # Replace with the relative file path on your system
    dictionary = load_dictionary(file_path)

    encrypted_data = input("Enter the encrypted data: ")
    results = decrypt(encrypted_data, dictionary)

    print("\nTop 10 Decryption Results:")
    for decrypted_data, shift, score in results[:10]:
        print(f"Shift: {shift:3} | Decrypted Data: {decrypted_data} | Score: {score:.4f}")

    show_all = input("Show all decrypted results? (y/n): ")
    if show_all.lower() == 'y':
        print("\nAll Decryption Results:")
        for decrypted_data, shift, score in results:
            print(f"Shift: {shift:3} | Decrypted Data: {decrypted_data} | Score: {score:.4f}")

if __name__ == "__main__":
    main()


'''
term frequency and term term distribution based keyword extraction
'''

import re
import os
import nltk
from nltk.stem.porter import *
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from collections import defaultdict

class TFD(object):
    def __init__(self, data_dir, rank = 36):
        self.data_dir = data_dir
        self.stopwords = stopwords.words('english')
        self.stemmer = PorterStemmer()
        self.lemmatizer = WordNetLemmatizer()
        self.rank = rank

    def pre_process(self, text):
        text = re.sub(r'[^a-z-]+', ' ', text)
        text = re.split(r'[^a-z-]+', text)
        text = [word for word in text if word not in self.stopwords]
        text = [self.stemmer.stem(word) for word in text]
        text = [self.lemmatizer.lemmatize(word) for word in text]
        text = [word for word in text if len(word) > 1]
        return text

    def get_doc(self, f):
        self.doc = f.read().lower()
        self.doc = self.pre_process(self.doc)

    def get_key(self, f):
        self.key = []
        for line in f.readlines():
            line = self.pre_process(line.lower())
            line = ' '.join(line)
            self.key.append(line)

    def append_ngrams(self, tokens):
        temp = []
        bigram = [' '.join(tokens[i:i+2]) for i in xrange(len(tokens) - 1)]
        trigram = [' '.join(tokens[i:i+3]) for i in xrange(len(tokens) - 2)]
        fourgram = [' '.join(tokens[i:i+4]) for i in xrange(len(tokens) - 3)]
        temp = tokens + bigram + trigram + fourgram
        return temp

    def partition_doc(self, n = 5):
        self.d = dict()
        bin = len(self.doc) / n  #number of terms in each part
        for i in xrange(n):
            self.d[i] = self.doc[i*bin : (i+1)*bin]
        self.d[n-1] = self.doc[(i+1)*bin : ]
        for i in xrange(n):
            self.d[i] = self.append_ngrams(self.d[i])
        return self.d

    def get_term_frequency(self):
        self.tf = dict()
        tokens = self.append_ngrams(self.doc)
        for term in tokens:
            self.tf[term] = self.tf.get(term, 0) + 1
        return self.tf

    def get_term_distribution(self):
        self.td = dict()
        self.partition_doc()
        for term in set(self.append_ngrams(self.doc)):
            for key in self.d:
                if term in self.d[key]:
                    self.td[term] = self.td.get(term, 0) + 1
        return self.td

    def get_term_score(self):
        if self.method == 'tf':
            self.score = self.get_term_frequency()
        if self.method == 'td':
            self.score = self.get_term_distribution()
        if self.method == 'tfd':
            self.get_term_distribution()
            self.get_term_frequency()
            self.score = {key:self.tf[key] + self.td[key] for key in self.td}

    def get_ans(self, ans_file):
        with open(self.data_dir + ans_file, 'r') as f:
            self.ans = []
            for line in f.readlines():
                line = self.pre_process(line.lower())
                line = ' '.join(line)
                self.ans.append(line)
            self.ans = [word for word in self.ans if word in self.append_ngrams(self.doc)]

    def fit(self, filename, ans_file):
        self.get_keywords(filename, self.method)
        self.get_ans(ans_file)

    def get_keywords(self, filename, method):
        self.method = method
        with open(self.data_dir + filename, 'r') as f:
            self.get_doc(f)
            self.get_term_score()
            doclen = len(set(self.append_ngrams(self.doc)))
            self.kw = [items[0] for items in sorted(self.score.items(), key = lambda item: item[1])[-doclen/self.rank:]]
            return self.kw

    def get_accuracy_recall(self):
        hit = len([word for word in self.kw if word in self.ans])
        accuracy = hit*1.0 / len(self.kw)
        recall = hit*1.0 / len(self.ans)
        return accuracy, recall

    def get_accuracy_recall2(self, method):
        self.method = method
        accuracy = recall = 0.0
        nfiles = 0.0
        for filename in os.listdir(self.data_dir):
            if filename.endswith('txt'):
                nfiles += 1
                ans_file = filename[:-3] + 'key'
                self.fit(filename, ans_file)
                a, r = self.get_accuracy_recall()
                accuracy += a
                recall += r
        return accuracy/nfiles, recall/nfiles


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report
import joblib

# Load the CSV file
csv_path = 'data/jobs.csv'
data_custom = pd.read_csv(csv_path)

# Add a new column as a unique identifier
data_custom['row_id'] = range(len(data_custom))

# Save new csv file with row_id
custom_data = data_custom
custom_csv = 'custom/moore_with_row_id.csv'
custom_data.to_csv(custom_csv, index= False)
print(f"New CSV has been saved: {custom_csv}")

# Read new custome csv file
custom_csv_path = 'custom/moore_with_row_id.csv'
data = pd.read_csv(custom_csv_path)

# Identify string columns that need encoding
string_columns = data.select_dtypes(include='object').columns

# Apply one-hot encoding for string columns
data = pd.get_dummies(data, columns=string_columns, drop_first=True)

X = data.drop('row_id', axis=1)
y = data['row_id']

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create and train the model
model = RandomForestClassifier(random_state=42)
model.fit(X, y)

# Save the trained model
model_path = 'training/moore_model.pkl'
joblib.dump(model, model_path)
print(f"Model trained and saved to {model_path}")


# coding: utf8

"""
This file contains a set of functional tests designed to check the correct execution of the pipeline and the
different functions available in Clinica
"""

import warnings
from os import fspath
from pathlib import Path
from test.nonregression.testing_tools import (
    configure_paths,
    similarity_measure,
    similarity_measure_large_nifti,
)

import numpy as np
import pytest
from numpy.testing import assert_array_almost_equal

# Determine location for working_directory
warnings.filterwarnings("ignore")

@pytest.mark.fast
def test_dwi_b0_flirt(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        b0_flirt_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIB0Flirt")
    b0_flirt = b0_flirt_pipeline(num_b0s=11)
    b0_flirt.inputs.inputnode.in_file = str(input_dir / "sub-01_ses-M000_dwi_b0.nii.gz")
    (tmp_path / "tmp").mkdir()
    b0_flirt.base_dir = str(tmp_path / "tmp")
    b0_flirt.run()

    out_file = fspath(
        tmp_path
        / "tmp"
        / "b0_coregistration"
        / "concat_ref_moving"
        / "merged_files.nii.gz"
    )
    ref_file = fspath(ref_dir / "merged_files.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.slow
def test_dwi_epi_pipeline(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        epi_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIEPIPipeline")
    (tmp_path / "tmp").mkdir()
    epi = epi_pipeline(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        ants_random_seed=42,
        use_double_precision=False,
        delete_cache=True,
    )
    epi.inputs.inputnode.t1_filename = str(input_dir / "sub-01_ses-M000_T1w.nii.gz")
    epi.inputs.inputnode.dwi_filename = str(input_dir / "sub-01_ses-M000_dwi.nii.gz")
    epi.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )

    epi.run()

    out_file = fspath(
        tmp_path / "tmp" / "rotated_b_vectors" / "sub-01_ses-M000_dwi_rotated.bvec"
    )
    ref_file = fspath(ref_dir / "sub-01_ses-M000_dwi_rotated.bvec")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=2)

    out_file = fspath(
        tmp_path
        / "tmp"
        / "epi_corrected_dwi_image"
        / "Jacobian_image_maths_thresh_merged.nii.gz"
    )
    ref_file = fspath(ref_dir / "Jacobian_image_maths_thresh_merged.nii.gz")

    similarity_measure_large_nifti(out_file, ref_file, 0.95)

@pytest.mark.slow
def test_dwi_perform_ants_registration(cmdopt, tmp_path):
    from test.nonregression.testing_tools import similarity_measure

    import nibabel as nib

    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        perform_ants_registration,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIANTSRegistration"
    )
    (tmp_path / "tmp").mkdir()
    ants_registration = perform_ants_registration(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        ants_random_seed=42,  # Set the random seed to avoid stochastic results (requires ants >= 2.3.0)
    )
    ants_registration.inputs.inputnode.t1_filename = str(
        input_dir / "sub-01_ses-M000_T1w.nii.gz"
    )
    ants_registration.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    ants_registration.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )

    ants_registration.run()

    out_file = fspath(
        tmp_path / "tmp" / "epi_correction_deformation_field" / "transform1Warp.nii.gz"
    )
    ref_file = fspath(ref_dir / "transform1Warp.nii.gz")

    assert nib.load(out_file).shape == nib.load(ref_file).shape

    out_file = fspath(
        tmp_path / "tmp" / "epi_correction_image_warped" / "transformWarped.nii.gz"
    )
    ref_file = fspath(ref_dir / "transformWarped.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.9)

    out_file = fspath(tmp_path / "tmp" / "merged_transforms" / "transform1Warp.nii.gz")
    ref_file = fspath(ref_dir / "merged_transform.nii.gz")

    assert nib.load(out_file).shape == nib.load(ref_file).shape

    out_file = fspath(
        tmp_path / "tmp" / "rotated_b_vectors" / "sub-01_ses-M000_dwi_rotated.bvec"
    )
    ref_file = fspath(ref_dir / "sub-01_ses-M000_dwi_rotated.bvec")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=2)

@pytest.mark.slow
def test_dwi_perform_dwi_epi_correction(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        perform_dwi_epi_correction,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIEPICorrection"
    )
    (tmp_path / "tmp").mkdir()
    epi_correction = perform_dwi_epi_correction(
        base_dir=str(base_dir),
        output_dir=str(tmp_path / "tmp"),
        delete_cache=True,
        use_double_precision=False,
    )
    epi_correction.inputs.inputnode.t1_filename = str(
        input_dir / "sub-01_ses-M000_T1w.nii.gz"
    )
    epi_correction.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    epi_correction.inputs.inputnode.merged_transforms = str(
        input_dir / "merged_transforms.nii.gz"
    )

    epi_correction.run(plugin="MultiProc", plugin_args={"n_procs": 8})

    out_file = fspath(
        tmp_path
        / "tmp"
        / "epi_corrected_dwi_image"
        / "Jacobian_image_maths_thresh_merged.nii.gz"
    )
    ref_file = fspath(ref_dir / "Jacobian_image_maths_thresh_merged.nii.gz")

    similarity_measure_large_nifti(out_file, ref_file, 0.95)

@pytest.mark.slow
def test_dwi_eddy_fsl(cmdopt, tmp_path):
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_workflows import (
        eddy_fsl_pipeline,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIEddyFSL")
    (tmp_path / "tmp").mkdir()
    eddy_fsl = eddy_fsl_pipeline(
        use_cuda=False,
        initrand=True,
        compute_mask=True,
        output_dir=str(tmp_path / "tmp"),
    )
    eddy_fsl.inputs.inputnode.total_readout_time = 0.0342002
    eddy_fsl.inputs.inputnode.phase_encoding_direction = "y-"
    eddy_fsl.inputs.inputnode.dwi_filename = str(
        input_dir / "sub-01_ses-M000_dwi.nii.gz"
    )
    eddy_fsl.inputs.inputnode.b_values_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bval"
    )
    eddy_fsl.inputs.inputnode.b_vectors_filename = str(
        input_dir / "sub-01_ses-M000_dwi.bvec"
    )
    eddy_fsl.inputs.inputnode.reference_b0 = str(
        input_dir / "sub-01_ses-M000_dwi_b0.nii.gz"
    )

    eddy_fsl.run()

    out_file = fspath(tmp_path / "tmp" / "out_corrected" / "eddy_corrected.nii.gz")
    ref_file = fspath(ref_dir / "eddy_corrected.nii.gz")

    assert similarity_measure(out_file, ref_file, 0.97)

    out_file = fspath(
        tmp_path / "tmp" / "out_rotated_bvecs" / "eddy_corrected.eddy_rotated_bvecs"
    )
    ref_file = fspath(ref_dir / "eddy_corrected.eddy_rotated_bvecs")
    out_bvecs = np.loadtxt(out_file)
    ref_bvecs = np.loadtxt(ref_file)

    assert_array_almost_equal(out_bvecs, ref_bvecs, decimal=3)

@pytest.mark.slow
def test_dwi_preprocessing_using_t1(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIPreprocessingUsingT1"
    )
    run_dwi_preprocessing_using_t1(input_dir, tmp_dir, ref_dir, working_dir)

@pytest.mark.slow
def test_dwi_compute_reference_b0(cmdopt, tmp_path):
    """Test step 1 of pipeline DWIPreprocessingUsingPhaseDiff.

    This is a slow test. Expect approximately a 2.5 hours runtime.

    We use the same input files as for the DWIPreprocessingUsingPhaseDiff pipeline.

    However, since we are testing the step 1 in isolation, we don't have the
    BIDS querying logic available to us. The compute_reference_b0 pipeline is
    not BIDS-aware, it just expects three files:
        - the DWI image
        - the associated b-values
        - and the associated b-vectors

    These files are directly available in the input_dir (no BIDS structure, files
    are directly at the root level).

    It also needs the phase encoding direction and total readout time which are
    available in the JSON file (also available in the input_dir).
    In the main pipeline, the input node is performing the extraction of these
    metadata from the JSON file. Here, we perform this extraction directly with
    the `extract_metadata_from_json` and `bids_dir_to_fsl_dir`functions.

    The compute_reference_b0 workflow produces two outputs that are compared
    against their reference values:
        - The reference B0 volume
        - The brain mask computed on the B0 volume
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        compute_reference_b0,
    )
    from clinica.utils.dwi import bids_dir_to_fsl_dir
    from clinica.utils.filemanip import (
        extract_metadata_from_json,
        handle_missing_keys_dwi,
    )

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIComputeReferenceB0"
    )
    (tmp_path / "tmp").mkdir()

    [total_readout_time, phase_encoding_direction] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_dwi.json"),
        ["TotalReadoutTime", "PhaseEncodingDirection"],
        handle_missing_keys=handle_missing_keys_dwi,
    )
    phase_encoding_direction = bids_dir_to_fsl_dir(phase_encoding_direction)

    wf = compute_reference_b0(
        b_value_threshold=5.0,
        use_cuda=False,
        initrand=False,
        output_dir=str(tmp_path / "tmp"),
        name="compute_reference_b0",
    )
    wf.inputs.inputnode.b_values_filename = str(input_dir / "sub-01_ses-M000_dwi.bval")
    wf.inputs.inputnode.b_vectors_filename = str(input_dir / "sub-01_ses-M000_dwi.bvec")
    wf.inputs.inputnode.dwi_filename = str(input_dir / "sub-01_ses-M000_dwi.nii.gz")
    wf.inputs.inputnode.image_id = "sub-01_ses-M000"
    wf.inputs.inputnode.total_readout_time = total_readout_time
    wf.inputs.inputnode.phase_encoding_direction = phase_encoding_direction

    wf.run()

    for folder, filename in zip(
        ["reference_b0", "brainmask"],
        ["sub-01_ses-M000_avg_b0_brain.nii.gz", "brainmask.nii.gz"],
    ):
        out_file = fspath(tmp_path / "tmp" / folder / filename)
        ref_file = fspath(ref_dir / folder / filename)

        assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.fast
def test_prepare_phasediff_fmap(cmdopt, tmp_path):
    """Test the pipeline prepare_phasediff_fmap which is part of
     step 2 of the pipeline DWIPreprocessingUsingPhaseDiff.

    This is a fast test which should run in less than a minute.
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        prepare_phasediff_fmap,
    )
    from clinica.utils.filemanip import extract_metadata_from_json

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWIPreparePhasediffFmap"
    )
    (tmp_path / "tmp").mkdir()

    [echo_time1, echo_time2] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_phasediff.json"), ["EchoTime1", "EchoTime2"]
    )
    delta_echo_time = abs(echo_time2 - echo_time1)

    wf = prepare_phasediff_fmap(output_dir=str(tmp_path / "tmp"))
    wf.inputs.input_node.fmap_mask = str(
        input_dir / "sub-01_ses-M000_magnitude1_corrected_brain_mask.nii.gz"
    )
    wf.inputs.input_node.fmap_phasediff = str(
        input_dir / "sub-01_ses-M000_phasediff.nii.gz"
    )
    wf.inputs.input_node.fmap_magnitude = str(
        input_dir / "sub-01_ses-M000_magnitude1_corrected_brain.nii.gz"
    )
    wf.inputs.input_node.delta_echo_time = str(delta_echo_time)

    wf.run()

    out_filename = (
        "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths.nii.gz"
    )
    out_file = fspath(tmp_path / "tmp" / "calibrated_fmap" / out_filename)
    ref_file = fspath(ref_dir / "calibrated_fmap" / out_filename)

    assert similarity_measure(out_file, ref_file, 0.99)

@pytest.mark.fast
def test_dwi_calibrate_and_register_fmap(cmdopt, tmp_path):
    """Test step 2 of pipeline DWIPreprocessingUsingPhaseDiff.

    This is a fast test which should run in about 1 minute.
    """
    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_workflows import (
        calibrate_and_register_fmap,
    )
    from clinica.utils.filemanip import extract_metadata_from_json

    base_dir = Path(cmdopt["input"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir, tmp_path, "DWICalibrateAndRegisterFmap"
    )
    (tmp_path / "tmp").mkdir()

    [echo_time1, echo_time2] = extract_metadata_from_json(
        str(input_dir / "sub-01_ses-M000_phasediff.json"), ["EchoTime1", "EchoTime2"]
    )
    delta_echo_time = abs(echo_time2 - echo_time1)

    wf = calibrate_and_register_fmap(output_dir=str(tmp_path / "tmp"))
    wf.inputs.inputnode.reference_b0 = str(
        input_dir / "sub-01_ses-M000_avg_b0_brain.nii.gz"
    )
    wf.inputs.inputnode.bias_magnitude_fmap = str(
        input_dir / "sub-01_ses-M000_magnitude1.nii.gz"
    )
    wf.inputs.inputnode.fmap_phasediff = str(
        input_dir / "sub-01_ses-M000_phasediff.nii.gz"
    )
    wf.inputs.inputnode.delta_echo_time = delta_echo_time

    wf.run()

    for folder, filename in zip(
        [
            "smooth_calibrated_fmap",
            "bet_magnitude_fmap_registered_onto_b0",
            "registered_calibrated_fmap",
        ],
        [
            "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths_flirt_smooth.nii.gz",
            "sub-01_ses-M000_magnitude1_corrected_brain_flirt.nii.gz",
            "sub-01_ses-M000_phasediff_rads_unwrapped_radsec_fieldmap_demean_maths_flirt.nii.gz",
        ],
    ):
        out_file = fspath(tmp_path / "tmp" / folder / filename)
        ref_file = fspath(ref_dir / folder / filename)

        assert similarity_measure(out_file, ref_file, 0.98)

@pytest.mark.slow
def test_dwi_preprocessing_using_phase_diff_field_map(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(
        base_dir,
        tmp_path,
        "DWIPreprocessingUsingPhaseDiffFieldmap",
    )
    run_dwi_preprocessing_using_phase_diff_field_map(
        input_dir, tmp_dir, ref_dir, working_dir
    )

@pytest.mark.slow
def test_dwi_dti(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIDTI")
    run_dwi_dti(input_dir, tmp_dir, ref_dir, working_dir)

@pytest.mark.slow
def test_dwi_connectome(cmdopt, tmp_path):
    base_dir = Path(cmdopt["input"])
    working_dir = Path(cmdopt["wd"])
    input_dir, tmp_dir, ref_dir = configure_paths(base_dir, tmp_path, "DWIConnectome")
    run_dwi_connectome(input_dir, tmp_dir, ref_dir, working_dir)

def run_dwi_preprocessing_using_t1(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    from clinica.pipelines.dwi_preprocessing_using_t1.dwi_preprocessing_using_t1_pipeline import (
        DwiPreprocessingUsingT1,
    )

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    parameters = {
        "initrand": True,
        "low_bval": 5,
        "use_cuda": False,
    }
    pipeline = DwiPreprocessingUsingT1(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Assert
    out_file = fspath(
        caps_dir
        / "subjects"
        / "sub-PREVDEMALS0010025PG"
        / "ses-M000"
        / "dwi"
        / "preprocessing"
        / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-T1w_preproc.nii.gz"
    )
    ref_file = fspath(
        ref_dir / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-T1w_preproc.nii.gz"
    )

    assert similarity_measure_large_nifti(out_file, ref_file, 0.99)

def run_dwi_preprocessing_using_phase_diff_field_map(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import warnings

    from clinica.pipelines.dwi_preprocessing_using_fmap.dwi_preprocessing_using_phasediff_fmap_pipeline import (
        DwiPreprocessingUsingPhaseDiffFMap,
    )

    warnings.filterwarnings("ignore")

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    parameters = {
        "initrand": True,
        "low_bval": 5,
        "use_cuda": False,
        "delete_cache": True,
    }
    pipeline = DwiPreprocessingUsingPhaseDiffFMap(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Assert
    out_file = fspath(
        caps_dir
        / "subjects"
        / "sub-PREVDEMALS0010025PG"
        / "ses-M000"
        / "dwi"
        / "preprocessing"
        / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-b0_preproc.nii.gz"
    )
    ref_file = fspath(
        ref_dir / "sub-PREVDEMALS0010025PG_ses-M000_dwi_space-b0_preproc.nii.gz"
    )

    assert similarity_measure(out_file, ref_file, 0.95)

def run_dwi_dti(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    import numpy as np
    import pandas as pds

    from clinica.pipelines.dwi_dti.dwi_dti_pipeline import DwiDti

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    # Copy necessary data from in to out
    shutil.copytree(input_dir / "caps", caps_dir, copy_function=shutil.copy)

    pipeline = DwiDti(
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Check files
    subject_id = "sub-PREVDEMALS0010025PG"
    maps = ["AD", "FA", "MD", "RD"]
    out_files = [
        fspath(
            caps_dir
            / "subjects"
            / subject_id
            / "ses-M000"
            / "dwi"
            / "dti_based_processing"
            / "atlas_statistics"
            / (
                subject_id
                + "_ses-M000_dwi_space-JHUDTI81_res-1x1x1_map-"
                + m
                + "_statistics.tsv"
            )
        )
        for m in maps
    ]
    ref_files = [
        fspath(
            ref_dir
            / (
                subject_id
                + "_ses-M000_dwi_space-JHUDTI81_res-1x1x1_map-"
                + m
                + "_statistics.tsv"
            )
        )
        for m in maps
    ]

    for i in range(len(out_files)):
        out_csv = pds.read_csv(out_files[i], sep="\t")
        out_mean_scalar = np.array(out_csv.mean_scalar)
        ref_csv = pds.read_csv(ref_files[i], sep="\t")
        ref_mean_scalar = np.array(ref_csv.mean_scalar)

        assert np.allclose(out_mean_scalar, ref_mean_scalar, rtol=0.025, equal_nan=True)

def run_dwi_connectome(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    from clinica.pipelines.dwi_connectome.dwi_connectome_pipeline import DwiConnectome

    caps_dir = output_dir / "caps"
    tsv = input_dir / "subjects.tsv"

    # Copy necessary data from in to out
    shutil.copytree(input_dir / "caps", caps_dir, copy_function=shutil.copy)

    parameters = {"n_tracks": 1000}
    pipeline = DwiConnectome(
        caps_directory=fspath(caps_dir),
        tsv_file=fspath(tsv),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Check files
    atlases = ["desikan", "destrieux"]
    subject_id = "sub-PREVDEMALS0010025PG"
    session_id = "ses-M000"

    out_fod_file = fspath(
        caps_dir
        / "subjects"
        / subject_id
        / session_id
        / "dwi"
        / "connectome_based_processing"
        / (subject_id + "_" + session_id + "_dwi_space-b0_model-CSD_diffmodel.nii.gz")
    )
    ref_fod_file = fspath(
        ref_dir
        / (subject_id + "_" + session_id + "_dwi_space-b0_model-CSD_diffmodel.nii.gz"),
    )

    out_parc_files = [
        fspath(
            caps_dir
            / "subjects"
            / subject_id
            / session_id
            / "dwi"
            / "connectome_based_processing"
            / (
                subject_id
                + "_"
                + session_id
                + "_dwi_space-b0_atlas-"
                + a
                + "_parcellation.nii.gz"
            )
        )
        for a in atlases
    ]
    ref_parc_files = [
        fspath(
            ref_dir
            / (
                subject_id
                + "_"
                + session_id
                + "_dwi_space-b0_atlas-"
                + a
                + "_parcellation.nii.gz"
            )
        )
        for a in atlases
    ]

    assert similarity_measure(out_fod_file, ref_fod_file, 0.97)

    for i in range(len(out_parc_files)):
        assert similarity_measure(out_parc_files[i], ref_parc_files[i], 0.955)


import pytest

import pyannote.core
from pyannote.core import Annotation
from pyannote.core import Segment
from pyannote.core import Timeline
from pyannote.metrics.diarization import DiarizationErrorRate
from pyannote.metrics.diarization import DiarizationPurity
from pyannote.metrics.diarization import DiarizationCoverage

import numpy.testing as npt

@pytest.fixture
def reference():
    reference = Annotation()
    reference[Segment(0, 10)] = 'A'
    reference[Segment(12, 20)] = 'B'
    reference[Segment(24, 27)] = 'A'
    reference[Segment(30, 40)] = 'C'
    return reference

@pytest.fixture
def reference_with_overlap():
    reference = Annotation()
    reference[Segment(0, 13)] = 'A'
    reference[Segment(12, 20)] = 'B'
    reference[Segment(24, 27)] = 'A'
    reference[Segment(30, 40)] = 'C'
    return reference

@pytest.fixture
def hypothesis():
    hypothesis = Annotation()
    hypothesis[Segment(2, 13)] = 'a'
    hypothesis[Segment(13, 14)] = 'd'
    hypothesis[Segment(14, 20)] = 'b'
    hypothesis[Segment(22, 38)] = 'c'
    hypothesis[Segment(38, 40)] = 'd'
    return hypothesis

def test_error_rate(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    error_rate = diarizationErrorRate(reference, hypothesis)
    npt.assert_almost_equal(error_rate, 0.5161290322580645, decimal=7)

def test_optimal_mapping(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    mapping = diarizationErrorRate.optimal_mapping(reference, hypothesis)
    assert mapping == {'a': 'A', 'b': 'B', 'c': 'C'}

def test_detailed(reference, hypothesis):
    diarizationErrorRate = DiarizationErrorRate()
    details = diarizationErrorRate(reference, hypothesis, detailed=True)

    confusion = details['confusion']
    npt.assert_almost_equal(confusion, 7.0, decimal=7)

    correct = details['correct']
    npt.assert_almost_equal(correct, 22.0, decimal=7)

    rate = details['diarization error rate']
    npt.assert_almost_equal(rate, 0.5161290322580645, decimal=7)

    false_alarm = details['false alarm']
    npt.assert_almost_equal(false_alarm, 7.0, decimal=7)

    missed_detection = details['missed detection']
    npt.assert_almost_equal(missed_detection, 2.0, decimal=7)

    total = details['total']
    npt.assert_almost_equal(total, 31.0, decimal=7)

def test_purity(reference, hypothesis):
    diarizationPurity = DiarizationPurity()
    purity = diarizationPurity(reference, hypothesis)
    npt.assert_almost_equal(purity, 0.8275, decimal=3)

def test_coverage(reference, hypothesis):
    diarizationCoverage = DiarizationCoverage()
    coverage = diarizationCoverage(reference, hypothesis)
    npt.assert_almost_equal(coverage, 0.7586, decimal=3)

def test_skip_overlap(reference_with_overlap, hypothesis):
    metric = DiarizationErrorRate(skip_overlap=True)
    total = metric(reference_with_overlap, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 32, decimal=3)

def test_leep_overlap(reference_with_overlap, hypothesis):
    metric = DiarizationErrorRate(skip_overlap=False)
    total = metric(reference_with_overlap, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 34, decimal=3)

def test_bug_16():
    reference = Annotation()
    reference[Segment(0, 10)] = 'A'
    hypothesis = Annotation()

    metric = DiarizationErrorRate(collar=1)
    total = metric(reference, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 9, decimal=3)

    metric = DiarizationErrorRate(collar=0)
    total = metric(reference, hypothesis, detailed=True)['total']
    npt.assert_almost_equal(total, 10, decimal=3)


# sunnyspeed studio
# GitHub: https://github.com/sunnyspeedstudio/favorite-focal-length
# YouTube: https://www.youtube.com/sunnyspeedstudio
# bilibili: http://space.bilibili.com/36609936
# Tested using python 3 under mac and windows

import subprocess
import os
import operator
import datetime
import pygal
import pickle

# all video folders need to be scanned (the script will go through all subfolders)
videoFolders = [  'path1',
                'path2' ]
# videoFolders = ['/Volumes/MEDIA/Camera - Old/2016-03 - Sony A6300', '/Volumes/MEDIA/Camera - Old/2016-12 - Sony A6500', '/Volumes/MEDIA/Camera - Old/2019-03 - Sony A6400', '/Volumes/MEDIA/Camera - Current/2018-03 - Sony A7R III']
# all video extensions need to be scanned
videoExtensions = ('.mp4', '.MP4')
# exiftool path
exiftoolPath = 'exiftool'

# funcions:
def formatDuration(s):
    hour = int(s / 3600)
    minute = int((s - hour * 3600) / 60)
    second = s - hour * 3600 - minute * 60
    return str(hour) + ':' + str(minute) + ':' + str(second)

def drawBarChart(title, data):
    value = []
    label = []
    for x in data:
        if ('(time in seconds)' in title):
            label.append(str(x[0]) + ' (' + formatDuration(x[1]) + ' - ' + str("{:.2f}".format(float(x[1]) * 100 / totalDuration)) + '%)')
        else:
            label.append(str(x[0]) + ' (' + str("{:.2f}".format(float(x[1]) * 100 / totalVideo)) + '%)')

        value.append(x[1])

    bar_chart = pygal.HorizontalBar(legend_at_bottom=True)
    bar_chart.x_labels =label
    bar_chart.add(title, value)
    bar_chart.render_to_file(svgPath + '/' + title.lower().replace(' ', '_') + '.svg')

def printResult(title, data, sort):
    print('\n')
    print(title)
    # sorting
    if sort == 'value':
        data = sorted(data.items(), key=operator.itemgetter(1), reverse=False)
    else:
        data = data.items()
    # printing
    for x in data:
        if ('(time in seconds)' in title):
            print(x[0], '=', x[1], '-', "{:.2f}".format(float(x[1]) * 100 / totalDuration), '%')
        else:
            print(x[0], '=', x[1], '-', "{:.2f}".format(float(x[1]) * 100 / totalVideo), '%')
    # generate chart
    drawBarChart(title, data)

def svgObject(title):
    return '<object type="image/svg+xml" data="svg/' + title.lower().replace(' ', '_') + '.svg"></object>'

def createHTML():
    separator = ', '

    html = """

Scanned number of videos: %s

Total size: %s, average size: %s

Total duration: %s, average duration: %s

Scanned video folders: %s

Powered by sunnyspeed studio @ GitHub
        YouTube

    """%(totalVideo, fileSize, averageSize, formatDuration(totalDuration), formatDuration(averageDuration), separator.join(videoFolders))
    html += '
'
    html += '
' + svgObject('Camera') + '	' + svgObject('Camera (time in seconds)') + '
'
    html += '
' + svgObject('Duration') + '	' + svgObject('Duration (time in seconds)') + '
'
    html += '
' + svgObject('Bitrate') + '	' + svgObject('Bitrate (time in seconds)') + '
'
    html += '
' + svgObject('Frame Rate') + '	' + svgObject('Frame Rate (time in seconds)') + '
'
    html += '
' + svgObject('Resolution') + '	' + svgObject('Resolution (time in seconds)') + '
'
    html += '
' + svgObject('Created Day') + '	' + svgObject('Created Day (time in seconds)') + '
'
    html += '
' + svgObject('Created Hour') + '	' + svgObject('Created Hour (time in seconds)') + '
'
    html += '
'
    Html_file= open(htmlPath + "/output.html", "w")
    Html_file.write(html)
    Html_file.close()

# pickle file path
pickleFilePath = 'video.pkl'
# load saved videos info from pickle file to save time
# please delete local pickle file for a fresh new scan
scannedVideo = {}
if os.path.exists(pickleFilePath):
    scannedVideo = pickle.load(open('video.pkl', 'rb'))

# variables
totalVideo = 0
fileSize = 0
averageSize = 0
totalDuration = 0
averageDuration = 0

 # counters
duration = {   'Unknown':0,
                '>1h':0,
                '45m~1h':0,
                '30m~45m':0,
                '20m~30m':0,
                '10m~20m':0,
                '5m~10m':0,
                '1m~5m':0,
                '<1m':0 }
durationT = {   'Unknown':0,
                '>1h':0,
                '45m~1h':0,
                '30m~45m':0,
                '20m~30m':0,
                '10m~20m':0,
                '5m~10m':0,
                '1m~5m':0,
                '<1m':0 }
createHour = {  '23':0,
                '22':0,
                '21':0,
                '20':0,
                '19':0,
                '18':0,
                '17':0,
                '16':0,
                '15':0,
                '14':0,
                '13':0,
                '12':0,
                '11':0,
                '10':0,
                '09':0,
                '08':0,
                '07':0,
                '06':0,
                '05':0,
                '04':0,
                '03':0,
                '02':0,
                '01':0,
                '00':0 }
createHourT = {  '23':0,
                '22':0,
                '21':0,
                '20':0,
                '19':0,
                '18':0,
                '17':0,
                '16':0,
                '15':0,
                '14':0,
                '13':0,
                '12':0,
                '11':0,
                '10':0,
                '09':0,
                '08':0,
                '07':0,
                '06':0,
                '05':0,
                '04':0,
                '03':0,
                '02':0,
                '01':0,
                '00':0 }
createDay = {   'Sunday':0,
                'Saturday':0,
                'Friday':0,
                'Thursday':0,
                'Wednesday':0,
                'Tuesday':0,
                'Monday':0,}
createDayT = {   'Sunday':0,
                'Saturday':0,
                'Friday':0,
                'Thursday':0,
                'Wednesday':0,
                'Tuesday':0,
                'Monday':0,}
resolution = {}
resolutionT = {}
camera = {}
cameraT = {}
bitrate= {}
bitrateT= {}
frameRate = {}
frameRateT = {}

# scan through all video files
for videoFolder in videoFolders:
    for subfolder, folders, files in os.walk(videoFolder):
        for f in files:
            videoFile = subfolder + os.sep + f

            if videoFile.endswith(videoExtensions):
                exifInfo = {}
                if videoFile in scannedVideo.keys():    # this video was already scanned
                    exifInfo = scannedVideo[videoFile]
                else:
                    # prepare the exif holder
                    exifInfo['File Size'] = 'Unknown'               # not for charting

                    # the following strange numbers, such as 99999, is for handling the edge cases
                    # if the exifInfo is changed below, please delete pickle file to do a fresh new scan
                    exifInfo['Duration'] = 'Unknown'        # sony
                    exifInfo['Create Date'] = 'Unknown'             # break down to hour and day
                    exifInfo['Video Size'] = 'Unknown'
                    exifInfo['Device Model Name'] = 'Unknown'
                    exifInfo['Video Avg Bitrate'] = 'Unknown'
                    exifInfo['Video Avg Frame Rate'] = 'Unknown'

                    # retrieve exif from each video
                    process = subprocess.Popen([exiftoolPath, videoFile, '-api', 'largefilesupport=1'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
                    # create the exif array for the video
                    for line in process.stdout:
                        kv = line.strip().split(':', 1)
                        if kv[0].strip() in exifInfo:   # only save the tags we need
                            exifInfo[kv[0].strip()] = kv[1].strip()
                    process.kill()

                    # update scannedVideo with raw exif info, and save it back to pickle file
                    scannedVideo[videoFile] = exifInfo
                    pickle.dump(scannedVideo, open(pickleFilePath, "wb"))

                # special treatments on the exif data
                # 1. file size: remove MB in file size, eg: 8.3 MB
                kv = exifInfo['File Size'].split(' ')
                fileSize += float(kv[0])

                # 3. duration: from 0:01:17 to seconds
                durationInSecond = 0
                if ('s' in exifInfo['Duration']):
                    kv = exifInfo['Duration'].split('.')
                    durationInSecond = kv[0]
                else:
                    kv = exifInfo['Duration'].split(':')
                    durationInSecond = int(kv[0]) * 3600 + int(kv[1]) * 60 + int(kv[2])
                totalDuration += int(durationInSecond)

                # 5. hour and day: grab hour and day of the week from the date, eg: 2020:05:02 13:57:57.072-07:00
                kv = exifInfo['Create Date'].split(' ')
                kv1 = kv[0].split(':')
                d = datetime.datetime(int(kv1[0]), int(kv1[1]), int(kv1[2]))
                exifInfo['Create Day'] = d.strftime('%A')   # day of the week
                kv2 = kv[1].split(':')
                exifInfo['Create Hour'] = kv2[0]    # hour

                # counting
                t = int(durationInSecond)

                # duration
                if exifInfo['Duration'] == 'Unknown':
                    duration['Unknown'] += 1
                    durationT['Unknown'] += t
                else:
                    if int(durationInSecond) <= 60:
                        duration['<1m'] += 1
                        durationT['<1m'] += t
                    elif int(durationInSecond) > 60 and int(durationInSecond) <= 300:
                        duration['1m~5m'] += 1
                        durationT['1m~5m'] += t
                    elif int(durationInSecond) > 300 and int(durationInSecond) <= 600:
                        duration['5m~10m'] += 1
                        durationT['5m~10m'] += t
                    elif int(durationInSecond) > 600 and int(durationInSecond) <= 1200:
                        duration['10m~20m'] += 1
                        durationT['10m~20m'] += t
                    elif int(durationInSecond) > 1200 and int(durationInSecond) <= 1800:
                        duration['20m~30m'] += 1
                        durationT['20m~30m'] += t
                    elif int(durationInSecond) > 1800 and int(durationInSecond) <= 2700:
                        duration['30m~45m'] += 1
                        durationT['30m~45m'] += t
                    elif int(durationInSecond) > 2700 and int(durationInSecond) <= 3600:
                        duration['45m~1h'] += 1
                        durationT['45m~1h'] += t
                    elif int(durationInSecond) > 3600:
                        duration['>1h'] += 1
                        durationT['>1h'] += t

                # camera
                if exifInfo['Device Model Name'] in camera.keys():
                    camera[exifInfo['Device Model Name']] += 1
                    cameraT[exifInfo['Device Model Name']] += t
                else:
                    camera[exifInfo['Device Model Name']] = 1
                    cameraT[exifInfo['Device Model Name']] = t

                # create hour
                createHour[exifInfo['Create Hour']] += 1
                createHourT[exifInfo['Create Hour']] += t

                # create day
                createDay[exifInfo['Create Day']] += 1
                createDayT[exifInfo['Create Day']] += t

                # resolution
                if exifInfo['Video Size'] in resolution.keys():
                    resolution[exifInfo['Video Size']] += 1
                    resolutionT[exifInfo['Video Size']] += t
                else:
                    resolution[exifInfo['Video Size']] = 1
                    resolutionT[exifInfo['Video Size']] = t

                # bitrate
                if exifInfo['Video Avg Bitrate'] in bitrate.keys():
                    bitrate[exifInfo['Video Avg Bitrate']] += 1
                    bitrateT[exifInfo['Video Avg Bitrate']] += t
                else:
                    bitrate[exifInfo['Video Avg Bitrate']] = 1
                    bitrateT[exifInfo['Video Avg Bitrate']] = t

                # frame rate
                if exifInfo['Video Avg Frame Rate'] in frameRate.keys():
                    frameRate[exifInfo['Video Avg Frame Rate']] += 1
                    frameRateT[exifInfo['Video Avg Frame Rate']] += t
                else:
                    frameRate[exifInfo['Video Avg Frame Rate']] = 1
                    frameRateT[exifInfo['Video Avg Frame Rate']] = t

                # total video counter
                totalVideo += 1

                # print each video's exif info
                print(  totalVideo, ':',
                        videoFile, ':',
                        exifInfo['Duration'], ':',
                        exifInfo['Create Day'], ':',
                        exifInfo['Create Hour'], ':',
                        exifInfo['Video Size'], ':',
                        exifInfo['Device Model Name'], ':',
                        exifInfo['Video Avg Bitrate'], ':',
                        exifInfo['Video Avg Frame Rate'])

# print out final results
print('\nTotal scanned number of videos:', totalVideo)
if (totalVideo == 0):
    print('\nNo Video is found.')
else:
    averageSize = str("{:.1f}".format(float(fileSize / totalVideo))) + 'MB'
    if fileSize < 1024:
        fileSize = str(int(fileSize)) + 'MB'
    else:
        fileSize = str(int(fileSize / 1024)) + 'GB'
    print('Total size:', fileSize)
    print('Average size:', averageSize)
    print('Total duration:', formatDuration(totalDuration))
    averageDuration = int(totalDuration / totalVideo)
    print('Average duration:', formatDuration(averageDuration))

    # create a new folder to store charts and html
    htmlPath = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    os.makedirs(htmlPath)
    svgPath = htmlPath + '/svg'
    os.makedirs(svgPath)

    printResult('Camera', camera, 'value')
    printResult('Duration', duration, 'no')
    printResult('Created Hour', createHour, 'no')
    printResult('Created Day', createDay, 'no')
    printResult('Resolution', resolution, 'value')
    printResult('Bitrate', bitrate, 'value')
    printResult('Frame Rate', frameRate, 'value')

    printResult('Camera (time in seconds)', cameraT, 'value')
    printResult('Duration (time in seconds)', durationT, 'no')
    printResult('Created Hour (time in seconds)', createHourT, 'no')
    printResult('Created Day (time in seconds)', createDayT, 'no')
    printResult('Resolution (time in seconds)', resolutionT, 'value')
    printResult('Bitrate (time in seconds)', bitrateT, 'value')
    printResult('Frame Rate (time in seconds)', frameRateT, 'value')

    createHTML()

    print('\n\nThe result is saved in folder ' + htmlPath)

print('Thank you for using this tool!\n\n')