from .models import Project
from productMarketingDwh.models import BoUp
from .helperFunctions import getProjectOverview, HighLevelProjectProblems
import time
from datetime import datetime

class bottomUpPersistor:

    finalRevenue = []
    finalVolume = []
    finalPrice = []
    finalGrossMargin = []
    finalGrossMarginPct = []
    final_revenue_month = []
    final_grossMargin_month = []
    final_grossMarginPct_month = []
    finalVhk = []
    final_revenue_FY = []
    final_grossProfit_FY = []
    final_grossProfitPct_FY = []
    final_volumes_FY = []
    vhkCy = []
    finalTotalCost = []
    asp = []
    weightedGrossMargin = []
    weightedGrossMarginPct = []
    weightedRevenue = []
    weightedVolume = []
    message = ""
    fxRate = 1.0

    def __init__(self, params):
        self.finalRevenue = params["finalRevenue"]
        self.finalVolume = params["finalVolume"]
        self.finalPrice = params["finalPrice"]
        self.finalGrossMargin = params["finalGrossMargin"]
        self.finalGrossMarginPct = params["finalGrossMarginPct"]
        self.final_revenue_month = params["final_revenue_month"]
        self.final_grossMargin_month = params["final_grossMargin_month"]
        self.final_grossMarginPct_month = params["final_grossMarginPct_month"]
        self.finalVhk = params["finalVhk"]
        self.final_revenue_FY = params["final_revenue_FY"]
        self.final_grossProfit_FY = params["final_grossProfit_FY"]
        self.final_grossProfitPct_FY = params["final_grossProfitPct_FY"]
        self.final_volumes_FY = params["final_volumes_FY"]
        self.vhkCy = params["vhkCy"]
        self.finalTotalCost = params["finalTotalCost"]
        self.asp = params["asp"]
        self.weightedGrossMargin = params["weightedGrossMargin"]
        self.weightedGrossMarginPct = params["weightedGrossMarginPct"]
        self.weightedRevenue = params["weightedRevenue"]
        self.weightedVolume = params["weightedVolume"]

    def persist(
        self,
        projectId,
        request,
        bulk,
        boUpObjectInput,
        probability,
        helperObjects,
        vhkUpdate,
        runDate,
    ):
        print(
            "persisting project", projectId, "boupobjinpu", boUpObjectInput, probability
        )
        project = Project.objects.get(id=projectId)
        errors = []

        user = None
        if request != None:
            user = request.user

        try:
            if bulk == False:
                probability = float(project.status.status) / 100.0
        except:
            errors.append(HighLevelProjectProblems.projectKeyFactsError)
            outputDictionary = {
                "revenue": [],
                "grossMargin": [],
                "years": [],
                "errors": errors,
                "sumRevenue": 0,
                "sumGrossMargin": 0,
                "sumCost": 0,
                "sumVolume": 0,
                "sumWeightedVolume": 0,
                "sumWeightedRevenue": 0,
                "averageAsp": 0,
                "sumWeightedGrossMargin": 0,
            }

            print("####### returning A")

            # case probability was not set yet
            message = "You are missing mandatory project key facts, as in this case the project probability (weighting). Please fill out before continuing."
            return False, outputDictionary, message

        if (len(self.finalVolume) == 0) and (bulk == False):

            (
                revenue,
                volumes,
                prices,
                grossMargin,
                grossMarginPct,
                revenue_month,
                grossMargin_month,
                grossMarginPct_month,
                vhk,
                revenue_FY,
                grossProfit_FY,
                grossProfitPct_FY,
                volumes_FY,
                VhkCy,
                totalCost,
                sumRevenue,
                sumGrossMargin,
                sumCost,
                years,
                errors,
                weightedGrossMargin,
                weightedGrossMarginPct,
                weightedRevenue,
                asp,
                weightedVolume,
                sumVolume,
                sumWeightedVolume,
                sumWeightedRevenue,
                averageAsp,
                sumWeightedGrossMargin,
                projectCurrency,
                fxRate,
            ) = getProjectOverview(projectId, user, probability, None)
            # transfer
            self.finalRevenue = revenue
            self.finalVolume = volumes
            self.finalPrice = prices
            self.finalGrossMargin = grossMargin
            self.finalGrossMarginPct = grossMarginPct
            self.final_revenue_month = revenue_month
            self.final_grossMargin_month = grossMargin_month
            self.final_grossMarginPct_month = grossMarginPct_month
            self.finalVhk = vhk
            self.final_revenue_FY = revenue_FY
            self.final_grossProfit_FY = grossProfit_FY
            self.final_grossProfitPct_FY = grossProfitPct_FY
            self.final_volumes_FY = volumes_FY
            self.vhkCy = VhkCy
            self.finalTotalCost = totalCost
            self.weightedGrossMargin = weightedGrossMargin
            self.weightedGrossMarginPct = weightedGrossMarginPct
            self.weightedRevenue = weightedRevenue
            self.asp = asp
            self.weightedVolume = weightedVolume
            self.fxRate = fxRate

        elif (len(self.finalVolume) == 0) and (bulk == True):
            print("input probability", probability)
            (
                revenue,
                volumes,
                prices,
                grossMargin,
                grossMarginPct,
                revenue_month,
                grossMargin_month,
                grossMarginPct_month,
                vhk,
                revenue_FY,
                grossProfit_FY,
                grossProfitPct_FY,
                volumes_FY,
                VhkCy,
                totalCost,
                sumRevenue,
                sumGrossMargin,
                sumCost,
                years,
                errors,
                weightedGrossMargin,
                weightedGrossMarginPct,
                weightedRevenue,
                asp,
                weightedVolume,
                sumVolume,
                sumWeightedVolume,
                sumWeightedRevenue,
                averageAsp,
                sumWeightedGrossMargin,
                projectCurrency,
                fxRate,
            ) = getProjectOverview(projectId, user, probability, helperObjects)
            # transfer
            self.finalRevenue = revenue
            self.finalVolume = volumes
            self.finalPrice = prices
            self.finalGrossMargin = grossMargin
            self.finalGrossMarginPct = grossMarginPct
            self.final_revenue_month = revenue_month
            self.final_grossMargin_month = grossMargin_month
            self.final_grossMarginPct_month = grossMarginPct_month
            self.finalVhk = vhk
            self.final_revenue_FY = revenue_FY
            self.final_grossProfit_FY = grossProfit_FY
            self.final_grossProfitPct_FY = grossProfitPct_FY
            self.final_volumes_FY = volumes_FY
            self.vhkCy = VhkCy
            self.finalTotalCost = totalCost
            self.weightedGrossMargin = weightedGrossMargin
            self.weightedGrossMarginPct = weightedGrossMarginPct
            self.weightedRevenue = weightedRevenue
            self.asp = asp
            self.weightedVolume = weightedVolume
            self.fxRate = fxRate
        if (len(self.finalVolume) == 0) | (len(self.finalPrice) == 0):

            errors.append(
                HighLevelProjectProblems.pricesAndVolumesMissingCompletely)

            outputDictionary = {
                "revenue": revenue,
                "grossMargin": grossMargin,
                "years": years,
                "errors": errors,
                "sumRevenue": sumRevenue,
                "sumGrossMargin": sumGrossMargin,
                "sumCost": sumCost,
                "sumVolume": sumVolume,
                "sumWeightedVolume": sumWeightedVolume,
                "sumWeightedRevenue": sumWeightedRevenue,
                "averageAsp": averageAsp,
                "sumWeightedGrossMargin": sumWeightedGrossMargin,
            }

            message = "An error ocurred while submitting the bottom up. You are either missing volume or price information or both. Until you complete the missing data, the project cannot be submitted."
            return False, outputDictionary, message, None

        """
        # clear project id from session

        try:
            del request.session["project_id"]
        except KeyError:
            pass
        """

        project.draft = False
        revenue = self.finalRevenue
        volumes = self.finalVolume
        prices = self.finalPrice
        grossMargin = self.finalGrossMargin
        grossMarginPct = self.finalGrossMarginPct
        revenue_month = self.final_revenue_month
        grossMargin_month = self.final_grossMargin_month
        grossMarginPct_month = self.final_grossMarginPct_month
        vhk = self.finalVhk
        revenue_FY = self.final_revenue_FY
        grossProfit_FY = self.final_grossProfit_FY
        grossProfitPct_FY = self.final_grossProfitPct_FY
        volumes_FY = self.final_volumes_FY
        VhkCy = self.vhkCy

        # create entry in BoUp TABLE
        start_time = time.time()
        if bulk == False:
            if not project.oem:
                oem = ""
            else:
                oem = project.oem  # oem = project.oem.oemName

        BoUpObject = None
        created = None
        # based on unique key
        if bulk == False:
            BoUpObject, created = BoUp.objects.get_or_create(
                applicationMain=project.applicationMain,
                applicationDetail=project.applicationDetail,
                mainCustomer=project.mainCustomer,
                endCustomer=project.finalCustomer,
                salesName=project.sales_name,
                syntheticProjectName=project.syntheticProjectName
            )
        else:
            BoUpObject = boUpObjectInput

        # except:
        #   print("... going to raise validation error")
        #    return False

        try:
            if bulk == False:
                BoUpObject.ID_APP = project
                BoUpObject.productMarketer = project.productMarketer
                BoUpObject.hfg = project.sales_name.rfp.hfg
                BoUpObject.ppos = project.sales_name.rfp.ppos
                BoUpObject.spNumber = project.spNumber
                BoUpObject.familyPriceApplicable = project.familyPriceApplicable
                BoUpObject.familyPriceDetails = project.familyPriceDetails
                #BoUpObject.priceType = project.priceType
                BoUpObject.comment = project.comment
                BoUpObject.applicationLine = (
                    project.applicationLine.applicationLineShortName if project.applicationLine else "N/A"
                )
                BoUpObject.Reviewed = project.projectReviewed
                BoUpObject.reviewDate = project.reviewDate
                # change all these to FK?
                BoUpObject.region = project.region.region
                BoUpObject.secondRegion = project.region.region
                BoUpObject.dcChannel = (
                    project.dcChannel.dcChannelDescription if project.dcChannel else ""
                )
                BoUpObject.priceType = (
                    project.priceType.priceTypeDisplay if project.priceType else ""
                )
                BoUpObject.plHfg = project.product.plHfg
                BoUpObject.endCustomerHelper = project.finalCustomer.finalCustomerName
                BoUpObject.mainCustomerHelper = project.mainCustomer.customerName
                BoUpObject.projectName = project.projectName
                BoUpObject.distributor = project.distributor
                BoUpObject.tier1 = project.tier1
                BoUpObject.ems = project.ems
                BoUpObject.vpaCustomer = project.vpaCustomer
                BoUpObject.salesContact = project.salesContact
                BoUpObject.statusProbability = project.status.statusDisplay
                BoUpObject.probability = probability
                BoUpObject.sop = project.estimatedSop
                BoUpObject.availablePGS = project.sales_name.rfp.availablePGS
                # project.user,#modifiedBy = project.user.username,
                BoUpObject.modifiedBy = request.user
                BoUpObject.modifiedDate = project.modifiedDate
                BoUpObject.creationDate = project.creationDate
                BoUpObject.package = project.sales_name.rfp.packageHelper
                BoUpObject.series = project.sales_name.rfp.seriesHelper
                BoUpObject.rfp = project.sales_name.rfp
                BoUpObject.dummy = project.dummy
                BoUpObject.basicType = project.sales_name.rfp.basicType
                BoUpObject.priceSource = project.otherPriceComments
                BoUpObject.gen = project.sales_name.rfp.familyHelper
                BoUpObject.genDetail = project.sales_name.rfp.familyDetailHelper
                BoUpObject.seriesLong = "N/A"
                BoUpObject.rfpHlp = project.sales_name.rfp.rfp
                BoUpObject.mcHlp = project.mainCustomer.customerName
                BoUpObject.ecHlp = project.finalCustomer.finalCustomerName
                BoUpObject.fxRate = self.fxRate
                BoUpObject.syntheticProjectName = project.syntheticProjectName
                BoUpObject.contractualCurrency = project.contractualCurrency
                BoUpObject.currency = "EUR"

                if vhkUpdate == True:
                    BoUpObject.vhkDate = runDate
                BoUpObject.save()
        except:

            errors.append(HighLevelProjectProblems.otherProjectIntegrityError)
            outputDictionary = {
                "revenue": revenue,
                "grossMargin": grossMargin,
                "years": years,
                "errors": errors,
                "sumRevenue": sumRevenue,
                "sumGrossMargin": sumGrossMargin,
                "sumCost": sumCost,
                "sumVolume": sumVolume,
                "sumWeightedVolume": sumWeightedVolume,
                "sumWeightedRevenue": sumWeightedRevenue,
                "averageAsp": averageAsp,
                "sumWeightedGrossMargin": sumWeightedGrossMargin,
            }

            print("... going to raise second validation error")
            message = "An unique constraint was violated! This can happen when there already is a project in the Bottom Up table with the same combination of Main and End Customers, Main and Detail applications and sales name."
            return False, outputDictionary, message, None

        """
        bis oem
        """

        try:
            BoUpObject.oem = oem
        except:
            print("could not set oem!")

        BoUpObject.gmLifeTime = sum(grossMargin)
        BoUpObject.revEurLifeTime = sum(revenue)
        BoUpObject.volLifeTime = sum(volumes)
        BoUpObject.volWeightedLifeTime = sum(volumes) * probability

        BoUpObject.vol2020 = volumes[0]
        BoUpObject.vol2021 = volumes[1]
        BoUpObject.vol2022 = volumes[2]
        BoUpObject.vol2023 = volumes[3]
        BoUpObject.vol2024 = volumes[4]
        BoUpObject.vol2025 = volumes[5]
        BoUpObject.vol2026 = volumes[6]
        BoUpObject.vol2027 = volumes[7]
        BoUpObject.vol2028 = volumes[8]
        BoUpObject.vol2029 = volumes[9]
        BoUpObject.vol2030 = volumes[10]
        BoUpObject.vol2031 = volumes[11]
        BoUpObject.vol2032 = volumes[12]
        BoUpObject.vol2033 = volumes[13]
        BoUpObject.vol2034 = volumes[14]
        BoUpObject.vol2035 = volumes[15]
        BoUpObject.vol2036 = volumes[16]
        BoUpObject.vol2037 = volumes[17]
        BoUpObject.vol2038 = volumes[18]
        BoUpObject.vol2039 = volumes[19]
        BoUpObject.vol2040 = volumes[20]
        BoUpObject.vol2041 = volumes[21]
        BoUpObject.vol2042 = volumes[22]
        BoUpObject.vol2043 = volumes[23]
        BoUpObject.vol2044 = 0.0

        # original code linked all volCustomer to first entry in volumes
        BoUpObject.volCustomer2020 = volumes[0]
        BoUpObject.volCustomer2021 = volumes[0]
        BoUpObject.volCustomer2022 = volumes[0]
        BoUpObject.volCustomer2023 = volumes[0]
        BoUpObject.volCustomer2024 = volumes[0]
        BoUpObject.volCustomer2025 = volumes[0]
        BoUpObject.volCustomer2026 = volumes[0]
        BoUpObject.volCustomer2027 = volumes[0]
        BoUpObject.volCustomer2028 = volumes[0]
        BoUpObject.volCustomer2029 = volumes[0]
        BoUpObject.volCustomer2030 = volumes[0]
        BoUpObject.volCustomer2031 = volumes[0]
        BoUpObject.volCustomer2032 = volumes[0]
        BoUpObject.volCustomer2033 = volumes[0]
        BoUpObject.volCustomer2034 = volumes[0]
        BoUpObject.volCustomer2035 = volumes[0]
        BoUpObject.volCustomer2036 = volumes[0]
        BoUpObject.volCustomer2037 = volumes[0]
        BoUpObject.volCustomer2038 = volumes[0]
        BoUpObject.volCustomer2039 = volumes[0]
        BoUpObject.volCustomer2040 = volumes[0]
        BoUpObject.volCustomer2041 = volumes[0]
        BoUpObject.volCustomer2042 = volumes[0]
        BoUpObject.volCustomer2043 = volumes[0]
        BoUpObject.volCustomer2044 = volumes[0]

        BoUpObject.price2020 = prices[0]
        BoUpObject.price2021 = prices[1]
        BoUpObject.price2022 = prices[2]
        BoUpObject.price2023 = prices[3]
        BoUpObject.price2024 = prices[4]
        BoUpObject.price2025 = prices[5]
        BoUpObject.price2026 = prices[6]
        BoUpObject.price2027 = prices[7]
        BoUpObject.price2028 = prices[8]
        BoUpObject.price2029 = prices[9]
        BoUpObject.price2030 = prices[10]
        BoUpObject.price2031 = prices[11]
        BoUpObject.price2032 = prices[12]
        BoUpObject.price2033 = prices[13]
        BoUpObject.price2034 = prices[14]
        BoUpObject.price2035 = prices[15]
        BoUpObject.price2036 = prices[16]
        BoUpObject.price2037 = prices[17]
        BoUpObject.price2038 = prices[18]
        BoUpObject.price2039 = prices[19]
        BoUpObject.price2040 = prices[20]
        BoUpObject.price2041 = prices[21]
        BoUpObject.price2042 = prices[22]
        BoUpObject.price2043 = prices[23]
        BoUpObject.price2044 = 0.0

        BoUpObject.vhk2020 = vhk[0]
        BoUpObject.vhk2021 = vhk[1]
        BoUpObject.vhk2022 = vhk[2]
        BoUpObject.vhk2023 = vhk[3]
        BoUpObject.vhk2024 = vhk[4]
        BoUpObject.vhk2025 = vhk[5]
        BoUpObject.vhk2026 = vhk[6]
        BoUpObject.vhk2027 = vhk[7]
        BoUpObject.vhk2028 = vhk[8]
        BoUpObject.vhk2029 = vhk[9]
        BoUpObject.vhk2030 = vhk[10]
        BoUpObject.vhk2031 = vhk[11]
        BoUpObject.vhk2032 = vhk[12]
        BoUpObject.vhk2033 = vhk[13]
        BoUpObject.vhk2034 = vhk[14]
        BoUpObject.vhk2035 = vhk[15]
        BoUpObject.vhk2036 = vhk[16]
        BoUpObject.vhk2037 = vhk[17]
        BoUpObject.vhk2038 = vhk[18]
        BoUpObject.vhk2039 = vhk[19]
        BoUpObject.vhk2040 = vhk[20]
        BoUpObject.vhk2041 = vhk[21]
        BoUpObject.vhk2042 = vhk[22]
        BoUpObject.vhk2043 = vhk[23]
        BoUpObject.vhk2044 = 0.0

        BoUpObject.gm2020 = grossMargin[0]
        BoUpObject.gm2021 = grossMargin[1]
        BoUpObject.gm2022 = grossMargin[2]
        BoUpObject.gm2023 = grossMargin[3]
        BoUpObject.gm2024 = grossMargin[4]
        BoUpObject.gm2025 = grossMargin[5]
        BoUpObject.gm2026 = grossMargin[6]
        BoUpObject.gm2027 = grossMargin[7]
        BoUpObject.gm2028 = grossMargin[8]
        BoUpObject.gm2029 = grossMargin[9]
        BoUpObject.gm2030 = grossMargin[10]
        BoUpObject.gm2031 = grossMargin[11]
        BoUpObject.gm2032 = grossMargin[12]
        BoUpObject.gm2033 = grossMargin[13]
        BoUpObject.gm2034 = grossMargin[14]
        BoUpObject.gm2035 = grossMargin[15]
        BoUpObject.gm2036 = grossMargin[16]
        BoUpObject.gm2037 = grossMargin[17]
        BoUpObject.gm2038 = grossMargin[18]
        BoUpObject.gm2039 = grossMargin[19]
        BoUpObject.gm2040 = grossMargin[20]
        BoUpObject.gm2041 = grossMargin[21]
        BoUpObject.gm2042 = grossMargin[22]
        BoUpObject.gm2043 = grossMargin[23]
        BoUpObject.gm2044 = 0.0

        BoUpObject.wVol2020 = float(weightedVolume[0])
        BoUpObject.wVol2021 = float(weightedVolume[1])
        BoUpObject.wVol2022 = float(weightedVolume[2])
        BoUpObject.wVol2023 = float(weightedVolume[3])
        BoUpObject.wVol2024 = float(weightedVolume[4])
        BoUpObject.wVol2025 = float(weightedVolume[5])
        BoUpObject.wVol2026 = float(weightedVolume[6])
        BoUpObject.wVol2027 = float(weightedVolume[7])
        BoUpObject.wVol2028 = float(weightedVolume[8])
        BoUpObject.wVol2029 = float(weightedVolume[9])
        BoUpObject.wVol2030 = float(weightedVolume[10])
        BoUpObject.wVol2031 = float(weightedVolume[11])
        BoUpObject.wVol2032 = float(weightedVolume[12])
        BoUpObject.wVol2033 = float(weightedVolume[13])
        BoUpObject.wVol2034 = float(weightedVolume[14])
        BoUpObject.wVol2035 = float(weightedVolume[15])
        BoUpObject.wVol2036 = float(weightedVolume[16])
        BoUpObject.wVol2037 = float(weightedVolume[17])
        BoUpObject.wVol2038 = float(weightedVolume[18])
        BoUpObject.wVol2039 = float(weightedVolume[19])
        BoUpObject.wVol2040 = float(weightedVolume[20])
        BoUpObject.wVol2041 = float(weightedVolume[21])
        BoUpObject.wVol2042 = float(weightedVolume[22])
        BoUpObject.wVol2043 = float(weightedVolume[23])
        BoUpObject.wVol2044 = 0.0

        BoUpObject.wRev2020 = weightedRevenue[0]
        BoUpObject.wRev2021 = weightedRevenue[1]
        BoUpObject.wRev2022 = weightedRevenue[2]
        BoUpObject.wRev2023 = weightedRevenue[3]
        BoUpObject.wRev2024 = weightedRevenue[4]
        BoUpObject.wRev2025 = weightedRevenue[5]
        BoUpObject.wRev2026 = weightedRevenue[6]
        BoUpObject.wRev2027 = weightedRevenue[7]
        BoUpObject.wRev2028 = weightedRevenue[8]
        BoUpObject.wRev2029 = weightedRevenue[9]
        BoUpObject.wRev2030 = weightedRevenue[10]
        BoUpObject.wRev2031 = weightedRevenue[11]
        BoUpObject.wRev2032 = weightedRevenue[12]
        BoUpObject.wRev2033 = weightedRevenue[13]
        BoUpObject.wRev2034 = weightedRevenue[14]
        BoUpObject.wRev2035 = weightedRevenue[15]
        BoUpObject.wRev2036 = weightedRevenue[16]
        BoUpObject.wRev2037 = weightedRevenue[17]
        BoUpObject.wRev2038 = weightedRevenue[18]
        BoUpObject.wRev2039 = weightedRevenue[19]
        BoUpObject.wRev2040 = weightedRevenue[20]
        BoUpObject.wRev2041 = weightedRevenue[21]
        BoUpObject.wRev2042 = weightedRevenue[22]
        BoUpObject.wRev2043 = weightedRevenue[23]
        BoUpObject.wRev2044 = 0.0

        # now ASP and weighted gross margin:   asp = weighted revenue / weighted volume. weighted gross margin = weighted revenue - weighted volume * cost per unit (aka VHK)
        BoUpObject.wGrossMargin2020 = weightedGrossMargin[0]
        BoUpObject.wGrossMargin2021 = weightedGrossMargin[1]
        BoUpObject.wGrossMargin2022 = weightedGrossMargin[2]
        BoUpObject.wGrossMargin2023 = weightedGrossMargin[3]
        BoUpObject.wGrossMargin2024 = weightedGrossMargin[4]
        BoUpObject.wGrossMargin2025 = weightedGrossMargin[5]
        BoUpObject.wGrossMargin2026 = weightedGrossMargin[6]
        BoUpObject.wGrossMargin2027 = weightedGrossMargin[7]
        BoUpObject.wGrossMargin2028 = weightedGrossMargin[8]
        BoUpObject.wGrossMargin2029 = weightedGrossMargin[9]
        BoUpObject.wGrossMargin2030 = weightedGrossMargin[10]
        BoUpObject.wGrossMargin2031 = weightedGrossMargin[11]
        BoUpObject.wGrossMargin2032 = weightedGrossMargin[12]
        BoUpObject.wGrossMargin2033 = weightedGrossMargin[13]
        BoUpObject.wGrossMargin2034 = weightedGrossMargin[14]
        BoUpObject.wGrossMargin2035 = weightedGrossMargin[15]
        BoUpObject.wGrossMargin2036 = weightedGrossMargin[16]
        BoUpObject.wGrossMargin2037 = weightedGrossMargin[17]
        BoUpObject.wGrossMargin2038 = weightedGrossMargin[18]
        BoUpObject.wGrossMargin2039 = weightedGrossMargin[19]
        BoUpObject.wGrossMargin2040 = weightedGrossMargin[20]
        BoUpObject.wGrossMargin2041 = weightedGrossMargin[21]
        BoUpObject.wGrossMargin2042 = weightedGrossMargin[22]
        BoUpObject.wGrossMargin2043 = weightedGrossMargin[23]
        BoUpObject.wGrossMargin2044 = 0.0

        BoUpObject.asp2020 = asp[0]
        BoUpObject.asp2021 = asp[1]
        BoUpObject.asp2022 = asp[2]
        BoUpObject.asp2023 = asp[3]
        BoUpObject.asp2024 = asp[4]
        BoUpObject.asp2025 = asp[5]
        BoUpObject.asp2026 = asp[6]
        BoUpObject.asp2027 = asp[7]
        BoUpObject.asp2028 = asp[8]
        BoUpObject.asp2029 = asp[9]
        BoUpObject.asp2030 = asp[10]
        BoUpObject.asp2031 = asp[11]
        BoUpObject.asp2032 = asp[12]
        BoUpObject.asp2033 = asp[13]
        BoUpObject.asp2034 = asp[14]
        BoUpObject.asp2035 = asp[15]
        BoUpObject.asp2036 = asp[16]
        BoUpObject.asp2037 = asp[17]
        BoUpObject.asp2038 = asp[18]
        BoUpObject.asp2039 = asp[19]
        BoUpObject.asp2040 = asp[20]
        BoUpObject.asp2041 = asp[21]
        BoUpObject.asp2042 = asp[22]
        BoUpObject.asp2043 = asp[23]
        BoUpObject.asp2044 = 0.0

        #########

        BoUpObject.fy_vol2020 = volumes_FY[0]
        BoUpObject.fy_vol2021 = volumes_FY[1]
        BoUpObject.fy_vol2022 = volumes_FY[2]
        BoUpObject.fy_vol2023 = volumes_FY[3]
        BoUpObject.fy_vol2024 = volumes_FY[4]
        BoUpObject.fy_vol2025 = volumes_FY[5]
        BoUpObject.fy_vol2026 = volumes_FY[6]
        BoUpObject.fy_vol2027 = volumes_FY[7]
        BoUpObject.fy_vol2028 = volumes_FY[8]
        BoUpObject.fy_vol2029 = volumes_FY[9]
        BoUpObject.fy_vol2030 = volumes_FY[10]
        BoUpObject.fy_vol2031 = volumes_FY[11]
        BoUpObject.fy_vol2032 = volumes_FY[12]
        BoUpObject.fy_vol2033 = volumes_FY[13]
        BoUpObject.fy_vol2034 = volumes_FY[14]
        BoUpObject.fy_vol2035 = volumes_FY[15]
        BoUpObject.fy_vol2036 = volumes_FY[16]
        BoUpObject.fy_vol2037 = volumes_FY[17]
        BoUpObject.fy_vol2038 = volumes_FY[18]
        BoUpObject.fy_vol2039 = volumes_FY[19]
        BoUpObject.fy_vol2040 = volumes_FY[20]
        BoUpObject.fy_vol2041 = volumes_FY[21]
        BoUpObject.fy_vol2042 = volumes_FY[22]
        BoUpObject.fy_vol2043 = volumes_FY[23]
        BoUpObject.fy_vol2044 = 0.0

        BoUpObject.fy_gm2020 = grossProfit_FY[0]
        BoUpObject.fy_gm2021 = grossProfit_FY[1]
        BoUpObject.fy_gm2022 = grossProfit_FY[2]
        BoUpObject.fy_gm2023 = grossProfit_FY[3]
        BoUpObject.fy_gm2024 = grossProfit_FY[4]
        BoUpObject.fy_gm2025 = grossProfit_FY[5]
        BoUpObject.fy_gm2026 = grossProfit_FY[6]
        BoUpObject.fy_gm2027 = grossProfit_FY[7]
        BoUpObject.fy_gm2028 = grossProfit_FY[8]
        BoUpObject.fy_gm2029 = grossProfit_FY[9]
        BoUpObject.fy_gm2030 = grossProfit_FY[10]
        BoUpObject.fy_gm2031 = grossProfit_FY[11]
        BoUpObject.fy_gm2032 = grossProfit_FY[12]
        BoUpObject.fy_gm2033 = grossProfit_FY[13]
        BoUpObject.fy_gm2034 = grossProfit_FY[14]
        BoUpObject.fy_gm2035 = grossProfit_FY[15]
        BoUpObject.fy_gm2036 = grossProfit_FY[16]
        BoUpObject.fy_gm2037 = grossProfit_FY[17]
        BoUpObject.fy_gm2038 = grossProfit_FY[18]
        BoUpObject.fy_gm2039 = grossProfit_FY[19]
        BoUpObject.fy_gm2040 = grossProfit_FY[20]
        BoUpObject.fy_gm2041 = grossProfit_FY[21]
        BoUpObject.fy_gm2042 = grossProfit_FY[22]
        BoUpObject.fy_gm2043 = grossProfit_FY[23]
        BoUpObject.fy_gm2044 = 0.0
        BoUpObject.fy_wVol2020 = float(volumes_FY[0]) * probability
        BoUpObject.fy_wVol2021 = float(volumes_FY[1]) * probability
        BoUpObject.fy_wVol2022 = float(volumes_FY[2]) * probability
        BoUpObject.fy_wVol2023 = float(volumes_FY[3]) * probability
        BoUpObject.fy_wVol2024 = float(volumes_FY[4]) * probability
        BoUpObject.fy_wVol2025 = float(volumes_FY[5]) * probability
        BoUpObject.fy_wVol2026 = float(volumes_FY[6]) * probability
        BoUpObject.fy_wVol2027 = float(volumes_FY[7]) * probability
        BoUpObject.fy_wVol2028 = float(volumes_FY[8]) * probability
        BoUpObject.fy_wVol2029 = float(volumes_FY[9]) * probability
        BoUpObject.fy_wVol2030 = float(volumes_FY[10]) * probability
        BoUpObject.fy_wVol2031 = float(volumes_FY[11]) * probability
        BoUpObject.fy_wVol2032 = float(volumes_FY[12]) * probability
        BoUpObject.fy_wVol2033 = float(volumes_FY[13]) * probability
        BoUpObject.fy_wVol2034 = float(volumes_FY[14]) * probability
        BoUpObject.fy_wVol2035 = float(volumes_FY[15]) * probability
        BoUpObject.fy_wVol2036 = float(volumes_FY[16]) * probability
        BoUpObject.fy_wVol2037 = float(volumes_FY[17]) * probability
        BoUpObject.fy_wVol2038 = float(volumes_FY[18]) * probability
        BoUpObject.fy_wVol2039 = float(volumes_FY[19]) * probability
        BoUpObject.fy_wVol2040 = float(volumes_FY[20]) * probability
        BoUpObject.fy_wVol2041 = float(volumes_FY[21]) * probability
        BoUpObject.fy_wVol2042 = float(volumes_FY[22]) * probability
        BoUpObject.fy_wVol2043 = float(volumes_FY[23]) * probability
        BoUpObject.fy_wVol2044 = 0.0

        BoUpObject.fy_wRev2020 = revenue_FY[0] * probability
        BoUpObject.fy_wRev2021 = revenue_FY[1] * probability
        BoUpObject.fy_wRev2022 = revenue_FY[2] * probability
        BoUpObject.fy_wRev2023 = revenue_FY[3] * probability
        BoUpObject.fy_wRev2024 = revenue_FY[4] * probability
        BoUpObject.fy_wRev2025 = revenue_FY[5] * probability
        BoUpObject.fy_wRev2026 = revenue_FY[6] * probability
        BoUpObject.fy_wRev2027 = revenue_FY[7] * probability
        BoUpObject.fy_wRev2028 = revenue_FY[8] * probability
        BoUpObject.fy_wRev2029 = revenue_FY[9] * probability
        BoUpObject.fy_wRev2030 = revenue_FY[10] * probability
        BoUpObject.fy_wRev2031 = revenue_FY[11] * probability
        BoUpObject.fy_wRev2032 = revenue_FY[12] * probability
        BoUpObject.fy_wRev2033 = revenue_FY[13] * probability
        BoUpObject.fy_wRev2034 = revenue_FY[14] * probability
        BoUpObject.fy_wRev2035 = revenue_FY[15] * probability
        BoUpObject.fy_wRev2036 = revenue_FY[16] * probability
        BoUpObject.fy_wRev2037 = revenue_FY[17] * probability
        BoUpObject.fy_wRev2038 = revenue_FY[18] * probability
        BoUpObject.fy_wRev2039 = revenue_FY[19] * probability
        BoUpObject.fy_wRev2040 = revenue_FY[20] * probability
        BoUpObject.fy_wRev2041 = revenue_FY[21] * probability
        BoUpObject.fy_wRev2042 = revenue_FY[22] * probability
        BoUpObject.fy_wRev2043 = revenue_FY[23] * probability
        BoUpObject.fy_wRev2044 = 0.0

        if bulk == False:
            BoUpObject.save()

        outputDictionary = {
            "revenue": revenue,
            "grossMargin": grossMargin,
            "years": years,
            "errors": errors,
            "sumRevenue": sumRevenue,
            "sumGrossMargin": sumGrossMargin,
            "sumCost": sumCost,
            "sumVolume": sumVolume,
            "sumWeightedVolume": sumWeightedVolume,
            "sumWeightedRevenue": sumWeightedRevenue,
            "averageAsp": averageAsp,
            "sumWeightedGrossMargin": sumWeightedGrossMargin,
        }

        message = "Project was submitted to the Bottom Up table and is now active."

        if bulk == True:
            return True, outputDictionary, message, BoUpObject
        else:
            return True, outputDictionary, message, None


# -*- coding: utf-8 -*-
"""
Created on Wed Mar  6 17:07:01 2019

@author: Dingqy
"""
import os

import numpy as np
import deepchem as dc
import pandas as pd

from rdkit import Chem
from rdkit.Chem import AllChem, MACCSkeys

from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.layers import Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ModelCheckpoint

from sklearn import linear_model
from sklearn.svm import LinearSVR

from scipy.stats import pearsonr
from sklearn.metrics import mean_squared_error
from math import sqrt
from sklearn import preprocessing

#---------------------------------------------
# baseline model training
def optimize_SVR(X_train, Y_train):
    # input: X_train --- each row a sample; each col a feature
    #        Y_train
    # output: a predictive model of support vector regression
    clf = LinearSVR(C=1.0, epsilon=0.2)
    try:
        clf.fit(X_train, Y_train)
    except Exception as e:
        print(e)
    return clf

def optimize_RidgeCV(X_train, Y_train):
    '''
    input: X_train --- each row a sample; each col a feature
           Y_train
    output: a predictive model of ridge regression with selected alpha by cross-validation
    '''
    alphas = np.logspace(start = -1, stop = 2, num = 20)
    reg = linear_model.RidgeCV(alphas)
    try:
        reg.fit(X_train, Y_train)
    except Exception as e:
        print(e)
    return reg

#---------------------------------------------
def prepare_dataset(fname, task, dataset_file, FP_type,
                    model_flag = 'ST', add_features = None,
                    smiles_field = 'salt_removed_smi', id_field = 'molregno'):
    '''
    input: fname --- name of the file of raw data containing chemicals and the value for each assay;
           task --- list of task names (supporting 1 or more, but must be a list)
           dataset_file --- name of the temperary file saving intermediate dataset containing only the data for a specific training task
           FP_type --- name of the fingprint type (one of 'Circular_2018', 'Circular_1024', 'Morgan', 'MACCS', 'RDKit_FP')
           model_flag --- type of the model (ST: single task, len(task)==1; MT: multitask, len(task)>=1)
           add_features --- must be None for ST; list of properties
           smiles_field --- the name of the field in fname defining the SMILES for chemicals
           id_field --- the name of the field in fname defining the identifier for chemicals
    output: dataset build by deepchem
    '''
    MT_df = pd.read_csv(fname)
    if model_flag == 'ST':
        df = extract_clean_dataset(task, MT_df, smiles_field = smiles_field, id_field = id_field)
    elif model_flag == 'MT':
        df = extract_clean_dataset(task, MT_df, add_features = add_features, smiles_field = smiles_field, id_field = id_field)
        if not add_features is None:
            task = task + add_features

    if FP_type == 'Circular_2048':
        df.to_csv(dataset_file)
        featurizer = dc.feat.CircularFingerprint(size=2048)
        loader = dc.data.CSVLoader(id_field=id_field,
                                   smiles_field=smiles_field,
                                   tasks = task,
                                   featurizer=featurizer)
        dataset = loader.featurize(dataset_file)

    elif FP_type == 'Circular_1024':
        df.to_csv(dataset_file)
        featurizer = dc.feat.CircularFingerprint(size=1024)
        loader = dc.data.CSVLoader(id_field=id_field,
                                   smiles_field=smiles_field,
                                   tasks = task,
                                   featurizer=featurizer)
        dataset = loader.featurize(dataset_file)

    elif FP_type == 'RDKit_FP':
        df.to_csv(dataset_file)
        featurizer = dc.feat.RDKitDescriptors()
        loader = dc.data.CSVLoader(id_field=id_field,
                                   smiles_field=smiles_field,
                                   tasks = task,
                                   featurizer=featurizer)
        dataset = loader.featurize(dataset_file)

    elif FP_type == 'Morgan': #2048
        smiles = df[smiles_field].tolist()
        mols = [Chem.MolFromSmiles(smi) for smi in smiles]
        fps = []
        for mol in mols:
            fps.append(np.matrix(AllChem.GetMorganFingerprintAsBitVect(mol,2)))
        fps = np.concatenate(fps)
        new_df = pd.DataFrame(fps)
        new_df.columns = ['V' + str(i) for i in range(fps.shape[1])]
        new_df[id_field] = df[id_field]
        for m in range(len(task)):
            new_df[task[m]] = df[task[m]]
        new_df.to_csv(dataset_file)
        user_specified_features = ['V' + str(i) for i in range(fps.shape[1])]
        featurizer = dc.feat.UserDefinedFeaturizer(user_specified_features)
        loader = dc.data.UserCSVLoader(
                tasks=task, smiles_field=smiles_field, id_field=id_field,
                featurizer=featurizer)
        dataset = loader.featurize(dataset_file)

    elif FP_type == 'MACCS': #167
        smiles = df[smiles_field].tolist()
        mols = [Chem.MolFromSmiles(smi) for smi in smiles]
        fps = []
        for mol in mols:
            fps.append(np.matrix(MACCSkeys.GenMACCSKeys(mol)))
        fps = np.concatenate(fps)
        new_df = pd.DataFrame(fps)
        new_df.columns = ['V' + str(i) for i in range(fps.shape[1])]
        new_df[id_field] = df[id_field]
        for m in range(len(task)):
            new_df[task[m]] = df[task[m]]
        new_df.to_csv(dataset_file)
        user_specified_features = ['V' + str(i) for i in range(fps.shape[1])]
        featurizer = dc.feat.UserDefinedFeaturizer(user_specified_features)
        loader = dc.data.UserCSVLoader(
                tasks = task, smiles_field=smiles_field, id_field=id_field,
                featurizer=featurizer)
        dataset = loader.featurize(dataset_file)

    else:
        print('Unsupported Fingerprint type!')

    return dataset, df
#----------------------------------------------
def extract_clean_dataset(subset_names, MT_df, add_features = None, id_field = 'molregno', smiles_field = 'salt_removed_smi'):
    '''
    input: subset_names --- a list of the task names
           MT_df --- pd.DataFrame of the total raw data
    output: subset of raw data， containing only the selected tasks
    '''
    extract_column = subset_names + [id_field, smiles_field]
    sub_df = MT_df[extract_column]

    n_tasks = len(subset_names)
    mask_mat = np.sum(np.isnan(np.matrix(MT_df[subset_names])), axis = 1).reshape(-1).flatten()
    mask_new = np.array(mask_mat < n_tasks)[0]

    if add_features is None:
        extract_df = sub_df.loc[mask_new]
    else:
        new_sub_df = MT_df[extract_column + add_features]
        extract_df = new_sub_df.loc[mask_new]

        # normalization!!
        all_fields = subset_names + add_features
        for i in range(len(all_fields)):
            temp_mask = ~np.isnan(np.array(extract_df[all_fields[i]].tolist()))
            extract_df[all_fields[i]].loc[temp_mask] = preprocessing.scale(extract_df[all_fields[i]].dropna())

    print('Extracted dataset shape: ' + str(extract_df.shape))
    return extract_df

#----------------------------------------------
def ST_model_builder(model_params, model_dir):
    model = dc.models.MultitaskRegressor(**model_params)
    return model

def ST_model(n_feature, layer_size, drop_prob,
            lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0, clipnorm=5):
    training_dat = Input(shape = (n_feature,), dtype = 'float32')
    X = Dense(layer_size[0], activation = 'relu')(training_dat)
    for i in range(1,len(layer_size)):
        X = Dropout(drop_prob)(X)
        X = Dense(layer_size[i], activation = 'relu')(X)
    model = Model(training_dat, X)
    optimizer = Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon)
    model.compile(loss = 'mean_squared_error', optimizer = optimizer)
    return model

def ST_model_hyperparam_screen(fname, task_names, FP_type, params_dict, log_path = './logs/',
                               smiles_field = 'salt_removed_smi', id_field='molregno'):
    '''
    hyperparameter screening using deepchem package
    input: fname --- name of the file of raw data containing chemicals and the value for each assay;
           task --- list of task names (supporting 1 or more, but must be a list)
           FP_type --- name of the fingprint type (one of 'Circular_2048', 'Circular_1024', 'Morgan', 'MACCS', 'RDKit_FP')
           params_dict --- dictionary containing the parameters along with the screening range
           log_path --- the directory saving the log file
    output: the log; log file saved in log_path
    '''
    log_output = []
    for task in task_names:
        print('----------------------------------------------')
        dataset_file = '%s/temp.csv' % (log_path)
        dataset, _ = prepare_dataset(fname, [task], dataset_file, FP_type,
                                    smiles_field = smiles_field,
                                    add_features = None,
                                    id_field = id_field, model_flag = 'ST')
        for cnt in range(3):
            print('Preparing dataset for %s of rep %d...' % (task, cnt + 1))
            splitter = dc.splits.RandomSplitter(dataset_file)
            train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(dataset)

            print('Hyperprameter screening ...')
            metric = dc.metrics.Metric(dc.metrics.r2_score)
            optimizer = dc.hyper.HyperparamOpt(ST_model_builder)
            best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(params_dict,
                                                                      train_dataset,
                                                                      valid_dataset, [],
                                                                      metric)
            # get the layer size and dropout rate of all_results
            for (key, value) in all_results.items():
                log_output.append('rep%d\t%s\t%s\t%s' % (cnt, task, str(key), str(value)))

            print('Generate performace report ...')
            with open('%s/hyperparam_log.txt' % (log_path), 'w') as f:
                for line in log_output:
                    f.write("%s\n" % line)
        os.system('rm %s' % dataset_file)

    return  log_output

def ST_model_training(fname, FP_type, best_hyperparams, result_path, epoch_num = 45,
                      frac_train = 0.8, N_test = None,
                      smiles_field = 'salt_removed_smi', id_field='molregno',
                      tot_iter_num = 51, repo_num = 10):
    '''
    Training the specific groups of single task (or multitask with only one task and a few chemical properties),
           using
    input: fname --- name of the file of raw data containing chemicals and the value for each assay;
           FP_type --- name of the fingprint type (one of 'Circular_2018', 'Circular_1024', 'Morgan', 'MACCS', 'RDKit_FP')
           best_hyperparams --- dictionary containing the parameters, with key the name of the task
           result_path --- the directory saving the trained models and merits file
    output: the log; log file saved in log_path
    '''
    task_names = list(best_hyperparams.keys())
    output_dat = np.zeros((len(task_names) * 3, 12))
    rep_labels = ['NA' for _ in range(len(task_names) * 3)]
    task_labels = ['NA' for _ in range(len(task_names) * 3)]
    nrow = 0
    for task in task_names:
        print('----------------------------------------------')
        dataset_file = '%s/temp.csv' % (result_path)
        dataset, _ = prepare_dataset(fname, [task], dataset_file, FP_type,
                                    smiles_field = smiles_field,
                                    add_features = None,
                                    id_field = id_field, model_flag = 'ST')

        for cnt in range(3):
            print('Preparing dataset for %s of rep %d...' % (task, cnt))
            splitter = dc.splits.RandomSplitter(dataset_file)
            if not N_test:
                train_dataset, test_dataset = splitter.train_test_split(dataset, frac_train = frac_train)
            else:
                frac = 1 - N_test / dataset.X.shape[0]
                train_dataset, test_dataset = splitter.train_test_split(dataset, frac_train = frac)

            print('Model training ...')
            n_features = train_dataset.X.shape[1]
            model = ST_model(n_features, best_hyperparams[task][0], best_hyperparams[task][1], lr=0.0001)
            for iteration in range(tot_iter_num):
                checkpoint_callback = ModelCheckpoint('%s/%s_rep%d_%d.hdf5' % (result_path, task,
                                                                                cnt, iteration),
                                          monitor='val_loss', verbose = 0, save_weights_only = False,
                                          mode = 'auto', period = 1)
                if iteration % repo_num == 0:
                    model.fit(train_dataset.X, train_dataset.y, epochs=5, callbacks = [checkpoint_callback])
                else:
                    model.fit(train_dataset.X, train_dataset.y, epochs=epoch_num, verbose=0)

            print('Training baseline models ...')
            X_new = np.c_[[1]*train_dataset.X.shape[0], train_dataset.X]
            clf = optimize_SVR(X_new, train_dataset.y)
            reg = optimize_RidgeCV(X_new, train_dataset.y)

            #print('Training TRACE models ...')
            # todo!

            print('Saving metrics ...')
            y_true = test_dataset.y.flatten()
            y_pred_ST = model.predict(test_dataset.X).flatten()
            y_pred_SVR = clf.predict(np.c_[[1]*test_dataset.X.shape[0],test_dataset.X]).flatten()
            y_pred_Rig = reg.predict(np.c_[[1]*test_dataset.X.shape[0],test_dataset.X]).flatten()

            output_dat[nrow, 0] = sqrt(mean_squared_error(y_pred_ST, y_true))
            output_dat[nrow, 1] = sqrt(mean_squared_error(y_pred_SVR, y_true))
            output_dat[nrow, 2] = sqrt(mean_squared_error(y_pred_Rig, y_true))
            output_dat[nrow, 3], _ = pearsonr(y_pred_ST, y_true)
            output_dat[nrow, 4], _ = pearsonr(y_pred_SVR, y_true)
            output_dat[nrow, 5], _ = pearsonr(y_pred_Rig, y_true)

            y_true = train_dataset.y.flatten()
            y_pred_ST = model.predict(train_dataset.X).flatten()
            y_pred_SVR = clf.predict(np.c_[[1]*train_dataset.X.shape[0],train_dataset.X]).flatten()
            y_pred_Rig = reg.predict(np.c_[[1]*train_dataset.X.shape[0],train_dataset.X]).flatten()

            output_dat[nrow, 6] = sqrt(mean_squared_error(y_pred_ST, y_true))
            output_dat[nrow, 7] = sqrt(mean_squared_error(y_pred_SVR, y_true))
            output_dat[nrow, 8] = sqrt(mean_squared_error(y_pred_Rig, y_true))
            output_dat[nrow, 9], _ = pearsonr(y_pred_ST, y_true)
            output_dat[nrow, 10], _ = pearsonr(y_pred_SVR, y_true)
            output_dat[nrow, 11], _ = pearsonr(y_pred_Rig, y_true)

            rep_labels[nrow] = cnt
            task_labels[nrow] = task
            nrow += 1
        os.system('rm %s' % dataset_file)

        print('Generate performace report ...')
        output_df = pd.DataFrame(output_dat)
        output_df.columns = ['test_rmse_ST', 'test_rmse_SVR', 'test_rmse_Rig',
                             'test_pearsonr_ST', 'test_pearsonr_SVR', 'test_pearsonr_Rig',
                             'train_rmse_ST', 'train_rmse_SVR', 'train_rmse_Rig',
                             'train_pearsonr_ST', 'train_pearsonr_SVR', 'train_pearsonr_Rig']
        output_df['rep_label'] = rep_labels
        output_df['task_label'] = task_labels
        os.system('rm %s' % dataset_file)
        output_df.to_csv('%s/performance_metrics.csv' % (result_path))

    return  output_df

#----------------------------------------------
def RobustMT_model_builder(model_params, model_dir):
    model = dc.models.RobustMultitaskRegressor(**model_params)
    return model

def RobustMT_model_hyperparam_screen(fname, task_names, FP_type, params_dict, log_path = './logs/',
                               smiles_field = 'salt_removed_smi', id_field='molregno'):
    '''
    hyperparameter screening using deepchem package
    input: fname --- name of the file of raw data containing chemicals and the value for each assay;
           task --- list of task names (supporting 1 or more, but must be a list)
           FP_type --- name of the fingprint type (one of 'Circular_2048', 'Circular_1024', 'Morgan', 'MACCS', 'RDKit_FP')
           params_dict --- dictionary containing the parameters along with the screening range
           log_path --- the directory saving the log file
    output: the log; log file saved in log_path
    '''
    log_output = []
    print('----------------------------------------------')
    dataset_file = '%s/temp.csv' % (log_path)
    dataset, _ = prepare_dataset(fname, task_names, dataset_file, FP_type,
                                 smiles_field = smiles_field,
                                 add_features = None,
                                 id_field = id_field, model_flag = 'MT')
    for cnt in range(3):
        print('Preparing dataset of rep %d...' % ((cnt + 1)))
        splitter = dc.splits.RandomSplitter(dataset_file)
        train_dataset, valid_dataset, test_dataset = splitter.train_valid_test_split(dataset)

        print('Hyperprameter screening ...')
        metric = dc.metrics.Metric(dc.metrics.r2_score, np.mean)
        optimizer = dc.hyper.HyperparamOpt(RobustMT_model_builder)
        best_dnn, best_hyperparams, all_results = optimizer.hyperparam_search(params_dict,
                                                                      train_dataset,
                                                                      valid_dataset, [],
                                                                      metric)
        # get the layer size and dropout rate of all_results
        for (key, value) in all_results.items():
            log_output.append('rep%d\t%s\t%s' % (cnt, str(key), str(value)))

        print('Generate performace report ...')
        with open('%s/hyperparam_log.txt' % (log_path), 'w') as f:
            for line in log_output:
                f.write("%s\n" % line)
    os.system('rm %s' % dataset_file)

    return  log_output

def RobustMT_model_training(MT_dat_name, FP_type, task_list, log_path,
                            n_features, layer_sizes, bypass_layer_sizes, bypass_dropouts, dropout, lr,
                            N_test = 500.0, add_features = None, n_epoch = 40, epoch_num = 10,
                            id_field = 'chembl_id', smiles_field = 'canonical_smiles'):

    dataset_file = '%s/raw_data.csv' % (log_path)
    if len(task_list) > 1:
        model_flag = 'MT'
    else:
        model_flag = 'ST'
    dataset, df = prepare_dataset(MT_dat_name, task_list, dataset_file, FP_type,
                              smiles_field = smiles_field,
                              add_features = add_features,
                              id_field = id_field, model_flag = model_flag)

    # calculate the ratio of missing values in the multitask setting
    weights = dataset.w
    true_cnt = sum(sum(weights))
    missing_ratio = 1 - (true_cnt / (weights.shape[0] * weights.shape[1]))
    print('Missing ratio of the dataset is %.2f' % missing_ratio)
    print('Number of valid samples is %d' % int(true_cnt))

    # split dataset
    frac_train = 1 - N_test / dataset.X.shape[0]
    splitter = dc.splits.RandomSplitter(dataset_file)
    train_dataset, test_dataset = splitter.train_test_split(dataset, frac_train = frac_train)

    metric = dc.metrics.Metric(
        dc.metrics.r2_score, np.mean, mode = 'regression')

    # model training
    n_tasks = len(task_list)
    model = dc.models.RobustMultitaskRegressor(n_tasks = n_tasks, n_features = n_features, layer_sizes = layer_sizes,
                                               bypass_layer_sizes=bypass_layer_sizes, bypass_dropouts = bypass_dropouts,
                                               dropouts = dropout, learning_rate = lr)
    model.save_file = log_path + '/model'

    train_evaluation = [dataset.get_task_names()]
    test_evaluation = [dataset.get_task_names()]
    for iteration in range(epoch_num):
        model.fit(train_dataset, n_epoch = n_epoch, max_checkpoints_to_keep = 1, checkpoint_interval=20)
        print('======== Iteration %d ======' % iteration)
        print("Evaluating model")
        train_scores = model.evaluate(train_dataset, [metric], [], per_task_metrics=metric)
        train_evaluation.append(train_scores[1]["mean-r2_score"])
        print("Training R2 score: %f" % train_scores[0]["mean-r2_score"])
        test_scores = model.evaluate(test_dataset, [metric], [], per_task_metrics=metric)
        test_evaluation.append(test_scores[1]["mean-r2_score"])
        print("Test R2 score: %f" % test_scores[0]["mean-r2_score"])

        # save evaluation scores (need test!)
        train_df = pd.DataFrame(np.array(train_evaluation))
        test_df = pd.DataFrame(np.array(test_evaluation))
        train_df.to_csv(model.save_file + '_train_log.csv')
        test_df.to_csv(model.save_file + '_test_log.csv')
    return

#------------------------------------------------------------------


#!/usr/bin/env/ python3
import math

def main():
   while True:
      imin1=0
      imax1=0
      imin2=0
      imax2=0
      smin1=input("Please enter first minimum:")
      if (smin1==''):
         break
      else:
         imin1=int(smin1)
      smax1=input("Please enter first maximum:")
      if (smax1==''):
         continue
      else:
         imax1=int(smax1)
      smin2=input("Please enter second minimum:")
      if (smin2==''):
         continue
      else:
         imin2=int(smin2)
      smax2=input("Please enter second maximum:")
      if (smax2==''):
         continue
      else:
         imax2=int(smax2)
      s=""
      for i in range(imin1,imax1):
         for j in range(imin2,imax2):
            s=s+str(j)+' x '+str(i)+' = '+str(round(math.sqrt(j*i),2))+'\t'
         print(s)
         s=''
   print("Goodbye!")

if __name__=='__main__':
   main()


# 백준 11729번 문제
# 하노이 탑 이동 순서
# https://www.acmicpc.net/problem/11729

import sys

def solve(n, a, b, c):    # a 출발, b 남는기둥, c 목적지
    if n == 1:    # 마지막 이동
        print(a, c)
    else:
        solve(n-1, a, c, b)    # n-1을 남는 기둥에 옮기기
        print(a, c)            # n에 해당하는 가장 큰 원판을 옮기기
        solve(n-1, b, a, c)    # 남는 기둥에 옮긴 n-1을 다시 목적지에 옮기기

n = int(sys.stdin.readline())
print(2 ** n -1)    # 실행 횟수는 2의 n승 -1
solve(n, 1, 2, 3)    #n개의 탑을 1에서 3으로 이동


class Carro:

    def __init__(self,marca,modelo,cor,combustivel):
        self.marca = marca
        self.modelo = modelo
        self.cor = cor
        self.combustivel = combustivel
        self.velocidade = 0

        self.ligado = False

    def ligar(self):
        if self.ligado:
            print("carro ja esta ligado. nada a fazer")
        else:
             print("Carro ligado")
             self.ligado = True

    def desligar(self):
         if self.ligado:
            self.ligado = False
            print("Carro desligado")
         else:
              print("carro ja esta ligado. nada a fazer")

    def acelerar(self):
        if self.ligado:
            self.velocidade += 1
            print(f"A velocidade {self.velocidade}")
        else:
            print("Nao pode acelerar, carro desligado")

    def freae(self):
        if self.ligado:
            self.velocidade -= 1
            print(f"A velocidade {self.velocidade}")
        else:
            print("Nao pode frear, carro desligado")


#!/usr/bin/env python
# coding: utf-8

# # Project Assignment 5 - Get different types of data
# ## Author: Jonathan Heard
# ## Class: CIS256DA

# Chapter 5 Asignment
#
# Search KaggleLinks to an external site. for something that you are interested in to see if there is any datasets related to that interest. If there is, download that dataset.  If not, just download one of the trending datasets in the trending section as shown below (2 pts.):
#
# Tasks
#
# 1. Read the data from the CSV file into a DataFrame (2 pts.).
# 2. Display the first five rows of data (2 pts.).
# 3. Display information about each of the columns (2 pts.).
#
# Question
#
# Why did you choose the downloaded dataset (2 pts.)?
#
#
# After completing the assignment, please submit your .ipynb file and the dataset you downloaded from Kaggle.
#
#

# ### The url for the site the data came from is:
#
# https://www.kaggle.com/datasets/taruntiwarihp/phishing-site-urls
#
# The download was a 9 MB zip file,
#     called "Malicious_and_Phishing_attacks_ulrs_archive.zip"

# ## Import required modules

# In[1]:

# Import the modules needed for this project.

# Import pandas and assign the alias pd
import pandas as pd

# Import ZipFile from the zipfile module
from zipfile import ZipFile

# ## Tasks list

# In[2]:

# Extract the files from the zip file and diplsy the file names

file_names = list()
with ZipFile('Malicious_and_Phishing_attacks_ulrs_archive.zip', mode = 'r') as zip:
    zip.extractall()
    for file in zip.infolist():
        file_names.append(file.filename)
        print(file.filename, file.compress_size, file.file_size)

# In[3]:

# Task 01 Read the data from the CSV file into a DataFrame (2 pts.).

phishing = pd.read_csv('phishing_site_urls.csv')

# In[4]:

# Task 02 Display the first five rows of data (2 pts.).

phishing.head()

# In[5]:

# Task 03 Display information about each of the columns (2 pts.).

phishing.info()

# ## Question

# ### Why did you choose the downloaded dataset (2 pts.)?
#
# The reason for my line of inquiry, is that I am always looking into Cybersecurity topics. I was not very happy with
#     my first few attempts at this assignment.
#
# I had to download 4 different zip files, before I found one that would let step 1 proceed.
#     "Read the data from the CSV file into a DataFrame."
#
# I finally decided to quit looking once, I was able to satisfy the requirements for this assignment.
#
# The "bad" csv files I tried, all gave the same error messages as in the Jonathan_Heard_PA-5_Bad_csv.ipynb. I am
#     include
#
#
#

# In[ ]:


import multiprocessing
from time import sleep
import logging

logging.basicConfig(level=logging.INFO)

def foo(i):
    sleep(3)
    logging.info('Called function in process: %s' % i)
    return

if __name__ == '__main__':
    process_jobs = []
    for i in range(5):
        process = multiprocessing.Process(target=foo, args=(i,))
        process_jobs.append(process)
        process.start()
        process.join()


import logging
from pathlib import Path
from typing import Tuple, Union

import numpy as np
from matlatzinca.core.bn import BayesianNetwork
from matlatzinca.io import get_table_text
from matlatzinca.ui.dialogs import NotificationDialog
from PyQt5 import QtWidgets

logger = logging.getLogger(__name__)

class Project:
    def __init__(self, mainwindow) -> None:
        self.mainwindow = mainwindow
        self.bn = BayesianNetwork()

        self.signals = self.mainwindow.signals

        self.signals.node_coordinate_selected.connect(self.add_node)
        self.signals.node_about_to_be_removed.connect(self.remove_node)

        self.signals.edge_nodes_selected.connect(self.add_edge)
        self.signals.edge_about_to_be_removed.connect(self.remove_edge)
        self.signals.edge_about_to_be_reversed.connect(self.reverse_edge)

        # self.signals.delete_coordinate_selected.connect(self.add_node)

        self.signals.observed_correlation_about_to_change.connect(self.change_observed_correlation)
        self.signals.correlation_about_to_change.connect(self.change_edge_correlation)
        self.signals.name_about_to_change.connect(self.change_node_name)

        # self.signals.sampling_order_about_to_change.connect(self.change_sampling_order)
        self.signals.node_order_about_to_change.connect(self.change_node_order)
        self.signals.parent_order_about_to_change.connect(self.change_parent_order)

    def update_bn(self) -> None:
        """Updates the BN model by calculating the correlation matrix, correlation bounds, and edges"""
        if len(self.bn.nodes) == 0:
            self.bn.R = np.empty((0, 0), dtype=np.float64)
        else:
            logger.info("Calculating correlation matrix and bounds.")
            # self.bn.check_sampling_order()
            self.bn.calculate_correlation_matrix()
            self.bn.calculate_correlation_bounds()
            self.bn.create_edge_overview()

    def add_node(self, crd: Union[Tuple[float, float], None]) -> None:
        """Add node at specific coordinate"""
        if crd is None:
            return None

        self.signals.lists_about_to_change.emit()
        # Add node to BN
        name = self.bn._get_unused_name()
        self.bn.add_node(name=name, parents=[], rank_corrs=[])
        # Add coordinates to Node
        self.update_bn()

        logger.info(f'Node "{name}" added.')
        self.signals.lists_changed.emit()

        # Get the node and add coordinates
        node = self.bn._get_node_by_name(name)
        node.x, node.y = crd

        # Emit the signal that will update the plots
        self.signals.node_added.emit(crd, node)

    def remove_node(self, name: str) -> None:
        """Remove node by name"""
        self.signals.lists_about_to_change.emit()
        self.bn.remove_node(name)
        logger.info(f'Node "{name}" removed.')
        self.update_bn()
        self.signals.node_removed.emit(name)
        self.signals.lists_changed.emit()

    def remove_edge(self, parent: str, child: str) -> None:
        """Remove edge by parent and child name"""
        self.signals.lists_about_to_change.emit()
        self.bn.remove_edge(parent, child)
        logger.info(f'Edge from "{parent}" to "{child}" removed.')
        self.update_bn()
        self.signals.edge_removed.emit(parent, child)
        self.signals.lists_changed.emit()

    def add_edge(self, edge: Union[Tuple[str, str], None]) -> None:
        """Add edge from parent and child string"""
        if edge is None:
            return None

        self.signals.lists_about_to_change.emit()
        succes = self.bn.add_edge(*edge)
        if succes:
            logger.info(f'Edge from "{edge[0]}" to "{edge[1]}" added.')
            self.update_bn()
            # Emit the signal that will update the plots
            self.signals.edge_added.emit(*edge)

        else:
            NotificationDialog("Edge was not added, as the BN would no longer be a DAG.")
        self.signals.lists_changed.emit()

    def reverse_edge(self, parent: str, child: str) -> None:
        """Reverse an existing edge (and recalculate conditional rank correlations)"""

        self.signals.lists_about_to_change.emit()
        succes = self.bn.reverse_edge(parent, child)
        if succes:
            logger.info(f'Edge from "{parent}" to "{child}" reversed.')
            self.update_bn()
            # Emit the signal that will update the plots
            self.signals.edge_reversed.emit(parent, child)

        else:
            NotificationDialog("Edge was not reversed, as the BN would no longer be a DAG.")
        self.signals.lists_changed.emit()

    def change_observed_correlation(self, parent: str, child: str, corr: str) -> None:
        """Change (non-conditional) correlation coefficient"""

        node = self.bn._get_node_by_name(child)
        iparent = node.parent_index(parent)
        oldvalue = node.edges[iparent].rank_corr

        node.edges[iparent].rank_corr = corr
        corr = float(corr)

        # Check if the correlation is within bounds
        edge = node.edges[iparent]
        if corr < edge.rank_corr_bounds[0] or corr > edge.rank_corr_bounds[-1]:
            node.edges[iparent].rank_corr = oldvalue
            raise ValueError("Choose a value in between {:.4g} and {:.4g}".format(*edge.rank_corr_bounds))

        # Calculate the conditional correlation from the observed correlation
        cond_corr = self.bn.calculate_conditional_correlation(parent, child, observed=corr)

        # Change the conditional correlation
        self.change_edge_correlation(parent, child, corr=cond_corr)

    def change_edge_correlation(self, parent: str, child: str, corr: str) -> None:
        """Change the edge correlation"""

        node = self.bn._get_node_by_name(child)
        iparent = node.parent_index(parent)
        node.edges[iparent].cond_rank_corr = corr

        logger.info(f'Conditional edge ("{parent}" to "{child}") correlation changed to {corr}.')
        self.signals.lists_about_to_change.emit()
        self.update_bn()
        self.signals.correlation_changed.emit()
        self.signals.lists_changed.emit()

    def change_node_name(self, oldname: str, newname: str) -> None:
        """Change a node name"""

        if newname == "":
            return None
            # raise ValueError("Name cannot be empty")

        self.signals.lists_about_to_change.emit()

        self.bn.change_node_name(oldname, newname)

        logger.info(f'Node "{oldname}" was renamed to "{newname}".')

        self.signals.name_changed.emit()

        # self.bn.create_edge_overview()

        self.signals.lists_changed.emit()

    def change_node_order(self, source_pos: int, target_pos: int) -> None:
        """Change the node order"""

        self.signals.lists_about_to_change.emit()

        # Change the node order
        current_order = self.bn._node_names
        self.bn.nodes.insert(target_pos, self.bn.nodes.pop(source_pos))
        requested_order = self.bn._node_names

        # Reorder the nodes
        logger.info(f"Node order changed to to ({', '.join(requested_order)})")

        self.update_bn()

        self.signals.node_order_changed.emit()

        self.signals.lists_changed.emit()

    # def change_sampling_order(self, source_pos: int, target_pos: int) -> None:

    #     self.signals.lists_about_to_change.emit()

    #     # Change the node order
    #     self.bn.nodes.insert(target_pos, self.bn.nodes.pop(source_pos))
    #     requested_order = self.bn._node_names

    #     # Reorder the nodes based on the sampling order
    #     self.bn.check_sampling_order()
    #     valid_order = self.bn._node_names

    #     logger.info(f"Sampling order changed to to ({', '.join(valid_order)})")

    #     self.update_bn()

    #     self.signals.sampling_order_changed.emit()

    #     self.signals.lists_changed.emit()

    #     if requested_order != valid_order:
    #         NotificationDialog(
    #             f"The given order ({', '.join(requested_order)}) is not valid sampling order. Therefore, the order is changed to ({', '.join(valid_order)})"
    #         )

    def change_parent_order(self, source_pos: int, target_pos: int) -> None:
        """Change order of parents, as a result of reordering the edges"""
        self.signals.lists_about_to_change.emit()

        # Get edge from source position
        edge = self.bn.edgelist[source_pos]
        # Update order
        self.bn.change_parent_order(source_pos, target_pos)
        # Update BN with correlations
        self.update_bn()

        self.signals.parent_order_changed.emit()

        self.signals.lists_changed.emit()
        # Select the new row with the edge position
        self.signals.selected.emit((edge.parent, edge.child))

    def new(self) -> None:
        """Empty the project"""
        for node in reversed(self.bn.nodes):
            self.signals.node_about_to_be_removed.emit(node.name)

    def open(self, *args, fname: Path = None) -> None:
        """
        Method that loads project file and builds gui
        """

        if fname is None:
            fname = self.get_project_file()
            if fname == "":
                return None
            else:
                fname = Path(fname)

        # Clear current project
        self.new()

        # Open project
        self.mainwindow.setCursorWait()

        # Load from file
        self.signals.lists_about_to_change.emit()
        loaded_bn = BayesianNetwork.parse_file(fname)
        self.bn.nodes.extend(loaded_bn.nodes)
        self.update_bn()

        # Add nodes to UI
        for node in self.bn.nodes:
            self.signals.node_added.emit((np.random.rand(1)[0], np.random.rand(1)[0]), node)

        # Add edged to UI
        for node in self.bn.nodes:
            for edge in node.edges:
                self.signals.edge_added.emit(edge.parent, edge.child)

        self.signals.lists_changed.emit()

        # save current dir
        self.mainwindow.appsettings.setValue("currentdir", str(fname.parent))
        self.mainwindow.update_projectname(fname.name)
        self.signals.set_window_modified.emit(False)
        self.mainwindow.setCursorNormal()

    def save_as(self, fname=None) -> None:
        """
        Asks the user for the file name to save the project to.
        If the file name is passed, the user is not asked the file
        and the project is saved directly.

        Parameters
        ----------
        fname : str, optional
            File name, by default None
        """

        # if fname is None:
        # Set open file dialog settings
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog
        options |= QtWidgets.QFileDialog.DontConfirmOverwrite

        # Set current dir
        currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

        # Open dialog to select file
        fname, ext = QtWidgets.QFileDialog.getSaveFileName(
            self.mainwindow, "Matlatzinca - Save project", currentdir, "JSON (*.json)", options=options
        )

        if fname == "":
            return None

        # Add extension if not given in file
        for end in [".json"]:
            if end in ext and not fname.endswith(end):
                fname += end

        # Convert to path
        fname = Path(fname)

        if fname.exists():
            reply = QtWidgets.QMessageBox.warning(
                self.mainwindow,
                "Matlatzinca - Save project",
                f"{fname.name} already exists.\nDo you want to overwrite it?",
                QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
            )
            if reply == QtWidgets.QMessageBox.No:
                return None

        # Save files
        self.mainwindow.update_projectname(fname.name)

        self.bn.to_json(fname)
        self.signals.set_window_modified.emit(False)

    def save(self) -> None:
        """
        Save current project, checks if the file name is already known.
        If not, the save as function is called and the user is asked to
        pick a file.
        """

        # If no current project, save as...
        currentproject = self.mainwindow.appsettings.value("currentproject", "", type=str)
        if currentproject == "":
            self.save_project_as()

        else:
            # Get current directory
            currentdir = Path(self.mainwindow.appsettings.value("currentdir", ".", type=str))

            # Else, save files directly
            self.bn.to_json(currentdir / currentproject)
            self.signals.set_window_modified.emit(False)

    def get_project_file(self) -> str:
        """
        Opens a dialog to select a project file to open.
        """
        # Set open file dialog settings
        options = QtWidgets.QFileDialog.Options()
        options |= QtWidgets.QFileDialog.DontUseNativeDialog

        # Set current dir
        currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

        # Open dialog to select file
        fname, _ = QtWidgets.QFileDialog.getOpenFileName(
            self.mainwindow,
            "Matlatzinca - Open project",
            currentdir,
            "JSON (*.json)",
            options=options,
        )

        return fname

    def export(self, what: str, how: str) -> None:
        """Exports correlation matrix or tables to csv or clipboard

        Args:
            what (str): correlation_matrix, nodes, or edges
            how (str): csv or clipboard
        """

        if not what in ["correlation_matrix", "nodes", "edges"]:
            raise KeyError(
                f'Choose one of "{", ".join(["correlation_matrix", "nodes", "edges"])}" for "what"'
            )

        if not how in ["csv", "clipboard"]:
            raise KeyError(f'Choose one of "{", ".join(["csv" , "clipboard"])}" for "how"')

        sep = "\t" if how == "clipboard" else ","

        # Gather data
        if what == "correlation_matrix":
            data = [self.bn._node_names] + self.bn.R[:, :].astype(str).tolist()
            data = [[name] + row for row, name in zip(data, [""] + self.bn._node_names)]
            text = "\n".join([sep.join(row) for row in data])

        elif what == "nodes":
            text = get_table_text(model=self.mainwindow.nodes_edges_widget.nodemodel, newline_replace=" ")

        elif what == "edges":
            text = get_table_text(model=self.mainwindow.nodes_edges_widget.edgemodel, newline_replace=" ")

        # Export data
        if how == "csv":
            # Set open file dialog settings
            options = QtWidgets.QFileDialog.Options() | QtWidgets.QFileDialog.DontUseNativeDialog

            # Set current dir
            currentdir = self.mainwindow.appsettings.value("currentdir", ".", type=str)

            # Open dialog to select file
            fname, ext = QtWidgets.QFileDialog.getSaveFileName(
                self.mainwindow, "BN Drawing - Save data", currentdir, "CSV (*.csv)", options=options
            )

            if fname == "":
                return None
            elif ".csv" in ext and not fname.endswith(".csv"):
                fname += ".csv"

            # Convert to path
            fname = Path(fname)
            with fname.open("w", encoding="utf-8", errors=" ") as f:
                f.write(text)

        elif how == "clipboard":
            QtWidgets.qApp.clipboard().setText(text)


import sklearn
import sklearn.cross_validation
import sklearn.preprocessing
import numpy

class BalancedShuffleSplit(object):
	"""docstring for BalancedShuffleSplit"""
	def __init__(self, y, n = 5, train_size = 0.25, random_state = numpy.random.RandomState()):
		super(BalancedShuffleSplit, self).__init__()
		self._nsamples = len(y)
		self._n = n
		self._rng = random_state
		if 0 < train_size < 1:
			self._trainsize = int(train_size*self._nsamples)
		else:
			self._trainsize = train_size

		self._labelencoder = sklearn.preprocessing.LabelEncoder()
		self._y = self._labelencoder.fit_transform(y)#make the labels in range [0,c), where c is number of labels
		self._nclasses = max(self._y)+1
		self._buckets = [numpy.where(self._y==i)[0] for i in numpy.unique(self._y)]

		self._samplesperclass = self._trainsize/self._nclasses
		self._excess = self._trainsize%self._nclasses

		classes_not_ok = []
		for i in numpy.unique(self._y):

			if not (sum(self._y==i)>(self._samplesperclass+self._excess)):
				classes_not_ok.append(i)

		#print self._trainsize
		#print self._samplesperclass+self._excess

		if len(classes_not_ok)>0:
			raise Exception('Need at least %d samples for classes = %s'%(self._samplesperclass+self._excess+1, self._labelencoder.inverse_transform(classes_not_ok)))

	def __iter__(self):
		for i in xrange(self._n):
			yield self._generateBalancedShuffleSplit()

	def _generateBalancedShuffleSplit(self):
		train = numpy.ones(0)
		test = numpy.ones(0)
		idx = 0
		for i in self._buckets:
			self._rng.shuffle(i)
			if idx<self._excess:
				trainsize = self._samplesperclass+1
			else:
				trainsize = self._samplesperclass
			#print trainsize
			train = numpy.append(train, i[:trainsize])
			test = numpy.append(test, i[trainsize:])
			idx+=1

		# print (train, test)
		return (train.astype(int), test.astype(int))


from django.shortcuts import render, get_object_or_404, redirect
from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin
from django.contrib.auth.models import User
from django.template import context
from django.views.generic import (
    ListView,
    DetailView,
    CreateView,
    UpdateView,
    DeleteView,
)
from django.http import Http404
from .models import Post
from django.contrib.auth.models  import User
from django.contrib import messages
from django.core.mail import send_mail
from django.conf import settings
from typing import Union
import logging
from django.contrib.auth.decorators import login_required

logger = logging.getLogger(__name__)

def home(request):
    #  lets create a dictionary and call it context, lets create a key named posts and the value of that key will be the data above in the posts list
    context = {
        'posts': Post.objects.all()
    }

    return render (request,'blog/home.html',context)

@login_required
def send_email(request, pk):
    user_email = User.objects.filter(username=request.user).first().email
    blog_post = Post.objects.filter(id=pk).first()

    if not user_email:
        logger.warning(f'Failed to send post {pk} via email due to missing user email')
        messages.warning(request, 'Make sure you are logged in as correct user with correct email address.')
    else:
        send_mail(
            subject=blog_post.title,
            message=blog_post.content,
            from_email=settings.EMAIL_HOST_USER,
            recipient_list=[user_email]
        )
        logger.info(f'Successfully sent post {pk} via email')
        messages.success(request, f'Successfully sent post to {user_email}')

    return redirect('post-detail', pk)

class PostListView(ListView):
    model = Post
    template_name = 'blog/home.html'   # <app>/<model>_<viewtype>.html
    context_object_name = 'posts'
    # date_posted order the list from oldest to newest, to reverse we add a - sign in the front
    ordering = ['-date_posted']
    paginate_by = 5

class UserPostListView(ListView):
    model = Post
    template_name = 'blog/user_posts.html'  # <app>/<model>_<viewtype>.html
    context_object_name = 'posts'
    paginate_by = 5

    def get_queryset(self):
        '''
            return query set consists of posts posted by user and ordered
            by posted date.

            Raise Not found error when user doesn't exist in User model
        '''
        try:
            user = get_object_or_404(User, username=self.kwargs.get('username'))
        except Http404:
            logger.error(f"Failed to find username {self.kwargs.get('username')} in User model.")
            messages.warning(self.request, "Something went wrong, can not view posts filtered by user.")
            raise Http404("User doesn't exist")
        return Post.objects.filter(author=user).order_by('-date_posted')

class PostDetailView(DetailView):
    model = Post
    fields = ["title", "content"]
    # When we are not defining template_name = <FILENAME>.html in our django defined class function, this means
    # that django will search for a template with default naming convention of # <app>/<model>_<viewtype>.html
    # In our PostDetailView function, it is looking for blog/post_detail.html
    # We had passed a template in PostListView since we had already created a template with another name and we
    # wanted django to search for it ,not the file with filename constructed using default naming convention.

class PostCreateView(LoginRequiredMixin, CreateView):
    # When we are not defining template_name = <FILENAME>.html in our django defined class function, this means
    # that django will search for a template with default naming convention of # <app>/<model>_form.html
    # In our PostCreateView & PostUpdateView function, it is looking for blog/post_form.html
    model = Post
    fields = ['title','content']

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

class PostUpdateView(LoginRequiredMixin, UserPassesTestMixin, UpdateView):
    model = Post
    fields = ['title','content']

    def form_valid(self, form):
        form.instance.author = self.request.user
        return super().form_valid(form)

    def test_func(self):
        return check_user_and_author_same(self)

class PostDeleteView(LoginRequiredMixin, UserPassesTestMixin, DeleteView):
    model = Post
    success_url = '/'

    def test_func(self):
        return check_user_and_author_same(self)

### Utility methods ###
def check_user_and_author_same(post_obj: Union[PostDeleteView, PostUpdateView]):
    '''
        Check user in request is same as author of post
    '''
    post = post_obj.get_object()
    if post_obj.request.user == post.author:
        return True
    return False


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 24 18:33:39 2018

@author: patel
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as pl
from sklearn import preprocessing,cross_validation,svm
from sklearn.cross_validation import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.externals.six import StringIO
from sklearn import tree
from matplotlib import style
from matplotlib.colors import ListedColormap
style.use("ggplot")

dataset = pd.read_csv('test.csv')
dataset = dataset.replace(['male','female'],
[0,1])

#print(dataset)

# Seperating value into two objects
#X = dataset.iloc[:, [2,3,4]].values
#A = dataset.iloc[:, 17].values
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, 21].values
#dataset[X[:, 17]] = dataset[X[:, 17]].replace(0, dataset[X[:, 17]].mean())
#dataset[A] = dataset[A].replace(0, dataset[A].mean)

print("This is new age column replace with male and female into 0 and 1.")
#print(A)
print(X)
print(y)

print("This is the new value of gender column with missing values.")
print(X[:, 18])

# Replacing nan value with median in gender column which is 18th column

from sklearn.preprocessing import Imputer
imputer = Imputer(missing_values='NaN', strategy = 'median', axis = 0)
#X[:, [18,19]] = X[:,[18,19]].reshape(1, -1)
imputer = imputer.fit(X[:, [18,19]])
X[:, [18,19]] = imputer.transform(X[:, [18,19]])
print("This is gender column after replacing missing values with median function")
print(X[:, 18])
print("\n")

# Split the data into training and testing part
from sklearn.cross_validation import train_test_split
# Giving 20% of total dataset in testing part and remaining in training part (80%)

x_train, x_test, y_train, y_test = train_test_split(X,y,test_size=0.20, random_state = 0)

# Implementing Naive-Bayes Classifier algorithm for our dataset
from sklearn.naive_bayes import GaussianNB

#Create a Gaussian Classifier
model = GaussianNB()

# Train the model using the training sets
model.fit(x_train,y_train)

#Predict Output
predicted= model.predict(x_test)
print("This is the predicted data for testing X which contains 2000 datasets")
print(predicted)
print("\n")

accuracy = model.score(x_test, y_test)
print("This is the accuracy of Naive Bayes Classifier algoritham")
print(accuracy)
print("\n")

# Calculating confusion matrix for machine learning prediction and
cm= confusion_matrix(y_test, predicted)
print("The confusion matrix is described below.")
print(cm)


"""Unit tests for the openai LLM class"""

import pytest

from pandasai.llm import GoogleVertexAI
from pandasai.exceptions import UnsupportedModelError

class MockedCompletion:
    def __init__(self, result: str):
        self.result = result

class TestGoogleVertexAI:
    def test_init_with_default_model(self):
        project_id = "your_project_id"
        location = "northamerica-northeast1"
        vertexai_instance = GoogleVertexAI(project_id, location)

        assert vertexai_instance.model == "text-bison@001"
        assert vertexai_instance.project_id == project_id
        assert vertexai_instance.location == location

    def test_init_with_custom_model(self):
        project_id = "test-project"
        location = "northamerica-northeast1"
        custom_model = "code-bison@001"

        vertexai_instance = GoogleVertexAI(project_id, location, model=custom_model)

        assert vertexai_instance.model == custom_model
        assert vertexai_instance.project_id == project_id
        assert vertexai_instance.location == location

    @pytest.fixture
    def google_vertexai(self):
        # Create an instance of YourClass for testing
        project_id = "test-project"
        location = "northamerica-northeast1"
        custom_model = "code-bison@001"
        return GoogleVertexAI(project_id, location, custom_model)

    def test_validate_with_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = "text-bison@001"
        google_vertexai._validate()  # Should not raise any errors

    def test_validate_with_invalid_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = "invalid-model"
        with pytest.raises(
            UnsupportedModelError,
            match=(
                "Unsupported model: The model 'invalid-model' doesn't exist "
                "or is not supported yet."
            ),
        ):
            google_vertexai._generate_text("Test prompt")

    def test_validate_without_model(self, google_vertexai: GoogleVertexAI):
        google_vertexai.model = None
        with pytest.raises(ValueError, match="model is required."):
            google_vertexai._validate()


from threading import Condition
from typing import List, Optional, Set

from ..backend import EventBackend

class StubBackend(EventBackend):
    def __init__(self) -> None:
        self.condition = Condition()
        self.events: Set[bytes] = set()

    def wait_many(self, keys: List[bytes], timeout: int) -> Optional[bytes]:
        with self.condition:
            if self.condition.wait_for(
                lambda: self._anyset(keys), timeout=timeout / 1000
            ):
                for key in keys:
                    if key in self.events:
                        self.events.remove(key)
                        return key
        return None

    def poll(self, key: bytes) -> bool:
        with self.condition:
            if key in self.events:
                self.events.remove(key)
                return True
        return False

    def notify(self, key: bytes, ttl: int) -> None:
        with self.condition:
            self.events.add(key)
            self.condition.notify_all()

    def _anyset(self, keys: List[bytes]) -> bool:
        return any(k in self.events for k in keys)


from _c import *
from _u import *
from dateutil.relativedelta import *
import os
import copy
import time
import datetime
import threading
import pyupbit

class BotUpbit():

    def __init__(self):

        self.is_aws = True
        self.access_key = UB_ACCESS_KEY_AWS if self.is_aws else UB_ACCESS_KEY_NAJU
        self.secret_key = UB_SECRET_KEY_AWS if self.is_aws else UB_SECRET_KEY_NAJU
        self.ubt = pyupbit.Upbit(self.access_key, self.secret_key)

        self.q_l = []
        self.b_l = []
        self.r_l = []
        self.o_l = {}

        self.time_order = None
        self.time_rebalance = None

        self.bool_start = False
        self.bool_balance = False
        self.bool_order = False

        self.prc_ttl = 0
        self.prc_lmt = 0
        self.prc_buy = 0

        self.const_up = 500000000
        self.const_up = 5500000
        self.const_dn = 5500

    def init_per_day(self):

        if self.bool_balance == False:

            tn = datetime.datetime.now()
            tn_0 = tn.replace(hour=0, minute=0, second=0)
            tn_d = int(((tn - tn_0).seconds) % 300)
            print(f'{tn_d} Second')

            if tn_d <= 150:
                time.sleep(300 - tn_d - 150)
            else:
                time.sleep(300 - tn_d + 150)

            self.bool_balance = True

        print('##############################')

        self.ubt = pyupbit.Upbit(self.access_key, self.secret_key)

        self.q_l = pyupbit.get_tickers("KRW")
        prc_ttl, prc_lmt, _, bal_lst  = self.get_balance_info(self.q_l)
        self.b_l = list(set(self.q_l + bal_lst))
        self.r_l = list(set(bal_lst).difference(self.q_l))
        self.prc_ttl = prc_ttl if prc_ttl < self.const_up else self.const_up
        self.prc_lmt = prc_lmt if prc_ttl < self.const_up else prc_lmt - (prc_ttl - self.const_up)
        prc_buy = self.prc_ttl / (len(self.q_l) * 4)
        self.prc_buy = prc_buy if prc_buy > self.const_dn else self.const_dn

        if os.path.isfile(FILE_URL_TIKR_3M):
            self.o_l = load_file(FILE_URL_TIKR_3M)
        else:
            self.o_l = {}
            save_file(FILE_URL_TIKR_3M, self.o_l)

        for tk in self.b_l:
            if not (tk in self.o_l):
                self.get_tiker_data_init(tk)

        if self.prc_lmt < self.prc_buy:
            line_message('BotUpbit Insufficient Balance !!!')

        int_prc_ttl = int(self.prc_ttl)
        int_prc_lmt = int(self.prc_lmt)
        len_bal_lst = len(self.b_l)

        line_message(f'BotUpbit \nTotal Price : {int_prc_ttl:,} KRW \nLimit Price : {int_prc_lmt:,} KRW \nSymbol List : {len_bal_lst}')

        __tn = datetime.datetime.now()
        __tn_min = __tn.minute % 5
        __tn_sec = __tn.second

        self.time_rebalance = threading.Timer(300 - (60 * __tn_min) - __tn_sec + 150, self.init_per_day)
        self.time_rebalance.start()

    def stock_order(self):

        if self.bool_order == False:

            tn = datetime.datetime.now()
            tn_0 = tn.replace(hour=0, minute=0, second=0)
            tn_d = int(((tn - tn_0).seconds) % 300)
            time.sleep(300 - tn_d)
            self.bool_order = True

        _tn = datetime.datetime.now()

        # self.get_remain_cancel(self.b_l)

        _, _, bal_lst, _ = self.get_balance_info(self.q_l)
        sel_lst = []

        for symbol in self.b_l:

            df = self.strategy_rsi(self.gen_ubt_df(symbol, 'minute5', 120))
            is_df = not (df is None)

            if is_df:

                df_h = df.tail(2).head(1)
                close = df_h['close'].iloc[-1]
                rsi = df_h['rsi'].iloc[-1]
                rsi_prev = df_h['rsi_prev'].iloc[-1]
                volume_osc = df_h['volume_osc'].iloc[-1]

                str_rsi = round(rsi, 2)
                str_rsi_prev = round(rsi_prev, 2)
                str_volume_osc = round(volume_osc, 2)
                print(f'{symbol} : RSI - {str_rsi}, RSI_P - {str_rsi_prev}, VO - {str_volume_osc}')

                cur_prc = float(close)

                is_symbol_bal = symbol in bal_lst
                is_psb_sel = (is_symbol_bal and (cur_prc * bal_lst[symbol]['b'] > self.const_dn))
                ol_bool_buy = copy.deepcopy(self.o_l[symbol]['bool_buy'])
                is_nothing = ol_bool_buy and ((not is_symbol_bal) or (is_symbol_bal and (cur_prc * bal_lst[symbol]['b'] < self.const_dn)))

                if is_nothing:
                    self.get_tiker_data_init(symbol)

                if is_psb_sel and ol_bool_buy:

                    bl_balance = copy.deepcopy(bal_lst[symbol]['b'])
                    ol_buy_price = float(copy.deepcopy(self.o_l[symbol]['buy_price']))
                    ol_quantity_ratio = copy.deepcopy(self.o_l[symbol]['quantity_ratio'])
                    ol_bool_sell = copy.deepcopy(self.o_l[symbol]['bool_sell'])
                    ol_70_position = copy.deepcopy(self.o_l[symbol]['70_position'])
                    sell_qty = bl_balance * (1 / ol_quantity_ratio)
                    is_psb_sel_div = (cur_prc * sell_qty) > self.const_dn

                    if (not is_psb_sel_div) and ol_quantity_ratio > 1:
                        sell_qty = bl_balance * (1 / ol_quantity_ratio - 1)

                    if rsi <= 50 and ol_bool_sell:
                        self.ubt.sell_market_order(symbol, bl_balance)
                        self.get_tiker_data_init(symbol)

                        _ror = get_ror(ol_buy_price, cur_prc)
                        print(f'Sell - Symbol: {symbol}, Profit: {round(_ror, 4)}')
                        sel_lst.append({'c': '[S] ' + symbol, 'r': round(_ror, 4)})

                    elif rsi >= 70 and ((ol_70_position == '70_down') or (ol_70_position == '70_up' and (rsi_prev <= rsi))):
                        self.ubt.sell_market_order(symbol, sell_qty)
                        self.o_l[symbol]['quantity_ratio'] = ol_quantity_ratio - 1
                        self.o_l[symbol]['bool_sell'] = True

                        if (not is_psb_sel_div) and ol_quantity_ratio > 1:
                            self.o_l[symbol]['quantity_ratio'] = ol_quantity_ratio - 2

                        if self.o_l[symbol]['quantity_ratio'] == 0:
                            self.get_tiker_data_init(symbol)

                        _ror = get_ror(ol_buy_price, cur_prc)
                        print(f'Sell - Symbol: {symbol}, Profit: {round(_ror, 4)}')
                        sel_lst.append({'c': '[S] ' + symbol, 'r': round(_ror, 4)})

                if (rsi <= 30) and (rsi_prev > rsi) and (volume_osc > 0):

                    is_psb_ord = self.prc_lmt > self.prc_buy
                    is_remain_symbol = symbol in self.r_l
                    buy_qty = float(self.prc_buy / cur_prc)

                    if is_psb_ord and (not is_remain_symbol):

                        self.ubt.buy_market_order(symbol, self.prc_buy)
                        ol_bool_buy = copy.deepcopy(self.o_l[symbol]['bool_buy'])
                        ol_quantity_ratio = copy.deepcopy(self.o_l[symbol]['quantity_ratio'])

                        if ol_bool_buy:
                            self.o_l[symbol]['buy_price'] = cur_prc
                            self.o_l[symbol]['quantity_ratio'] = ol_quantity_ratio + 1
                        else:
                            self.o_l[symbol] = {
                                'bool_buy': True,
                                'buy_price': cur_prc,
                                'quantity_ratio': 2,
                                'bool_sell': False,
                                '70_position': ''
                            }

                        print(f'Buy - Symbol: {symbol}, Balance: {buy_qty}')
                        sel_lst.append({'c': '[B] ' + symbol, 'r': (buy_qty)})

                self.o_l[symbol]['70_position'] = '70_down' if rsi < 70 else '70_up'

        save_file(FILE_URL_TIKR_3M, self.o_l)
        for o in self.o_l:
            if self.o_l[o]['bool_buy'] == True:
                print(o, self.o_l[o])

        sel_txt = ''
        for sl in sel_lst:
            sel_txt = sel_txt + '\n' + str(sl['c']) + ' : ' + str(sl['r'])

        __tn = datetime.datetime.now()
        __tn_min = __tn.minute % 5
        __tn_sec = __tn.second

        self.time_backtest = threading.Timer(300 - (60 * __tn_min) - __tn_sec, self.stock_order)
        self.time_backtest.start()

        int_prc_ttl = int(self.prc_ttl)
        str_start = _tn.strftime('%Y/%m/%d %H:%M:%S')
        str_end = __tn.strftime('%Y/%m/%d %H:%M:%S')

        line_message(f'BotUpbit \nStart : {str_start}, \nEnd : {str_end}, \nTotal Price : {int_prc_ttl:,} KRW {sel_txt}')

    # Tiker Data Init
    def get_tiker_data_init(self, tk):
        self.o_l[tk] = {
            'bool_buy': False,
            'buy_price': 0,
            'quantity_ratio': 0,
            'bool_sell': False,
            '70_position': ''
        }

    # Strategy RSI
    def strategy_rsi(self, df):
        if not (df is None):
            df['rsi'] = indicator_rsi(df['close'], 14)
            df['rsi_prev'] = df['rsi'].shift()
            df['volume_osc'] = indicator_volume_oscillator(df['volume'], 5, 10)
            return df

    # Generate DataFrame
    def gen_ubt_df(self, tk, tf, lm):
        ohlcv = pyupbit.get_ohlcv(ticker=tk, interval=tf, count=lm)
        if not (ohlcv is None) and len(ohlcv) >= lm:
            return ohlcv

    # Balance Code List
    def get_balance_info(self, tks):
        bal_cur = pyupbit.get_current_price(tks)
        bal_lst = self.ubt.get_balances()
        bal_krw = 0
        prc = 0
        obj = {}
        lst = []
        if len(bal_lst) > 0:
            for bl in bal_lst:
                avgp = float(bl['avg_buy_price'])
                blnc = float(bl['balance'])
                tikr = bl['unit_currency'] + '-' + bl['currency']
                if tikr != 'KRW-KRW':
                    obj[tikr] = {
                        'a': avgp,
                        'b': blnc
                    }
                    if tikr in bal_cur:
                        prc = prc + (bal_cur[tikr] * blnc)
                    lst.append(tikr)
                else:
                    prc = prc + blnc
                    bal_krw = blnc

        return prc, bal_krw, obj, lst

    # Not Signed Cancel Order
    def get_remain_cancel(self, l):
        for _l in l:
            rmn_lst = self.ubt.get_order(_l)
            if len(rmn_lst) > 0:
                for rmn in rmn_lst:
                    self.ubt.cancel_order(rmn['uuid'])

    # All Sell
    def all_sell_order(self):
        _, _, bal_lst, _  = self.get_balance_info(self.q_l)
        for bl in bal_lst:
            resp = self.ubt.sell_market_order(bl, bal_lst[bl]['b'])
            print(resp)
            time.sleep(0.25)

if __name__ == '__main__':

    bu = BotUpbit()
    # bu.init_per_day()
    # bu.stock_order()
    # bu.all_sell_order()

    while True:

        try:

            tn = datetime.datetime.now()
            tn_start = tn.replace(hour=0, minute=0, second=0)

            if tn >= tn_start and bu.bool_start == False:
                bu.init_per_day()
                bu.stock_order()
                bu.bool_start = True

        except Exception as e:

            line_message(f"BotUpbit Error : {e}")
            break


from django.shortcuts import render, HttpResponse
from app01 import models

# Create your views here.

def home(request):

    return HttpResponse("没错，我就是首页！")

def delete(request, title, id):
    # 删除语句
    # models.Book.objects.get(id=id).delete()

    # 数据库中保存的类名都是首字母大写的
    title = title.capitalize()

    # 从另一个文件根据字符串反射到具体的变量
    if hasattr(models, title):
        title_class = getattr(models, title)
        try:
            title_class.objects.get(id=id).delete()
        except Exception as e:
            print(str(e))
            print("id值不存在")
            return HttpResponse("名称：{} id:{}".format(title, id))
    else:
        return HttpResponse("表{}不存在".format(title))