# Todo: Directing Customers To Subscription Through Financial App Behavior Analysis

# todo: Import essential libraries
import numpy as np  # for numeric calculation
import pandas as pd  # for data analysis and manupulation
import matplotlib.pyplot as plt  # for data visualization
import seaborn as sns  # for data visualization
from dateutil import parser  # convert time in date time data type

# todo: Load dataset
fineTech_appData = pd.read_csv("FineTech_appData.csv")
print(fineTech_appData.shape)
fineTech_appData.head(6)  # show first 6 rows of fineTech_appData DataFrame
fineTech_appData.tail(6)  # show last 6 rows of fineTech_appData DataFrame

# todo: Showing first 5 rows from screen list column
for i in [1, 2, 3, 4, 5]:
    print(fineTech_appData.loc[i, 'screen_list'], '\n')

fineTech_appData.isnull().sum()  # take summation of null values
fineTech_appData.info()  # brief inforamtion about Dataset
fineTech_appData.describe()  # give the distribution of numerical variables

# todo: Get the unique value of each columns and it's length
features = fineTech_appData.columns
for i in features:
    print("""Unique value of {}\n{}\nlen is {} \n........................\n
          """.format(i, fineTech_appData[i].unique(), len(fineTech_appData[i].unique())))

# todo: get data type of each columns
print(fineTech_appData.dtypes)

# todo: hour data convert string to int
fineTech_appData['hour'] = fineTech_appData.hour.str.slice(1, 3).astype(int)

# todo: get data type of each columns
print(fineTech_appData.dtypes)

# todo: column names
print(fineTech_appData.columns)

# todo: drop object dtype columns
fineTech_appData2 = fineTech_appData.drop(['user', 'first_open', 'screen_list', 'enrolled_date'], axis=1)
fineTech_appData2.head(6)  # head of numeric dataFrame

# todo: Data visualization

# todo: Heatmap using the correlation matrix
plt.figure(figsize=(16, 9))  # heatmap size in ratio 16:9
sns.heatmap(fineTech_appData2.corr(), annot=True, cmap='coolwarm')  # show heatmap
plt.title("Heatmap using correlation matrix of fineTech_appData2", fontsize=25)  # title of heatmap

# todo: Pailplot of fineTech_appData2 Dataset
sns.pairplot(fineTech_appData2, hue='enrolled')
plt.show()

# todo: Show counterplot of 'enrolled' feature
sns.countplot(fineTech_appData.enrolled)
plt.show()

# todo: value enrolled and not enrolled customers
print("Not enrolled user = ", (fineTech_appData.enrolled < 1).sum(), "out of 50000")
print("Enrolled user = ", 50000 - (fineTech_appData.enrolled < 1).sum(), "out of 50000")

# todo: plot histogram
plt.figure(figsize=(16, 9))  # figure size in ratio 16:9
features = fineTech_appData2.columns  # list of columns name
for i, j in enumerate(features):
    plt.subplot(3, 3, i + 1)  # create subplot for histogram
    plt.title("Histogram of {}".format(j), fontsize=15)  # title of histogram
    bins = len(fineTech_appData2[j].unique())  # bins for histogram
    plt.hist(fineTech_appData2[j], bins=bins, rwidth=0.8, edgecolor="y", linewidth=2, )  # plot histogram
plt.subplots_adjust(hspace=0.5)  # space between horixontal axes (subplots)
plt.show()

# todo: showing all feature from dataset
for i, j in enumerate(features):
    print(i, j)

# todo: Correlation barplot with ‘enrolled’ feature
sns.set()  # set background dark grid
plt.figure(figsize=(14, 5))
plt.title("Correlation all features with 'enrolled' ", fontsize=20)
fineTech_appData3 = fineTech_appData2.drop(['enrolled'], axis=1)  # drop 'enrolled' feature
ax = sns.barplot(fineTech_appData3.columns, fineTech_appData3.corrwith(fineTech_appData2.enrolled))  # plot barplot
ax.tick_params(labelsize=15, labelrotation=20, color="k")  # decorate x & y ticks font

# todo: parsing object datatype into datatime dtype
fineTech_appData['first_open'] = [parser.parse(i) for i in fineTech_appData['first_open']]
fineTech_appData['enrolled_date'] = [parser.parse(i) if isinstance(i, str) else i for i in
                                     fineTech_appData['enrolled_date']]
print(fineTech_appData.dtypes)

# todo: creating new feature that has time of customer used for register for free version
fineTech_appData['time_to_enrolled'] = (fineTech_appData.enrolled_date - fineTech_appData.first_open).astype(
    'timedelta64[h]')
plt.hist(fineTech_appData['time_to_enrolled'].dropna())
plt.show()
plt.hist(fineTech_appData['time_to_enrolled'].dropna(), range=(0, 100))
plt.show()

# todo: Those customers have enrolled after 48 hours set as 0
fineTech_appData.loc[fineTech_appData.time_to_enrolled > 48, 'enrolled'] = 0
print(fineTech_appData)

# todo: droping new created features
fineTech_appData.drop(columns=['time_to_enrolled', 'enrolled_date', 'first_open'], inplace=False)
print(fineTech_appData)

# todo: read csv file and convert it into numpy array
fineTech_app_screen_Data = pd.read_csv("top_screens.csv").top_screens.values
print(fineTech_app_screen_Data)
type(fineTech_app_screen_Data)

# todo: Add ',' at the end of each string of 'screen_list' for further operation.
fineTech_appData['screen_list'] = fineTech_appData.screen_list.astype(str) + ','
print(fineTech_appData)

# todo: string into to number
for screen_name in fineTech_app_screen_Data:
    fineTech_appData[screen_name] = fineTech_appData.screen_list.str.contains(screen_name).astype(int)
    fineTech_appData['screen_list'] = fineTech_appData.screen_list.str.replace(screen_name + ",", "")

# todo: test
fineTech_appData.screen_list.str.contains('Splash').astype(int)
fineTech_appData.screen_list.str.replace('Splash' + ",", "")

print(fineTech_appData.shape)
print(fineTech_appData.head(6))
print(fineTech_appData.screen_list.str.count(",").head(6))

# todo: remain screen in 'screen_list'
print(fineTech_appData.loc[0, 'screen_list'])

# todo: count remain screen list and store counted number in 'remain_screen_list'
fineTech_appData['remain_screen_list'] = fineTech_appData.screen_list.str.count(",")

# todo: Drop the 'screen_list'
fineTech_appData.drop(columns=['screen_list'], inplace=False)

# todo: total columns
print(fineTech_appData)
print(fineTech_appData.columns)

# todo: take sum of all saving screen in one place
saving_screens = ['Saving1',
                  'Saving2',
                  'Saving2Amount',
                  'Saving4',
                  'Saving5',
                  'Saving6',
                  'Saving7',
                  'Saving8',
                  'Saving9',
                  'Saving10']
fineTech_appData['saving_screens_count'] = fineTech_appData[saving_screens].sum(axis=1)
fineTech_appData.drop(columns=saving_screens, inplace=False)
print(fineTech_appData)

credit_screens = ['Credit1',
                  'Credit2',
                  'Credit3',
                  'Credit3Container',
                  'Credit3Dashboard']
fineTech_appData['credit_screens_count'] = fineTech_appData[credit_screens].sum(axis=1)
fineTech_appData.drop(columns=credit_screens, axis=1, inplace=False)
print(fineTech_appData)

cc_screens = ['CC1',
              'CC1Category',
              'CC3']
fineTech_appData['cc_screens_count'] = fineTech_appData[cc_screens].sum(axis=1)
fineTech_appData.drop(columns=cc_screens, inplace=False)
print(fineTech_appData)

loan_screens = ['Loan',
                'Loan2',
                'Loan3',
                'Loan4']
fineTech_appData['loan_screens_count'] = fineTech_appData[loan_screens].sum(axis=1)
fineTech_appData.drop(columns=loan_screens, inplace=False)
print(fineTech_appData)

print(fineTech_appData.shape)
print(fineTech_appData.info())
print(fineTech_appData.describe())

# todo: Heatmap with correlation matrix of new fineTech_appData
plt.figure(figsize=(25, 16))
sns.heatmap(fineTech_appData.corr(), annot=True, linewidth=2)  # *****code 13
plt.show()

print(fineTech_appData.columns)
fineTech_appData['ProfileChildren '].unique()

corr_matrix = fineTech_appData.corr()
print(corr_matrix['ProfileChildren '])
print(fineTech_appData['ProfileChildren '])

# todo: Data Preprocessing

# todo: Split dataset in features and target
clean_fineTech_appData = fineTech_appData
target = fineTech_appData['enrolled']
fineTech_appData.drop(columns='enrolled', inplace=False)

# todo: Split dataset in Train and Test
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(fineTech_appData, target, test_size=0.2, random_state=0)
print('Shape of X_train = ', X_train.shape)
print('Shape of X_test = ', X_test.shape)
print('Shape of y_train = ', y_train.shape)
print('Shape of y_test = ', y_test.shape)

# todo: take User ID in another variable
train_userID = X_train['user']
X_train.drop(columns='user', inplace=False)
test_userID = X_test['user']
X_test.drop(columns='user', inplace=False)

print('Shape of X_train = ', X_train.shape)
print('Shape of X_test = ', X_test.shape)
print('Shape of train_userID = ', train_userID.shape)
print('Shape of test_userID = ', test_userID.shape)

# todo: Feature Scaling
from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
X_train_sc = sc.fit_transform(X_train)
X_test_sc = sc.transform(X_test)

# todo: Model Building
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score

# todo: Decision Tree Classifier
from sklearn.tree import DecisionTreeClassifier

dt_model = DecisionTreeClassifier(criterion='entropy', random_state=0)
dt_model.fit(X_train, y_train)
y_pred_dt = dt_model.predict(X_test)
accuracy_score(y_test, y_pred_dt)

# todo: train with Standard Scaling dataset
dt_model2 = DecisionTreeClassifier(criterion='entropy', random_state=0)
dt_model2.fit(X_train_sc, y_train)
y_pred_dt_sc = dt_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_dt_sc)

# todo: K-NN
from sklearn.neighbors import KNeighborsClassifier

knn_model = KNeighborsClassifier(n_neighbors=5, metric='minkowski', p=2, )
knn_model.fit(X_train, y_train)
y_pred_knn = knn_model.predict(X_test)
accuracy_score(y_test, y_pred_knn)

# todo: train with Standard Scaling dataset
knn_model2 = KNeighborsClassifier(n_neighbors=5, metric='minkowski', p=2, )
knn_model2.fit(X_train_sc, y_train)
y_pred_knn_sc = knn_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_knn_sc)

# todo: Naive Bayes
from sklearn.naive_bayes import GaussianNB

nb_model = GaussianNB()
nb_model.fit(X_train, y_train)
y_pred_nb = nb_model.predict(X_test)
accuracy_score(y_test, y_pred_nb)

# todo: train with Standard Scaling dataset
nb_model2 = GaussianNB()
nb_model2.fit(X_train_sc, y_train)
y_pred_nb_sc = nb_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_nb_sc)

# todo: Random Forest Classifier
from sklearn.ensemble import RandomForestClassifier

rf_model = RandomForestClassifier(n_estimators=10, criterion='entropy', random_state=0)
rf_model.fit(X_train, y_train)
y_pred_rf = rf_model.predict(X_test)
accuracy_score(y_test, y_pred_rf)

# todo: train with Standard Scaling dataset
rf_model2 = RandomForestClassifier(n_estimators=10, criterion='entropy', random_state=0)
rf_model2.fit(X_train_sc, y_train)
y_pred_rf_sc = rf_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_rf_sc)

# todo: Logistic Regression
from sklearn.linear_model import LogisticRegression

lr_model = LogisticRegression(random_state=0, penalty='l1')
lr_model.fit(X_train, y_train)
y_pred_lr = lr_model.predict(X_test)
accuracy_score(y_test, y_pred_lr)

# todo: train with Standard Scaling dataset
lr_model2 = LogisticRegression(random_state=0, penalty='l1')
lr_model2.fit(X_train_sc, y_train)
y_pred_lr_sc = lr_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_lr_sc)

# todo: Support Vector Machine
from sklearn.svm import SVC

svc_model = SVC()
svc_model.fit(X_train, y_train)
y_pred_svc = svc_model.predict(X_test)
accuracy_score(y_test, y_pred_svc)

# todo: train with Standard Scaling dataset
svc_model2 = SVC()
svc_model2.fit(X_train_sc, y_train)
y_pred_svc_sc = svc_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_svc_sc)

"""from sklearn.svm import SVC
grid_para = {'C': [1, 10, 100], 'gamma': [1, 0.01, 0.001], 'kernel': ['rbf']}
from sklearn.model_selection import GridSearchCV
grid_lr = GridSearchCV(SVC(), param_grid=grid_para, refit=True, verbose=4, n_jobs=-1)
grid_lr.fit(X_train, y_train)
grid_pred_lr = grid_lr.predict(X_test)
cm_grid_lr = confusion_matrix(y_test, grid_pred_lr)
sns.heatmap(cm_grid_lr, annot=True, fmt='g')
accuracy_score(y_test, grid_pred_lr)"""

# todo: XGBoost Classifier
from xgboost import XGBClassifier
xgb_model = XGBClassifier()
xgb_model.fit(X_train, y_train)
y_pred_xgb = xgb_model.predict(X_test)
accuracy_score(y_test, y_pred_xgb)

# todo: train with Standard Scaling dataset
xgb_model2 = XGBClassifier()
xgb_model2.fit(X_train_sc, y_train)
y_pred_xgb_sc = xgb_model2.predict(X_test_sc)
accuracy_score(y_test, y_pred_xgb_sc)

# todo: XGB classifier with parameter tuning
xgb_model_pt1 = XGBClassifier(
    learning_rate=0.01,
    n_estimators=5000,
    max_depth=4,
    min_child_weight=6,
    gamma=0,
    subsample=0.8,
    colsample_bytree=0.8,
    reg_alpha=0.005,
    objective='binary:logistic',
    nthread=4,
    scale_pos_weight=1,
    seed=27)
xgb_model_pt1.fit(X_train, y_train)
y_pred_xgb_pt1 = xgb_model_pt1.predict(X_test)
accuracy_score(y_test, y_pred_xgb_pt1)

# todo: XGB classifier with parameter tuning and train with Standard Scaling dataset
xgb_model_pt2 = XGBClassifier(
    learning_rate=0.01,
    n_estimators=5000,
    max_depth=4,
    min_child_weight=6,
    gamma=0,
    subsample=0.8,
    colsample_bytree=0.8,
    reg_alpha=0.005,
    objective='binary:logistic',
    nthread=4,
    scale_pos_weight=1,
    seed=27)
xgb_model_pt2.fit(X_train_sc, y_train)
y_pred_xgb_sc_pt2 = xgb_model_pt2.predict(X_test_sc)
accuracy_score(y_test, y_pred_xgb_sc_pt2)

# todo: confussion matrix
cm_xgb_pt2 = confusion_matrix(y_test, y_pred_xgb_sc_pt2)
sns.heatmap(cm_xgb_pt2, annot=True, fmt='g')
plt.title("Confussion Matrix", fontsize=20)  # *****code 14

# todo: Clasification Report
cr_xgb_pt2 = classification_report(y_test, y_pred_xgb_sc_pt2)
print("Classification report >>> \n", cr_xgb_pt2)

# todo: Cross validation
from sklearn.model_selection import cross_val_score
cross_validation = cross_val_score(estimator=xgb_model_pt2, X=X_train_sc, y=y_train, cv=10)
print("Cross validation of XGBoost model = ", cross_validation)
print("Cross validation of XGBoost model (in mean) = ", cross_validation.mean())

# todo: Mapping predicted output to the target
final_result = pd.concat([test_userID, y_test], axis=1)
final_result['predicted result'] = y_pred_xgb_sc_pt2
print(final_result)

# todo: Save the Model Using Pickle
import pickle

# todo: save model
pickle.dump(xgb_model_pt2, open('FineTech_app_ML_model.pickle', 'wb'))

# todo: load model
ml_model_pl = pickle.load(open('FineTech_app_ML_model.pickle', 'rb'))

# todo: predict the output
y_pred_pl = ml_model_pl.predict(X_test_sc)

# todo: confusion matrix
cm_pl = confusion_matrix(y_test, y_pred_pl)
print('Confussion matrix = \n', cm_pl)

# todo: show the accuracy
print("Accuracy of model = ", accuracy_score(y_test, y_pred_pl))

# todo: Save the Model Using Joblib
import joblib

# todo: save model
joblib.dump(xgb_model_pt2, 'FineTech_app_ML_model.joblib')

# todo: load model
ml_model_jl = joblib.load('FineTech_app_ML_model.joblib')

# todo: predict the output
y_pred_jl = ml_model_jl.predict(X_test_sc)

# todo: confusion matrix
cm_jl = confusion_matrix(y_test, y_pred_jl)
print('Confussion matrix = \n', cm_jl)

# todo: show the accuracy
print("Accuracy of model = ", accuracy_score(y_test, y_pred_jl))

Python/Machine Learning/Indian AI Production/ML Algorithm/Directing Customers To Subscription.py


# QDesktopWidget
import sys
from PyQt5.QtWidgets import QDesktopWidget,QMainWindow,QApplication
from PyQt5.QtGui import QIcon

class CenterForm(QMainWindow):
    def __init__(self):
        super(CenterForm,self).__init__()

        # 设置主窗口的标题
        self.setWindowTitle('让窗口居中')

        # 设置窗口的尺寸
        self.resize(400,300)

    def center(self):
        # 获取屏幕坐标系
        screen = QDesktopWidget().screenGeometry()
        # 获取窗口坐标系
        size = self.geometry()
        newLeft = (screen.width() - size.width()) / 2
        newTop = (screen.height() - size.height()) / 2
        self.move(newLeft,newTop)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    main = CenterForm()
    main.show()

    sys.exit(app.exec_())


import pandas as pd
import streamlit as st
from joblib import load

st.set_page_config(
    page_title="Agri Sense",
    page_icon="ðŸŒ¾"
)

# add background image

# def add_bg_from_local(image_file):
#     with open(image_file, "rb") as image_file:
#         encoded_string = base64.b64encode(image_file.read())
#     st.markdown(
#     f"""
#
#     """,
#     unsafe_allow_html=True
#     )
# add_bg_from_local('images/background_image.png')

def main():

    dtc_model = load('crop_recommendation.json')

    html_temp="""

Crop Recommendation using DTC

    """
    st.markdown(html_temp,unsafe_allow_html=True)
    st.write('')
    st.write('')
    st.markdown("##### Do you want us to recommend crops? \n ")

    p1 = st.number_input("What is N (Nitrogen) value?",0,140,step=1)
    p2 = st.number_input("What is P (Phosphorus) value?",5,145,step=1)
    p3 = st.number_input("What is K (Potassium) value?",5,205,step=1)
    p4 = st.number_input("How much temperature?",8.82,44.67,step=1.00)
    p5 = st.number_input("How much humidity?",14.25,99.98,step=1.00)
    p6 = st.number_input("What is the pH value?",3.50,9.93,step=1.00)
    p7 = st.number_input("How much rainfall?",20.21,213.90,step=1.00)

    data_new = pd.DataFrame({
        'N':p1,
        'P':p2,
        'K':p3,
        'temperature':p4,
        'humidity':p5,
        'ph':p6,
        'rainfall':p7,
    },index=[0])

    try:
        if st.button('Predict'):
            pred = dtc_model.predict(data_new)
            st.success("You can cultivate {}".format(pred[0]))
            #st.balloons()
    except:
        st.warning("You can't cultivate crops in this land")

if __name__ == "__main__":
    main()


from __future__ import absolute_import,division,print_function
import sys
import os
import numpy as np
import pandas as pd
import scipy.stats as st
from scipy.interpolate import interp1d
import h5py
import corner
import emcee
import progressbar
import extinction
from priors import BrokenPrior,LogNormalPrior,PowerLawPrior

#---------------- Matplotlib ----------
import matplotlib
matplotlib.use('PDF')
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib import gridspec
from matplotlib.ticker import NullFormatter

class posterior_variate:
    """
    This class provides flexibility to infer the posterior distribution of the variate given the absolute magnitudes
    It also infers the extinction nu
    """
    def __init__(self,observed,uncert,N_bands,variate2phot,waves,prior,hyper,
                    initial_hyper=None,walkers_ratio=2,
                    min_variate=0.1,max_variate=10,
                    max_Rv=10.0,
                    burnin_frac=0.2,quantiles=[2.5,97.5]):

        self.burnin_frac = burnin_frac
        self.ndim        = 7
        self.nwalkers    = walkers_ratio * self.ndim
        self.max_variate = max_variate
        self.min_variate = min_variate
        self.N_bands     = N_bands
        self.variate2phot = variate2phot
        self.waves       = waves
        self.max_Rv      = max_Rv

        #------------------------------------------------------
        idx     = np.ix_(np.where(np.isfinite(observed))[0])[0]
        o_phot  = observed[idx]
        u_phot  = uncert[idx]

        self.idx = idx
        self.obs = o_phot
        self.unc = u_phot

        ################################# PRIORS ######################################################
        #------------- Mass -------------------------------------------------------
        if prior["variate"]=="Uniform":
            prior_Ms = st.uniform(loc=min_variate,scale=max_variate-min_variate)
        elif prior["variate"] == "Half-Cauchy":
            prior_Ms = st.halfcauchy(loc=0.0,scale=100.0)
        elif prior["variate"] == "LogNorm":
            prior_Ms = st.lognorm(s=0.55,loc=np.log(0.2))
        elif prior["variate"] == "Chabrier":
            # Chabrier 2004, Eq. 2
            prior_Ms = BrokenPrior(components=[LogNormalPrior(np.log(0.25), 0.55 * np.log(10)),
                                               PowerLawPrior(-1.35, (1.0, 100.0))],
                                   breakpoints=[1.0],
                                   bounds=[min_variate,max_variate])
        else:
            sys.exit("Incorrect prior distribution")
        #------------------------------------------------------------------------

        #------------- Extinction ------------------------------------------------
        if prior["Av"] == "Gamma":
            prior_Av   = st.gamma(a=hyper["loc_Av"],scale=hyper["scl_Av"])
        elif prior["Av"] == "Uniform":
            prior_Av   = st.uniform(loc=hyper["loc_Av"],scale=hyper["scl_Av"])
        elif prior["Av"] == "Gaussian":
            prior_Av   = st.norm(loc=hyper["loc_Av"],scale=hyper["scl_Av"])
        else:
            sys.exit("Incorrect prior distribution")
        #------------------------------------------------------------------------

        #------------- Rv ------------------------------------------------
        if prior["Rv"] == "Gaussian":
            prior_Rv   = st.norm(loc=hyper["loc_Rv"],scale=hyper["scl_Rv"])
        elif prior["Rv"] == "Uniform":
            prior_Rv   = st.uniform(loc=hyper["loc_Rv"],scale=hyper["scl_Rv"])
        else:
            sys.exit("Incorrect prior distribution")
        #------------------------------------------------------------------------

        #--------------- Extra ---------------------------------------------
        prior_Pb   = st.dirichlet(alpha=hyper["alpha_Pb"])
        prior_Yb   = st.norm(loc=np.mean(o_phot),scale=5.00*np.std(o_phot))
        prior_sd_b = st.gamma(a=2.0,scale=hyper["beta_sd_b"])
        prior_sd_m = st.gamma(a=2.0,scale=hyper["beta_sd_m"])
        #-------------------------------------------------------------------

        def lnprior(theta):
            lp_Ms   = prior_Ms.logpdf(theta[0])
            lp_Av   = prior_Av.logpdf(theta[1])
            lp_Rv   = prior_Rv.logpdf(theta[2])
            lp_Pb   = prior_Pb.logpdf(np.array([theta[3],1.-theta[3]]))
            lp_Yb   = prior_Yb.logpdf(theta[4])
            lp_sd_b = prior_sd_b.logpdf(theta[5])
            lp_sd_m = prior_sd_m.logpdf(theta[6])
            return lp_Ms+lp_Av+lp_Rv+lp_Pb+lp_Yb+lp_sd_b+lp_sd_m

        #------------ Initial positions --------------------------------------
        a = (min_variate - initial_hyper["loc_variate"]) / initial_hyper["scl_variate"]
        b = (max_variate - initial_hyper["loc_variate"]) / initial_hyper["scl_variate"]
        self.pos0 = [
                np.array([
                st.truncnorm.rvs(a,b,loc=initial_hyper["loc_variate"] ,
                            scale=initial_hyper["scl_variate"],size=1)[0],
                st.uniform(loc=initial_hyper["loc_Av"],
                            scale=initial_hyper["scl_Av"]).rvs(size=1)[0],
                st.norm(loc=initial_hyper["loc_Rv"],
                            scale=initial_hyper["scl_Rv"]).rvs(size=1)[0],
                st.uniform(loc=0.0,scale=0.05).rvs(size=1)[0],
                prior_Yb.rvs(size=1)[0],
                prior_sd_b.rvs(size=1)[0],
                prior_sd_m.rvs(size=1)[0]
                ])
                for i in range(self.nwalkers)]

        self.lnprior = lnprior
        #-------------------------------------------------------------------

        #---------- Prior RVS ------------------------------------------------
        def prior_sample(n):
            smp = np.array([prior_Ms.rvs(size=n),
                            prior_Av.rvs(size=n),
                            prior_Rv.rvs(size=n),
                            prior_Pb.rvs(size=n)[:,0],
                            prior_Yb.rvs(size=n),
                            prior_sd_b.rvs(size=n),
                            prior_sd_m.rvs(size=n)])
            return smp.T

        self.prior_rvs = prior_sample
        ############################################################################################

    ############################# LIKELIHOOD ###############################################

    def log_likelihood(self,parameters):
        '''
        This log likelihood depends on the variate, photometry and uncertainty of the later
        '''
        variate = parameters[0] # The variate of the star
        Av   = parameters[1] # Extinction
        Rv   = parameters[2] # Ratio total to selective
        Pb   = parameters[3] # The probability of being an outlier
        Yb   = parameters[4] # The mean position of the outlier distribution
        sd_b = parameters[5] # The variance of the outlier distribution
        sd_m = parameters[6] # The variance added to the photometry

        true_phot = np.array([self.variate2phot[i](variate) for i in range(self.N_bands)])

        redden_phot = true_phot + extinction.ccm89(self.waves, Av, Rv)

        t_phot    = redden_phot[self.idx]

        good = (1-Pb)*(1.0/np.sqrt(2.0*np.pi*(self.unc**2+sd_m**2)))*np.exp(-((self.obs-t_phot)**2)/(2.0*(self.unc**2 + sd_m**2)))
        bad  = (Pb)*(1.0/np.sqrt(2.0*np.pi*(self.unc**2 + sd_b**2)))*np.exp(-((self.obs-Yb)**2)/(2.0*(self.unc**2 + sd_b**2))) +1e-200

        return np.sum(np.log(good+bad))

    ################ POSTERIOR#######################
    def lnprob(self,theta):
        not_good_values = (theta[0] > self.max_variate or theta[0] < self.min_variate or
                           theta[1] < 0.0 or
                           theta[2] < 0.0 or theta[2] >= self.max_Rv or
                           theta[3] < 0.0 or theta[3] >= 1.0 or
                           theta[5] < 0.0 or
                           theta[6] < 0.0 )
        if not_good_values:
            return -np.inf

        return self.lnprior(theta) + self.log_likelihood(theta)

    #################### RUN THE SAMPLER ####################################

    def run(self,N_iter):
        sampler = emcee.EnsembleSampler(self.nwalkers,self.ndim, self.lnprob)
        sampler.run_mcmc(self.pos0,N_iter,skip_initial_state_check=True)
        sample = sampler.chain[:,int(self.burnin_frac*N_iter):,:]

        #-----MAP ------------------------------------------------------------
        ind_map = np.unravel_index(np.argmax(sampler.lnprobability),
                                   np.shape(sampler.lnprobability))
        MAP  = sampler.chain[ind_map[0],ind_map[1],:]
        #-------------------------------------------------------------

        #-------- Remove outcast walkers------------------------
        sd = np.std(sample,axis=(0,1))
        idx = np.where(np.abs(sample[:,-1,0] - MAP[0]) < 5.*sd[0])[0]
        sample = sample[idx]
        #-----------------------------------------------------

        #-------- Prior sample ---------------------------------
        prior = self.prior_rvs(sample.shape[0]*sample.shape[1])
        #--------------------------------------------------------

        return MAP,sample,prior,np.mean(sampler.acceptance_fraction)

class Sakam:
    def __init__(self,file_samples,prior,hyperparameters=None,initial_hyper=None,
                quantiles=[0.16,0.84],name_variate=r"Mass $[\mathrm{M_{\odot}}]$"):

        self.name_parameters = [name_variate,r"Av", r"Rv", r"$Pb$", r"$Yb$",r"$Sdb$",r"$Sdm$"]
        self.quantiles       = quantiles
        self.prior           = prior
        self.hyper           = hyperparameters
        self.initial_hyper   = initial_hyper
        self.n_parameters    = len(self.name_parameters)

        #-------- Hyper-parameters --------------
        if self.hyper is None:
            self.hyper = {  "loc_Av":1.0,
                            "scl_Av":2.0,
                            "loc_Rv":3.1,
                            "scl_Rv":0.1,
                            "alpha_Pb":[1,19],
                            "beta_sd_b":1.0,
                            "beta_sd_m":0.1}

        if self.initial_hyper is None:
            self.initial_hyper = {
                            "loc_variate":1.0,
                            "scl_variate":0.1,
                            "loc_Av":0.05,
                            "scl_Av":0.01,
                            "loc_Rv":3.1,
                            "scl_Rv":0.5,
                            }

        #-------------- Files --------------------------
        self.file_samples    = file_samples

    def load_isochrone(self,file_isochrone,variate,covariates,waves,upper_limit_variate=10.0):
        cols_isoc    = sum([[variate],covariates],[])

        print("Loading isochrone")
        isochrone = pd.read_csv(file_isochrone,usecols=cols_isoc,dtype=np.float64)
        print("Dropping duplicated values")
        isochrone = isochrone.drop_duplicates(subset=[variate])
        print("Dropping variate values larger than upper limit: {0:2.1f}".format(upper_limit_variate))
        idx_valid = np.where(isochrone[variate]< upper_limit_variate)[0]
        isochrone = isochrone.iloc[idx_valid,:]
        ####################################################################################################

        ########################### INTERPOLATING FUNCTION ###########################
        '''
        This is intended to give a smooth representation of the variate.
        For a given value of the variate it returns a set of true photometric values.
        '''

        self.variate2phot = [interp1d(isochrone[variate],isochrone[cov],kind="cubic") for cov in covariates]

        self.min_variate  = np.min(isochrone[variate])
        self.max_variate  = np.max(isochrone[variate])

        print("The range of the variate is [{0:2.2f},{1:2.2f}].".format(self.min_variate,self.max_variate))
        ############################################################################################

        self.waves = np.array(waves)

    def load_data(self,file_data,identifier,bands,errors,
                    nan_threshold=3,init_variate=None,nan_values=99.0,*args,**kwargs):
        if init_variate is None:
            columns_data = sum([[identifier],bands,errors],[])
        else:
            columns_data = sum([[identifier],bands,errors,[init_variate]],[])

        observables  = sum([bands,errors],[])
        data         = pd.read_csv(file_data,usecols=columns_data,*args,**kwargs)
        data         = data.reindex(columns=columns_data)

        #----- Nan ----
        data.replace(to_replace=nan_values,value=np.nan,inplace=True)

        #------- index as string ------
        data[identifier] = data[identifier].astype('str')

        #----- put ID as row name-----
        data.set_index(identifier,inplace=True)

        self.data  = data.dropna(thresh=nan_threshold*2,subset=observables)

        self.N       = self.data.shape[0]
        self.n_bands = len(bands)
        self.bands   = bands
        self.errors  = errors
        self.identifier = identifier
        self.init_variate = init_variate

        if init_variate is None:
            self.init_variate = "init_variate"
            self.data.insert(loc=self.data.shape[1],column=self.init_variate,
                            value=self.initial_hyper["loc_variate"])
        ############################################################################################

    def run(self,iterations=4000,walkers_ratio=4,burnin_fraction=0.5):
        #--------- Use existing sources ------------------
        if os.path.exists(self.file_samples):
            print("Using existing samples")
            fh5       = h5py.File(self.file_samples,'r')
            ids       = fh5.keys()
            self.data.drop(ids,inplace=True)
            fh5.close()
            self.fh5  = h5py.File(self.file_samples,'a')

        else:
            self.fh5  = h5py.File(self.file_samples,'w')

        print("Sampling the posterior")

        bar = progressbar.ProgressBar(maxval=self.N).start()
        i   = 0

        for ID,datum in self.data.iterrows():
            grp = self.fh5.create_group(ID)

            init_hyper = self.initial_hyper

            init_hyper["loc_variate"] = datum[self.init_variate]

            #------ Initialize the module --------
            Module = posterior_variate(
                    observed=datum[self.bands].values,
                    uncert=datum[self.errors].values,
                    N_bands=self.n_bands,
                    variate2phot=self.variate2phot,
                    waves=self.waves,
                    prior=self.prior,
                    hyper=self.hyper,
                    initial_hyper=init_hyper,
                    walkers_ratio=walkers_ratio,
                    min_variate=self.min_variate,
                    max_variate=self.max_variate,
                    burnin_frac=burnin_fraction)

            #------- run the module -------------------------------------------------------------
            MAP,pos_smp,pri_smp,acc_fraction = Module.run(N_iter=iterations)

            #----------- Statistics ----------------------------------
            Median = np.median(pos_smp,axis=(0,1))
            SD  = np.std(pos_smp,axis=(0,1))
            CI  = np.quantile(pos_smp,axis=(0,1),q=self.quantiles)
            #---------------------------------------------------------

            #------- Save ------------------------------------
            dset = grp.create_dataset("MAP",    data=MAP)
            dset = grp.create_dataset("Median", data=Median)
            dset = grp.create_dataset("SD",     data=SD)
            dset = grp.create_dataset("CI",     data=CI)
            dset = grp.create_dataset("sample", data=pos_smp)
            dset = grp.create_dataset("prior",  data=pri_smp)
            dset = grp.create_dataset("acc",    data=acc_fraction)
            self.fh5.flush()

            #----- update bar ----
            bar.update(i+1)
            i += 1

        self.fh5.close()

    def plots(self,dir_plots,scale="lin"):
        #------- Plot parameters ---------------------------
        self.nullfmt = plt.NullFormatter()
        left, width = 0.1, 0.4
        bottom, height = 0.1, 0.4
        bottom_h = left_h = left + width + 0.0
        self.rect_scatter = [left, bottom, width, height]
        self.rect_histx   = [left, bottom_h, width, 0.4]
        self.rect_histy   = [left_h, bottom, 0.1, height]
        #---------------------------------------------------

        print("Plotting posterior samples")
        fh5  = h5py.File(self.file_samples,'r')
        for ID,datum in self.data.iterrows():
            grp = fh5.get(ID)
            MAP     = np.array(grp.get("MAP"))
            CI      = np.array(grp.get("CI"))
            sample  = np.array(grp.get("sample"))
            prior   = np.array(grp.get("prior"))
            acc     = np.array(grp.get("acc"))

            if acc < 0.2:
                print("Warning: source {0} has low acceptance fraction!".format(ID))

            self.plot_source(dir_plots,ID,
                    datum[self.bands].values,
                    datum[self.errors].values,
                    sample,prior,MAP,CI,scale=scale)

        fh5.close()

    def statistics(self,file_statistics):
        #----------- Compute statistics -----------------------
        fh5  = h5py.File(self.file_samples,'r')
        ids  = list(fh5.keys())

        #------------ Intitialize arrays and directory ----------------
        N = len(ids)
        maps      = np.zeros((N,self.n_parameters))
        medians   = np.zeros((N,self.n_parameters))
        cis       = np.zeros((N,2,self.n_parameters))

        for i,key in enumerate(ids):
            grp = fh5.get(key)
            maps[i]    = np.array(grp.get("MAP"))
            medians[i] = np.array(grp.get("Median"))
            cis[i]     = np.array(grp.get("CI"))
        fh5.close()

        V_tot = np.sum(maps[:,0])
        V_low = V_tot - np.sum(cis[:,0,0])
        V_up  = np.sum(cis[:,1,0]) - V_tot

        print("Total variate: {0:3.1f}_{1:3.1f}^{2:3.1f}".format(V_tot,V_low,V_up))

        #---------- output -----------
        data = {
                self.identifier:ids,
                "lower_variate": cis[:,0,0],
                "map_variate":    maps[:,0],
                "upper_variate": cis[:,1,0],
                "lower_av":   cis[:,0,1],
                "map_av":      maps[:,1],
                "upper_av":   cis[:,1,1],
                "lower_rv":   cis[:,0,2],
                "map_rv":      maps[:,2],
                "upper_rv":   cis[:,1,2]
                }
        #-------- Save data -------------------------------------
        out = pd.DataFrame(data)
        out.to_csv(path_or_buf=file_statistics,index=False)

    def plot_source(self,dir_plots,ID,observed,uncert,sample,prior,MAP,CI,scale):
        file_plot = dir_plots+"/source_{0}.pdf".format(str(ID))

        pdf = PdfPages(filename=file_plot)
        y_min,y_max= 0.95*np.min(sample[:,:,0]),1.05*np.max(sample[:,:,0])

        fig = plt.figure(221, figsize=(10, 10))
        ax0 = fig.add_subplot(223, position=self.rect_scatter)
        if scale == "log":
            print("Plotting log scale")
            ax0.set_yscale("log")
        ax0.set_xlabel("Iteration")
        ax0.set_ylabel(self.name_parameters[0])
        ax0.set_ylim(y_min,y_max)
        ax0.plot(sample[:,:,0].T, '-', color='black', alpha=0.3,linewidth=0.3)
        ax0.axhline(MAP[0],  color='blue',ls="-",linewidth=0.5,label="MAP")
        ax0.axhline(CI[0,0], color='blue',ls=":",linewidth=0.5,label="CI")
        ax0.axhline(CI[1,0], color='blue',ls=":",linewidth=0.5)
        ax0.legend(loc="upper left",ncol=4,fontsize=4)

        ax1 = fig.add_subplot(224, position=self.rect_histy)
        if scale == "log":
            ax1.set_yscale("log")
        ax1.set_ylim(y_min,y_max)
        ax1.axhline(MAP[0],     color='blue',ls="-",linewidth=0.5,label="MAP")
        ax1.axhline(CI[0,0],    color='blue',ls=":",linewidth=0.5,label="CI")
        ax1.axhline(CI[1,0],    color='blue',ls=":",linewidth=0.5)

        ax1.set_xlabel("Density")
        ax1.yaxis.set_ticks_position('none')

        xticks = ax1.xaxis.get_major_ticks()
        xticks[0].label1.set_visible(False)

        ax1.yaxis.set_major_formatter(self.nullfmt)
        ax1.yaxis.set_minor_formatter(self.nullfmt)

        ax1.hist(sample[:,:,0].flatten(),bins=100,density=True,
            color="k",orientation='horizontal', fc='none', histtype='step',lw=0.5)

        pdf.savefig(bbox_inches='tight')  # saves the current figure into a pdf page
        plt.close()

        plt.figure()
        true_phot = np.array([self.variate2phot[j](MAP[0]) for j in range(self.n_bands)])
        true_phot += extinction.ccm89(self.waves, MAP[1], MAP[2])
        x  = np.arange(self.n_bands)

        # plt.scatter(x,true_phot-true_phot,yerr=,color="grey",label="Model")
        plt.errorbar(x,observed-true_phot,yerr=uncert,fmt=".",label="Observed")
        plt.xticks(x,self.bands,rotation='vertical')
        plt.margins(0.2)
        # Tweak spacing to prevent clipping of tick-labels
        plt.subplots_adjust(bottom=0.15)
        plt.legend(loc="upper right",fontsize=4)
        plt.ylabel("$\\Delta$ Magnitude (observed-modelled)")
        plt.xlabel("Filter")
        pdf.savefig(bbox_inches='tight')
        plt.close()

        #============================ Corner plot ===========================================
        sample_flatten = sample.reshape((sample.shape[0]*sample.shape[1],sample.shape[2]))

        figure = corner.corner(sample_flatten, labels=self.name_parameters,truths=MAP,
                    truth_color="red",
                    quantiles=self.quantiles,
                    show_titles=True,
                    title_kwargs={"fontsize": 12},
                    hist_kwargs={"density":True})

        #---------- Plot prior distributions --------------------
        axes = np.array(figure.axes).reshape((self.n_parameters,self.n_parameters))

        # Loop over the diagonal
        for i in range(self.n_parameters):
            ax = axes[i, i]
            xlim = ax.get_xlim()
            ylim = ax.get_ylim()
            ax.hist(prior[:,i],density=True,histtype="step",color="g",zorder=-1)
            ax.set_xlim(xlim)
            ax.set_ylim(ylim)

        #-------------------------------------------------------
        pdf.savefig(bbox_inches='tight')
        plt.close()
        #===================================================================================
        pdf.close()


from typing import List

class Solution:
    @staticmethod
    def topKFrequent(self, nums: List[int], k: int) -> List[int]:
        """
            Topics: Bucket list
            Solution: Storing the numbers in the bucket list indices which represent the number of
            occurrence of the number. List inside the index represent the numbers that occurred that many times.
            Time complexity: O(N)
            Space complexity: O(N)
        """
        count = {}
        freq = [[] for i in range(len(nums) + 1)]  # + 1 because of the indexing done in the freq list

        for n in nums:
            count[n] = 1 + count.get(n, 0)

        for n, c in count.items():
            freq[c].append(n)

        res = []
        for i in range(len(freq) - 1, 0, -1):
            for n in freq[i]:
                res.append(n)
                if len(res) == k:
                    return res

if __name__ == "__main__":
    print(Solution.topKFrequent(self=Solution, nums=[1, 1, 1, 2, 2, 3], k=1))


class Solution(object):
    def minTotalDistance(self, grid):
        """
        :type grid: List[List[int]]
        :rtype: int

        beats 95.87%
        """
        row_sum = map(sum, grid)
        col_sum = map(sum, zip(*grid))  # syntax sugar learned from stefan :-)

        def minTotalDistance1D(vec):
            i, j = -1, len(vec)
            d = left = right = 0
            while i != j:
                if left < right:
                    d += left
                    i += 1
                    left += vec[i]
                else:
                    d += right
                    j -= 1
                    right += vec[j]
            return d

        return minTotalDistance1D(row_sum) + minTotalDistance1D(col_sum)


import numpy as np
import tensorflow as tf
import tensorflow.contrib.slim as slim
import math

class monodepth(object):
    """monodepth model"""

    def __init__(self, x, model):
        self.use_deconv = False

        self.output = self.build_model(x, model)

    def get_output(self):
        return self.output

    def scale_pyramid(self, img, num_scales):
        scaled_imgs = [img]
        s = tf.shape(img) # img.get_shape().as_list()
        h = s[1]
        w = s[2]
        for i in range(num_scales - 1):
            ratio = 2 ** (i + 1)
            nh = h / ratio
            nw = w / ratio
            scaled_imgs.append(tf.image.resize_area(img, [math.ceil(nh), math.ceil(nw)]))
        return scaled_imgs

    def conv(self, x, num_out_layers, kernel_size, stride, activation_fn=tf.nn.elu):
        p = np.floor((kernel_size - 1) / 2).astype(np.int32)
        p_x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]])
        return slim.conv2d(p_x, num_out_layers, kernel_size, stride, 'VALID', activation_fn=activation_fn)

    def conv_block(self, x, num_out_layers, kernel_size):
        conv1 = self.conv(x,     num_out_layers, kernel_size, 1)
        conv2 = self.conv(conv1, num_out_layers, kernel_size, 2)
        return conv2

    def upconv(self, x, num_out_layers, kernel_size, scale):
        upsample = self.upsample_nn(x, scale)
        conv = self.conv(upsample, num_out_layers, kernel_size, 1)
        return conv

    def deconv(self, x, num_out_layers, kernel_size, scale):
        p_x = tf.pad(x, [[0, 0], [1, 1], [1, 1], [0, 0]])
        conv = slim.conv2d_transpose(p_x, num_out_layers, kernel_size, scale, 'SAME')
        return conv[:,3:-1,3:-1,:]

    def get_disp(self, x):
        disp = 0.3 * self.conv(x, 2, 3, 1, tf.nn.sigmoid)
        return disp

    def upsample_nn(self, x, ratio):
        s = tf.shape(x)
        h = s[1]
        w = s[2]
        return tf.image.resize_nearest_neighbor(x, [h * ratio, w * ratio])

    def maxpool(self, x, kernel_size):
        p = np.floor((kernel_size - 1) / 2).astype(np.int32)
        p_x = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]])
        return slim.max_pool2d(p_x, kernel_size)

    def resconv(self, x, num_layers, stride):
        do_proj = tf.shape(x)[3] != num_layers or stride == 2
        shortcut = []
        conv1 = self.conv(x,         num_layers, 1, 1)
        conv2 = self.conv(conv1,     num_layers, 3, stride)
        conv3 = self.conv(conv2, 4 * num_layers, 1, 1, None)
        if do_proj:
            shortcut = self.conv(x, 4 * num_layers, 1, stride, None)
        else:
            shortcut = x
        return tf.nn.elu(conv3 + shortcut)

    def resblock(self, x, num_layers, num_blocks):
        out = x
        for i in range(num_blocks - 1):
            out = self.resconv(out, num_layers, 1)
        out = self.resconv(out, num_layers, 2)
        return out

    def build_resnet50(self, x):
        # set convenience functions
        conv = self.conv
        if self.use_deconv:
            upconv = self.deconv
        else:
            upconv = self.upconv

        with tf.variable_scope('encoder'):
            conv1 = conv(x, 64, 7, 2)  # H/2  -   64D
            pool1 = self.maxpool(conv1, 3)  # H/4  -   64D
            conv2 = self.resblock(pool1, 64, 3)  # H/8  -  256D
            conv3 = self.resblock(conv2, 128, 4)  # H/16 -  512D
            conv4 = self.resblock(conv3, 256, 6)  # H/32 - 1024D
            conv5 = self.resblock(conv4, 512, 3)  # H/64 - 2048D

        with tf.variable_scope('skips'):
            skip1 = conv1
            skip2 = pool1
            skip3 = conv2
            skip4 = conv3
            skip5 = conv4

        # DECODING
        with tf.variable_scope('decoder'):
            upconv6 = upconv(conv5, 512, 3, 2)  # H/32
            concat6 = tf.concat([upconv6, skip5], 3)
            iconv6 = conv(concat6, 512, 3, 1)

            upconv5 = upconv(iconv6, 256, 3, 2)  # H/16
            concat5 = tf.concat([upconv5, skip4], 3)
            iconv5 = conv(concat5, 256, 3, 1)

            upconv4 = upconv(iconv5, 128, 3, 2)  # H/8
            concat4 = tf.concat([upconv4, skip3], 3)
            iconv4 = conv(concat4, 128, 3, 1)
            disp4 = self.get_disp(iconv4)
            udisp4 = self.upsample_nn(disp4, 2)

            upconv3 = upconv(iconv4, 64, 3, 2)  # H/4
            concat3 = tf.concat([upconv3, skip2, udisp4], 3)
            iconv3 = conv(concat3, 64, 3, 1)
            disp3 = self.get_disp(iconv3)
            udisp3 = self.upsample_nn(disp3, 2)

            upconv2 = upconv(iconv3, 32, 3, 2)  # H/2
            concat2 = tf.concat([upconv2, skip1, udisp3], 3)
            iconv2 = conv(concat2, 32, 3, 1)
            disp2 = self.get_disp(iconv2)
            udisp2 = self.upsample_nn(disp2, 2)

            upconv1 = upconv(iconv2, 16, 3, 2)  # H
            concat1 = tf.concat([upconv1, udisp2], 3)
            iconv1 = conv(concat1, 16, 3, 1)
            disp1 = self.get_disp(iconv1)

        # Set name for output
        out = tf.expand_dims(disp1[:, :, :, 0], 3, name='output_depth')

        return out

    def build_vgg(self, x):
        # set convenience functions
        conv = self.conv
        if self.use_deconv:
            upconv = self.deconv
        else:
            upconv = self.upconv

        with tf.variable_scope('encoder'):
            conv1 = self.conv_block(x,     32,  7)  # H/2
            conv2 = self.conv_block(conv1, 64,  5)  # H/4
            conv3 = self.conv_block(conv2, 128, 3)  # H/8
            conv4 = self.conv_block(conv3, 256, 3)  # H/16
            conv5 = self.conv_block(conv4, 512, 3)  # H/32
            conv6 = self.conv_block(conv5, 512, 3)  # H/64
            conv7 = self.conv_block(conv6, 512, 3)  # H/128
            # conv_block(x, num_out_layers, kernel_size):

        with tf.variable_scope('skips'):
            skip1 = conv1
            skip2 = conv2
            skip3 = conv3
            skip4 = conv4
            skip5 = conv5
            skip6 = conv6

        with tf.variable_scope('decoder'):
            upconv7 = upconv(conv7, 512, 3, 2)  # H/64
            concat7 = tf.concat([upconv7, skip6], 3)
            iconv7 = conv(concat7, 512, 3, 1)

            upconv6 = upconv(iconv7, 512, 3, 2)  # H/32
            concat6 = tf.concat([upconv6, skip5], 3)
            iconv6 = conv(concat6, 512, 3, 1)

            upconv5 = upconv(iconv6, 256, 3, 2)  # H/16
            concat5 = tf.concat([upconv5, skip4], 3)
            iconv5 = conv(concat5, 256, 3, 1)

            upconv4 = upconv(iconv5, 128, 3, 2)  # H/8
            concat4 = tf.concat([upconv4, skip3], 3)
            iconv4 = conv(concat4, 128, 3, 1)
            disp4 = self.get_disp(iconv4)
            udisp4 = self.upsample_nn(disp4, 2)

            upconv3 = upconv(iconv4, 64, 3, 2)  # H/4
            concat3 = tf.concat([upconv3, skip2, udisp4], 3)
            iconv3 = conv(concat3, 64, 3, 1)
            disp3 = self.get_disp(iconv3)
            udisp3 = self.upsample_nn(disp3, 2)

            upconv2 = upconv(iconv3, 32, 3, 2)  # H/2
            concat2 = tf.concat([upconv2, skip1, udisp3], 3)
            iconv2 = conv(concat2, 32, 3, 1)
            disp2 = self.get_disp(iconv2)
            udisp2 = self.upsample_nn(disp2, 2)

            upconv1 = upconv(iconv2, 16, 3, 2)  # H
            concat1 = tf.concat([upconv1, udisp2], 3)
            iconv1 = conv(concat1, 16, 3, 1)
            disp1 = self.get_disp(iconv1)
            # batch_size, height, width, channels = disp1.get_shape().as_list()

        # Set name for output
        out = tf.expand_dims(disp1[:, :, :, 0], 3, name='output_depth')

        return out

    def build_model(self, x, model):
        with slim.arg_scope([slim.conv2d, slim.conv2d_transpose], activation_fn=tf.nn.elu):
            with tf.variable_scope('model', reuse=None):

                #build model
                if model == 'vgg':
                    return self.build_vgg(x)
                elif model == 'resnet':
                    return self.build_resnet50(x)
                else:
                    return None


#!/usr/bin/env python
# coding: utf-8

# In[10]:

from Crypto.Hash import MD5

class ListNode:
    def __init__(self, val):
        self.val = val
        self.next = None

class MyHashSet:
    def __init__(self, capacity=5):
        self.capacity = capacity
        self.data = [None] * capacity

    def add(self, key):

        code = MD5.new()
        code.update(key.encode("utf-8"))
        number = int(code.hexdigest(),16)
        index = number % self.capacity

        if self.data[index] == None:
            self.data[index] = ListNode(number)

        else:
            new_node = ListNode(number)
            new_node.next = self.data[index]
            self.data[index] = new_node

    def remove(self, key):

        while self.contains(key) != False:
            code = MD5.new()
            code.update(key.encode("utf-8"))
            number = int(code.hexdigest(),16)
            index = number % self.capacity
            blank = self.data[index]

            if blank.val == number:
                self.data[index] = blank.next
                self.remove(key)
                return

            pointer = None
            while blank:
                if blank.val == number and blank.next:
                    pointer.next = blank.next
                    break

                elif blank.val == number and blank.next is None:
                    pointer.next = None
                    break

                elif blank.val != number:
                    pointer = blank
                    blank = blank.next

            self.remove(key)

        if self.contains(key) == False:
            return

    def contains(self, key):

        code = MD5.new()
        code.update(key.encode("utf-8"))
        number = int(code.hexdigest(),16)
        index = number % self.capacity
        blank = self.data[index]

        if blank == None:
            return False

        else:
            while blank:
                if blank.val == number:
                    return True

                elif blank.val != number:
                    blank = blank.next

                    if blank == None:
                        return False

# 參考資料：

# https://www.cs.wcupa.edu/rkline/ds/hash-sets.html

# https://www.runoob.com/python/func-number-abs.html #絕對值

# https://blog.csdn.net/qq_40587575/article/details/82431806 #return用法

# https://github.com/yunghsin615/little_sun/blob/master/LeetCode/linked_list.py #我自己的linked-list


import argparse
import os

import numpy as np
import pyecvl.ecvl as ecvl
import pyeddl.eddl as eddl

from lib import utils
from lib.models import Unet, SegNet, SegNetBN

def inference(phase, dataset, epoch, args, num_batches, net, best_miou=0):
    current_path = os.path.join(args.out_dir, phase, f'Epoch_{epoch + 1}')
    evaluator = utils.Evaluator()
    out = eddl.getOut(net)[0]
    dataset.Start()
    out_string = f'- epoch [{epoch + 1}/{args.epochs}] ' if phase == "Validation" else ''

    for b in range(num_batches):
        print(f'{phase} {out_string}- batch [{b + 1}/{num_batches}]')
        samples, x, y = dataset.GetBatch()
        eddl.forward(net, [x])
        output = eddl.getOutput(out)
        current_bs = x.shape[0]
        for bs in range(current_bs):
            img = output.select([str(bs)])
            gt = y.select([str(bs)])
            img_np = np.array(img, copy=False)
            gt_np = np.array(gt, copy=False)
            iou = evaluator.BinaryIoU(img_np, gt_np, thresh=0.5)
            print(f' - IoU: {iou:.3f}', end='', flush=True)
            if args.out_dir:
                os.makedirs(current_path, exist_ok=True)
                # Original image
                orig_img = x.select([str(bs)])
                orig_img.mult_(255.)
                orig_img.normalize_(0., 255.)
                orig_img_t = ecvl.TensorToView(orig_img)
                orig_img_t.colortype_ = ecvl.ColorType.RGB
                orig_img_t.channels_ = 'xyc'
                # Draw the Predicted mask
                img_t = ecvl.TensorToView(img)
                img_t.colortype_ = ecvl.ColorType.GRAY
                img_t.channels_ = 'xyc'
                ecvl.Threshold(img_t, img_t, 0.5, 255)
                tmp, labels = ecvl.Image.empty(), ecvl.Image.empty()
                ecvl.ConvertTo(img_t, tmp, ecvl.DataType.uint8)
                ecvl.ConnectedComponentsLabeling(tmp, labels)
                ecvl.ConvertTo(labels, tmp, ecvl.DataType.uint8)
                contours = ecvl.FindContours(tmp)
                ecvl.ConvertTo(orig_img_t, tmp, ecvl.DataType.uint8)
                tmp_np = np.array(tmp, copy=False)
                for cseq in contours:
                    for c in cseq:
                        tmp_np[c[0], c[1], 0] = 255
                        tmp_np[c[0], c[1], 1] = 0
                        tmp_np[c[0], c[1], 2] = 0
                filename = samples[bs].location_[0]
                head, tail = os.path.splitext(os.path.basename(filename))
                bname = '%s.png' % head
                output_fn = os.path.join(current_path, bname)
                ecvl.ImWrite(output_fn, tmp)
                # Ground truth mask
                gt_t = ecvl.TensorToView(gt)
                gt_t.colortype_ = ecvl.ColorType.GRAY
                gt_t.channels_ = 'xyc'
                gt.mult_(255.)
                gt_filename = samples[bs].label_path_
                gt_fn = os.path.join(current_path,
                                     os.path.basename(gt_filename))
                ecvl.ImWrite(gt_fn, gt_t)
        print()

    dataset.Stop()  # Stop validation split generator

    last_miou = evaluator.MIoU()
    print(f'{phase} {out_string}- Total MIoU: {last_miou:.3f}')

    if phase == "Validation" and last_miou > best_miou:
        best_miou = last_miou
        filepath = os.path.join(args.weights, f'isic_segm_{args.model}_epoch_{epoch + 1}.onnx')
        eddl.save_net_to_onnx_file(net, filepath)
        print('Weights saved')

    return best_miou

def main(args):
    image_size = args.size, args.size

    if args.weights:
        os.makedirs(args.weights, exist_ok=True)

    training_augs = ecvl.SequentialAugmentationContainer([
        ecvl.AugResizeDim(image_size, ecvl.InterpolationType.cubic,
                          gt_interp=ecvl.InterpolationType.nearest),
        ecvl.AugMirror(.5),
        ecvl.AugFlip(.5),
        ecvl.AugRotate([-180, 180]),
        ecvl.AugAdditivePoissonNoise([0, 10]),
        ecvl.AugGammaContrast([0.5, 1.5]),
        ecvl.AugGaussianBlur([0, 0.8]),
        ecvl.AugCoarseDropout([0, 0.03], [0.02, 0.05], 0.25),
        ecvl.AugToFloat32(255, divisor_gt=255),
        ecvl.AugNormalize([0.6681, 0.5301, 0.5247],
                          [0.1337, 0.1480, 0.1595]),  # isic stats

    ])
    validation_test_augs = ecvl.SequentialAugmentationContainer([
        ecvl.AugResizeDim(image_size, ecvl.InterpolationType.cubic,
                          gt_interp=ecvl.InterpolationType.nearest),
        ecvl.AugToFloat32(255, divisor_gt=255),
        ecvl.AugNormalize([0.6681, 0.5301, 0.5247], [
                          0.1337, 0.1480, 0.1595]),  # isic stats

    ])
    dataset_augs = ecvl.DatasetAugmentations([training_augs, validation_test_augs, validation_test_augs])

    print('Reading dataset')
    d = ecvl.DLDataset(args.in_ds, args.batch_size,
                       dataset_augs, ctype=ecvl.ColorType.RGB,
                       num_workers=args.datagen_workers,
                       queue_ratio_size=args.queue_ratio_size)
    num_classes = d.n_channels_gt_
    size = d.n_channels_, args.size, args.size

    if args.ckpts:
        net = eddl.import_net_from_onnx_file(args.ckpts, size)
    else:
        in_ = eddl.Input(size)
        if args.model == 'SegNet':
            out = SegNet(in_, num_classes)
        elif args.model == 'SegNetBN':
            out = SegNetBN(in_, num_classes)
        elif args.model == 'Unet':
            out = Unet(in_, num_classes)
        else:
            raise Exception(f'Invalid model name provided ({args.model})')
        out_sigm = eddl.Sigmoid(out)
        net = eddl.Model([in_], [out_sigm])

    loss_name = 'binary_cross_entropy'
    metric_name = 'mean_squared_error'
    eddl.build(
        net,
        eddl.adam(args.learning_rate),
        [loss_name],
        [metric_name],
        eddl.CS_GPU(args.gpu, mem='low_mem') if args.gpu else eddl.CS_CPU(),
        False if args.ckpts else True  # Initialize weights only with new model
    )

    eddl.summary(net)
    os.makedirs('logs', exist_ok=True)
    eddl.setlogfile(net, 'logs/skin_lesion_segmentation')

    best_miou = 0.
    if args.train:
        num_batches_train = d.GetNumBatches("training")
        num_batches_val = d.GetNumBatches("validation")
        evaluator = utils.Evaluator()

        print('Starting training')
        for e in range(args.epochs):
            d.SetSplit(ecvl.SplitType.training)
            d.ResetBatch(ecvl.SplitType.training, shuffle=True)
            d.Start()
            eddl.reset_loss(net)
            for b in range(num_batches_train):
                _, x, y = d.GetBatch()
                eddl.train_batch(net, [x], [y])
                losses = eddl.get_losses(net)
                metrics = eddl.get_metrics(net)

                print(f'Train - epoch [{e + 1}/{args.epochs}] - batch [{b + 1}/{num_batches_train}]'
                      f' - {loss_name}={losses[0]:.3f} - {metric_name}={metrics[0]:.3f}', flush=True)
            d.Stop()

            d.SetSplit(ecvl.SplitType.validation)
            d.ResetBatch(ecvl.SplitType.validation, shuffle=False)
            evaluator.ResetEval()
            eddl.set_mode(net, 0)

            best_miou = inference("Validation", dataset=d, epoch=e, args=args, num_batches=num_batches_val, net=net,
                             best_miou=best_miou)

    if args.test:
        evaluator = utils.Evaluator()
        evaluator.ResetEval()

        d.SetSplit(ecvl.SplitType.test)
        num_batches_test = d.GetNumBatches("test")
        eddl.set_mode(net, 0)
        inference("Test", dataset=d, epoch=args.epochs, args=args, num_batches=num_batches_test, net=net)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('in_ds', metavar='INPUT_DATASET')
    parser.add_argument('--ckpts', type=str, help='Load an existing ONNX')
    parser.add_argument('--model', type=str, default='Unet',
                        choices=['Unet', 'SegNet', 'SegNetBN'],
                        help='Model to use for training from scratch')
    parser.add_argument('--epochs', type=int, metavar='INT', default=30)
    parser.add_argument('--batch-size', type=int, metavar='INT', default=24)
    parser.add_argument('--learning-rate', type=float, default=1e-5)
    parser.add_argument('--size', type=int, metavar='INT', default=224,
                        help='Size of input slices')
    parser.add_argument('--gpu', nargs='+', type=int, required=False,
                        help='`--gpu 1 1` to use two GPUs')
    parser.add_argument('--out-dir', metavar='DIR', default="",
                        help='if set, save images in this directory')
    parser.add_argument('--weights', metavar='DIR', type=str, default='weights',
                        help='save weights in this directory')

    # Set the pipeline mode: training, testing or both
    parser.add_argument('--train-val', dest='train', action='store_true')
    parser.add_argument('--no-train-val', dest='train', action='store_false')
    parser.add_argument('--test', dest='test', action='store_true')
    parser.add_argument('--no-test', dest='test', action='store_false')
    parser.set_defaults(train=True)
    parser.set_defaults(test=False)

    # Data generator parallelization
    parser.add_argument('--datagen-workers', default=1, type=int,
                        help='Number of worker threads to use for loading the batches')
    parser.add_argument('--queue-ratio-size', default=1, type=int,
                        help=('The producers-consumer queue of the data '
                              'generator will have a maximum size equal to '
                              'batch_size x queue_ratio_size x datagen_workers'))
    main(parser.parse_args())


import matplotlib.pyplot as plt
import numpy
x = numpy.random.randn(1000)
y = numpy.sin(x)
z = numpy.cos(x)
#plt.hist(x,bins="auto",edgecolor="k",alpha=0.5)
#plt.hist(y,bins="auto",ec="k")
#plt.hist(z,bins="auto",ec="k")
plt.hist([x,y,z],bins="auto",ec="k",stacked=True)
plt.xlabel("Age")
plt.ylabel("Population count")
plt.title("Population age count")
plt.show()


import cv2
import numpy as np

hsv_lower = (48, 116, 28)
hsv_upper = (255, 255, 255)

img = cv2.imread('screenshot.png')
blurred = cv2.GaussianBlur(img, (11, 11), 0)  # apply blur
hsv_frame = cv2.cvtColor(blurred, cv2.COLOR_BGR2HSV)  # convert from RGB to HSV colorspace
mask = cv2.inRange(hsv_frame, hsv_lower, hsv_upper)  # create a mask over the object to track
mask = cv2.erode(mask, None, iterations=5)
mask = cv2.dilate(mask, None, iterations=5)

contours, hierarchy = cv2.findContours(mask.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
print("Number of contours = " + str(len(contours)))

if len(contours) > 0:  # make sure at least 1 contour is obtained
    largest_contour = max(contours, key=cv2.contourArea)  # get the largest contour
    bbox = cv2.minAreaRect(largest_contour)  # create a rectangle using the largest found contour
    bbox = cv2.boxPoints(bbox)
    bbox = np.int0(bbox)

cv2.drawContours(img, contours, 1, (0, 255, 0), 3)
frame = cv2.drawContours(img, [bbox], 0, (0, 255, 0), 3)
# cv2.imwrite('bbox.png', img)

cv2.imshow('Image', img)
cv2.waitKey(0)
cv2.destroyAllWindows()


#!/usr/bin/python3

# The exact solution is graphed.  For the derivation of the solution,
# please see the handwritten notes.

from scipy.constants import *
from math import *

from matplotlib import pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import animation

from matplotlib.patches import FancyArrowPatch
from mpl_toolkits.mplot3d import proj3d

class Arrow3D(FancyArrowPatch):
    def __init__(self, xs, ys, zs, *args, **kwargs):
        FancyArrowPatch.__init__(self, (0,0), (0,0), *args, **kwargs)
        self._verts3d = xs, ys, zs

    def draw(self, renderer):
        xs3d, ys3d, zs3d = self._verts3d
        xs, ys, zs = proj3d.proj_transform(xs3d, ys3d, zs3d, renderer.M)
        self.set_positions((xs[0],ys[0]),(xs[1],ys[1]))
        FancyArrowPatch.draw(self, renderer)

gamma=physical_constants['neutron gyromag. ratio'][0]
gamma=-gamma/1e6 # rad/s/uT
# gamma is positive in the physical constants library
# to make it negative, you have to add a minus sign
# I think I've corrected signs below so that either sign can be selected
# After cheating by doing RK, I fixed the signs again.

# Set the field and rate at which it should change, here.
B1=1 # uT
T1=1 # s, time to go around once
omega1=2*pi/T1 # Hz
# At 1 uT, gamma*B1 = 2*pi*(30 Hz)
# So pick an omega1 that's a bit slower than that so we're fairly adiabatic
# I picked omega1=2*pi*(1 Hz) here.

Beff=sqrt(B1**2+(omega1/gamma)**2)
sintheta=omega1/gamma/Beff
costheta=B1/Beff

omega=gamma*Beff

# The 100 in the line below should make sure we get about 100 points
# in each of the revolutions in the rotating frame.
N=abs(int(omega/omega1*100))
print(N)

def spin_rot(t):
    sx=sintheta**2*np.cos(omega*t)+costheta**2
    sy=sintheta*np.sin(omega*t)
    sz=sintheta*costheta*(np.cos(omega*t)-1)
    return sx,sy,sz

def spin(t): # non-rotating frame
    sx=(sintheta**2*np.cos(omega*t)+costheta**2)*np.cos(omega1*t)+sintheta*np.sin(omega*t)*np.sin(omega1*t)
    sy=-(sintheta**2*np.cos(omega*t)+costheta**2)*np.sin(omega1*t)+sintheta*np.sin(omega*t)*np.cos(omega1*t)
    sz=sintheta*costheta*(np.cos(omega*t)-1)
    return sx,sy,sz

def spin_perfect(t): # non-rotating frame, but assuming perfect adiabatic
    sx=np.cos(omega1*t)
    sy=-np.sin(omega1*t)
    sz=0*t
    return sx,sy,sz

fig = plt.figure()
ax = fig.add_subplot(projection='3d')

t=np.linspace(0,T1,N)
sx,sy,sz=spin(t)
sx_perfect,sy_perfect,sz_perfect=spin_perfect(t)
ax.set_xlim(-1,1)
ax.set_ylim(-1,1)
ax.set_zlim(-1,1)

line,=ax.plot(sx[:1],sy[:1],sz[:1])
line2,=ax.plot(sx_perfect[:1],sy_perfect[:1],sz_perfect[:1])
a=Arrow3D([0,sx[0]],[0,sy[0]],[0,sz[0]],mutation_scale=20,arrowstyle="-|>",color="r")
a2=Arrow3D([0,sx_perfect[0]],[0,sy_perfect[0]],[0,sz_perfect[0]],mutation_scale=20,arrowstyle="-|>",color="b")
ax.add_artist(a)
ax.add_artist(a2)

def update(num):
    line.set_xdata(sx[:num])
    line.set_ydata(sy[:num])
    line.set_3d_properties(sz[:num])
    line2.set_xdata(sx_perfect[:num])
    line2.set_ydata(sy_perfect[:num])
    line2.set_3d_properties(sz_perfect[:num])
    a._verts3d=[0,sx[num]],[0,sy[num]],[0,sz[num]]
    a2._verts3d=[0,sx_perfect[num]],[0,sy_perfect[num]],[0,sz_perfect[num]]
    return [line,line2,a2,a]

ani=animation.FuncAnimation(fig,update,frames=N,interval=1,blit=False,repeat=True)
ani.save('asr.mp4')
plt.show()


# coding: utf-8

from __future__ import division, unicode_literals, print_function

class ScalingCaseMaker(object):
    '''A test vector generator for scaling tests'''

    def __init__(self, ref_mem_mb, ref_nnodes, min_mem, max_mem):
        self.min_mem = min_mem
        self.max_mem = max_mem
        self.ref_mem = ref_mem
        self.multiplier = 2

    def make_strong_scaling_stress_cases(self, min_nodes, max_nodes,
                                         max_steps):
        '''Generate cases for strong scaling limit test

        Strong scaling limit tests uses a series of strong scaling tests from
        the maximum memory per node to minimum memory per node, and scales from
        the starting node count to two orders of mangnitidue to identify the
        strong scaling limit of all possible cases. For example, we want to
        investigate the strong scaling from 4 nodes on, and the reference case
        occupies 1/4 of node memory on 4 nodes. The cases are:

            [[(1, 4), (1, 8), ..., (1, 512)],
             [(0, 4), (0, 8), ..., (0, 512)],
             [(-1, 4), (-1, 8), ..., (-1, 512)],
             [(-2, 4), (-2, 8), ..., (-2, 512)],
             [(-3, 4), (-3, 8), ..., (-3, 512)]]

        '''
        cases = []

        mem = self.ref_mem
        nnodes = self.ref_nnodes
        mpn = old_div(mem, nnodes)
        while mpn <= self.max_mpn:
            ratio += 1
            mpn *= 2
        ration -= 1

        mpn = max_mpn
        ratio = 0
        while mpn >= min_mpn:
            cases.extend((ratio, min_nodes * 2**x)
                         for x in range(0, max_node_multipler, steping))
            ratio = ratio - steping
            mpn = max_mpn * 2.0**ratio
        #print(cases)

    def make_strong_weak_matrix_cases(self, min_nodes, max_nodes, max_steps):
        # then generate all expanding cases
        ratio = steping
        nnodes = min_nodes * 2**steping
        # Find the maximal possible starting nnodes so that the largest case won't
        # exceed max_nodes.
        max_start_nnodes = min_nodes
        while max_start_nnodes * 2**(max_node_multipler -
                                     steping) <= max_nodes:
            max_start_nnodes = max_start_nnodes * 2**steping
        max_start_nnodes = old_div(max_start_nnodes, 2**steping)
        while nnodes <= max_start_nnodes:
            cases.extend((ratio, nnodes * 2**x)
                         for x in range(0, max_node_multipler, steping))
            ratio = ratio + steping
            nnodes = nnodes * 2**steping

    def make_weak_scaling_stress_cases(self, min_nodes, max_nodes, max_steps):
        # then put more into weak scaling cases
        all_weak = []
        mpn = max_mpn
        ratio = 0
        while mpn >= min_mpn:
            all_weak.append((ratio, min_nodes))
            ratio = ratio - steping
            mpn = max_mpn * 2.0**ratio
        chosen = [
            int(i * float(len(all_weak) - 1) / float(nweaks))
            for i in range(nweaks)
        ]
        chosen = [all_weak[i] for i in chosen]
        max_weak_portion = 0
        while min_nodes * 2**max_weak_portion <= max_nodes:
            max_weak_portion = max_weak_portion + steping
        max_weak_portion = max_weak_portion - steping
        for _ in chosen:
            cases.extend((ratio + n, min_nodes * 2**n)
                         for n in range(0, max_weak_portion + 1, steping))

        cases = sorted(list(set(cases)))

def main():
    pass

if __name__ == "__main__":
    main()


"""
Leia um valor em hectares (H) e apresente-o convertido em metros quadrados m² (M).
A fórmula de conversão é:
M = H * 10000

"""

# Entradas

h = float(input('Informe o valor em hectares: '))

# Processamento

m = h * 10000

# Saída

print(f'{h} hectares equivalem a {m} m²')


import pytest

from dsw_sdk.common.attributes import AttributeNotSetError
from dsw_sdk.high_level_api.dto.template import TemplateFormat
from dsw_sdk.high_level_api.models.templates.template import Template
from dsw_sdk.high_level_api.models.templates.template_asset import (
    TemplateAsset,
)
from dsw_sdk.high_level_api.models.templates.template_file import TemplateFile

def test_create_template(dsw_sdk, template_data):
    template = Template(dsw_sdk, **template_data)
    template.save()

    assert template.uuid is not None
    assert isinstance(template.formats[0], TemplateFormat)

def test_create_template_missing_attr(template_data):
    del template_data['description']
    template = Template(..., **template_data)
    with pytest.raises(AttributeNotSetError) as e:
        template.save()
    assert e.match(e.value.msg.format('description'))

def test_create_template_with_files(dsw_sdk, template_data):
    template = Template(dsw_sdk, **template_data)
    # When assigning to the list attribute, we can
    # pass dict representation of the object.
    template.files = [{
        'content': 'Some content',
        'file_name': 'some_filename2.html',
    }]
    # While extending an existing list, we must pass the actual object
    template.files.append(
        TemplateFile(dsw_sdk, content='123', file_name='foo2.txt'),
    )

    template.save()
    assert len(template.files) == 2
    for file in template.files:
        assert isinstance(file, TemplateFile)
        assert file.uuid is not None

def test_create_template_with_files_with_id():
    template = Template(..., files=[{
        'content': 'Some content',
        'file_name': 'some_filename.html',
        'template_id': 'some_template_id'
    }])
    with pytest.raises(ValueError):
        template.save()

def test_update_template_with_files_with_id():
    template = Template(..., __update_attrs={'files': [{
        'content': 'Some content',
        'file_name': 'some_filename.html',
        'template_id': 'some_template_id'
    }]})
    template.name = 'some change'
    with pytest.raises(ValueError):
        template.save()

def test_create_template_with_assets(dsw_sdk, template_data):
    template = Template(dsw_sdk, **template_data)
    # When assigning to the list attribute, we can
    # pass dict representation of the object.
    template.assets = [{
        'content': 'Some content',
        'contentType': 'text/html',
        'file_name': 'some_filename.html',
    }]
    # While extending an existing list, we must pass the actual object
    template.assets.append(
        TemplateAsset(
            dsw_sdk,
            content='123',
            content_type='text/plain',
            file_name='foo.txt',
        ),
    )

    template.save()
    assert len(template.assets) == 2
    for asset in template.assets:
        assert isinstance(asset, TemplateAsset)
        assert asset.uuid is not None

def test_create_template_with_assets_with_id():
    template = Template(..., assets=[{
        'content': 'Some content',
        'content_type': 'text/html',
        'file_name': 'some_filename.html',
        'template_id': 'some_template_id'
    }])
    with pytest.raises(ValueError):
        template.save()

def test_update_template_with_assets_with_id():
    template = Template(..., __update_attrs={'assets': [{
        'content': 'Some content',
        'content_type': 'text/html',
        'file_name': 'some_filename.html',
        'template_id': 'some_template_id'
    }]})
    template.name = 'some change'
    with pytest.raises(ValueError):
        template.save()


import shutil, os, send2trash

# PATHS
#############################################################
python_folder = '/home/nfonseca/Pythonprograms'
sample_folder = '/home/nfonseca/Pythonprograms/sample_folder'
bacon_file = sample_folder + '/bacon.txt'
inner_folder = sample_folder + '/inner_folder'
waffles_folder = inner_folder + '/delicious/walnut/waffles'
#os.makedirs(waffles_folder)
#############################################################
print(os.getcwd())
os.chdir(sample_folder)

#os.makedirs(waffles_folder)

# NOTES
#############################################################
# permanently deleting files and folders
#SHUTILS: COPY, MOVE AND REMOVE TREE
# shutil.move() will rename a file if you do not choose a specific folder to move your file.
# shutil.copy() can also rename the file being removed IF YOU WISH SO.
# shutil.rmtree() - will remove folders and the files it contains.

########################
'''def go_to_folder_folder(path):
	os.chdir(path)
	print(os.getcwd())'''
########################

########################
'''
os.chdir(sample_folder)
print(os.getcwd())

for filename in os.listdir(sample_folder):
	if filename.endswith('.txt'):
		os.unlink(filename) # always make sure that we are working in the current directory
'''
########################

# SEND2TRASH MODULE
########################
'''create_bacon = open('bacon.txt', 'w')
create_bacon.write('bacon')
create_bacon.close()
create_spam = open('spam.txt', 'w')
create_spam.write('spam')
create_spam.close()
create_eggs = open('eggs.txt', 'w')
create_eggs.write('eggs')
create_eggs.close()

create_bacon = open('bacon.txt', 'a')
create_bacon.write('\nbacon is not a vegetable')
create_bacon.close()
send2trash.send2trash('bacon.txt')'''
########################


import re
from mylib.aoc_frame import Day

class PartA(Day):
    def parse(self, text, d):  # store puzzle parsing result data into attributes of d
        lines = text.splitlines()
        d.int_lines = [[int(n) for n in re.findall(r"[0-9]+", line)] for line in lines]

    def compute(self, d):  # return puzzle result, get parsing data from attributes of d
        return sum(1 if (a_from <= b_from and b_to <= a_to or
                         b_from <= a_from and a_to <= b_to)
                   else 0
                   for a_from, a_to, b_from, b_to in d.int_lines)

    def tests(self):  # yield testcases as tuple: (test_input, correct_result [, test_name])
        yield self.test_solve(example), 2, "example"

class PartB(PartA):
    def compute(self, d):  # return puzzle result, get parsing data from attributes of d
        return sum(0 if a_to < b_from or b_to < a_from
                   else 1
                   for a_from, a_to, b_from, b_to in d.int_lines)

    def tests(self):  # yield testcases as tuple: (test_input, correct_result [, test_name])
        yield self.test_solve(example), 4, "example"

example = '''
2-4,6-8
2-3,4-5
5-7,7-9
2-8,3-7
6-6,4-6
2-6,4-8
'''

Day.do_day(day=4, year=2022, part_a=PartA, part_b=PartB)


class Solution:
    def removeDuplicates(self, nums: list[int]) -> int:
        res = 0
        p = 0

        while p + 1 < len(nums):
            print(nums)
            if nums[p] == nums[p+1]:
                nums.remove(nums[p+1])
            else:
                res +=1
                p+=1
        print(nums)
        return res

s = Solution()

print(s.removeDuplicates([1,1,2]))


from OrdinaryLeastSquares.PolinomApproximation import PolinomApproximate
from OrdinaryLeastSquares.ExpApproximate import ExpApproximate
from OrdinaryLeastSquares.LogApproximate import LogApproximate
from OrdinaryLeastSquares.PowApproximate import PowApproximate

if __name__ == '__main__':
    print("Введите колв-во точек:")
    n = int(input())
    print("Введите точки (по одной точке на каждой строке, разделяя координаты пробелом)")
    points: list[tuple] = []
    for _ in range(n):
        points.append(tuple(map(float, input().split())))

    approximations = dict()
    approximations['Линейная'] = PolinomApproximate(points, 1)
    approximations['Квадратичная'] = PolinomApproximate(points, polinom_degree=2)
    approximations['Кубическая'] = PolinomApproximate(points, polinom_degree=3)
    approximations['Экспоненциальная'] = ExpApproximate(points)
    approximations['Логарифмическая'] = LogApproximate(points)
    approximations['Степенная'] = PowApproximate(points)

    best_approx_key = "Линейная"
    for key in approximations:
        print(f"{key} апроксимация:")
        print(f"\t f(x) = {approximations[key]}")
        print(f"\t Среднеквадратичное отклонение: {approximations[key].metrix['sigma']}")
        print(f"\t Все характеристики: {approximations[key].metrix}")

        if approximations[best_approx_key].metrix["sigma"] > approximations[key].metrix['sigma']:
            best_approx_key = key

    print(f"{best_approx_key} аппроксимация --- лучшая")


# simple program that distributes 'compute' function' to each node running 'dispynode'
def compute(n):
    import time
    time.sleep(n)
    # dispy_node_name is name of server where this computation is being executed
    return (dispy_node_name, n)

if __name__ == '__main__':
    import dispy, random
    cluster = dispy.JobCluster(compute)
    jobs = []
    for i in range(10):
        # schedule execution of 'compute' on a node (running 'dispynode')
        # with a parameter (random number in this case)
        job = cluster.submit(random.randint(5,20))
        jobs.append(job)
    # cluster.wait() # wait for all scheduled jobs to finish
    for job in jobs:
        host, n = job() # waits for job to finish and returns results
        print('%s (%s) executed job %s at %s with %s' % (host, job.ip_addr, job.id,
                                                         job.start_time, n))
        # other fields of 'job' that may be useful:
        # print(job.stdout, job.stderr, job.exception, job.ip_addr, job.start_time, job.end_time)
    cluster.print_status()


import unittest
from type_system import Type, PolymorphicType, PrimitiveType, Arrow, List, UnknownType, INT, BOOL, STRING
from program import Program, Function, Variable, BasicPrimitive, New
from program_as_list import evaluation_from_compressed, reconstruct_from_compressed

from dsl import DSL
from DSL.deepcoder import semantics,primitive_types

from Algorithms.heap_search import heap_search
# from Algorithms.ray_parallel import make_parallel_pipelines, start

import grammar_splitter

class TestSum(unittest.TestCase):

    # TODO: Uncomment to test Ray
    # def test_ray_parallel_and_splitting(self):
    #     """
    #     Check if ray_parallel + splitting algorithm does not miss any program.
    #     """
    #     t0 = PolymorphicType("t0")
    #     t1 = PolymorphicType("t1")
    #     semantics = {
    #         "RANGE": (),
    #         "HEAD": (),
    #         "TAIL": (),
    #         "SUCC": (),
    #         "PRED": (),
    #         "MAP": (),
    #     }
    #     primitive_types = {
    #         "HEAD": Arrow(List(INT), INT),
    #         "TAIL": Arrow(List(INT), INT),
    #         "RANGE": Arrow(INT, List(INT)),
    #         "SUCC": Arrow(INT, INT),
    #         "PRED": Arrow(INT, INT),
    #         "MAP": Arrow(Arrow(t0, t1), Arrow(List(t0), List(t1))),
    #     }
    #     toy_DSL = DSL(semantics, primitive_types)
    #     type_request = Arrow(List(INT), List(INT))
    #     deepcoder_CFG = toy_DSL.DSL_to_CFG(type_request)
    #     deepcoder_PCFG = deepcoder_CFG.CFG_to_Random_PCFG(alpha=0.8)
    #     n = 100

    #     def insert_prefix(prefix, prog):
    #         try:
    #             head, tail = prog
    #             return (head, insert_prefix(prefix, tail))
    #         except:
    #             return prefix

    #     def bounded_generator(prefix, pcfg):
    #         # It should be easier for
    #         def new_gen():
    #             for p in heap_search(pcfg):
    #                 yield p
    #         return new_gen
    #     make_generators = [bounded_generator(
    #         prefix, pcfg) for prefix, pcfg in grammar_splitter.split(deepcoder_PCFG, 10)[0]]
    #     make_filter = lambda: lambda x: True

    #     producers, filters, _, out = make_parallel_pipelines(make_generators, make_filter, 2, 1000, 10000, 10)
    #     start(producers)
    #     start(filters)

    #     seen_programs = set()
    #     while len(seen_programs) < n:
    #         program = out.get()
    #         # prog = reconstruct_from_compressed(program, r)
    #         self.assertNotIn(program, seen_programs)
    #         seen_programs.add(program)

    def test_splitter(self):
        """
        Check if the grammar splitter generate disjoint valid grammars
        """
        t0 = PolymorphicType("t0")
        t1 = PolymorphicType("t1")
        semantics = {
            "RANGE": (),
            "HEAD": (),
            "TAIL": (),
            "SUCC": (),
            "PRED": (),
            "MAP": (),
        }
        primitive_types = {
            "HEAD": Arrow(List(INT), INT),
            "TAIL": Arrow(List(INT), INT),
            "RANGE": Arrow(INT, List(INT)),
            "SUCC": Arrow(INT, INT),
            "PRED": Arrow(INT, INT),
            "MAP": Arrow(Arrow(t0, t1), Arrow(List(t0), List(t1))),
        }
        toy_DSL = DSL(semantics, primitive_types)
        type_request = Arrow(List(INT), List(INT))
        deepcoder_CFG = toy_DSL.DSL_to_CFG(type_request)
        deepcoder_PCFG = deepcoder_CFG.CFG_to_Uniform_PCFG()

        r = type_request.returns()

        n = 1_0000
        splits = 4
        programs_per_split = n // splits

        def insert_prefix(prefix, prog):
            try:
                head, tail = prog
                return (head, insert_prefix(prefix, tail))
            except:
                return prefix

        seen_programs = set()
        preceding_sets = set()
        for pcfg in grammar_splitter.split(deepcoder_PCFG, splits, alpha=1.05)[0]:
            j = 0
            # If there is one single program for this PCFG SBSUR won't generate it because there is no need for sampling
            if all([len(pcfg.rules[S]) == 1 for S in pcfg.rules]):
                continue
            for p in heap_search(pcfg):
                cp = p
                program = cp #reconstruct_from_compressed(cp, r)
                j += 1
                if j >= programs_per_split or program is None:
                    break
                self.assertNotIn(program, preceding_sets)
                self.assertNotIn(program, seen_programs)
                seen_programs.add(program)
                try:
                    p = deepcoder_PCFG.probability_program(deepcoder_PCFG.start, program)
                except:
                    self.assertFalse(True, "Failed computing probability")
            assert len(seen_programs) > 0
            preceding_sets = preceding_sets.union(seen_programs)
            seen_programs = set()

if __name__ == "__main__":
    unittest.main(verbosity=2)


NFZ = {1234, 3476, 4544, 3423, 3254, 8769, 5436, 2345, 6532, 1243, 6435, 1298, 6732, 7688, 7648, 2345, 2356}
chorzy_rok = set([1234, 3476, 4544, 3423, 3254, 8769, 5436])
chorzy_miesiac = set([1234, 3476, 3098, 4544, 3423])
krzyki = {4544, 3423, 3254, 8769, 5436, 2345, 6532}
centrum = {7648, 2345, 2356, 1243, 3987, 3476, 3254}
wszystkie_zbiory = {'NFZ': NFZ, 'chorzy_rok': chorzy_rok, 'chorzy_miesiac': chorzy_miesiac, 'krzyki': krzyki, 'centrum': centrum}

def czy_ok(pesel):
    return pesel>=10**3 and pesel<=10**4

def wprowadz_pesel():
    while True:
        p=input('podaj pesel ').strip()
        try:
            moze_pesel=int(p)
            if not czy_ok(moze_pesel):
                print('pesel musi miec 4 cyfry')
                continue
            return moze_pesel
        except ValueError:
            print('prosze podac dobry pesel')
pesel=wprowadz_pesel()
if pesel % 2 == 0:
    print('podany pesel ', pesel, ' należy do kobiety')
else:
    print('podany pesel ', pesel, ' należy do mężczyzny')

def sprawdz_PESEL(pesel):
    for name,zbior in wszystkie_zbiory.items():
        if pesel in zbior:
            print(f"Podany pesel to: {pesel} jest z {name}")

print(sprawdz_PESEL(pesel))


from skvideo.datasets import bigbuckbunny

import mediapy as mp

def test_audio():
    """
    读取mp4的音频并保存
    """
    meta, data = mp.read("../imgs/taylor.mp4", media_type='audio')
    print(data.shape, data.dtype)
    # 把音频文件单独写出去，然后播放
    mp.write("taylor.mp3", data, meta.audio_stream.sample_rate, show_log=False)
    print(mp.get_meta('taylor.mp3'))

def test_video():
    filepath = bigbuckbunny()
    meta, a = mp.read(filepath)
    mp.write('a.mp4', a, meta.frame_rate())
    """
    ffmpeg -y -f rawvideo -pix_fmt rgb24 -s 680x480 -i - outputvideo.mp4
    """


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 23 14:47:57 2019

@author: AvantikaDG
"""
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
df = pd.read_csv(r"C:\Users\AvantikaDG\Desktop\Wage Gap\genderpaygay1718.csv")

l = list()
comm = list()
for index, row in df.iterrows():
    for col in df.columns:
        l.append(len(str(row[col])))
    comm.append(len(row))
print(max(l))
print("Min ",min(comm), "Max ", max(comm))

gpg1718 = pd.read_csv(r"C:\Users\AvantikaDG\Desktop\Wage Gap\output.csv")
gpg1819 = pd.read_csv(r"C:\Users\AvantikaDG\Desktop\Wage Gap\genderpaygap1819.csv")
bonusdiff1718 = gpg1718.MaleBonusPercent - gpg1718.FemaleBonusPercent
bonusdiff1819 = gpg1819.MaleBonusPercent - gpg1819.FemaleBonusPercent
colors = ["red"]* len(bonusdiff1718)
colors.extend(["blue"]* len(bonusdiff1819))
a = range(0,len(bonusdiff1718))
plt.scatter(list(a), bonusdiff1718, c = "red")
a = range(0,len(bonusdiff1819))
plt.scatter(list(a), bonusdiff1819, c = "blue")
plt.show()

df = pd.read_csv(r"C:\Users\AvantikaDG\Desktop\Wage Gap\genderpaygap1819.csv")
df.Address = df.Address.str.replace("\r\n", " ")
df.SicCodes = df.SicCodes.str.replace(",\r\n", " ")
df.EmployerName = df.EmployerName.str.replace(",", " ")
df.Address = df.Address.str.replace(",", " ")
df.SicCodes = df.SicCodes.str.replace(",", " ")
df.ResponsiblePerson = df.ResponsiblePerson.str.replace(",", " ")
df.EmployerSize = df.EmployerSize.str.replace(",", " ")
df.CurrentName = df.CurrentName.str.replace(",", " ")

df.to_csv(r"C:\Users\AvantikaDG\Desktop\Wage Gap\wagegap1819.csv", index = False)


import requests
from bs4 import BeautifulSoup
from datetime import datetime
import time

def clock():
    current = datetime.now()
    return(str(current) + " EST")

clock()
headers = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36',
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9'
}

url = 'https://www.yahoo.com/elections'

def president():
    while True:
        page = requests.get(url, headers=headers)

        soup = BeautifulSoup(page.text, 'html.parser')

        for info in soup.find_all('div', attrs={"class":"balance balance--presidential balance--counting"}):
            percent = info.find_all('div', class_="balance__below")
            #Gets percentage of Biden votes
            joe_percent = percent[0].find_all('span')[0].text
            joe_votes = percent[0].find_all('span')[1].text

            # Gets percentage of Trump votes
            donald_percent = percent[0].find_all('span')[3].text
            donald_votes = percent[0].find_all('span')[2].text
            #print(percent)

            # Gets Biden name and electoral votes
            biden = info.find_all("div", class_="side__content")[0]
            biden_name = biden.find("div").text
            biden_elect = biden.find(class_="side__num").text

            #Gets trump name and electoral votes
            trump = info.find_all("div", class_="side__content")[1]
            trump_name = trump.find("div").text
            trump_elect = trump.find(class_="side__num").text

            print(clock(), '::: The candidates need 270 electoral votes to win')
            print(clock(), ':::',biden_name, 'has',biden_elect,'electoral votes with',joe_percent, 'of the popular vote and', joe_votes, 'total')
            print(clock(), ':::',trump_name, 'has',trump_elect,'electoral votes with',donald_percent, 'of the popular vote and', donald_votes, 'total')
            #refeshes page every 30 seconds
            time.sleep(30)
            print()
president()


"""
给你一个由 '1'（陆地）和 '0'（水）组成的的二维网格，请你计算网格中岛屿的数量。

岛屿总是被水包围，并且每座岛屿只能由水平方向和/或竖直方向上相邻的陆地连接形成。

此外，你可以假设该网格的四条边均被水包围。

来源：力扣（LeetCode）
链接：https://leetcode-cn.com/problems/number-of-islands
著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。

类似的题 542 01矩阵 面试题13
"""

class Solution:
    def numIslands(self, grid: list) -> int:
        # 64ms 96.81%
        count = 0
        queue = []
        # 获取grid的长宽
        m = len(grid)
        if m:
            n = len(grid[0])
        else:
            return 0
        for i in range(m):
            for j in range(n):
                if grid[i][j] == '1':
                    # 找到了一块岛屿
                    count += 1
                    # 将这块岛屿置为0
                    queue.append((i, j))
                    while len(queue) > 0:
                        i_t, j_t = queue.pop()
                        grid[i_t][j_t] = '0'
                        # 四个方向扩张
                        if 0 <= i_t-1 < m and grid[i_t-1][j_t] == '1':  # 上
                            queue.append((i_t-1, j_t))
                        if 0 <= j_t+1 < n and grid[i_t][j_t+1] == '1':  # 右
                            queue.append((i_t, j_t+1))
                        if 0 <= i_t+1 < m and grid[i_t+1][j_t] == '1':  # 下
                            queue.append((i_t+1, j_t))
                        if 0 <= j_t-1 < n and grid[i_t][j_t-1] == '1':  # 左
                            queue.append((i_t, j_t-1))
        return count

    def numIslands2(self, grid: List[List[str]]) -> int:
        import collections
        count = 0
        for row in range(len(grid)):
            for col in range(len(grid[0])):
                if grid[row][col] == '1':  # 发现陆地
                    count += 1  # 结果加1
                    grid[row][col] = '0'  # 将其转为 ‘0’ 代表已经访问过
                    # 对发现的陆地进行扩张即执行 BFS，将与其相邻的陆地都标记为已访问
                    # 下面还是经典的 BFS 模板
                    land_positions = collections.deque()
                    land_positions.append([row, col])
                    while len(land_positions) > 0:
                        x, y = land_positions.popleft()
                        for new_x, new_y in [[x, y + 1], [x, y - 1], [x + 1, y], [x - 1, y]]:  # 进行四个方向的扩张
                            # 判断有效性
                            if 0 <= new_x < len(grid) and 0 <= new_y < len(grid[0]) and grid[new_x][new_y] == '1':
                                grid[new_x][new_y] = '0'  # 因为可由 BFS 访问到，代表同属一块岛，将其置 ‘0’ 代表已访问过
                                land_positions.append([new_x, new_y])
        return count

if __name__ == "__main__":
    test = [['1', '1', '0', '0', '0'],
            ['1', '1', '0', '0', '0'],
            ['0', '0', '1', '0', '0'],
            ['0', '0', '0', '1', '1']]
    test2 = [["1", "1", "1"],
             ["0", "1", "0"],
             ["1", "1", "1"]]

    s = Solution()
    print(s.numIslands(test))