import numpy as np
import pandas as pd
from plotnine import ggplot, aes, geom_bar, geom_line, theme, element_text, coord_flip, ylab, xlab, scale_x_datetime
from mizani.breaks import date_breaks
from mizani.formatters import date_format
import re

# Genere le graphe distribution des films par catégorie et le sauvegarde sous format png
def generate_graph_type_occurrences(movies):

    # Generer  une série pour l'apparition de chaque catégorie dans un film
    types = []
    for movie in movies:
        types += movie.types;

    # Generer un dataframe à partir de la série types et faire le counts des occurences de chaque type
    df = pd.DataFrame(types, columns=['type'])
    result = df["type"].value_counts().rename_axis('type').reset_index(name='occurrences')

    # Dessiner le graphe
    p = (
        ggplot(result, aes(x="reorder(type, -occurrences)", y="occurrences"))
        + geom_bar(size=20, stat = "identity")
        + theme(axis_text_x = element_text(angle = 90, hjust = 1))
        + xlab("Category")
        + ylab("Occurrences")
    )
    p.save("static/generated_graph_type_occurrences.png", dpi = 600)

# Genere le graphe top 50 des budgets des films et le sauvegarde sous format png
def generate_graph_top50_budget(movies):
    # Filtrer la liste des films pour avoir uniquement ceux avec budget en dollar
    # Dans le process de scrapping les films avec budget en dollar sont recuperés sans l'unité ($)
    movies = list(filter(lambda x: re.search("^[0-9]+$", x.budget), movies))

    # Convertir la colonne budget en int et la stocker dans une nouvelle colonne budget_int
    for movie in movies:
        movie.budget_int = int(movie.budget)

    # Trier les films par budget et prendre les 50 premiers
    movies = sorted(movies, key=lambda x: x.budget_int, reverse=True)[0:50]

    # Generer un dataframe avec les deux colonnes name et budget_int
    columns = ["name", "budget_int"]
    df = pd.DataFrame([[getattr(movie, columns) for columns in columns] for movie in movies], columns = columns)

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='reorder(name, budget_int)', y="budget_int"))
        + geom_bar(size=20, stat = "identity", position="dodge")
        + theme(axis_text_y = element_text( size = 5))
        + coord_flip()
        + xlab("Title")
        + ylab("Budget")
    )
    p.save("static/generated_graph_top50_budget.png", dpi = 600)

# Genere le graphe top 20 films ayant générés le plus de revenu et le sauvegarder sous format png
def generate_graph_top20_cumulative_worldwide_gross(movies):
    # Filter la liste des films pour avoir uniquement ceux avec cumulative_worldwide_gross en dollar
    # Dans le process de scrapping les films avec cumulative_worldwide_gross en dollar sont recuperés sans l'unité ($)
    movies = list(filter(lambda x: re.search("^[0-9]+$", x.cumulative_worldwide_gross), movies))

    # Convertir la colonne cumulative_worldwide_gross en int et la stocker dans une nouvelle colonne cumulative_worldwide_gross_int
    for movie in movies:
        movie.cumulative_worldwide_gross_int = int(movie.cumulative_worldwide_gross)

    # Trier les films par cumulative_worldwide_gross et prendre les 20 premiers
    movies = sorted(movies, key=lambda x: x.cumulative_worldwide_gross_int, reverse=True)[0:20]

    # Generer un data frame avec les deux colonnes name et cumulative_worldwide_gross_int
    columns = ["name", "cumulative_worldwide_gross_int"]
    df = pd.DataFrame([[getattr(movie, columns) for columns in columns] for movie in movies], columns = columns)

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='reorder(name, cumulative_worldwide_gross_int)', y="cumulative_worldwide_gross_int"))
        + geom_bar(size=20, stat = "identity", position="dodge")
        + theme(axis_text_y = element_text( size = 5))
        + coord_flip()
        + xlab("Title")
        + ylab("Cumulative worldwide gross")
    )
    p.save("static/generated_graph_top20_cumulative_worldwide_gross.png", dpi = 600)

# Genere le graphe top 20 films ayant générés le moins de revenu et le sauvgarde sous format png
def generate_graph_less20_cumulative_worldwide_gross(movies):
    # Filter la liste des films pour avoir uniquement ceux avec cumulative_worldwide_gross en dollar
    # Dans le process de scrapping les films avec cumulative_worldwide_gross en dollar sont recuperés sans l'unité ($)
    movies = list(filter(lambda x: re.search("^[0-9]+$", x.cumulative_worldwide_gross), movies))

    # Convertir la colonne cumulative_worldwide_gross en int et la stocker dans une nouvelle colonne cumulative_worldwide_gross_int
    for movie in movies:
        movie.cumulative_worldwide_gross_int = int(movie.cumulative_worldwide_gross)

    # Trier les films par cumulative_worldwide_gross (ascendants) et prendre les 20 premiers
    movies = sorted(movies, key=lambda x: x.cumulative_worldwide_gross_int, reverse=False)[0:20]

    # Generer un dataframe avec les deux colonnes name et cumulative_worldwide_gross_int
    columns = ["name", "cumulative_worldwide_gross_int"]
    df = pd.DataFrame([[getattr(movie, columns) for columns in columns] for movie in movies], columns = columns)

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='reorder(name, -cumulative_worldwide_gross_int)', y="cumulative_worldwide_gross_int"))
        + geom_bar(size=20, stat = "identity", position="dodge")
        + theme(axis_text_y = element_text( size = 5))
        + coord_flip()
        + xlab("Title")
        + ylab("Cumulative worldwide gross")
    )
    p.save("static/generated_graph_less20_cumulative_worldwide_gross.png", dpi = 600)

# Genere le graph top 10 des films par retour sur investissement et le sauvgarde sous format png
def generate_graph_top10_benefit(movies):
    # Filter la liste des films pour avoir uniquement ceux avec cumulative_worldwide_gross et budget en dollar
    # Dans le process de scrapping les films avec cumulative_worldwide_gross (ou budget) en dollar sont recuperés sans l'unité ($)
    movies = list(filter(lambda x: re.search("^[0-9]+$", x.cumulative_worldwide_gross + x.budget), movies))

    # Calculer le bénifices generer par le film (cumulative_worldwide_gross - budget)
    for movie in movies:
        movie.benefit = int(movie.cumulative_worldwide_gross) - int(movie.budget)

    # Trier les films par benefit et prendre les 10 premiers
    movies = sorted(movies, key=lambda x: x.benefit, reverse=True)[0:10]

    # Generer un data frame avec les deux colonnes name et benefit
    columns = ["name", "benefit"]
    df = pd.DataFrame([[getattr(movie, columns) for columns in columns] for movie in movies], columns = columns)

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='reorder(name, benefit)', y="benefit"))
        + geom_bar(size=20, stat = "identity", position="dodge")
        + theme(axis_text_y = element_text( size = 5))
        + coord_flip()
        + xlab("Title")
        + ylab("Benifit")
    )
    p.save("static/generated_graph_top10_benefit.png", dpi = 600)

# Genere le graphe note moyenne par année de sortie de film et le sauvgarde sous format png
def generate_graph_average_rating_per_release_year(movies):
    # Filter et garder uniquement les film avec les deux données release_date et rating presentes
    movies = list(filter(lambda x: x.release_date != "XXX" and x.rating != "XXX", movies))

    # Convertir le rating en float (rating_float) et release_date en release_year (garder uniquement l'année)
    for movie in movies:
        movie.release_year = movie.release_date[-4:]
        movie.rating_float = float(movie.rating)

    # Créer le dataframe avec les deux colonne release_year et rating_float
    columns = ["release_year", "rating_float"]
    df = pd.DataFrame([[getattr(movie, columns) for columns in columns] for movie in movies], columns = columns)

    # convertir la colonne release_year et datetime
    df["release_year"] = pd.to_datetime(df['release_year'], format="%Y")

    # Generer un dataframe du groupby appliqué sur la colonne release_year et la moyenne appliquée sur la colonne rating_float
    df = df.groupby('release_year', sort=True).mean()["rating_float"].rename_axis('release_year').reset_index(name='rating_float')

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='release_year', y="rating_float"))
        + geom_line()
        + scale_x_datetime(breaks=date_breaks('2 year'), labels=date_format('%Y'))
        + theme(axis_text_x = element_text(angle = 90, hjust = 1, size=5))
        + xlab("Year")
        + ylab("Rating average")
    )
    p.save("static/generated_graph_average_rating_per_release_year.png", dpi = 600)

# Genere le graphe distribution des revenus générés par genre de film et le sauvegarde sous format png
def generate_graph_cumulative_worldwide_gross_peer_type(movies):
    # Filter la liste des films pour avoir uniquement ceux avec cumulative_worldwide_gross en dollar
    # Dans le process de scrapping les films avec cumulative_worldwide_gross en dollar sont recuperés sans l'unité ($)
    movies = list(filter(lambda x: re.search("^[0-9]+$", x.cumulative_worldwide_gross), movies))

    # Convertir la colonne cumulative_worldwide_gross en int et la stocker dans une nouvelle colonne cumulative_worldwide_gross_int
    for movie in movies:
        movie.cumulative_worldwide_gross_int = int(movie.cumulative_worldwide_gross)

    # Generer les deux series des apparitions des types et de leurs cumulative_worldwide_gross (divisé par le nombre de type de chaque film)
    data = {"type":[], "cumulative_worldwide_gross":[]}
    for movie in movies:
        for type in movie.types:
            data["cumulative_worldwide_gross"].append(movie.cumulative_worldwide_gross_int / len(movie.types))
            data["type"].append(type)

    # Generer un dataframe avec les deux series type cumulative_worldwide_gross
    df = pd.DataFrame(data, columns = ["type", "cumulative_worldwide_gross"])

    # Generer un dataframe du groupby appliqué sur la colonne type et la somme appliquée sur la colonne cumulative_worldwide_gross
    df = df.groupby('type', sort=True).sum()["cumulative_worldwide_gross"].rename_axis('type').reset_index(name='cumulative_worldwide_gross')

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='reorder(type, cumulative_worldwide_gross)', y="cumulative_worldwide_gross"))
        + geom_bar(size=20, stat = "identity")
        + coord_flip()
        + theme(axis_text_x = element_text(angle = 90, hjust = 1, size=5))
        + xlab("Category")
        + ylab("Cumulative worldwide gross")
    )
    p.save("static/generated_graph_cumulative_worldwide_gross_peer_type.png", dpi = 600)

# Genere le graphe nombre de films sortie par décennie et le sauvgarde sous format png
def generate_graph_number_of_films_released_per_decade(movies):
    # Generer une serie des décennies
    decades = []
    for movie in movies:
        if (movie.release_date != "XXX"): decades.append(movie.release_date[-4:-1] + "0")

    # Generer un dataframe à partir de la séries décennies et faire le counts des occurences de chaque décennies
    df = pd.DataFrame(decades, columns =['decades'])
    df = df["decades"].value_counts().rename_axis('decades').reset_index(name='occurrences')

    # Convertir decades en datetime
    df["decades"] = pd.to_datetime(df['decades'], format="%Y")

    # Dessiner le graphe
    p = (
        ggplot(df, aes(x='decades', y="occurrences"))
        + geom_line()
        + scale_x_datetime(breaks=date_breaks('10 year'), labels=date_format('%Y'))
        + theme(axis_text_x = element_text(angle = 90, hjust = 1, size=5))
        + xlab("Decades")
        + ylab("Nb films released")
    )
    p.save("static/generated_graph_number_of_films_released_per_decade.png", dpi = 600)


# coding: utf-8

# ### import spacy
# import requests
# import it_core_news_sm
# from bs4 import BeautifulSoup as bs
# import json
#
# def ecrire_json(chemin , contenu):
#     w = open(chemin , "w", encoding="utf-8")
#     w.write(json.dumps(contenu , indent=2, ensure_ascii=False))
#     w.close()

# In[42]:

import os

repertoire_json_resultatSpacy = "resultatSpacy"
if not os.path.exists(repertoire_json_resultatSpacy):
    os.makedirs(repertoire_json_resultatSpacy)

repertoire_json_resultatBabelfy = "resultatBabelfy"
if not os.path.exists(repertoire_json_resultatBabelfy):
    os.makedirs(repertoire_json_resultatBabelfy)

repertoire_json_resultatSpacy = "resultatSpacy"
if not os.path.exists(repertoire_json_resultatSpacy):
    os.makedirs(repertoire_json_resultatSpacy)

# In[43]:

URL ="https://babelfy.io/v1/disambiguate?"    #babelfly

lang = 'IT'

key  = ""

if not key:
    raise ValueError("No Babelfy key provided, please modify source code")

# In[48]:

import ast,re # module pour détecter le format de liste dans un string

with open("wikiItalien.xml", "r", encoding="utf-8") as file:
    content=file.read()
    textes= []
    for texte in re.finditer('<text .*>((\n|.|\t)*?)</text>', content):
        contenu=texte.group(1)
        match=re.search("==", contenu)
        if match!=None:
            contenu = texte.group(1)[0:match.start()]
        textes.append(contenu)
    par = {
    "text" : "",
    "lang" : "IT",
    "key" : key
}
    print(textes[6])# à enlever
    nlp = it_core_news_sm.load()
    for cpt, i in enumerate(textes):
        dico={}
        dicBabel={}
        if len(i) < 6000:  # vérifier que le fichier n'est pas trop grand (valeur maximale à vérifier)
            par['text']= i
            r = requests.get(url = URL, params = par)
            dicBabel = json.dumps(r.json())
            dicBabel = ast.literal_eval(dicBabel) # transformation de string en liste
            doc=nlp(i)
            nom = "DOC_%s" % (cpt+1)
            for ent in doc.ents:
                resultatSpacy=ent.start_char,ent.end_char,ent.label_
                entite=list(resultatSpacy)
                dico[ent.text] = entite
            ecrire_json("resultatSpacy/Spacy_%s.json" % nom,dico)
            ecrire_json("resultatBabelfy/Babel_%s.json" % nom,dicBabel)

# In[ ]:


'''

PERCEPTRON MULTICOUCHE  - MLP

'''
#%%
import numpy.random as rd
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split , GridSearchCV
import numpy as np
from sklearn.metrics import accuracy_score
import random as r
from sklearn.datasets import load_digits
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score
import matplotlib.pyplot as plt
from sklearn import tree, datasets
from sklearn.svm import SVC
from svm_source import *
from sklearn import svm
#%%

def randX(n):
    x=rd.randint(0,2,(n,2))
    return(x)

#%%
def YAND(X):
    C = []
    for i in range(0,X.shape[0]):
        if X[i,0]==X[i,1]==1:
            C.append(1)
        else:
            C.append(0)
    return(C)

#%%
def YOR(X):
    D = []
    for i in range(0,X.shape[0]):
        if X[i,0]==1 or X[i,1]==1:
            D.append(1)
        else:
            D.append(0)
    return(D)
#%%
def YXOR(X):
    E = []
    for i in range(0,X.shape[0]):
        if (X[i,0]==1 and X[i,1]==0) or (X[i,0]==0 and X[i,1]==1):
            E.append(1)
        else:
            E.append(0)
    return(E)

# %%

X = randX(100)
Y1 = YAND(X)
Y2 = YOR(X)
Y3 = YXOR(X)

# %% question 2

X_train1, X_test1, y_train1, y_test1 = train_test_split(X, Y1,test_size=0.1)
clfAND = MLPClassifier(hidden_layer_sizes=(),activation='identity',solver='lbfgs').fit(X_train1, y_train1)

print("score : ",accuracy_score(y_test1,clfAND.predict(X_test1)))

#%% question 3

X_train2, X_test2, y_train2, y_test2 = train_test_split(X, Y2,test_size=0.1)

clfO = MLPClassifier(hidden_layer_sizes=(),activation='identity',solver='lbfgs').fit(X_train2, y_train2)
accuracy_score(y_test2,clfO.predict(X_test2))

# %% question 4
# a
X_train3, X_test3, y_train3, y_test3 = train_test_split(X, Y3,test_size=0.1)
clfXOR = MLPClassifier(hidden_layer_sizes=(),activation='identity',solver='lbfgs').fit(X_train3, y_train3)
accuracy_score(y_test3,clfXOR.predict(X_test3))
# %%
#b
X_train4, X_test4, y_train4, y_test4 = train_test_split(X, Y3,test_size=0.1)

clfXOR = MLPClassifier(hidden_layer_sizes=(4,2),activation='identity',solver='lbfgs').fit(X_train3, y_train3)
accuracy_score(y_test4,clfXOR.predict(X_test4))
# %%
#c

X_train5, X_test5, y_train5, y_test5 = train_test_split(X, Y3,test_size=0.1)
score4 = np.zeros(100)
for j in range (0,len(score4)):
    clfXOR = MLPClassifier(hidden_layer_sizes=(4,2),activation='tanh',solver='lbfgs').fit(X_train5, y_train5)
    score4[j]=clfXOR.score(X_test5, y_test5)

print("score ",score4) #
# %%
digits = load_digits()

n_samples = len(digits.data)
x_train, x_test, Y_train, Y_test = train_test_split(digits.data, digits.target, test_size=0.1)

#%% couche
test = 100
score = 0
param = [0,'','','']
for parameter in range(test):

    hls = r.randint(100, 1000)
    activ = r.choice(['identity', 'logistic', 'tanh', 'relu'])
    lr = r.choice(['constant', 'invscaling', 'adaptive'])
    solv = r.choice(['lbfgs', 'sgd', 'adam'])
    clfdig= MLPClassifier(hidden_layer_sizes=hls,activation=activ,learning_rate=lr,solver=solv).fit(x_train,Y_train)
    scoreclf = accuracy_score(Y_test,clfdig.predict(x_test))
    if scoreclf > score:
        param[0]=hls
        param[1]=activ
        param[2]=lr
        param[3]=solv
        score = scoreclf
#%%
param1=[472, 'logistic', 'lbfgs']

#MLPClassifier(hidden_layer_sizes=(472),activation=('logistic'),solver='lbfgs')

# %%

test = 10
score2 = score
param2 = [0,'','','']
score3=0
for parameter in range(test):

    hls = r.randint(100, 1000)
    activ = r.choice(['identity', 'logistic', 'tanh', 'relu'])
    lr = r.choice(['constant', 'invscaling', 'adaptive'])
    solv = r.choice(['lbfgs', 'sgd', 'adam'])
    clfdig1= MLPClassifier(hidden_layer_sizes=(param1[0],hls),activation=(param1[1]),learning_rate=(param1[2]),solver=(param1[3])).fit(x_train,Y_train)
    scoreclf = accuracy_score(Y_test,clfdig1.predict(x_test))
    if scoreclf > score2:
        param[0]=hls
        param[1]=activ
        param[2]=lr
        param[3]=solv
        score3 = scoreclf

#Useless 2 couches

#%%

df=MLPClassifier(hidden_layer_sizes=(param1[0]),activation=(param1[1]),solver=(param1[2]))
acc = cross_val_score(df, x_train, Y_train, cv=5, n_jobs=-1)
print("The error using MLP method equals {:.3f}%.".format((1 - acc.mean()) * 100))
#%%

#linear classifier SVM

parameters = {'kernel': ['linear'], 'C': list(np.logspace(-3, 3, 20))}
svr = svm.SVC()
acc = cross_val_score(svr, x_train, Y_train, cv=5)
clf_linear = GridSearchCV(svr, parameters)
clf_linear.fit(x_train, Y_train)
print('Generalization score for linear kernel: %s' %((1-max(sorted(clf_linear.cv_results_['mean_test_score'])))*100),'%')

# %%
# fit a classifier (poly) and test all the Cs
parameters = {'kernel': ['poly'], 'C': list(np.logspace(-3, 3, 20))}
svr = svm.SVC()
clf_linear = GridSearchCV(svr, parameters)
clf_linear.fit(x_train, Y_train)

print('Generalization score for linear kernel: %s' %((1-max(sorted(clf_linear.cv_results_['mean_test_score'])))*100),'%')
# %%
# fit a classifier (rbf) and test all the Cs
parameters = {'kernel': ['rbf'], 'C': list(np.logspace(-3, 3, 20))}
svr = svm.SVC()
clf_linear = GridSearchCV(svr, parameters)
clf_linear.fit(x_train, Y_train)

print('Generalization score for linear kernel: %s' %((1-max(sorted(clf_linear.cv_results_['mean_test_score'])))*100),'%')
# %%
# fit a classifier (sigmoid) and test all the Cs
parameters = {'kernel': ['sigmoid'], 'C': list(np.logspace(-3, 3, 20))}
svr = svm.SVC()
clf_linear = GridSearchCV(svr, parameters)
clf_linear.fit(x_train, Y_train)

print('Generalization score for linear kernel: %s' %((1-max(sorted(clf_linear.cv_results_['mean_test_score'])))*100),'%')

###### ANNEXE ############

# %%
L = []
for k in range(10) :
    s = 0
    for i in range(len(digits.target)) :
        if digits.target[i] == k :
            s = s +1
    L.append(s)
# %%
inter = [-0.5, 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5]

plt.hist(digits.target, bins=inter, rwidth=0.8)  # Création de l'histogramme
plt.xlabel('digits')
plt.xticks(np.arange(0, 10))
plt.ylabel('Effectif total')
plt.title("Répartition des digits par labels")
plt.show()

# %%
#%%
def transfo_color(Y):
    A = []
    for i in range(len(Y)):
        if Y[i]==0:
            A+=["#FF3030"]
        if Y[i]==1:
            A+=["#3D9140"]
    return A

X = np.array([
    [0,0],
    [0,1],
    [1,0],
    [1,1]
]
)

Y = [
    transfo_color(YOR(X)),
    transfo_color(YAND(X)),
    transfo_color(YXOR(X))
    ]

Title = ["OR", "AND", "XOR"]
u = np.linspace(0,1,10)
lines = [[0.5-u],[1.5-u],[]]

for i in range(len(Y)):
    plt.xlim([-0.2, 1.2])
    plt.ylim([-0.2, 1.2])
    plt.xticks([0,1])
    plt.yticks([0,1])
    plt.axis('off')
    #plt.title(Title[i])
    plt.scatter(
        X[:,0],
        X[:,1],
        s= 500,
        c = Y[i],
        marker="o")
    for j in range(len(Y[i])):
        plt.text(X[j][0]-0.04,X[j][1]-0.15,"("+str(X[j][0])+","+str(X[j][1])+")")
    for l in range(len(lines[i])):
        plt.plot(u,lines[i][l],color="b")
    plt.show()

# %%


from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponseRedirect
from django.http import HttpResponse, JsonResponse
from django.contrib import messages
from .forms import AdminRegistrationForm, UserLoginForm, AppointmentForm, AppointmentRequestForm, AppointmentResponseForm, MessageForm
from django.contrib.auth.decorators import login_required
from .models import *
from . import forms, models
from django.db.models import Sum
from django.contrib.auth.models import Group
from django.contrib.auth.decorators import login_required, user_passes_test
from datetime import datetime, timedelta, date

# Create your views here.

def homePage(request):
    categories = ConsultancyCategories.objects.all()
    context = {'categories' : categories}
    return render(request, 'baseapp/homePage.html', context)

def consultantsList(request):
    consultants = Consultant.objects.all()
    context = {'consultants' : consultants}
    return render(request, 'baseapp/consultants_list.html', context)

#---------------------------------BASE PRELOGIN VIEWS--------------------------#
def home_view(request):
    if request.user.is_authenticated:
        return HttpResponseRedirect('afterlogin')
    return render(request,'baseapp/homePage.html')

#for showing signup/login button for admin(by sumit)
def adminclick_view(request):
    if request.user.is_authenticated:
        return HttpResponseRedirect('afterlogin')
    return render(request,'baseapp/adminclick.html')

#for showing signup/login button for doctor(by sumit)
def consultantclick_view(request):
    if request.user.is_authenticated:
        return HttpResponseRedirect('afterlogin')
    return render(request,'baseapp/consultantclick.html')

#for showing signup/login button for patient(by sumit)
def clientclick_view(request):
    if request.user.is_authenticated:
        return HttpResponseRedirect('afterlogin')
    return render(request,'baseapp/clientclick.html')

#-------------Registration Views--------#
def admin_signup_view(request):
    form=forms.AdminRegistrationForm()
    context = {'form': form}
    if request.method=='POST':
        form=forms.AdminRegistrationForm(request.POST)
        if form.is_valid():
            user=form.save()
            user.set_password(user.password)
            user.save()
            my_admin_group = Group.objects.get_or_create(name='ADMIN')
            my_admin_group[0].user_set.add(user)
            return HttpResponseRedirect('adminlogin')
    return render(request,'baseapp/adminsignup.html', context)

def consultant_signup_view(request):
    userForm=forms.ConsultantUserForm()
    context = {'userForm':userForm}
    if request.method=='POST':
        userForm=forms.ConsultantUserForm(request.POST)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            my_consultant_group = Group.objects.get_or_create(name='CONSULTANT')
            my_consultant_group[0].user_set.add(user)
        return HttpResponseRedirect('consultantlogin')
    return render(request,'baseapp/consultantsignup.html', context)

def client_signup_view(request):
    userForm=forms.ClientUserForm()
    context = {'userForm' : userForm}
    if request.method=='POST':
        userForm = forms.ClientUserForm(request.POST)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            my_client_group = Group.objects.get_or_create(name='CLIENT')
            my_client_group[0].user_set.add(user)
        return HttpResponseRedirect('clientlogin')
    return render(request,'baseapp/clientsignup.html', context)

#-----------for checking user is CONSULTANT , CLIENT or admin(by sumit)
def is_admin(user):
    return user.groups.filter(name='ADMIN').exists()
def is_consultant(user):
    return user.groups.filter(name='CONSULTANT').exists()
def is_client(user):
    return user.groups.filter(name='CLIENT').exists()

#---------AFTER ENTERING CREDENTIALS WE CHECK WHETHER USERNAME AND PASSWORD IS OF ADMIN,CONSULTANT OR CLIENT
def afterlogin_view(request):
    if is_admin(request.user):
        return redirect('admin-dashboard')
    elif is_consultant(request.user):
        accountapproval=models.Consultant.objects.all().filter(user_id=request.user.id,availability_status=True)
        if accountapproval:
            return redirect('consultant-dashboard')
        else:
            return render(request,'baseapp/consultant_wait_for_approval.html')
    elif is_client(request.user):
        accountapproval=models.Client.objects.all().filter(user_id=request.user.id,status=True)
        if accountapproval:
            return redirect('client-dashboard')
        else:
            return render(request,'baseapp/client_wait_for_approval.html')

#---------------------------------------------------------------------------------
#------------------------ ADMIN RELATED VIEWS START ------------------------------
#---------------------------------------------------------------------------------

def admin_dashboard_view(request):
    # for both table in admin dashboard
    consultants=models.Consultant.objects.all().order_by('-id')
    clients=models.Client.objects.all().order_by('-id')
    #for three cards
    consultantcount=models.Consultant.objects.all().filter(status=True).count()
    pendingconsultantcount=models.Consultant.objects.all().filter(status=False).count()

    clientcount=models.Client.objects.all().filter(status=True).count()
    pendingclientcount=models.Client.objects.all().filter(status=False).count()

    appointmentcount=models.Appointment.objects.all().filter(status=True).count()
    pendingappointmentcount=models.Appointment.objects.all().filter(status=False).count()
    context = {
    'consultants':consultants,
    'clients':clients,
    'consultantcount':consultantcount,
    'pendingconsultantcount':pendingconsultantcount,
    'clientcount':clientcount,
    'pendingclientcount':pendingclientcount,
    'appointmentcount':appointmentcount,
    'pendingappointmentcount':pendingappointmentcount,
    }
    return render(request,'baseapp/admin_dashboard.html', context)

# this view for sidebar click on admin page

def admin_consultant_view(request):
    return render(request,'baseapp/admin_consultant.html')

def admin_view_consultant_view(request):
    consultants = models.Consultant.objects.all().filter(status=True)
    context = {'consultants':consultants}
    return render(request,'baseapp/admin_view_consultant.html', context)

def delete_consultant_from_hospital_view(request,pk):
    consultant=models.Consultant.objects.get(id=pk)
    user=models.User.objects.get(id=consultant.user_id)
    user.delete()
    consultant.delete()
    return redirect('admin-view-consultant')

def update_consultant_view(request,pk):
    consultant=models.Consultant.objects.get(id=pk)
    user=models.User.objects.get(id=consultant.user_id)

    userForm=forms.ConsultantUserForm(instance=user)
    context = {'userForm':userForm}
    if request.method=='POST':
        userForm=forms.ConsultantUserForm(request.POST,instance=user)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            return redirect('admin-view-consultant')
    return render(request,'baseapp/admin_update_consultant.html',context)

def admin_add_consultant_view(request):
    userForm=forms.ConsultantUserForm()
    context = {'userForm':userForm}
    if request.method=='POST':
        userForm=forms.ConsultantUserForm(request.POST)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            my_consultant_group = Group.objects.get_or_create(name='CONSULTANT')
            my_consultant_group[0].user_set.add(user)
        return HttpResponseRedirect('admin-view-consultant')
    return render(request,'baseapp/admin_add_consultant.html',context)

def admin_approve_consultant_view(request):
    #those whose approval are needed
    consultant = models.Consultant.objects.all().filter(status=False)
    context = {'consultant':consultant}
    return render(request,'baseapp/admin_approve_consultant.html', context)

def approve_consultant_view(request,pk):
    consultant = models.Consultant.objects.get(id=pk)
    consultant.status=True
    consultant.save()
    return redirect(reverse('admin-approve-consultant'))

def reject_consultant_view(request,pk):
    consultant = models.Consultant.objects.get(id=pk)
    user=models.User.objects.get(id=consultant.user_id)
    user.delete()
    consultant.delete()
    return redirect('admin-approve-consultant')

def admin_view_consultant_specialisation_view(request):
    consultant = models.Consultant.objects.all().filter(status=True)
    context = {'consultant':consultant}
    return render(request,'baseapp/admin_view_consultant_specialisation.html', context)

def admin_client_view(request):
    return render(request,'baseapp/admin_client.html')

def admin_view_client_view(request):
    clients = models.Client.objects.all().filter(status=True)
    context = {'clients':clients}
    return render(request,'baseapp/admin_view_patient.html', context)

def delete_client_from_hospital_view(request,pk):
    client=models.Client.objects.get(id=pk)
    user=models.User.objects.get(id=client.user_id)
    user.delete()
    client.delete()
    return redirect('admin-view-client')

def update_client_view(request,pk):
    client=models.Client.objects.get(id=pk)
    user=models.User.objects.get(id=client.user_id)

    userForm=forms.ClientUserForm(instance=user)

    context = {'userForm':userForm}
    if request.method=='POST':
        userForm=forms.ClientUserForm(request.POST,instance=user)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            return redirect('admin-view-client')
    return render(request,'baseapp/admin_update_client.html', context)

def admin_add_client_view(request):
    userForm=forms.ClientUserForm()
    context = {'userForm':userForm}
    if request.method=='POST':
        userForm=forms.ClientUserForm(request.POST)
        if userForm.is_valid():
            user=userForm.save()
            user.set_password(user.password)
            user.save()
            my_client_group = Group.objects.get_or_create(name='CLIENT')
            my_client_group[0].user_set.add(user)
        return HttpResponseRedirect('admin-view-client')
    return render(request,'baseapp/admin_add_client.html',context)

def admin_inbox(request):
    adminRecivedmessages = Message.objects.filter(reciever=request.user)
    context = {'messages' : adminRecivedmessages}
    return render(request, 'baseapp/admin_inbox.html', context)

def admin_notifications(request):
    adminNotifcations =Notification.objects.filter(user=request.user).order_by('-timestamp')
    context = {'notifications' : adminNotifcations}
    return render(request, 'baseapp/admin_notifications.html', context)

#------------------FOR APPROVING CLIENTS BY ADMIN----------------------

def admin_approve_client_view(request):
    #those whose approval are needed
    clients = models.Client.objects.all().filter(status=False)
    context = {'clients':clients}
    return render(request,'baseapp/admin_approve_client.html', context)

def approve_client_view(request,pk):
    client=models.Client.objects.get(id=pk)
    client.status=True
    client.save()
    return redirect(reverse('admin-approve-client'))

def reject_client_view(request,pk):
    client=models.Client.objects.get(id=pk)
    user=models.User.objects.get(id=client.user_id)
    user.delete()
    client.delete()
    return redirect('admin-approve-client')

#-----------------APPOINTMENT START----------------------------------------------------------------#####

def admin_appointment_view(request):
    return render(request,'baseapp/admin_appointment.html')

def admin_view_appointment_view(request):
    appointments=models.Appointment.objects.all().filter(status=True)
    context = {'appointments':appointments}
    return render(request,'baseapp/admin_view_appointment.html', context)

def admin_add_appointment_view(request):
    appointmentForm=forms.AppointmentForm()
    context = {'appointmentForm':appointmentForm,}
    if request.method=='POST':
        appointmentForm=forms.AppointmentForm(request.POST)
        if appointmentForm.is_valid():
            appointment=appointmentForm.save(commit=False)
            appointment.consultantID=request.POST.get('consultantID')
            appointment.clientID=request.POST.get('clientID')
            appointment.appointmentRequestCategory=request.POST.get('category')
            appointment.description=request.POST.get('description')
            appointment.notes=request.POST.get('notes')
            appointment.consultantName=models.User.objects.get(id=request.POST.get('consultantID')).first_name
            appointment.clientName=models.User.objects.get(id=request.POST.get('clientID')).first_name
            appointment.status=True
            appointment.save()
        return HttpResponseRedirect('admin-view-appointment')
    return render(request,'hospital/admin_add_appointment.html',context)

def admin_approve_appointment_view(request):
    #those whose approval are needed
    appointments=models.Appointment.objects.all().filter(status=False)
    context = {'appointments':appointments}
    return render(request,'baseapp/admin_approve_appointment.html', context)

def approve_appointment_view(request,pk):
    appointment=models.Appointment.objects.get(id=pk)
    appointment.status=True
    appointment.save()
    return redirect(reverse('admin-approve-appointment'))

def reject_appointment_view(request,pk):
    appointment=models.Appointment.objects.get(id=pk)
    appointment.delete()
    return redirect('admin-approve-appointment')
#---------------------------------------------------------------------------------
#------------------------ ADMIN RELATED VIEWS END ------------------------------
#---------------------------------------------------------------------------------

#---------------------------------------------------------------------------------
#------------------------ CONSULTANT RELATED VIEWS START ------------------------------
#---------------------------------------------------------------------------------

def consultant_dashboard_view(request):
    # for three cards
    clientcount=models.Client.objects.all().filter(status=True,assignedConsultantID=request.user.id).count()
    appointmentcount=models.Appointment.objects.all().filter(status=True,consultantID=request.user.id).count()
    clientdischarged=models.ClientDischargeDetails.objects.all().distinct().filter(assignedConsultantName=request.user.first_name).count()

    #for  table in doctor dashboard
    appointments=models.Appointment.objects.all().filter(status=True,consultantID=request.user.id).order_by('-id')
    clientID=[]
    for appointment in appointments:
        clientID.append(appointment.clientID)
    clients=models.Client.objects.all().filter(status=True,user_id__in=clientID).order_by('-id')
    appointments=zip(appointments,clients)
    context = {
    'clientcount':clientcount,
    'appointmentcount':appointmentcount,
    'clientdischarged':clientdischarged,
    'appointments':appointments,
    'consultant':models.Consultant.objects.get(user_id=request.user.id), #for profile picture of doctor in sidebar
    }
    return render(request,'baseapp/consultant_dashboard.html', context)

def consultant_client_view(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id)
    context =   {'consultant': consultant}
   #for profile picture of CONSULTANT in sidebar}
    return render(request,'baseapp/consultant_client.html',context)

def consultant_view_client_view(request):
    clients = models.Client.objects.all().filter(status=True,assignedConsultantID=request.user.id)
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of CONSULTANT in sidebar
    context = {'clients':clients,'consultant':consultant}
    return render(request,'baseapp/consultant_view_client.html', context)

def consultant_view_discharge_client_view(request):
    dischargedclients = models.ClientDischargeDetails.objects.all().distinct().filter(assignedConsultantName=request.user.first_name)
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of cosultant in sidebar
    context = {'dischargedclients':dischargedclients,'consultant':consultant}
    return render(request,'baseapp/consultant_view_discharge_client.html', context)

def consultant_appointment_view(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of COSULTANT in sidebar
    context = {'consultant':consultant}
    return render(request,'baseapp/consultant_appointment.html', context)

def consultant_view_appointment_view(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of CONSULTANT in sidebar
    appointments=models.Appointment.objects.all().filter(status=True,consultantID=request.user.id)
    clientID=[]
    for appiontment in appointments:
        clientID.append(appiontment.clientID)
    clients=models.Client.objects.all().filter(status=True,user_id__in=clientID)
    appointments=zip(appointments, clients)
    context = {'appointments':appointments,'consultant':consultant}
    return render(request,'baseapp/consultant_view_appointment.html', context)

def consultant_delete_appointment_view(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of CONSULTANT in sidebar
    appointments=models.Appointment.objects.all().filter(status=True,doctorId=request.user.id)
    clientID=[]
    for appointment in appointments:
        clientID.append(appointment.clientID)
    clients = models.Client.objects.all().filter(status=True,user_id__in=clientID)
    appointments=zip(appointments, clients)
    context = {'appointments':appointments,'consultant':consultant}
    return render(request,'baseapp/consultant_delete_appointment.html', context)

def delete_appointment_view(request,pk):
    appointment=models.Appointment.objects.get(id=pk)
    appointment.delete()
    consultant = models.Consultant.objects.get(user_id=request.user.id) #for profile picture of CONSULTANT in sidebar
    appointments=models.Appointment.objects.all().filter(status=True,consultantID=request.user.id)
    clientID=[]
    for appointment in appointments:
        clientID.append(appointment.clientID)
    clients = models.Client.objects.all().filter(status=True,user_id__in=clientID)
    appointments=zip(appointments, clients)
    context = {'appointments':appointments,'consultant':consultant}
    return render(request,'baseapp/consultant_delete_appointment.html', context)

def consultant_inbox(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id)
    consultantRecivedmessages = Message.objects.filter(reciever=request.user)
    context = {'messages' : consultantRecivedmessages, 'consultant' : consultant}
    return render(request, 'baseapp/consultant_inbox.html', context)

def consultant_notifications(request):
    consultant = models.Consultant.objects.get(user_id=request.user.id)
    consultantNotifcations =Notification.objects.filter(user=request.user).order_by('-timestamp')
    context = {'notifications' : consultantNotifcations}
    return render(request, 'baseapp/consultant_notifications.html', context)

def consultantServicesView(request, username):
    services = ConsultancyService.objects.filter(consultant__username=username)
    context = {'services': services}
    return render(request, 'baseapp/consultantServices.html', context)

def consultancyServiceDetail(request, serviceID):
    service = get_object_or_404(ConsultancyService, id=serviceID)
    if request.method == 'POST':
        ConsultationRequest.objects.create(
            client=request.user,
            consultant=service.consultant,
            service=service
        )
        return redirect('profile', username=request.user.username)
    context = {'service' : service}
    return render(request, 'baseapp/serviceDetail.html', context)

def manageAppointments(request):
    appointments = Appointment.objects.filter(client=request.user.client)
    context = {'appiontments' : appointments}
    return render(request, 'baseapp/manage_appiontments.html', context)

def acceptDeclineRequest(request, requestID, action):
    consultationRequest = get_object_or_404(ConsultationRequest, id=requestID)
    if action == 'accept':
        consultationRequest.requestStatus = 'accepted'
        consultationRequest.save()
    elif action == 'decline':
        consultationRequest.requestStatus = 'declined'
        consultationRequest.save()
    return redirect('consultation_request')

def respond_appointment_request(request, appointment_id):
    appointment = Appointment.objects.get(id=appointment_id)
    client = appointment.client

    if request.method == 'POST':
        form = AppointmentResponseForm(request.POST)
        if form.is_valid():
            response = form.cleaned_data['response']

            # Update the appointment status
            appointment.status = response
            appointment.save()

            # Create a notification for the client
            message = f"Your appointment request has been {response}."
            notification = Notification(user=client, message=message)
            notification.save()

            return redirect('appointment_response_confirmation')  # Redirect to a confirmation page
    else:
        form = AppointmentResponseForm()

    return render(request, 'consultancy/respond_appointment_request.html', {'form': form, 'appointment': appointment})

def respond_to_client(request, client_id):
    client = User.objects.get(id=client_id)

    if request.method == 'POST':
        response_message = request.POST['response_message']  # Adjust this based on your form
        # Your response logic here...

        # Create a notification for the client
        message = f"A response has been received from the consultant: {response_message}"
        notification = Notification(user=client, message=message)
        notification.save()

        return redirect('response_confirmation')  # Redirect to a confirmation page

    return render(request, 'consultancy/respond_to_client.html', {'client': client})

#---------------------------------------------------------------------------------
#------------------------ CONSULTANT RELATED VIEWS END ------------------------------
#---------------------------------------------------------------------------------

#---------------------------------------------------------------------------------
#------------------------ CLIENT RELATED VIEWS START ------------------------------
#---------------------------------------------------------------------------------

def client_dashboard_view(request):
    client = models.Client.objects.get(user_id=request.user.id)
    consultant = models.Consultant.objects.get(user_id=client.assignedConsultantID)
    context = {'client': client,'consultantName': consultant.get_name, 'consultantEmail' : consultant.email, 'consultantAvailability' : consultant.availability_status}
    return render(request,'baseapp/client_dashboard.html',context)

def client_appointment_view(request):
    client = models.Client.objects.get(user_id=request.user.id) #for profile picture of Client in sidebar
    context = {'client':client}
    return render(request,'baseapp/client_appointment.html', context)

def client_book_appointment_view(request):
    appointmentForm=forms.AppointmentForm()
    client = models.Client.objects.get(user_id=request.user.id) #for profile picture of Client in sidebar
    context = {'appointmentForm':appointmentForm,'client': client}
    if request.method=='POST':
        appointmentForm=forms.AppointmentForm(request.POST)
        if appointmentForm.is_valid():
            appointment=appointmentForm.save(commit=False)
            appointment.consultantID=request.POST.get('consultantID')
            appointment.cientID=request.user.id #----user can choose any client but only their info will be stored
            appointment.appointmentRequestCategory=request.POST.get('category')
            appointment.description=request.POST.get('description')
            appointment.notes=request.POST.get('notes')
            appointment.consultantName=models.User.objects.get(id=request.POST.get('consultantID')).first_name
            appointment.clientName=request.user.first_name #----user can choose any CLIENT but only their info will be stored
            appointment.status=False
            appointment.save()
        return HttpResponseRedirect('client-view-appointment')
    return render(request,'baseapp/client_book_appointment.html',context)

def client_view_appointment_view(request):
    client = models.Client.objects.get(user_id=request.user.id) #for profile picture of Client in sidebar
    appointments=models.Appointment.objects.all().filter(clientID=request.user.id)
    context = {'appointments':appointments,'client':client}
    return render(request,'baseapp/client_view_appointment.html', context)

def consultant_detail(request, consultant_id):
    consultant = get_object_or_404(Consultant, id=consultant_id)
    context = {'consultant' : consultant}
    return render(request, 'baseapp/consultant_detail.html', context)

def client_inbox(request):
    client = models.Client.objects.get(user_id=request.user.id)
    clientRecivedmessages = Message.objects.filter(reciever=request.user)
    context = {'messages' : clientRecivedmessages}
    return render(request, 'baseapp/client_inbox.html', context)

def client_notifications(request):
    client = models.Client.objects.get(user_id=request.user.id)
    clientnNotifcations =Notification.objects.filter(user=request.user).order_by('-timestamp')
    context = {'notifications' : clientnNotifcations}
    return render(request, 'baseapp/client_notifications.html', context)

def scheduleAppointment(request, consultant_id):
    consultant = get_object_or_404(Consultant, id=consultant_id)

    if request.method == 'POST':
        form = AppointmentForm(request.POST)
        if form.is_valid():
            appointment = form.save(commit=False)
            appointment.client = request.user
            appointment.consultant = consultant
            appointment.save()

            return redirect('appointment_confirmation')
    else:
        form = AppointmentForm()
    context = {'form' : form, 'consultant' : consultant}
    return render(request, 'baseapp/schedule_appointment.html', context)

def cancelAppointment(request, appointment_id):
    appiontment = get_object_or_404(Appointment, id=appointment_id)

    if request.user.client == appiontment.client:
        appiontment.delete()

    return redirect('manage_appointments')

def sendMessage(request, reciever_id):
    if request.method == 'POST':
        content = request.POST.get('content')
        reciever = User.objects.get(id=reciever_id)
        message = Message.objects.create(sender=request.user, reciever=reciever, content=content)
        message.save()

        notification = Notification.objects.create(user=reciever, message=f'You have a New Message from {request.user}')
        notification.save()

        return redirect('inbox')

#------------------------ CLIENT RELATED VIEWS END ------------------------------
#---------------------------------------------------------------------------------

#---------------------------------------------------------------------------------
#------------------------ ABOUT US AND CONTACT US VIEWS START ------------------------------
#---------------------------------------------------------------------------------
def aboutus_view(request):
    return render(request,'baseapp/aboutus.html')

def contactus_view(request):
    subscribeForm = forms.ContactusForm()
    if request.method == 'POST':
        subscribeForm = forms.ContactusForm(request.POST)
        if subscribeForm.is_valid():
            email = subscribeForm.cleaned_data['Email']
            name=subscribeForm.cleaned_data['Name']
            message = subscribeForm.cleaned_data['Message']
            send_mail(str(name)+' || '+str(email),message, EMAIL_HOST_USER, EMAIL_RECEIVING_USER, fail_silently = False)
            return render(request, 'baseapp/contactussuccess.html')
        context = {'form':subscribeForm}
    return render(request, 'baseapp/contactus.html', context)

#\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\#####

def consultationRequest(request):
    consultationRequests = ConsultationRequest.objects.filter(consultant=request.user)
    context = {'consultationRequests' : consultationRequests}
    return render(request, 'baseapp/consultationRequests.html', context)

def request_appointment(request, consultant_id):
    consultant = User.objects.get(id=consultant_id)

    if request.method == 'POST':
        form = AppointmentRequestForm(request.POST)
        if form.is_valid():
            # Your appointment request logic here...

            # Create a notification for the consultant
            message = f"A client has requested an appointment with you."
            notification = Notification(user=consultant, message=message)
            notification.save()

            return redirect('appointment_request_confirmation')  # Redirect to a confirmation page
    else:
        form = AppointmentRequestForm()

    return render(request, 'consultancy/request_appointment.html', {'form': form, 'consultant': consultant})

def send_message(request, recipient_id):
    recipient = User.objects.get(id=recipient_id)

    if request.method == 'POST':
        form = MessageForm(request.POST)
        if form.is_valid():
            message = form.cleaned_data['message']

            # Create a notification for the recipient
            notification = Notification(user=recipient, message=message)
            notification.save()

            return redirect('message_sent_confirmation')  # Redirect to a confirmation page
    else:
        form = MessageForm()

    return render(request, 'consultancy/send_message.html', {'form': form, 'recipient': recipient})

def appointmentConfirmation(request):
    return render(request, 'baseapp/appointment_confirmation.html')


######################################  PANDAS ###################################################

# ************************************  019 SERİLER
# ************************************  158 SERİ ELEMAN İŞLEMLERİ
# ************************************  205 DATA FRAME
# ************************************  298 DATA FRAME ELEMAN İŞLEMLERİ
# ************************************  350 GÖZLEM VE DEĞİŞKEN SEÇİMLERİ (LOC- İLOC)
# ************************************  385 KOŞULLU ELEMAN İŞLEMLERİ
# ************************************  425 BİRLEŞTİRME (JOIN) İŞLEMLERİ
# ************************************  523 TOPLULAŞTIRMA VE GRUPLAMA
# ************************************  530 TOPLULAŞTIRMA İŞLEMLERİ
# ************************************  589 GRUPLAMA İŞLEMLERİ
# ************************************  626 İLERİ TOPLULAŞTIRMA İŞLEMLERİ(AGGREGATE,APPLY,FİLTER vs.)
# ************************************  744 PİVOT TABLOLAR
# ************************************  792 DIŞ KAYNAKLI VERİ OKUMA

import pandas as pd

############################# SERİLER
#####SERİ OLUŞTURMA:
pd.Series([2,4,6,8,10])
# 0     2
# 1     4
# 2     6
# 3     8
# 4    10
# dtype: int64

# Listeden Oluşturma:
sayilar = list(np.random.randint(0,20,3))
numpy_dizi =np.random.randint(0,20,10)

seri1 = pd.Series(sayilar)
seri1
# 0     3
# 1     3
# 2    19
# dtype: int64

# Sözlükten Oluşturma:
sozluk = {"5" : 52,
          "10": 62,
          "15": 85}
seri3 = pd.Series(sozluk)
seri3
# 5     52
# 10    62
# 15    85
# dtype: int64

#Veri Seti Üzerinden Oluşturma(Dış kaynaklı veri setini okutma işlemi en aşağıda daha detaylı anlatılmıştır.)
tips_data = pd.read_csv("Data_Sets/tips.csv", nrows = 30)
tips_total_bill = tips_data["total_bill"]
tips_days = tips_data["day"]

# 0     16.99
# 1     10.34
# 2     21.01
# 3     23.68
# Name: total_bill, dtype: float64

##### Tip Sorgulama:
type(tips_total_bill)
# pandas.core.series.Series

seri = pd.Series([2,4,6,8,10])
type(seri)
# pandas.core.series.Series

##### İndex bilgilerine erişme:
seri.axes
# [RangeIndex(start=0, stop=5, step=1)]

##### Seri ile ilgili tip bilgisi için:
seri.dtype
# dtype('int64')

##### Eleman sayısı:
seri.size
#5

##### Boyut bilgisi:
seri.ndim
#1

##### Belirli bir sütuna vektör olarak erişmek(veya array yapısında erişmek)
seri.values
# array([ 2,  4,  6,  8, 10], dtype=int64)

##### Seriye başka veri yapısı şeklinde erişmek istersek:
dict(seri.items())
# {0: 2, 1: 4, 2: 6, 3: 8, 4: 10}
list(seri.items())
# [(0, 2), (1, 4), (2, 6), (3, 8), (4, 10)]

##### İlk 3 gözleme erişmek:
seri.head(3)
# 0    2
# 1    4
# 2    6
# dtype: int64

##### Sondan 2 gözleme erişmek:
seri.tail(2)
# 3     8
# 4    10
# dtype: int64

###### İndex isimlendirmesi:
seri1 = pd.Series([85,222,356,41,25],index= [10,20,30,"Kırk","Elli"])
# 10       85
# 20      222
# 30      356
# Kırk     41
# Elli     25
# dtype: int64

##### İndex ile elemanlara erişmek
seri1["Kırk"]
#41
### Slice
seri1["Kırk":"Elli" ]
# Kırk    41
# Elli    25
# dtype: int64

##### Sözlük üzerinden seri oluşturma:
sozluk = pd.Series({"reg" : 10, "log":22,"cart":54})
sozluk
# reg     10
# log     22
# cart    54
# dtype: int64

# 2. Yol
sozluk = {"reg" : 10, "log":22,"cart":54}
seri2 = pd.Series(sozluk)
seri2
# reg     10
# log     22
# cart    54
# dtype: int64

##### İki seri birleştirerek seri oluşturma:
pd.concat([seri,seri1])
# 0         2
# 1         4
# 2         6
# 3         8
# 4        10
# 10       85
# 20      222
# 30      356
# Kırk     41
# Elli     25
# dtype: int64

############## Eleman İşlemleri ############
import numpy as np
a = np.array([32,258,5,6])
seri = pd.Series(a)
seri

seri[0]
#32

seri[0:2]
# 0     32
# 1    258
# dtype: int32

seri1 = pd.Series([24,35,148,56], index=["ab","bc","cd","de"])
seri1
# ab     24
# bc     35
# cd    148
# de     56
# dtype: int64

seri1.keys
# <bound method Series.keys of ab     24
# bc     35
# cd    148
# de     56
# dtype: int64>

list(seri1.items())
# Out[53]: [('ab', 24), ('bc', 35), ('cd', 148), ('de', 56)]

"ab" in seri1
#True

#Fancy
seri1[["ab","de"]]
# ab    24
# de    56
# dtype: int64

#Değer atama:
seri1["bc"] = 250
seri1[["bc"]]
# bc    250
# dtype: int64

#################################  DATA FRAME  ####################################
# Numpy sabit tipli olduğundan işlem yapmada çok fonksiyonel olmadığından, veri manüplasyonunda pandas df kullanılır.
#Yapılandırılmış bir veri formudur.

#Listeden Oluşturma:
liste = [55,2,3,36,5]
pd.DataFrame(liste, columns=["Sayılar"])
   # Sayılar
# 0       55
# 1        2
# 2        3
# 3       36
# 4        5

#Numpy Arrayden Oluşturma:
m = np.arange(1,28,3).reshape(3,3)
m
pd.DataFrame(m,columns=["var1","var2","var3"])
     # var1  var2  var3
# 0     1     4     7
# 1    10    13    16
# 2    19    22    25

#Sözlükten oluşturma:
degisken_isimleri = ["var1","var2","var3"]
degerler = [np.random.randint(10,size=5)for _ in range(3)]
#-> [array([1, 6, 3, 7, 5]), array([7, 0, 0, 4, 6]), array([4, 5, 6, 6, 5])]
sozluk = dict(zip(degisken_isimleri,degerler))
sozlukten_df= pd.DataFrame(sozluk)
sozlukten_df
   # var1  var2  var3
# 0     2     7     1
# 1     2     8     4
# 2     5     8     1
# 3     0     3     4
# 4     4     9     8

#Seriden oluşturma:
seri = pd.Series([2,4,6,8,10])
df = pd.DataFrame(seri, columns=["Seri"])
df
   # Seri
# 0     2
# 1     4
# 2     6
# 3     8
# 4    10

# Değişken ismi değiştirme:
m = np.arange(1,28,3).reshape(3,3)
m
df = pd.DataFrame(m,columns=["var1","var2","var3"])
df.columns = ("deg1","deg2","deg3")
df.head(1)
   # deg1  deg2  deg3
# 0     1     4     7

df.columns
# Index(['deg1', 'deg2', 'deg3'], dtype='object')

df.dtypes
# deg1    int32
# deg2    int32
# deg3    int32
# dtype: object

df.ndim
#2

df.shape
#(3, 3)

df.size
#9

df.values   #Değerleri çekip array formuna çevirir.
# array([[ 1,  4,  7],
#        [10, 13, 16],
#        [19, 22, 25]])

df.info()
# <class 'pandas.core.frame.DataFrame'>
# RangeIndex: 3 entries, 0 to 2
# Data columns (total 3 columns):
#    Column  Non-Null Count  Dtype
# ---  ------  --------------  -----
#  0   deg1    3 non-null      int32
#  1   deg2    3 non-null      int32
#  2   deg3    3 non-null      int32
# dtypes: int32(3)

##### Eleman işlemleri :
s1 = np.random.randint(20, size=5)
s2 = np.random.randint(20, size=5)
s3 = np.random.randint(20, size=5)
sozluk = {"var1": s1 , "var2": s2, "var3" : s3}
df = pd.DataFrame(sozluk)
df

df.index = ["a","b","c","d","e"]
df
   # var1  var2  var3
# a    14    19     4
# b     9     1     9
# c     4    17     7
# d    15    19     3
# e    12     6    11

########  Silme işlemi  ###############
df.drop("c",axis =0)  #KALICI İŞLEM DEĞİL
   # var1  var2  var3
# a    14    19     4
# b     9     1     9
# d    15    19     3
# e    12     6    11

df.drop("c",axis =0, inplace=True) # KALICI OLARAK SİLER

######Fancy ile silme
sil = ["a","b"]
df.drop(sil,axis=0)  # inplace = True yaparsak kalıcı olarak silecektir.
   # var1  var2  var3
# d    15    19     3
# e    12     6    11

###### Değişkenleri silme:
sil2 = ["var1","var2"]
df.drop(sil2, axis=1)  # inplace = True yaparsak kalıcı olarak silecektir.
   # var3
# a     4
# b     9
# d     3
# e    11

# Değişken oluşturma
df["var4"] = df["var1"] * df["var3"]
df
   # var1  var2  var3  var4
# a    14    19     4    56
# b     9     1     9    81
# d    15    19     3    45
# e    12     6    11   132

############ GÖZLEM VE DEĞİŞKEN SEÇİMİ ##############
ma = np.random.randint(1,30, size=(5,3))
df = pd.DataFrame(ma, columns=["deg1","deg2","deg3"])
df
#### loc : Tanımlandığı şekliyle seçim yapmak için kullanılır.

df.loc[0:2]
   # deg1  deg2  deg3
# 0    18    16    18
# 1    26    15     3
# 2    25    24    21

#### iloc : Alışık olduğumuz indeksleme mantığı('e kadar).
df.iloc[0:2]
   # deg1  deg2  deg3
# 0    18    16    18
# 1    26    15     3

df.iloc[0,0]
# 18
df.iloc[:3,:2]
   # deg1  deg2
# 0    18    16
# 1    26    15
# 2    25    24

df.loc[0:3,"deg2"] #iloc değişken veya indexlerde mutlak bir değer işaretlemesi yapacaksak çalışmaz.
# 0    16
# 1    15
# 2    24
# 3     6
# Name: deg2, dtype: int32

# İNDEKSLERE BAĞLI KALACAKSAK LOC KALMAYACAKSAK İLOC

########## Koşullu Eleman İşlemleri  ##############
ma = np.random.randint(1,30, size=(5,3))
df = pd.DataFrame(ma, columns=["deg1","deg2","deg3"])
df

df[0:2][["deg1","deg2"]] #Birden fazla değişken girmek istedğimizde fancy yardımıyla [[deg]] yapılmalı...
   # deg1  deg2
# 0    20    13
# 1     1    28

# 2. değişkendeki değerlerin 10 dan büyük olması:
df[df.deg2 > 15]
   # deg1  deg2  deg3
# 1     7    27    24
# 2     2    20    15
# 3     3    16    12
df[df.deg2 > 15]["deg1"]
# 1    7
# 2    2
# 3    3
# Name: deg1, dtype: int32

# Birden fazla koşul gerçekleştirmek istersek
df[(df.deg2 > 15) & (df.deg3 < 20)]
   # deg1  deg2  deg3
# 2     2    20    15
# 3     3    16    12

df.loc[(df.deg2 > 15), ["deg1","deg2"]]
   # deg1  deg2
# 1     7    27
# 2     2    20
# 3     3    16

df[(df.deg2 > 15)][["deg1","deg3"]]
   # deg1  deg3
# 1     7    24
# 2     2    15
# 3     3    12

# Birleştirme (Join) işlemleri:
ma = np.random.randint(1,30, size=(5,3))
df1 = pd.DataFrame(ma, columns=["deg1","deg2","deg3"])
df1

##  Concat:
df2 = df1+ 85
df2
pd.concat([df1,df2])
# (   deg1  deg2  deg3
#  0    22     1    23
#  1    24    22     2
#  2    19     9     3
#  3    23    20    21
#  4     3     4    19,
#     deg1  deg2  deg3
#  0   107    86   108
#  1   109   107    87
#  2   104    94    88
#  3   108   105   106
#  4    88    89   104)

pd.concat([df1,df2],ignore_index= True)
# ?pd.concat
   # deg1  deg2  deg3
# 0    22     1    23
# 1    24    22     2
# 2    19     9     3
# 3    23    20    21
# 4     3     4    19
# 5   107    86   108
# 6   109   107    87
# 7   104    94    88
# 8   108   105   106
# 9    88    89   104

df2.columns = ["deg1","deg2","var3"]
df2
pd.concat([df1,df2])
   # deg1  deg2  deg3   var3
# 0    22     1  23.0    NaN
# 1    24    22   2.0    NaN
# 2    19     9   3.0    NaN
# 3    23    20  21.0    NaN
# 4     3     4  19.0    NaN
# 0   107    86   NaN  108.0
# 1   109   107   NaN   87.0
# 2   104    94   NaN   88.0
# 3   108   105   NaN  106.0
# 4    88    89   NaN  104.0

pd.concat([df1,df2], join="inner")
   # deg1  deg2
# 0    22     1
# 1    24    22
# 2    19     9
# 3    23    20
# 4     3     4
# 0   107    86
# 1   109   107
# 2   104    94
# 3   108   105
# 4    88    89

# İleri birleştirme işlemleri:

df1 = pd.DataFrame({"Çalışanlar" : ["Ali","Veli","Ayşe","Fatma"],
                    "Grup": ["Muhasebe","Mühendislik","Mühendislik","IK"]})
df1
df2 = pd.DataFrame({"Çalışanlar" : ["Ayşe","Ali","Veli","Fatma"],
                    "Ilk giriş": [2010,2009,2014,2018]})
df2
pd.merge(df1,df2)
  # Çalışanlar         Grup  Ilk giriş
# 0        Ali     Muhasebe       2009
# 1       Veli  Mühendislik       2014
# 2       Ayşe  Mühendislik       2010
# 3      Fatma           IK       2018

#Çoktan teke,
df3 = pd.merge(df1,df2)
df3
df4=pd.DataFrame({"Grup" : ["Muhasebe","Mühendislik","IK"],
                  "Müdür": ["Caner" , "Mustafa","Berkcan"]})
pd.merge(df3,df4)
 # Çalışanlar         Grup  Ilk giriş    Müdür
# 0        Ali     Muhasebe       2009    Caner
# 1       Veli  Mühendislik       2014  Mustafa
# 2       Ayşe  Mühendislik       2010  Mustafa
# 3      Fatma           IK       2018  Berkcan

#Çoktan çoka,
df5= pd.DataFrame({"Grup": ["Muhasebe","Muhasebe","Mühendislik","Mühendislik","IK","IK"],
                   "Yetenekler" : ["mat","excel","kodlama","linux","excel","yönetim"]})
df5
df1
pd.merge(df1,df5)

################### TOPLULAŞTIRMA VE GRUPLAMA ##################
import pandas as pd
import seaborn as sns
df = sns.load_dataset("planets")
df.head(10)
df.shape

####### TOPLULAŞTIRMA İŞLEMLERİ
# count() Gözlem birimlerinin sayısını gösterir.
df.count()
# method            1035
# number            1035
# orbital_period     992
# mass               513
# distance           808
# year              1035
# dtype: int64

# median() Gözlem birimlerinin medyanını gösterir.
df.median()
# number               1.0000
# orbital_period      39.9795
# mass                 1.2600
# distance            55.2500
# year              2010.0000
# dtype: float64

# min() Değişken içindeki minimum değeri getirir.
df["mass"].min()
# 0.0036

# max() Değişken içindeki maximum değer
df["distance"].max()
# 8500.0

# sum() Değişkenlerin değerlerini toplar.
df["mass"].sum()
# 1353.37638

# std() Değişkenlerin Standart sapma değerini gösterir.
df["mass"].std()
# 3.8186166509616046

# var() Değişkenlerin varyans değerini gösterir.
df["mass"].var()
# 14.58183312700122

# mean()  ORTALAMA ALMA İÇİN KULLANILIR.
df.mean()
# number               1.785507
# orbital_period    2002.917596
# mass                 2.638161
# distance           264.069282
# year              2009.070531
# dtype: float64

df["mass"].mean()
# 2.6381605847953233

# describe() Betimsel istatistiklerin tamamına erişmek
df.describe().T
      # count  mean       std   min   25%   50%   75%   max
# deg1    5.0   9.2  5.118594   3.0   7.0   9.0  10.0  17.0
# deg2    5.0  20.2  6.099180  11.0  20.0  20.0  22.0  28.0
# deg3    5.0  18.4  7.668116   5.0  19.0  22.0  23.0  23.0

################# GRUPLAMA İŞLEMLERİ

df = pd.DataFrame({"Gruplar" : ["A","B","C","A","B","C"],
                   "Veri": [10,11,52,23,43,55]}, columns=["Gruplar","Veri"])
df
df.groupby("Gruplar")
# <pandas.core.groupby.generic.DataFrameGroupBy object at 0x000001A9BA6E7160>
df.groupby("Gruplar").mean() #A,B,C gruplarının değerlerinin ortalmasını aldı
         # Veri
# Gruplar
# A        16.5
# B        27.0
# C        53.5

import seaborn as sns
df = sns.load_dataset("planets")
df.head(3)

df.groupby("method")
# <pandas.core.groupby.generic.DataFrameGroupBy object at 0x000001A9BA6E7190>
df.groupby("method")["orbital_period"] #Aggregation fonk yazmamız lazım yoksa alttaki çıktıyı vermeye devam eder.
# <pandas.core.groupby.generic.SeriesGroupBy object at 0x000001A9BA7148E0>
df.groupby("method")["orbital_period"].mean()
# method
# Astrometry                          631.180000
# Eclipse Timing Variations          4751.644444
# Imaging                          118247.737500
# Microlensing                       3153.571429
# Orbital Brightness Modulation         0.709307
# Pulsar Timing                      7343.021201
# Pulsation Timing Variations        1170.000000
# Radial Velocity                     823.354680
# Transit                              21.102073
# Transit Timing Variations            79.783500
# Name: orbital_period, dtype: float64

################### İLERİ TOPLULAŞTIRMA
#AGGREGATE,FİLTER,TRANSFORM,APPLY

df = pd.DataFrame({"Gruplar" : ["A","B","C","A","B","C"],
                   "Degisken1": [10,11,52,23,11,55],
                   "Degisken2" : [100,253,33,262,11,969]},
                  columns=["Gruplar","Degisken1","Degisken2"])
df

# Aggregate:
df.groupby("Gruplar").mean()
         # Degisken1  Degisken2
# Gruplar
# A             16.5      181.0
# B             11.0      132.0
# C             53.5      501.0
# ***************************
import numpy as np
df.groupby("Gruplar").aggregate(["min", np.median,max])
 #                 Degisken1            Degisken2
 #              min median max       min median  max
# Gruplar
# A              10   16.5  23       100    181  262
# B              11   11.0  11        11    132  253
# C              52   53.5  55        33    501  969
#Veri setini gruplara göre bölüp, aggregate fonk sayesinde kendi istediğimiz fonksiyonları çalıştımamızı sağladı.

df.groupby("Gruplar").aggregate({"Degisken1": "min",
                                 "Degisken2": "max"})
         # Degisken1  Degisken2
# Gruplar
# A               10        262
# B               11        253
# C               52        969

# FILTER:
df = pd.DataFrame({"Gruplar" : ["A","B","C","A","B","C"],
                   "Degisken1": [10,11,52,23,85,55],
                   "Degisken2" : [100,253,33,262,11,969]},
                  columns=["Gruplar","Degisken1","Degisken2"])
def filter_func(x):
    return x["Degisken1"].std()>9
df.groupby("Gruplar").std()
         # Degisken1   Degisken2
# Gruplar
# A         9.192388  114.551299
# B        52.325902  171.119841
# C         2.121320  661.851947

df.groupby("Gruplar").filter(filter_func)
  # Gruplar  Degisken1  Degisken2
# 0       A         10        100
# 1       B         11        253
# 3       A         23        262
# 4       B         85         11

#Mevcut pandas fonksiyonlarıiçinde yer almayan,
# bizim tanımladığımız bir fonksiyonu df değişkenlerinin üzerinde sorgulamamızı sağlar.

### TRANSFORM:
df["Degisken1"]*9
# 0     90
# 1     99
# 2    468
# 3    207
# 4    765
# 5    495
# Name: Degisken1, dtype: int64

df_a =df.iloc[:,1:]
df_a.transform(lambda x: x - x.mean())
   # Degisken1   Degisken2
# 0 -29.333333 -171.333333
# 1 -28.333333  -18.333333
# 2  12.666667 -238.333333
# 3 -16.333333   -9.333333
# 4  45.666667 -260.333333
# 5  15.666667  697.666667

df_a.transform(lambda x: (x -x.mean()) / x.std())
   # Degisken1  Degisken2
# 0  -0.986438  -0.478483
# 1  -0.952809  -0.051200
# 2   0.425962  -0.665594
# 3  -0.549267  -0.026065
# 4   1.535705  -0.727033
# 5   0.526848   1.948374

### APPLY:
df = pd.DataFrame({"Degisken1": [10,11,52,23,85,55],
                   "Degisken2" : [100,253,33,262,11,969]},
                  columns=["Degisken1","Degisken2"])
df

df.apply(np.sum)
# Degisken1     236
# Degisken2    1628
# dtype: int64

df.apply(np.mean)
# Degisken1     39.333333
# Degisken2    271.333333
# dtype: float64

df = pd.DataFrame({"Gruplar" : ["A","B","C","A","B","C"],
                   "Degisken1": [10,11,52,23,85,55],
                   "Degisken2" : [100,253,33,262,11,969]},
                  columns=["Gruplar","Degisken1","Degisken2"])

df.groupby("Gruplar").apply(np.sum)
        # Gruplar  Degisken1  Degisken2
# Gruplar
# A            AA         33        362
# B            BB         96        264
# C            CC        107       1002

# df'in değişkenleri üzerinde gezinmeye ve toplulaştırmaya yarayan fonksiyon.

######################### PİVOT TABLOLAR #############################
titanic = sns.load_dataset("titanic")
titanic.head()

titanic.groupby("sex")["survived"].mean()
# sex
# female    0.742038
# male      0.188908
# Name: survived, dtype: float64

titanic.groupby(["sex","class"])["survived"].aggregate("mean").unstack()
# class      First    Second     Third
# sex
# female  0.968085  0.921053  0.500000
# male    0.368852  0.157407  0.135447

# Cinsiyet ve class kategorik değişkenini sınıflarına göre bölüp kesişiminde işlem yapmış olduk.

#Pivot table ile işlem:
titanic.pivot_table("survived", index="sex",columns="class")
# class      First    Second     Third
# sex
# female  0.968085  0.921053  0.500000
# male    0.368852  0.157407  0.135447

titanic.age.head()
# 0    22.0
# 1    38.0
# 2    26.0
# 3    35.0
# 4    35.0
age = pd.cut(titanic["age"],[0,18,90])
age.head()
# 0    (18, 90]
# 1    (18, 90]
# 2    (18, 90]
# 3    (18, 90]
# 4    (18, 90]
# Name: age, dtype: category

titanic.pivot_table("survived", ["sex",age],"class")
# class               First    Second     Third
# sex    age
# female (0, 18]   0.909091  1.000000  0.511628
#        (18, 90]  0.972973  0.900000  0.423729
# male   (0, 18]   0.800000  0.600000  0.215686
#        (18, 90]  0.375000  0.071429  0.133663

###########################  DIŞ KAYNAKLI VERİ OKUMA(PYCHARM) #####
#Ayar işlemi
pd.set_option("display.max_columns",None)  # Bütün sütun isimlerini eksiksiz göster.(None yerine 5 dersek 5sütun gösterir.)
pd.set_option("display.expand_frame_repr",False)  # Bütün değişkenleri yan yana yazdırmak için kullanılır.

#### .csv  #####

tips_data = pd.read_csv("Data_Sets/tips.csv") #Bunu bulamadım diyecektir.
#Project içinden csv dosyasını bulup sağ tık -> Copy Path -> Path From Content Root a basıp yukarı kopyaladık.

tips_data = pd.read_csv("Data_Sets/tips_II.csv")
tips_data.head(2)
  # total_bill;"tip";"sex";"smoker";"day";"time";"size"
# 0          16.99;1.01;"Female";"No";"Sun";"Dinner";2
# 1            10.34;1.66;"Male";"No";"Sun";"Dinner";3

#Bu durumu düzletmek için sep parametresini kullanırız.
tips_data = pd.read_csv("Data_Sets/tips_II.csv", sep= ";")
tips_data.head(2)
   # total_bill   tip     sex smoker  day    time  size
# 0       16.99  1.01  Female     No  Sun  Dinner     2
# 1       10.34  1.66    Male     No  Sun  Dinner     3

# 100 satırını almak:
tips_data_100= pd.read_csv("Data_Sets/tips.csv", nrows=100) #Belirtlien sayı kadar alır
tips_data_100

#100-150 arasına almak istersek:
tips_data_100_150= pd.read_csv("Data_Sets/tips.csv", skiprows=100, nrows=150, header = None)
tips_data_100_150

diabetes = pd.read_csv("Data_Sets/diabetes.csv")
#... lı değerler oluşmuştu. Yukarıdaki ayar kodlarından ilkini çalıştırarak sorunu giderdik.
# Bazı değişkenler aşağı kaymıştı.Yukarıdaki ayar kodlarından ikincisini çalıştırarak sorunu giderdik.
diabetes.head(10)

##Veri setlerindeki boş değerleri veri setini alırken tanıtmak Örn:insulin değerleri 0 gözüküyor.
   # Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  DiabetesPedigreeFunction  Age  Outcome
# 0            6      148             72             35        0  33.6                     0.627   50        1
# 1            1       85             66             29        0  26.6                     0.351   31        0
# 2            8      183             64              0        0  23.3                     0.672   32        1
# 3            1       89             66             23       94  28.1
diabetes = pd.read_csv("Data_Sets/diabetes.csv", na_values={"Glucose": 0,                                                                                "BloodPressure": 0,
                                                           "SkinThickness": 0,
                                                           "Insulin": 0,
                                                           "BMI":0})
diabetes.head(3)
   # Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  DiabetesPedigreeFunction  Age  Outcome
# 0            6    148.0           72.0           35.0      NaN  33.6                     0.627   50        1
# 1            1     85.0           66.0           29.0      NaN  26.6                     0.351   31        0
# 2            8    183.0           64.0            NaN      NaN  23.3                     0.672   32        1

# 0 olan gözlemleri NaN olarak dolduruldu ve boş olarak gözüküyor

#### .xlsx ####
import pandas as pd
online_retail = pd.read_excel("Data_Sets/online_retail_GER.xlsx")
online_retail

#excel üstünde ki sütunların ismine göre çekmek
online_retail = pd.read_excel("Data_Sets/online_retail_GER.xlsx", usecols="B:E, G")
online_retail.head(3)
 # StockCode                     Description  Quantity         InvoiceDate  Customer ID
# 0    85049E       SCANDINAVIAN REDS RIBBONS        12 2009-12-01 11:50:00        12533
# 1     21976  PACK OF 60 MUSHROOM CAKE CASES        24 2009-12-01 11:50:00        12533
# 2     21498                RED SPOTS  WRAP         25 2009-12-01 11:50:00        12533

# Aynı veri setinden İki farklı veri setini birleştime istediğimizde:

online_retail_2009_2010 = pd.read_excel("Data_Sets/online_retail_GER.xlsx", sheet_name="Year 2009-2010")
online_retail_2010_2011 = pd.read_excel("Data_Sets/online_retail_GER.xlsx", sheet_name="Year 2010-2011")

online_retail_all = pd.concat(([online_retail_2009_2010,online_retail_2010_2011]))
online_retail_all

#Döngü aracılığıyla sonda hangi yıl aralığından geldiğini belirtecek bir sütun ile birleştirme

online_retail_all_new = pd.DataFrame()

for sheet_name, frame in pd.read_excel("Data_Sets/online_retail_GER.xlsx", sheet_name=None).items():
    frame["Year"] = sheet_name
    online_retail_all_new = online_retail_all_new.append(frame)
online_retail_all_new.head(2)
# Invoice StockCode                     Description  Quantity         InvoiceDate  Price  Customer ID  Country            Year
# 0  489526    85049E       SCANDINAVIAN REDS RIBBONS        12 2009-12-01 11:50:00   1.25        12533  Germany  Year 2009-2010
# 1  489526     21976  PACK OF 60 MUSHROOM CAKE CASES        24 2009-12-01 11:50:00   0.55        12533  Germany  Year 2009-2010


class Morph:
    def __init__(self, *args) -> None:
        self.lst_words = list(args)

    def add_word(self, word: str) -> None:
        if word not in self.lst_words:
            self.lst_words.append(word)

    def get_words(self) -> tuple:
        return tuple(self.lst_words)

    def __eq__(self, other) -> bool:
        return True if other.lower() in self.lst_words else False

def main() -> None:
    dict_words = [Morph('связь', 'связи', 'связью', 'связи', 'связей', 'связям', 'связями', 'связях'),
                  Morph('формула', 'формулы', 'формуле', 'формулу', 'формулой', 'формул', 'формулам', 'формулами',
                        'формулах'),
                  Morph('вектор', 'вектора', 'вектору', 'вектором', 'векторе', 'векторы', 'векторов', 'векторам',
                        'векторами', 'векторах'),
                  Morph('эффект', 'эффекта', 'эффекту', 'эффектом', 'эффекте', 'эффекты', 'эффектов', 'эффектам',
                        'эффектами', 'эффектах'),
                  Morph('день', 'дня', 'дню', 'днем', 'дне', 'дни', 'дням', 'днями', 'днях')]

    text = 'Мы будем устанавливать связь завтра днем.'
    count = 0
    for i in text.lower().strip('.,!? :').split():
        for j in dict_words:
            if i in j.get_words():
                count += 1
    print(count)

if __name__ == '__main__':
    main()


'''
# -*- coding: utf-8 -*-
@FileName: bot.py
'''

import time
import logging
from surprise import Reader, Dataset, dump
from telegram.ext.updater import Updater
from telegram.update import Update
from telegram.ext.callbackcontext import CallbackContext
from telegram.ext.commandhandler import CommandHandler
from telegram.ext.messagehandler import MessageHandler
from telegram.ext.filters import Filters
from collections import defaultdict
from redis import StrictRedis
import pandas as pd
import random

logging.basicConfig(
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
)

def start(update: Update, context: CallbackContext):
    update.message.reply_text("Hello sir, Welcome to IEMS5780_1155161034_bot.")

def load_dataset():
    '''
    Load the dataset of the anime_ratings.dat
    :return:
    '''
    reader = Reader(rating_scale=(1, 10))

    anime_ratings = pd.read_table('anime/anime_ratings.dat')
    ratings_dataset = Dataset.load_from_df(anime_ratings[['User_ID', 'Anime_ID', 'Feedback']], reader)
    animeID_name = {}
    with open('anime/anime_info.dat', 'r',encoding='utf-8') as File:
        for file in File.readlines():
            line = file.strip()
            context = line.split('	')

            # for row in line.strip(" "):
            if context[0] != 'anime_ids':
                animeID = int(context[0])
                anime_name = context[1]
                animeID_name[animeID] = anime_name
    return (ratings_dataset, animeID_name)

def help(update: Update, context: CallbackContext):
    update.message.reply_text("""Available Commands :-
    /rate_and_recommend: Rate ten animes and get the recommendation.
    """)

class State:
    def __init__(self):
        self.statenum = 'general'
        self.res = []
        self.temp = ''
        self.animeID = 0
        self.sendContext = ""

def general(update: Update, context: CallbackContext):
    if s.statenum == "start_predict":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question2"
            s.animeID = rand_num

        return
    if s.statenum == "question2":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question3"
            s.animeID = rand_num

        return

    if s.statenum == "question3":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question4"
            s.animeID = rand_num

        return
    if s.statenum == "question4":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question5"
            s.animeID = rand_num

        return

    if s.statenum == "question5":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question6"
            s.animeID = rand_num

        return

    if s.statenum == "question6":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question7"
            s.animeID = rand_num

        return

    if s.statenum == "question7":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question8"
            s.animeID = rand_num

        return

    if s.statenum == "question8":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "question9"
            s.animeID = rand_num

        return

    if s.statenum == "question9":
        rand_num = random.randint(1, 7390)

        update.message.reply_text(
            "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: ")
        else:
            s.res.append([s.animeID, response])
            s.statenum = "predict"
            s.animeID = rand_num

        return

    if s.statenum == "predict":
        response = int(update.message.text)
        if int(response) < 0 or int(response) > 10:
            update.message.reply_text("Please input the integer from 1 to 10: "
                                      "")
        else:
            s.res.append([s.animeID, response])
            queue = StrictRedis(host='localhost', port=6379)
            message_send = ""
            for i in range(9):
                for j in range(2):
                    message_send += str(s.res[i][j])
                    message_send += ","
            message_send += str(s.res[9][0])
            message_send += ","
            message_send += str(s.res[9][1])

            queue.publish("anime_request", message_send.encode("utf-8"))

            flag = True
            pubsub = StrictRedis(host='localhost', port=6379).pubsub()
            pubsub.subscribe('anime_response')
            # The first message you receive will be a confirmation of subscription
            message = pubsub.get_message()
            while flag:

                message = pubsub.get_message()
                print(message)
                if message and message['data'] != 1:
                    msg = message['data'].decode("utf-8")
                    update.message.reply_text(msg)
                    flag = False

                else:
                    time.sleep(1)

            s.statenum = "general"
        return

def rate_and_recommend(update: Update, context: CallbackContext):
    rand_num = random.randint(1, 7390)
    update.message.reply_text(
        "Rating the ten anime with score with 1 to 10 \n" + animeID_to_name[rand_num])
    s.statenum = "start_predict"
    s.animeID = rand_num

if __name__ == "__main__":
    # Provide your bot's token
    updater = Updater("5253385256:AAFpLJLoS4bG2YT4UxX0Msmrr-YdWJWEpsM", use_context=True)
    dispatcher = updater.dispatcher
    s = State()
    dataset, animeID_to_name = load_dataset()

    dispatcher.add_handler(CommandHandler('start', start))
    dispatcher.add_handler(CommandHandler('help', help))
    dispatcher.add_handler(CommandHandler('rate_and_recommend', rate_and_recommend))
    dispatcher.add_handler(MessageHandler(Filters.text, general))
    updater.start_polling()
    updater.idle()


# -*- coding: utf-8 -*-
import scrapy
import re
from bs4 import BeautifulSoup
from ..items import PoemsScraperItem

class PoemsSpider(scrapy.Spider):
    name = 'poems'
    # start_urls = ['https://www.poets.org/poetsorg/poems?field_poem_themes_tid=886']
    page_no = 1
    theme_id = 0
    custom_settings = {
        # specifies exported fields and order
        'FEED_EXPORT_FIELDS': ["theme","poem"],
    }
    def __init__(self, *args, **kwargs):
        super(PoemsSpider, self).__init__(*args, **kwargs)

        PoemsSpider.theme_id = kwargs.get('theme_id')
        self.start_urls = ["https://www.poets.org/poetsorg/poems?field_poem_themes_tid={}".format(kwargs.get('theme_id'))]

    def parse(self, response):
        max_pages = int(getattr(self,'max_pages'))
        theme = response.css(".title")[0].xpath("text()")[0].extract()
        bsObj = BeautifulSoup(response.text,"lxml")
        poems_tds = bsObj.find_all("td")
        for p_link in poems_tds:
            try:
                link = p_link.find("a")['href']
                if "poem" in link:
                    poem_link = "https://www.poets.org{}".format(link)
                    yield response.follow(poem_link, callback=self.parse_poem, meta = {'theme': theme})
            except:
                pass
        next_link = "https://www.poets.org/poetsorg/poems?field_poem_themes_tid={}&page={}".format(PoemsSpider.theme_id,PoemsSpider.page_no)
        if PoemsSpider.page_no < max_pages:
            PoemsSpider.page_no += 1
            yield response.follow(next_link, callback=self.parse)

    def parse_poem(self, response):
        items = PoemsScraperItem()
        items['theme'] = response.meta.get("theme")
        poem = response.css("pre::text").extract()
        if(poem == ''):
            poem = response.css("#poem-content .even").extract()
        poem = "".join(poem)
        poem = re.sub(" +"," ",poem)
        poem = poem.replace("\n"," ").replace("\t","").replace("\r","")
        items['poem'] = poem
        yield items


def jumping_number(number):
    x = str(number)
    if len(x) == 1:
        return "Jumping!!"
    c = 0
    for i in x:
        if i.isdigit():
            c += 1

    for i in range(c):
        if i == c - 1:
            break
        z = int(x[i]) - int(x[i + 1])
        k = abs(z)
        if k != 1:
            return "Not!!"

    return "Jumping!!"

def jumping_number2(number):
    arr = list(map(int, str(number)))
    return ('Not!!', 'Jumping!!')[all(map(lambda a, b: abs(a - b) == 1, arr, arr[1:]))]


#
from input3 import _INPUT

def main(*, int=int, input=input):
    n = int(input())
    cars = [int(input()) for _ in range(n)]
    cycle = 0
    for i in range(n - 1):
        if cars[i] > cars[i + 1]:
            cycle += 1
    print(cycle)

if __name__ == "__main__":
    import io, sys

    sys.stdin = io.StringIO(_INPUT)

    main()


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from optparse import OptionParser
import os
import sys
import codecs
import collections
import json
import re
"""
Module for converting tsv output into penn sentence by line format for dynet bilstm tagger

"""

sys.stdout = codecs.getwriter('utf-8')(sys.__stdout__)
sys.stderr = codecs.getwriter('utf-8')(sys.__stderr__)
sys.stdin = codecs.getreader('utf-8')(sys.__stdin__)

CLASSDIST = {
  "Allgemein":14066,
  "Zugfahrt":3583,
  "Sonstige_Unregelm\u00e4ssigkeiten":3361,
  "Atmosph\u00e4re":2135,
  "Sicherheit":1140,
  "Ticketkauf":1005,
  "Service_und_Kundenbetreuung":670,
  "DB_App_und_Website":570,
  "Informationen":491,
  "Connectivity":441,
  "Auslastung_und_Platzangebot":431,
  "Komfort_und_Ausstattung":214,
  "Gastronomisches_Angebot":131,
  "Barrierefreiheit":103,
  "Image":93,
  "Reisen_mit_Kindern":68,
  "Design":60,
  "Toiletten":56,
  "Gep\u00e4ck":16,
  "O":0
}
VALIDCLASS = set(CLASSDIST)
VALIDCLASS.update({c +p for c in CLASSDIST for p in [':neutral',':positive',':negative']})

def select_most_freq_cat(categories, options):
    """Return most frequent class if any or None"""
    mostfreq = (None,-1)
    for c in categories:
        if c not in VALIDCLASS:
            print >> sys.stderr,'# IGNORED-UNKNOWN-CAT',c
            continue
        if c  in CLASSDIST:
            count = CLASSDIST[c]
        else:
            if ":" in c and not options.polarity:
                _c = re.sub(r':(neutral|positive|negative)','',c)
            count = CLASSDIST[_c]

        if count > mostfreq[1]:
            mostfreq = (c,count)

    if mostfreq[0] == None:
        print >> sys.stderr, 'MOSTFREQ-WAS-NONE',categories
        return 'O'
    else:
        return mostfreq[0]

def get_best_category(categories, options):
    """

    """
    valid_categories = [c for c in categories if c in VALIDCLASS]
    if valid_categories == []:
        print >> sys.stderr, '#NO-VALID-CATEGORY-FOUND',categories
        return 'O'
    if len(valid_categories) == 1 :
        return valid_categories[0]
    else:
        category = select_most_freq_cat(valid_categories, options)
        print >> sys.stderr, 'CHOSEN-CATEGORY', category, 'FROM', categories
        return category

def print_segment(l):
    """Output penn style format: one sent per line, POS/TAG."""

    for (w, t) in l:
        print >> sys.stdout,"%s/%s" % (w, t),
    print

def process_freq(args, options):
    """

    """
    global CLASSDIST, VALIDCLASS
    catcounter = collections.Counter()
    with codecs.open(options.freq_file, 'r', encoding="utf-8") as f:
        # 3 columns: ID, Word, Category list
        for l in f:
            F = l.strip().split('\t')
            for c in F[2].split():
                if not options.polarity:
                    c = re.sub(r':(neutral|positive|negative)', '', c)
                if '#' in c:  # support for old format with full categories
                    maincat, subcat = c.split('#',2)
                    catcounter[maincat] += 1
                else:
                    catcounter[c] += 1
    print >> sys.stderr, catcounter
    CLASSDIST = dict(catcounter)
    VALIDCLASS = set(CLASSDIST)

def process(options=None, args=None):
    """Do the processing"""
    global CLASSDIST, VALIDCLASS
    if options.debug:
        print >>sys.stderr, options

    process_freq(args, options)

    linenb = -1
    segment = []
    characters = collections.Counter()
    with codecs.open(args[0], 'r', encoding="utf-8") as f:
        for l in f:
            F = l.strip().split('\t')
            if len(F) < 3:
                print >> sys.stderr, '#WARNING: Empty line', l
                continue
            current_linenb = int(F[0].split('-')[0])
            if not options.polarity:
                F[2] = re.sub(r':(neutral|positive|negative)', '', F[2])
            categories = F[2].split(' ')

            F[2] = get_best_category(categories, options)

            characters.update(F[1])
            if current_linenb != linenb:
                if segment != []:
                    print_segment(segment)
                segment = [(F[1],F[2])]
            else:
                segment.append((F[1],F[2]))
            linenb = current_linenb

    if segment != []:
        print_segment(segment)

    for k,c in characters.most_common():
        print >> sys.stderr, "%s\t%d" %(k,c)

def main():
    """
    Invoke this module as a script
    """
# global options
    parser = OptionParser(
        usage = '%prog [OPTIONS] TSVFILE',
        version='%prog 0.99', #
        description='Convert ABSA tokenized format into penn',
        epilog='Contact simon.clematide@uzh.ch'
        )
    parser.add_option('-l', '--logfile', dest='logfilename',
                      help='write log to FILE', metavar='FILE')
    parser.add_option('-q', '--quiet',
                      action='store_true', dest='quiet', default=False,
                      help='do not print status messages to stderr')
    parser.add_option('-d', '--debug',
                      action='store_true', dest='debug', default=False,
                      help='print debug information')
    parser.add_option('-t', '--tsv',
                      action='store', dest='tsv', default=False,
                      help='produce tsv data')
    parser.add_option('-P', '--polarity',
                      action='store_true', dest='polarity', default=False,
                      help='keep polarity labels on categories')
    parser.add_option('-F', '--freq_file',
                      action='store', dest='freq_file', default= 'data/train_v1.4.xml.tsv',
                      help='read file for collecting the frequency distribution (TRAIN) for selecting the best category (default=%default)')

    (options, args) = parser.parse_args()
    if options.debug:
        print >> sys.stderr, "options=",options

    process(options=options,args=args)

if __name__ == '__main__':
    main()


import logging
from mongo_connector import MongoDataStore

__author__ = "Andrew Gafiychuk"

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    logging.debug("[+]Test App MongoDataStore...")

    db = MongoDataStore(db_name="test", table="test")
    record = ("SomeAuthor", "SomeAuthor Title", "http://google2000.com/search", "....Some Data....")

    db.insert_unique(*record)

    logging.debug("[+]Data inserting, try to get back...")
    res_list = db.get_all_data()

    logging.debug("[+]Try to get all docs in collection...")
    for record in res_list:
        print("User: {0}\n"
              "Title: {1}\n"
              "Link: {2}\n"
              "Post: {3}\n"
              .format(*record))

    logging.debug("[+]Try to get all docs by author...")
    res_list = db.get_data_by_author("SomeAuthor")

    for record in res_list:
        print(*record)

    logging.debug("[+]App Done !!!")


import lzma
import os

def save_sentences_to_dir(dataset_path, output_name, limit=-1):
    file = lzma.open(dataset_path, mode="rt",encoding="utf-8")
    os.mkdir(output_name)
    text_data = []
    file_count = 0
    for sentence in file:
        text_data.append(sentence)
        if len(text_data) == 10_000:
            with open(f'{output_name}/text_{file_count}.txt', 'w', encoding='utf-8') as fp:
                fp.write('\n'.join(text_data))
            text_data = []
            file_count += 1
            if file_count == limit:
                break

def parse_data(doc):
    sentences = doc.strip().split('\n')
    sentences = [sentence.lower() for sentence in sentences]
    return sentences

if __name__ == '__main__':
    save_sentences_to_dir("data\\cc100_data\\el.txt.xz", "data\\cc100_data\\cc100_el", 20000)