from pyod.models.iforest import IForest
import matplotlib.pyplot as plt
from scipy import stats
import seaborn as sns
import pandas as pd
import numpy as np

import warnings
warnings.filterwarnings('ignore')

plt.rcParams['font.sans-serif']=['SimHei']
plt.rcParams['axes.unicode_minus']=False

path = 'C:/users/youcai/Data'
data = pd.read_excel(path + 'default of credit card clients.xlsx',index_col=0)

###Missing Value View
missing = data.isnull().sum()
print(missing)


###Demonstration of kernel density plots for continuous data
def destiny_plot_all(modeldata,col,a,b):

    plt.figure(figsize=(15,9))
    count = 0
    for i in col:
        count = count + 1
        plt.subplot(a,b,count)
        sns.distplot(modeldata[i], rug=False, hist=True)
        plt.xlabel(i, fontsize=16)
        plt.ylabel('destiny', fontsize=16)
        plt.tick_params(labelsize=13)
    plt.show()

###Histogram distribution of the number of category values
def bar_count(modeldata,name):

    print('\033[;34m{0:*^100}\033[0m'.format('Bar charting'))
    plt.figure(figsize=(15, 9))
    v = pd.Series(modeldata[name]).value_counts()
    vv = pd.Series(modeldata[name]).value_counts() / modeldata.shape[0]
    ax = sns.countplot(x=name,data=modeldata,order=v.index.to_list())
    for x, y in enumerate(v):
        text = ax.text(x, y, round(list(vv)[x],2),fontsize=15)
        text.set_ha('center')
    plt.ylabel('count', fontsize=18)
    plt.xlabel(name, fontsize=18)
    plt.tick_params(labelsize=15)
    plt.show()


###Histogram distribution of the number of category values
def bar_count_all(modeldata,name_ls):

    print('\033[;34m{0:*^100}\033[0m'.format('Bar charting'))
    plt.figure(figsize=(22, 15))
    count = 0
    for name in name_ls:
        count = count + 1
        plt.subplot(3,2,count)
        v = pd.Series(modeldata[name]).value_counts()
        vv = pd.Series(modeldata[name]).value_counts() / modeldata.shape[0]
        ax = sns.countplot(x=name,data=modeldata,order=v.index.to_list())
        for x, y in enumerate(v):
            text = ax.text(x, y, round(list(vv)[x],2),fontsize=15)
            text.set_ha('center')
        plt.ylabel('count', fontsize=20)
        plt.xlabel(name, fontsize=20)
        plt.tick_params(labelsize=18)
    plt.show()


##box plot (math.)
def box_plot(modeldata,name):

    # Create a canvas and axis object containing two subplots

    print('\033[1;34m{0:*^100}\033[0m'.format('Violin charting'))
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(15,9))


    sns.violinplot(y=name[0], data=modeldata[name], inner='quartile', linewidth=1, saturation=0.8, ax=axes[0])
    axes[0].set_ylabel(name[0],fontsize=16)
    axes[0].set_title('deal data',fontsize=20)
    axes[0].tick_params(labelsize=15)

    sns.violinplot(y=name[1], data=modeldata[name], inner='quartile', linewidth=1, saturation=0.8, ax=axes[1])
    axes[1].set_ylabel(name[1],fontsize=16)
    axes[1].set_title('deal data',fontsize=20)

    plt.tight_layout()
    axes[1].tick_params(labelsize=13)

    plt.show()

###pie chart
def pie_figure(modeldata,name):

    print('\033[1;34m{0:*^100}\033[0m'.format('饼图绘制'))
    share_ls = []
    area_ls = set(list(modeldata[name]))
    for i in area_ls:
        l = modeldata[modeldata.eval(name) == i].shape[0]
        share_ls.append(l / modeldata.shape[0])

    plt.figure(figsize=(15, 9))

    patches, l_text, p_text = plt.pie(share_ls,labels = area_ls, autopct = '%.1f%%',startangle = 180, shadow = True)
    for t in l_text:
        t.set_size(15)
    for t in p_text:
        t.set_size(15)
    # Set the x- and y-axis scales to be the same so that the pie chart is round.
    plt.axis('equal')
    plt.legend(labels=area_ls, fontsize=15)
    plt.title('%s pie'%name,fontsize=20)
    plt.show()



def model_iforest(modeldata):

    clf = IForest(behaviour='new',contamination=0.05,random_state=1234)
    clf.fit(modeldata)
    y_test_scores = clf.decision_function(modeldata)
    y_test_scores = pd.Series(y_test_scores)
    threshold = clf.threshold_
    print('thresholds:', threshold)
    return modeldata,y_test_scores,threshold,

def visual_iforest(modeldata,yts,thre):

    plt.figure(figsize=(15,9))
    plt.hist(yts, bins='auto')
    plt.title('iforest outlier distribution result',fontsize=20)
    plt.tick_params(labelsize=15)
    plt.show()
    modeldata['distance'] = list(yts)
    modeldata['cluster'] = np.where(modeldata['distance']<=thre, 0, 1)
    print(modeldata['cluster'].value_counts())
    return modeldata

##One-way analysis of discrete data on labels
def y_discrete(modeldata,y,col):

    plt.figure(figsize=(22,12))
    count = 0
    for i in col:
        count = count + 1
        plt.subplot(1,3,count)
        g = modeldata.groupby(y)[i].value_counts()
        l = modeldata.groupby(y)[y].value_counts()
        all_y = set(modeldata[y])
        for j in all_y:
            res_ls = list(g[j] / modeldata[i].value_counts())
            g[j] = res_ls

        g.plot(kind='bar')
        plt.tick_params(labelsize=15)
        plt.ylabel('proportion',fontsize=15)
        plt.xlabel(i,fontsize=15)
        plt.title('%s single factor analysis'%i,fontsize=20)
    plt.show()


##One-way analysis of continuous data on labels
def y_continous(modeldata,y,col,a,b):

    g = modeldata.groupby(y).mean()[col]
    axes = g.plot(kind='bar',figsize=(25,15),subplots=True,fontsize=20,layout=(a, b))

    for ax in axes[0]:
        # title is used to get the total from bu_total
        title = ax.get_title()
        ax.legend(fontsize=15)
        ax.set_xticklabels(ax.get_xticklabels(), fontdict={'fontsize': 25})
        ax.set_title(title, fontdict={'fontsize': 25})
        ax.set_xlabel(ax.get_xlabel(), fontdict={'fontsize': 25})
        ax.set_ylabel(ax.get_ylabel(), fontdict={'fontsize': 25})

    plt.show()

##correlation scatter plot
def corr_heatmap(modeldata,col,name):

    print('\033[1;34m{0:*^100}\033[0m'.format('相关性热图'))
    plt.figure(figsize=(15, 9))
    correlation = modeldata[col].corr()
    sns.heatmap(correlation, square=True, vmax=0.8,annot=True)
    plt.title('%s correlation analysis'%name,fontsize=20)
    plt.yticks(fontsize=14)
    plt.xticks(fontsize=14)
    plt.show()


box_plot(data,['AGE','LIMIT_BAL'])
pie_figure(data,'default_payment_next_month')
pie_figure(data,'SEX')
bar_count(data,'default_payment_next_month')
bar_count(data,'SEX')
bar_count(data,'EDUCATION')
bar_count(data,'MARRIAGE')
bar_count_all(data,['PAY_1','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6'])
destiny_plot_all(data,['LIMIT_BAL','AGE'],1,2)
destiny_plot_all(data,['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'],2,3)
destiny_plot_all(data,['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],2,3)

data = data[(data.EDUCATION!=0)&(data.EDUCATION!=6)]
data = data[(data.MARRIAGE!=0)]

x = data.iloc[:,:-1]
y = data.iloc[:,-1]
print(data)

model_iforest_res = model_iforest(x)
fin_data = visual_iforest(model_iforest_res[0],model_iforest_res[1],model_iforest_res[2])
fin_data = fin_data[fin_data.cluster==0].iloc[:,:-2]
label_ls = list(data.loc[fin_data.index.to_list()]['default_payment_next_month'])
fin_data['label'] = label_ls
fin_data.index = [i for i in range(fin_data.shape[0])]
print(fin_data)

#View the distribution after removing outliers through an isolated forest
box_plot(fin_data,['AGE','LIMIT_BAL'])
destiny_plot_all(fin_data,['LIMIT_BAL','AGE'],1,2)
destiny_plot_all(fin_data,['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'],2,3)
destiny_plot_all(fin_data,['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],2,3)
###Re-doing outliers for single factors of amounts after isolated forests

cash = ['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'] + ['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']

fin_data[cash] = fin_data[cash].applymap(lambda x: 0 if x < 0 else x)

for i in ['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6']:
    fin_data = fin_data[(fin_data.eval(i)>=0)&(fin_data.eval(i)<=(fin_data[i].mean()+3.7*fin_data[i].std()))]

for i in ['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']:
    fin_data = fin_data[(fin_data.eval(i)>=0)&(fin_data.eval(i)<=(fin_data[i].mean()+3.8*fin_data[i].std()))]


pie_figure(fin_data,'label')
destiny_plot_all(fin_data,['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'],2,3)
destiny_plot_all(fin_data,['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],2,3)

y_discrete(fin_data,'label',['SEX','EDUCATION','MARRIAGE'])
y_continous(fin_data,'label',['LIMIT_BAL','AGE'],1,2)
y_continous(fin_data,'label',['PAY_1','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6'],1,6)
y_continous(fin_data,'label',['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],1,6)
y_continous(fin_data,'label',['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'],1,6)
corr_heatmap(fin_data,['PAY_1','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6'],'PAY')
corr_heatmap(fin_data,['BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6'],'BILL_AMT')
corr_heatmap(fin_data,['PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],'PAY_AMT')
corr_heatmap(fin_data,['PAY_1','PAY_2','PAY_3','PAY_4','PAY_6','BILL_AMT1','BILL_AMT6','PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'],'ALL')


result_df = pd.DataFrame(columns=['Variable', 'KW_statistic', 'P_value'])

for column in ['LIMIT_BAL','AGE','PAY_1','PAY_2','PAY_3','PAY_4','PAY_6','BILL_AMT1','BILL_AMT6','PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6']:
    x_y0 = fin_data.loc[fin_data['label'] == 0, column]
    x_y1 = fin_data.loc[fin_data['label'] == 1, column]
    kw_stat, p_value = stats.kruskal(x_y0, x_y1)
    result_df = pd.concat([result_df, pd.DataFrame({'Variable': [column], 'KW_statistic': [kw_stat], 'P_value': [p_value]})], ignore_index=True)

print(result_df)
print(fin_data.shape)

fin_col = ['LIMIT_BAL','SEX','EDUCATION','MARRIAGE','PAY_1','PAY_2','PAY_3','PAY_4','BILL_AMT1','PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6','label']
fin_data_out = fin_data[fin_col]
fin_data_out.to_csv(path + '建模数据.csv')