Kaggleで学ぶ機械学習（分類）-特徴量エンジニアリング-

# ファイルの読み込み・データ確認
application_train = pd.read_csv("../input/home-credit-default-risk/application_train.csv")
print(application_train.shape)
application_train.head()

'''仮説に基づく特徴量生成'''
# [仮説]　所得金額が同じでも就労期間が短い方がハイステータスな人材だと予想できるため、貸し倒れリスクが低いと考える。
application_train['INCOME_div_EMPLOYED'] = application_train['AMT_INCOME_TOTAL'] / application_train['DAYS_EMPLOYED']

# [仮説]　外部機関によるスコアが平均的に高い方が貸し倒れリスクが低いと考えられる。
application_train["EXT_SOURCE_mean"] = application_train[["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3"]].mean(axis=1)
application_train["EXT_SOURCE_max"] = application_train[["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3"]].max(axis=1)
application_train["EXT_SOURCE_min"] = application_train[["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3"]].min(axis=1)
application_train["EXT_SOURCE_std"] = application_train[["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3"]].std(axis=1)
application_train["EXT_SOURCE_count"] = application_train[["EXT_SOURCE_1", "EXT_SOURCE_2", "EXT_SOURCE_3"]].notnull().sum(axis=1)


# [仮説]　所得金額が同じでも家族人数が多いほうが経済的な負担が大きいため貸し倒れリスクが高いと考えられる。
application_train['INCOME_div_PERSON'] = application_train['AMT_INCOME_TOTAL'] / application_train['CNT_FAM_MEMBERS']

#データセットの作成
x_train = application_train.drop(columns=["TARGET", "SK_ID_CURR"])
y_train = application_train["TARGET"]
id_train = application_train[["SK_ID_CURR"]]

for col in x_train.columns:
    if x_train[col].dtype=="O":
        x_train[col] = x_train[col].astype("category")

# ハイパーパラメータの設定
params = {
    'boosting_type': 'gbdt',
    'objective': 'binary', 
    'metric': 'auc',
    'learning_rate': 0.05,
    'num_leaves': 32,
    'n_estimators': 100000,
    "random_state": 123,
    "importance_type": "gain",
} 　 

list_nfold=[0,1,2,3,4]
n_splits=5
train_oof = np.zeros(len(x_train))
metrics = []
imp = pd.DataFrame()

# cross-validation
cv = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=123).split(x_train, y_train))
for nfold in list_nfold:
    print("-"*20, nfold, "-"*20)
        
    # make dataset
    idx_tr, idx_va = cv[nfold][0], cv[nfold][1]
    x_tr, y_tr, id_tr = x_train.loc[idx_tr, :], y_train[idx_tr], id_train.loc[idx_tr, :]
    x_va, y_va, id_va = x_train.loc[idx_va, :], y_train[idx_va], id_train.loc[idx_va, :]
    print(x_tr.shape, x_va.shape)
        
    # train
    model = lgb.LGBMClassifier(**params)
    model.fit(x_tr,
              y_tr,
              eval_set=[(x_tr, y_tr), (x_va, y_va)],
              early_stopping_rounds=100,
              verbose=100
             )
    fname_lgb = "model_lgb_fold{}.pickle".format(nfold)
    with open(fname_lgb, "wb") as f:
        pickle.dump(model, f, protocol=4)
        
    #evaluate
    y_tr_pred = model.predict_proba(x_tr)[:,1]
    y_va_pred = model.predict_proba(x_va)[:,1]
    metric_tr = roc_auc_score(y_tr, y_tr_pred)
    metric_va = roc_auc_score(y_va, y_va_pred)
    metrics.append([nfold, metric_tr, metric_va])
    print("[auc] tr:{:.4f}, va:{:.4f}".format(metric_tr, metric_va))
        
    # oof
    train_oof[idx_va] = y_va_pred
        
    # imp
    _imp = pd.DataFrame({"col":x_train.columns, "imp":model.feature_importances_, "nfold":nfold})
    imp = pd.concat([imp, _imp])
      
print("-"*20, "result", "-"*20)
# metric
metrics = np.array(metrics)
print(metrics)
print("[cv] tr:{:.4f}+-{:.4f}, va:{:.4f}+-{:.4f}".format(
    metrics[:,1].mean(), metrics[:,1].std(),
    metrics[:,2].mean(), metrics[:,2].std(),
))
print("[oof] {:.4f}".format(
    roc_auc_score(y_train, train_oof)
))
    
# oof
train_oof = pd.concat([
    id_train,
    pd.DataFrame({"pred":train_oof})
], axis=1)
    
# importance
imp = imp.groupby("col")["imp"].agg(["mean", "std"]).reset_index(drop=False)
imp.columns = ["col", "imp", "imp_std"]
    

#説明変数の重要度の確認
imp.sort_values("imp", ascending=False)[:10]