6 回归集成：xgb、lgb、cat

 这个代码是从kaggle上拷贝过来的：

1. 如何使用三个树模型模块化训练；
2. 文本特征如何做，如何挖掘；
3. 时间特征的处理；
4. 模型权重集成；

import pandas as pd 
import math
import numpy as np 
import joblib 
import optuna
from lightgbm import LGBMRegressor
from catboost import CatBoostRegressor
from xgboost import XGBRegressor
from sklearn.preprocessing import *
from sklearn.metrics import *
from sklearn.model_selection import *
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
import datetime
import gc
from sklearn.base import clone
pd.set_option('display.max_columns', None)
import warnings
warnings.filterwarnings("ignore")
d_s = pd.read_csv('/kaggle/input/rohlik-orders-forecasting-challenge/solution_example.csv')
te_d = pd.read_csv('/kaggle/input/rohlik-orders-forecasting-challenge/test.csv')
tr_d = pd.read_csv('/kaggle/input/rohlik-orders-forecasting-challenge/train.csv')
tr_d.drop('id',axis=1,inplace=True)
te_d.drop('id',axis=1,inplace=True)
tr_d['holiday_name'].fillna('None', inplace=True)
te_d['holiday_name'].fillna('None', inplace=True)
def Process_Date(Df):
    Df['date'] = pd.to_datetime(Df['date'])
    Df['year'] = Df['date'].dt.year
    Df['day'] = Df['date'].dt.day
    Df['month'] = Df['date'].dt.month
    Df['month_name'] = Df['date'].dt.month_name()
    Df['day_of_week'] = Df['date'].dt.day_name()
    Df['week'] = Df['date'].dt.isocalendar().week
    
    Df['year_sin'] = np.sin(2 * np.pi * Df['year'])
    Df['year_cos'] = np.cos(2 * np.pi * Df['year'])
    Df['month_sin'] = np.sin(2 * np.pi * Df['month'] / 12) 
    Df['month_cos'] = np.cos(2 * np.pi * Df['month'] / 12)
    Df['day_sin'] = np.sin(2 * np.pi * Df['day'] / 31)  
    Df['day_cos'] = np.cos(2 * np.pi * Df['day'] / 31)
    Df['group']=(Df['year']-2020)*48+Df['month']*4+Df['day']//7
    
    Df['total_holidays_month'] = Df.groupby(['year', 'month'])['holiday'].transform('sum')
    Df['total_shops_closed_week'] = Df.groupby(['year', 'week'])['shops_closed'].transform('sum')
    Df['group_sin'] = np.sin(2 * np.pi * Df['group'] / Df['group'].max())
    Df['group_cos'] = np.cos(2 * np.pi * Df['group'] / Df['group'].max())
    return Df
tr_d = Process_Date(tr_d)
te_d = Process_Date(te_d)
tr_d = tr_d[['warehouse', 'date', 'holiday_name', 'holiday', 'shops_closed',
       'winter_school_holidays', 'school_holidays', 'year', 'day', 'month',
       'month_name', 'day_of_week', 'week', 'year_sin', 'year_cos',
       'month_sin', 'month_cos', 'day_sin', 'day_cos', 'group',
       'total_holidays_month', 'total_shops_closed_week',
       'group_sin', 'group_cos',
       'orders']]
le_month = LabelEncoder()
le_week = LabelEncoder()
le_war = LabelEncoder()
tr_d['month_name'] = le_month.fit_transform(tr_d['month_name'])
tr_d['day_of_week'] = le_week.fit_transform(tr_d['day_of_week'])
tr_d['warehouse'] = le_war.fit_transform(tr_d['warehouse'])
te_d['month_name'] = le_month.transform(te_d['month_name'])
te_d['day_of_week'] = le_week.transform(te_d['day_of_week'])
te_d['warehouse'] = le_war.transform(te_d['warehouse'])
def apply_tfidf_svd(df, text_column, max_features=1000, n_components=10):
    vectorizer = TfidfVectorizer(max_features=max_features, stop_words='english')
    vectors = vectorizer.fit_transform(df[text_column])
    svd = TruncatedSVD(n_components)
    x_sv = svd.fit_transform(vectors)
    tfidf_df = pd.DataFrame(x_sv)
    cols = [(text_column + "_tfidf_" + str(f)) for f in tfidf_df.columns.to_list()]
    tfidf_df.columns = cols
    df = df.reset_index(drop=True)
    df = pd.concat([df, tfidf_df], axis="columns")
    return df
tr_d = apply_tfidf_svd(tr_d,'holiday_name')
te_d = apply_tfidf_svd(te_d,'holiday_name')
tr_d.drop(['date','holiday_name'],axis=1,inplace=True)
te_d.drop(['date','holiday_name'],axis=1,inplace=True)
print(f"Shape Of Train Data is {tr_d.shape}")
print(f"Shape Of Test Data is {te_d.shape}")
%%time 
X = tr_d.drop('orders',axis=1)
y =tr_d['orders']
def cross_validate(model, n_splits=15):
    
    scores = []
    test_preds = np.zeros(len(te_d))
    
    groups = X['group']
    
    kfold = GroupKFold(n_splits=n_splits)
    
    for fold, (train_index, valid_index) in enumerate(kfold.split(X, y, groups=groups)):
        
        X_train = X.iloc[train_index]
        y_train = y.iloc[train_index]
        X_val = X.iloc[valid_index]
        y_val = y.iloc[valid_index]
                    
        m = clone(model)
        m.fit(X_train, y_train, eval_set=[(X_val, y_val)])
        
        y_pred = m.predict(X_val)
        score = mean_absolute_percentage_error(y_val, y_pred)
        
        scores.append(score)
        
        test_preds += m.predict(te_d) / n_splits
        gc.collect()
    print(f" MAPE mean: {np.array(scores).mean():.7f} (+- {np.array(scores).std():.7f})")
    return test_preds
%%time
SEED = 2375
cat = CatBoostRegressor(verbose=0,learning_rate=0.01,iterations=2000,
    random_state = SEED)
cat_test_preds = cross_validate(cat)
SEED = 1023
xgb = XGBRegressor(n_estimators=1000,learning_rate=0.05,verbosity=0,
            random_state=SEED)
xgb_test_preds = cross_validate(xgb)
%%time
lgb = LGBMRegressor(verbose=-1,
                    random_state = SEED
                   )
lgb_test_preds = cross_validate(lgb)
%%time 
weights = {
    'cat_test_preds': 0.45,  
    'lgb_test_preds': 0.45,
    'xgb_test_preds': 0.1,
    
}
cat_test_preds_weighted = cat_test_preds * weights['cat_test_preds']
lgb_test_preds_weighted = lgb_test_preds * weights['lgb_test_preds']
xgb_test_preds_weighted = xgb_test_preds * weights['xgb_test_preds']
ensemble_preds = cat_test_preds_weighted + lgb_test_preds_weighted + xgb_test_preds_weighted
d_s['orders'] = ensemble_preds
d_s['id'] = d_s['id']
d_s.to_csv('Submission.csv', index=False)
print(d_s.head())