天池竞赛特征工程方法总结

import numpy as npimport pandas as pdimport matplotlib.pyplot as plt import seaborn as snsfrom scipy import statsimport gcfrom collections import Counter import copyimport warnings warnings.filterwarnings("ignore")%matplotlib inline

test_data = pd.read_csv('./data_format1/test_format1.csv') train_data = pd.read_csv('./data_format1/train_format1.csv')user_info = pd.read_csv('./data_format1/user_info_format1.csv')user_log = pd.read_csv('./data_format1/user_log_format1.csv')

def reduce_mem_usage(df, verbose=True):    start_mem = df.memory_usage().sum() / 1024**2    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']    for col in df.columns: col_type = df[col].dtypes if col_type in numerics:        c_min = df[col].min()        c_max = df[col].max()        if str(col_type)[:3] == 'int':            if c_min > np.iinfo(np.int8).min and c_max < np.iinfo( np.int8).max:                df[col] = df[col].astype(np.int8)            elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(                np.int16).max:                df[col] = df[col].astype(np.int16)            elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo( np.int32).max:                df[col] = df[col].astype(np.int32)            elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(                np.int64).max:                df[col] = df[col].astype(np.int64)            else:                if c_min > np.finfo(np.float16).min and c_max < np.finfo(                    np.float16).max:                    df[col] = df[col].astype(np.float16)                elif c_min > np.finfo(np.float32).min and c_max < np.finfo( np.float32).max:                    df[col] = df[col].astype(np.float32)                 else:                    df[col] = df[col].astype(np.float64)                    end_mem = df.memory_usage().sum() / 1024**2                    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))

all_data = train_data.append(test_data)all_data = all_data.merge(user_info,on=['user_id'],how='left') del train_data, test_data, user_infogc.collect()

# 用户日志数据按时间排序user_log = user_log.sort_values(['user_id', 'time_stamp'])# 合并用户日志数据各字段，新字段名为item_idlist_join_func = lambda x: " ".join([str(i) for i in x])agg_dict = {    'item_id': list_join_func,    'cat_id': list_join_func,    'seller_id': list_join_func,    'brand_id': list_join_func,    'time_stamp': list_join_func,    'action_type': list_join_func}rename_dict = {    'item_id': 'item_path',    'cat_id': 'cat_path',    'seller_id': 'seller_path',    'brand_id': 'brand_path',    'time_stamp': 'time_stamp_path',    'action_type': 'action_type_path'}
def merge_list(df_ID, join_columns, df_data, agg_dict, rename_dict):    df_data = df_data.groupby(join_columns).agg(agg_dict).reset_index().rename(        columns=rename_dict)    df_ID = df_ID.merge(df_data, on=join_columns, how="left")    return df_IDall_data = merge_list(all_data, 'user_id', user_log, agg_dict, rename_dict)del user_log gc.collect()

def cnt_(x):     try:        return len(x.split(' '))     except:        return -1
def nunique_(x):     try:        return len(set(x.split(' ')))     except:        return -1
def max_(x):     try:        return np.max([float(i) for i in x.split(' ')])     except:        return -1
def min_(x):     try:        return np.min([float(i) for i in x.split(' ')])     except:        return -1
def std_(x):     try:        return np.std([float(i) for i in x.split(' ')])     except:        return -1
def most_n(x, n):     try:        return Counter(x.split(' ')).most_common(n)[n-1][0]     except:        return -1
def most_n_cnt(x, n):     try:        return Counter(x.split(' ')).most_common(n)[n-1][1]     except:        return -1

def user_cnt(df_data, single_col, name):    df_data[name] = df_data[single_col].apply(cnt_)    return df_data
def user_nunique(df_data, single_col, name):    df_data[name] = df_data[single_col].apply(nunique_)    return df_data
def user_max(df_data, single_col, name):    df_data[name] = df_data[single_col].apply(max_)    return df_data
def user_min(df_data, single_col, name):    df_data[name] = df_data[single_col].apply(min_)    return df_data
def user_std(df_data, single_col, name):    df_data[name] = df_data[single_col].apply(std_)    return df_data
def user_most_n(df_data, single_col, name, n=1):    func = lambda x: most_n(x, n)    df_data[name] = df_data[single_col].apply(func)    return df_data
def user_most_n_cnt(df_data, single_col, name, n=1):    func = lambda x: most_n_cnt(x, n)    df_data[name] = df_data[single_col].apply(func)    return df_data

# 取2000条数据举例all_data_test = all_data.head(2000)# 总次数all_data_test = user_cnt(all_data_test, 'seller_path', 'user_cnt')# 不同店铺个数all_data_test = user_nunique(all_data_test, 'seller_path', 'seller_nunique ')# 不同品类个数all_data_test = user_nunique(all_data_test, 'cat_path', 'cat_nunique')# 不同品牌个数all_data_test = user_nunique(all_data_test, 'brand_path',                             'brand_nunique')  # 不同商品个数all_data_test = user_nunique(all_data_test, 'item_path', 'item_nunique')# 活跃天数all_data_test = user_nunique(all_data_test, 'time_stamp_path',                             'time_stamp _nunique')# 不同用户行为种数all_data_test = user_nunique(all_data_test, 'action_type_path',                             'action_ty pe_nunique')

# 用户最喜欢的店铺all_data_test = user_most_n(all_data_test, 'seller_path', 'seller_most_1', n=1)# 最喜欢的类目all_data_test = user_most_n(all_data_test, 'cat_path', 'cat_most_1', n=1)# 最喜欢的品牌all_data_test = user_most_n(all_data_test, 'brand_path', 'brand_most_1', n= 1)# 最常见的行为动作all_data_test = user_most_n(all_data_test, 'action_type_path', 'action_type _1', n=1)

from sklearn.feature_extraction.text import CountVectorizerfrom sklearn.feature_extraction.text import TfidfVectorizerfrom sklearn.feature_extraction.text import ENGLISH_STOP_WORDSfrom scipy import sparsetfidfVec = TfidfVectorizer(stop_words=ENGLISH_STOP_WORDS,                           ngram_range=(1, 1),                           max_features=100)columns_list = ['seller_path']for i, col in enumerate(columns_list):    tfidfVec.fit(all_data_test[col])    data_ = tfidfVec.transform(all_data_test[col])    if i == 0:        data_cat = data_    else:        data_cat = sparse.hstack((data_cat, data_))

import gensim
model = gensim.models.Word2Vec(    all_data_test['seller_path'].apply(lambda x: x.split(' ')),    size=100,    window=5,    min_count=5,    workers=4)
def mean_w2v_(x, model, size=100):    try:        i = 0        for word in x.split(' '):            if word in model.wv.vocab:                i += 1            if i == 1:                vec = np.zeros(size)                vec += model.wv[word]        return vec / i    except:        return np.zeros(size)
def get_mean_w2v(df_data, columns, model, size):    data_array = []    for index, row in df_data.iterrows():        w2v = mean_w2v_(row[columns], model, size)        data_array.append(w2v)    return pd.DataFrame(data_array)
df_embeeding = get_mean_w2v(all_data_test, 'seller_path', model, 100)df_embeeding.columns = ['embeeding_' + str(i) for i in df_embeeding.columns]

# 1、使用 5 折交叉验证from sklearn.model_selection import StratifiedKFold, KFold folds = 5seed = 1kf = KFold(n_splits=5, shuffle=True, random_state=0)# 2、选择 lgb 和 xgb 分类模型作为基模型clf_list = [lgb_clf, xgb_clf] clf_list_col = ['lgb_clf', 'xgb_clf']# 3、获取 Stacking 特征clf_list = clf_list column_list = [] train_data_list=[] test_data_list=[] for clf in clf_list:      train_data,test_data,clf_name=clf(x_train, y_train, x_valid, kf, label_ split=None)      train_data_list.append(train_data)      test_data_list.append(test_data)train_stacking = np.concatenate(train_data_list, axis=1)test_stacking = np.concatenate(test_data_list, axis=1)