【NLP】基于深度学习的文本分类应用

# coding: utf-8from __future__ import unicode_literals
from keras.models import Sequentialfrom keras.layers import Embeddingfrom keras.layers import GlobalAveragePooling1Dfrom keras.layers import Dense
VOCAB_SIZE = 2000EMBEDDING_DIM = 100MAX_WORDS = 500CLASS_NUM = 5

def build_fastText():    model = Sequential()    # 将词汇数VOCAB_SIZE映射为EMBEDDING_DIM维    model.add(Embedding(VOCAB_SIZE, EMBEDDING_DIM, input_length=MAX_WORDS))    # 平均文档中所有词的embedding    model.add(GlobalAveragePooling1D())    # softmax分类    model.add(Dense(CLASS_NUM, activation='softmax'))    # 定义损失函数、优化器、分类度量指标    model.compile(loss='categorical_crossentropy', optimizer='SGD', metrics=['accuracy'])    return model
if __name__ == '__main__':    model = build_fastText()    print(model.summary())

import timeimport numpy as npimport fasttextimport pandas as pd
from sklearn.metrics import f1_scorefrom sklearn.utils import shufflefrom sklearn.model_selection import StratifiedKFold

def fasttext_model(nrows, train_num, lr=1.0, wordNgrams=2, minCount=1, epoch=25, loss='hs', dim=100):    start_time = time.time()
    # 转换为FastText需要的格式    train_df = pd.read_csv('../input/train_set.csv', sep='\t', nrows=nrows)
    # shuffle    train_df = shuffle(train_df, random_state=666)
    train_df['label_ft'] = '__label__' + train_df['label'].astype('str')    train_df[['text', 'label_ft']].iloc[:train_num].to_csv('../input/fastText_train.csv', index=None, header=None, sep='\t')
    model = fasttext.train_supervised('../input/fastText_train.csv', lr=lr, wordNgrams=wordNgrams, verbose=2,                                       minCount=minCount, epoch=epoch, loss=loss, dim=dim)
    train_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[:train_num]['text']]    print('Train f1_score:', f1_score(train_df['label'].values[:train_num].astype(str), train_pred, average='macro'))    val_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[train_num:]['text']]    print('Val f1_score:', f1_score(train_df['label'].values[train_num:].astype(str), val_pred, average='macro'))    train_time = time.time()    print('Train time: {:.2f}s'.format(train_time - start_time))
     # 预测并保存    test_df = pd.read_csv('../input/test_a.csv')
    test_pred = [model.predict(x)[0][0].split('__')[-1] for x in test_df['text']]    test_pred = pd.DataFrame(test_pred, columns=['label'])    test_pred.to_csv('../input/test_fastText_ridgeclassifier.csv', index=False)    print('Test predict saved.')    end_time = time.time()    print('Predict time:{:.2f}s'.format(end_time - train_time))

if __name__ == '__main__':      nrows = 200000    train_num = int(nrows * 0.7)    lr=0.01    wordNgrams=2    minCount=1    epoch=25    loss='hs'
    fasttext_model(nrows, train_num)

Train f1_score: 0.998663548149514Val f1_score: 0.911468448971427Train time: 257.32sTest predict saved.Predict time:13.40s

models = []scores = []pred_list = []
# K折交叉验证skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=666)for train_index, test_index in skf.split(train_df['text'], train_df['label_ft']):
    train_df[['text', 'label_ft']].iloc[train_index].to_csv('../input/fastText_train.csv', index=None, header=None, sep='\t')
    model = fasttext.train_supervised('../input/fastText_train.csv', lr=lr, wordNgrams=wordNgrams, verbose=2,                                           minCount=minCount, epoch=epoch, loss=loss)    models.append(model)
    val_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[test_index]['text']]    score = f1_score(train_df['label'].values[test_index].astype(str), val_pred, average='macro')    print('score', score)    scores.append(score)
print('mean score: ', np.mean(scores))train_time = time.time()print('Train time: {:.2f}s'.format(train_time - start_time))

def fasttext_kfold_model(nrows, train_num, n_splits, lr=1.0, wordNgrams=2, minCount=1, epoch=25, loss='hs', dim=100):    start_time = time.time()
    # 转换为FastText需要的格式    train_df = pd.read_csv('../input/train_set.csv', sep='\t', nrows=nrows)
    # shuffle    train_df = shuffle(train_df, random_state=666)
    train_df['label_ft'] = '__label__' + train_df['label'].astype('str')
    models = []    train_scores = []    val_scores = []
    # K折交叉验证    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=666)    for train_index, test_index in skf.split(train_df['text'], train_df['label_ft']):        train_df[['text', 'label_ft']].iloc[train_index].to_csv('../input/fastText_train.csv', index=None, header=None, sep='\t')
        model = fasttext.train_supervised('../input/fastText_train.csv', lr=lr, wordNgrams=wordNgrams, verbose=2,                                           minCount=minCount, epoch=epoch, loss=loss)        models.append(model)
        train_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[train_index]['text']]        train_score = f1_score(train_df['label'].values[train_index].astype(str), train_pred, average='macro')        # print('Train length: ', len(train_pred))        print('Train score: ', train_score)        train_scores.append(train_score)
        val_pred = [model.predict(x)[0][0].split('__')[-1] for x in train_df.iloc[test_index]['text']]        val_score = f1_score(train_df['label'].values[test_index].astype(str), val_pred, average='macro')        # print('Val length: ', len(val_pred))        print('Val score', val_score)        val_scores.append(val_score)
    print('mean train score: ', np.mean(train_scores))    print('mean val score: ', np.mean(val_scores))    train_time = time.time()    print('Train time: {:.2f}s'.format(train_time - start_time))
    return models
def fasttext_kfold_predict(models, n_splits):
    pred_list = []
    start_time = time.time()    # 预测并保存    test_df = pd.read_csv('../input/test_a.csv')
    # 消耗时间较长    for model in models:        test_pred = [model.predict(x)[0][0].split('__')[-1] for x in test_df['text']]        pred_list.append(test_pred)
    test_pred_label = pd.DataFrame(pred_list).T.apply(lambda row: np.argmax(np.bincount([row[i] for i in range(n_splits)])), axis=1)    test_pred_label.columns='label'
    test_pred_label.to_csv('../input/test_fastText_ridgeclassifier.csv', index=False)    print('Test predict saved.')    end_time = time.time()    print('Predict time:{:.2f}s'.format(end_time - start_time))

if __name__ == '__main__':  nrows = 200000  train_num = int(nrows * 0.7)  n_splits = 3  lr=0.1  wordNgrams=2  minCount=1  epoch=25  loss='hs'  dim=200
    """    Train score:  0.9635013320936988    Val score 0.9086640111428032    Train score:  0.9623510782430645    Val score 0.9094998879044359    Train score:  0.9628121318772955    Val score 0.9096191534698315    mean train score:  0.9628881807380196    mean val score:  0.9092610175056901    Train time: 740.60s    """
    models = fasttext_kfold_model(nrows, train_num, n_splits, lr=lr, wordNgrams=wordNgrams, minCount=minCount, epoch=epoch, loss=loss, dim=dim)    fasttext_kfold_predict(models, n_splits=n_splits)