# -*- coding: utf-8 -*- import pandas as pd import numpy as np import jieba import multiprocessing # import keras from tensorflow import keras from gensim.models.word2vec import Word2Vec from gensim.corpora.dictionary import Dictionary from tensorflow.keras.preprocessing import sequence # from tensorflow.keras.layers import Dense # from tensorflow.keras.models import Model from sklearn.model_selection import train_test_split # from sklearn.cross_validation import train_test_split from tensorflow.keras.models import Sequential from tensorflow.python.keras.layers.embeddings import Embedding from tensorflow.python.keras.layers.recurrent import LSTM, GRU from tensorflow.python.keras.layers.core import Dense,Dropout,Activation from tensorflow.keras.models import model_from_yaml from tensorflow.keras.layers import Bidirectional from tensorflow.keras.layers import SimpleRNN import warnings # warnings.filterwarnings('ignore') np.random.seed(1337) # For Reproducibility import sys sys.setrecursionlimit(1000000) import yaml # 设置参数 cpu_count = multiprocessing.cpu_count() # 16 vocab_dim = 100 n_iterations = 5 n_exposures = 5 window_size = 7 n_epoch = 16 input_length = 100 maxlen = 100 batch_size = 128 def loadfile(): neg=pd.read_csv('./data/neg.csv',header=None,index_col=None,error_bad_lines=False) pos=pd.read_csv('./data/pos.csv',header=None,index_col=None,error_bad_lines=False) neu=pd.read_csv('./data/neu.csv', header=None, index_col=None,error_bad_lines=False) combined = np.concatenate((pos[0], neu[0], neg[0])) y = np.concatenate((np.ones(len(pos), dtype=int), np.zeros(len(neu), dtype=int), -1*np.ones(len(neg),dtype=int))) return combined,y #获取停用词 def loadStopWords(): stop = [line.strip() for line in open('./data/stopWords.txt', 'r', encoding='utf-8').readlines() ] #print("type(loadStopWords_stop)",type(stop)) return stop #对句子经行分词，并去掉换行符 def cutWords(text): text = [jieba.lcut(document.replace('\n', '')) for document in text] f_o = open('./data/cutWords.txt', 'w', encoding='utf-8') stopWords = loadStopWords() for line in text: leftWords = [] for i in line: if (i not in stopWords): leftWords.append(i) text_str = ' '.join(leftWords) f_o.write(text_str) f_o.write('\n') f_o.close() return text #创建词语字典，并返回每个词语的索引，词向量，以及每个句子所对应的词语索引 def create_dictionaries(model=None, combined=None): if (combined is not None) and (model is not None): gensim_dict = Dictionary() gensim_dict.doc2bow(model.wv.vocab.keys(), allow_update=True) w2indx = {v: k+1 for k, v in gensim_dict.items()} w2vec = {word: model[word] for word in w2indx.keys()} def parse_dataset(combined): data=[] for sentence in combined: new_txt = [] for word in sentence: try: new_txt.append(w2indx[word]) except: new_txt.append(0) # freqxiao10->0 data.append(new_txt) return data # word=>index combined=parse_dataset(combined) combined= sequence.pad_sequences(combined, maxlen=maxlen) return w2indx, w2vec,combined else: print ('没有提供数据...') #词向量训练 def word2vec_train(combined): model = Word2Vec(size=vocab_dim, min_count=n_exposures, window=window_size, workers=cpu_count, iter=n_iterations ) model.build_vocab(combined) # input: list model.train(combined,total_examples=model.corpus_count,epochs=model.iter) model.save('./model/Word2vec_model.pkl') index_dict, word_vectors,combined = create_dictionaries(model=model,combined=combined) return index_dict, word_vectors,combined def get_data(index_dict,word_vectors,combined,y): n_symbols = len(index_dict) + 1 embedding_weights = np.zeros((n_symbols, vocab_dim)) for word, index in index_dict.items(): embedding_weights[index, :] = word_vectors[word] x_train, x_test, y_train, y_test = train_test_split(combined, y, test_size=0.1) y_train = keras.utils.to_categorical(y_train,num_classes=3) y_test = keras.utils.to_categorical(y_test,num_classes=3) return n_symbols,embedding_weights,x_train,y_train,x_test,y_test def train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test): model = Sequential() # or Graph or whatever model.add(Embedding(output_dim=vocab_dim, input_dim=n_symbols, mask_zero=True, weights=[embedding_weights], input_length=input_length)) # 输入序列的长度 model.add(Bidirectional(LSTM(units=50, return_sequences=True))) model.add(Bidirectional(LSTM(units=50, return_sequences=False))) model.add(Dropout(0.5)) model.add(Dense(3)) model.add(Activation('softmax')) print ('编译模型...') model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy']) print ("训练...") model.fit(x_train, y_train, batch_size=batch_size, epochs=n_epoch,verbose=1) print ("评估...") score, acc = model.evaluate(x_test, y_test, batch_size=batch_size) print ('Test score:', score) print ('Test accuracy:', acc) # #以YAML字符串的形式返回模型，不包括权重，只包括结构 yaml_string = model.to_yaml() with open('./model/lstm.yml', 'w') as outfile: outfile.write( yaml.dump(yaml_string, default_flow_style=True) ) model.save_weights('./model/lstm.h5') if __name__=='__main__': print ('加载数据...') combined,y=loadfile() print ("数据集：",len(combined),"，数据集标签：",len(y)) print ('分词...') combined = cutWords(combined) print ('训练Word2vec模型...') index_dict, word_vectors,combined=word2vec_train(combined) print ('获取网络参数...') n_symbols,embedding_weights,x_train,y_train,x_test,y_test=get_data(index_dict, word_vectors,combined,y) print ("训练集：",x_train.shape,"，训练集标签：",y_train.shape) print ("测试集：",x_test.shape,"，测试集标签：",y_test.shape) train_lstm(n_symbols,embedding_weights,x_train,y_train,x_test,y_test)