# encoding: utf-8

#载入接下来分析用的库

import pandas as pd

import numpy as np

import xgboost as xgb

from tqdm import tqdm

from sklearn.svm import SVC

from keras.models import Sequential

from keras.layers.recurrent import LSTM, GRU

from keras.layers.core import Dense, Activation, Dropout

from keras.layers.embeddings import Embedding

from keras.layers.normalization import BatchNormalization

from keras.utils import np_utils

from sklearn import preprocessing, decomposition, model_selection, metrics, pipeline

from sklearn.model_selection import GridSearchCV

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

from sklearn.decomposition import TruncatedSVD

from sklearn.linear_model import LogisticRegression

from sklearn.model_selection import train_test_split

from sklearn.metrics import classification_report

from sklearn.naive_bayes import MultinomialNB

from keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D

from keras.preprocessing import sequence, text

from keras.callbacks import EarlyStopping

from nltk import word_tokenize

text = pd.read_csv('./competeDataForA.csv',sep = '\t', encoding ='utf-8')

test = pd.read_csv('./evaluationDataForA.csv',sep = '\t', encoding ='utf-8')

# print(text['id'].head())

# print(text['ocr'].head())

# print(text['label'].head())

print (text.info())

print (text.label.unique())

import jieba

# jieba.enable_parallel() #并行分词开启

text['文本分词'] = text['ocr'].apply(lambda i:jieba.cut(i) )

text['文本分词'] =[' '.join(i) for i in text['文本分词']]

test['文本分词'] = test['ocr'].apply(lambda i:jieba.cut(i) )

test['文本分词'] =[' '.join(i) for i in test['文本分词']]

print (text.head())

lbl_enc = preprocessing.LabelEncoder()

y = lbl_enc.fit_transform(text.label.values)

xtrain, xvalid, ytrain, yvalid = train_test_split(text.文本分词.values, y,

                                                  stratify=y,

                                                  random_state=42,

                                                  test_size=0.1, shuffle=True)

print (xtrain.shape)

print (xvalid.shape)

xtest = test.文本分词.values

X=text['文本分词']

X=[i.split() for i in X]

X[:2]

# ##################  LSTM 尝试  ##############################

# # 使用 keras tokenizer

from keras.preprocessing import sequence, text

token = text.Tokenizer(num_words=None)

max_len = 500

token.fit_on_texts(list(xtrain) + list(xvalid))

xtrain_seq = token.texts_to_sequences(xtrain)

xvalid_seq = token.texts_to_sequences(xvalid)

xtest_seq = token.texts_to_sequences(xtest)

#对文本序列进行zero填充

xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)

xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)

xtest_pad = sequence.pad_sequences(xtest_seq, maxlen=max_len)

word_index = token.word_index

import gensim

model = gensim.models.Word2Vec(X,min_count =5,window =8,size=100)   # X是经分词后的文本构成的list，也就是tokens的列表的列表

embeddings_index = dict(zip(model.wv.index2word, model.wv.vectors))

print('Found %s word vectors.' % len(embeddings_index))

print (len(word_index))

embedding_matrix = np.zeros((len(word_index) + 1, 100))

for word, i in tqdm(word_index.items()):

    embedding_vector = embeddings_index.get(word)

    if embedding_vector is not None:

        embedding_matrix[i] = embedding_vector

# 基于前面训练的Word2vec词向量，使用1个两层的LSTM模型

ytrain_enc = np_utils.to_categorical(ytrain)

yvalid_enc = np_utils.to_categorical(yvalid)

model = Sequential()

model.add(Embedding(len(word_index) + 1,

                     100,

                     weights=[embedding_matrix],

                     input_length=max_len,

                     trainable=False))

model.add(SpatialDropout1D(0.3))

model.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))

model.add(Dense(1024, activation='relu'))

model.add(Dropout(0.8))

model.add(Dense(1024, activation='relu'))

model.add(Dropout(0.8))

model.add(Dense(2))

model.add(Activation('softmax'))

model.compile(loss='categorical_crossentropy', optimizer='adam')

#在模型拟合时，使用early stopping这个回调函数（Callback Function）

earlystop = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto')

model.fit(xtrain_pad, y=ytrain_enc, batch_size=512, epochs=35,

          verbose=1, validation_data=(xvalid_pad, yvalid_enc), callbacks=[earlystop])

pred_lstm_2 = model.predict_classes(xtest_pad)

pred_lstm_2 = pd.DataFrame(pred_lstm_2)

pred_lstm_2_res = pd.concat([test['id'],pred_lstm_2], axis=1)

pred_lstm_2_res.rename(columns={0:'label'},inplace=True)

pred_lstm_2_res.to_csv('pred_lstm_2_res.csv',sep = ',', index = False, encoding = 'utf-8')

keras.preprocessing.text.Tokenizer(num_words=None,

                                   filters='!"#$%&()*+,-./:;<=>?@[\]^_`{|}~\t\n',

                                   lower=True,

                                   split=" ",

                                   char_level=False)

>>>from keras.preprocessing.text import Tokenizer

Using TensorFlow backend.

#  创建分词器 Tokenizer 对象

>>>tokenizer = Tokenizer()

#  text

>>>text = ["今天 北京 下 雨 了", "我 今天 加班"]

#  fit_on_texts 方法

>>>tokenizer.fit_on_texts(text)

#  word_counts属性

>>>tokenizer.word_counts

OrderedDict([('今天', 2),

             ('北京', 1),

             ('下', 1),

             ('雨', 1),

             ('了', 2),

             ('我', 1),

             ('加班', 1)])

#  word_docs属性

>>>tokenizer.word_docs

defaultdict(int, {'下': 1, '北京': 1, '今天': 2, '雨': 1, '了': 2, '我': 1, '加班': 1})

#  word_index属性

>>>tokenizer.word_index

{'今天': 1, '了': 2, '北京': 3, '下': 4, '雨': 5, '我': 6, '加班': 7}

#  document_count属性

>>>tokenizer.document_count

2

>>>tokenizer.texts_to_sequences(["下 雨 我 加班"])

[[4, 5, 6, 7]]

>>>keras.preprocessing.sequence.pad_sequences(tokenizer.texts_to_sequences(["下 雨 我 加班"]), maxlen=20)

array([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, 7]],dtype=int32)