LSTM を書いてみた

出典元)Python 機械学習プログラミング https://www.amazon.co.jp/dp/4295003379/

tar 解凍

import tarfile
with tarfile.open('aclImdb_v1.tar.gz', 'r:gz') as tar:
    tar.extractall()

ai.stanford.edu

データ作成

import pandas as pd
import os

base_path = 'aclImdb'
labels = {'pos':1, 'neg':0}
from tqdm import tqdm_notebook

df = pd.DataFrame()
for s in ('test', 'train'):
    for l in labels.keys():
        path = os.path.join(base_path, s, l)
        for file in tqdm_notebook(os.listdir(path)):
            with open(os.path.join(path, file), 'r', encoding='utf-8') as infile:
                txt = infile.read()
            df = df.append([[txt, labels[l]]], ignore_index=True)
# データシャッフル
from sklearn.utils import shuffle
df = shuffle(df).reset_index(drop=True)

# char2int
from string import punctuation
from collections import Counter

counts = Counter()
for i, review in tqdm_notebook(enumerate(df['review'])):
    text = ''.join([c if c not in punctuation else ' '+c+' ' for c in review]).lower()
    df.loc[i, 'review'] = text
    counts.update(text.split())

word_counts = sorted(counts, key=counts.get, reverse=True)
print(word_counts[:5])
word_to_int = {word: ii for ii, word in enumerate(word_counts, 1)}

mapped_reviews = []
for review in tqdm_notebook(df['review']):
    mapped_reviews.append([word_to_int[word] for word in review.split()])

# 200 文字で 0padding
sequence_length = 200
sequences = np.zeros((len(mapped_reviews), sequence_length), dtype=int)

for i, row in tqdm_notebook(enumerate(mapped_reviews)):
    review_arr = np.array(row)
    sequences[i, -len(row):] = review_arr[-sequence_length:]

LSTM

X_train = sequences[:25000, :]
y_train = df.loc[:25000, 'sentiment'].values
X_test = sequences[25000:, :]
y_test = df.loc[25000:, 'sentiment'].values

np.random.seed(123)

# バッチジェネレータ
def create_batch_generator(x, y=None, batch_size=64):
    n_batch = len(x) // batch_size
    x = x[:n_batch*batch_size]
    if y is not None:
        y = y[:n_batch*batch_size]
    for ii in range(0, len(x), batch_size):
        if y is not None:
            yield x[ii:ii+batch_size], y[ii:ii+batch_size]
        else:
            yield x[ii:ii+batch_size]
import tensorflow as tf

class SentimentRNN(object):
    def __init__(self, n_words, seq_len=200,
                lstm_size=256, num_layers=1, batch_size=64,
                learning_rate=0.0001, embed_size=200):
        self.n_words = n_words
        self.seq_len = seq_len
        self.lstm_size = lstm_size  # 隠れユニットの個数
        self.num_layers = num_layers
        self.batch_size = batch_size
        self.learning_rate = learning_rate
        self.embed_size = embed_size
        
        self.g = tf.Graph()
        with self.g.as_default():
            tf.set_random_seed(123)
            self.build()
            self.saver = tf.train.Saver()
            self.init_op = tf.global_variables_initializer()
            
    def build(self):
        # プレースホルダ
        tf_x = tf.placeholder(tf.int32,
                             shape=(self.batch_size, self.seq_len),
                             name='tf_x')
        tf_y = tf.placeholder(tf.float32,
                             shape=(self.batch_size),
                             name='tf_y')
        tf_keepprob = tf.placeholder(tf.float32,
                             name='tf_keepprob')
        
        embedding = tf.Variable(tf.random_uniform((self.n_words, self.embed_size), minval=-1, maxval=1))
        
        embed_x = tf.nn.embedding_lookup(embedding, tf_x, name='embeded_x')
        
        cells = tf.contrib.rnn.MultiRNNCell(
            [tf.contrib.rnn.DropoutWrapper(
                tf.contrib.rnn.BasicLSTMCell(self.lstm_size),
                output_keep_prob=tf_keepprob)
             for i in range(self.num_layers)
            ]
        )
        
        # 初期状態を定義
        
        self.initial_state = cells.zero_state(self.batch_size, tf.float32)
        print('   << initial state >>  ', self.initial_state)
        
        lstm_outputs, self.final_state = \
            tf.nn.dynamic_rnn(cells, embed_x, initial_state=self.initial_state)
            
        print('\n  << lstm_output  >>  ', lstm_outputs)
        print('\n  << final_state  >>  ', self.final_state)
        
        logits = tf.layers.dense(inputs=lstm_outputs[:, -1],
                                units=1, activation=None, name='logits')
        logits = tf.squeeze(logits, name='logits_squeeze')
        
        print('\n  <<  logits          >>', logits)
        
        y_proba = tf.nn.sigmoid(logits, name='probabilities')
        predictions = {
            'probabilities': y_proba,
            'labels': tf.cast(tf.round(y_proba), tf.int32, name='labels')
        }
        
        print('\n  <<  predictions      >>', predictions)
        
        cost = tf.reduce_mean(
            tf.nn.sigmoid_cross_entropy_with_logits(labels=tf_y,
                                                   logits=logits),
            name='cost'
        )
        optimizer = tf.train.AdamOptimizer(self.learning_rate)
        train_op = optimizer.minimize(cost, name='train_op')
    
    def train(self, X_train, y_train, num_epochs):
        with tf.Session(graph=self.g) as sess:
            sess.run(self.init_op)
            iteration = 1
            for epoch in range(num_epochs):
                state = sess.run(self.initial_state)
                
                for batch_x, batch_y in create_batch_generator(X_train, y_train, self.batch_size):
                    feed = {'tf_x:0': batch_x,
                           'tf_y:0': batch_y,
                           'tf_keepprob:0': 0.5,
                           self.initial_state: state}
                    loss, _, state = sess.run(
                        ['cost:0', 'train_op', self.final_state],
                        feed_dict=feed
                    )
                    
                    if iteration % 20 == 0:
                        print("Epoch: %d / %d Iteration: %d | Train loss: %.5f"
                             % (epoch + 1, num_epochs, iteration, loss))
                    iteration += 1
                
                if (epoch + 1) % 10 == 0:
                    self.saver.save(sess, "model/sentiment-%d.ckpt" % epoch)
    
    def predict(self, X_data,  return_proba=False):
        preds = []
        with tf.Session(graph=self.g) as sess:
            self.saver.restore(sess, tf.train.latest_checkpoint('./model/'))
            test_state = sess.run(self.initial_state)

            for ii, batch_x in enumerate(create_batch_generator(X_data, None, batch_size=self.batch_size), 1):
                feed = {'tf_x:0': batch_x,
                           'tf_keepprob:0': 1.0,
                           self.initial_state: test_state}
                if return_proba:
                    pred, test_state = sess.run(['probabilities:0', self.final_state],
                                               feed_dict=feed)
                else:
                    pred, test_state = sess.run(['labels:0', self.final_state],
                                               feed_dict=feed)
                preds.append(pred)

            return np.concatenate(preds)

train+test

n_words = max(list(word_to_int.values())) + 1
rnn = SentimentRNN(
    n_words=n_words,
    seq_len=sequence_length,
    embed_size=256,
    lstm_size=128,
    num_layers=1,
    batch_size=100,
    learning_rate=0.001
)
   << initial state >>   (LSTMStateTuple(c=<tf.Tensor 'MultiRNNCellZeroState/DropoutWrapperZeroState/BasicLSTMCellZeroState/zeros:0' shape=(100, 128) dtype=float32>, h=<tf.Tensor 'MultiRNNCellZeroState/DropoutWrapperZeroState/BasicLSTMCellZeroState/zeros_1:0' shape=(100, 128) dtype=float32>),)

  << lstm_output  >>   Tensor("rnn/transpose_1:0", shape=(100, 200, 128), dtype=float32)

  << final_state  >>   (LSTMStateTuple(c=<tf.Tensor 'rnn/while/Exit_3:0' shape=(100, 128) dtype=float32>, h=<tf.Tensor 'rnn/while/Exit_4:0' shape=(100, 128) dtype=float32>),)

  <<  logits          >> Tensor("logits_squeeze:0", shape=(100,), dtype=float32)

  <<  predictions      >> {'probabilities': <tf.Tensor 'probabilities:0' shape=(100,) dtype=float32>, 'labels': <tf.Tensor 'labels:0' shape=(100,) dtype=int32>}
rnn.train(X_train, y_train, num_epochs=40)
Epoch: 1 / 40 Iteration: 20 | Train loss: 0.68427
Epoch: 1 / 40 Iteration: 40 | Train loss: 0.64362
Epoch: 1 / 40 Iteration: 60 | Train loss: 0.67976
Epoch: 1 / 40 Iteration: 80 | Train loss: 0.60497
...
preds = rnn.predict(X_test)
y_true = y_test[:len(preds)]
print(np.sum(preds == y_true) / len(y_true))
INFO:tensorflow:Restoring parameters from ./model/sentiment-39.ckpt
0.854

データ分析の流れ

準備

  1. Prepare Problem a) Load libraries b) Load dataset
  2. Summarize Data a) Descriptive statistics b) Data visualizations
  3. Prepare Data a) Data Cleaning b) Feature Selection c) Data Transforms (Normalize,...)

分析

  1. Evaluate Algorithms a) Split-out validation dataset b) Test options and evaluation metric c) Spot Check Algorithms d) Compare Algorithms
  2. Improve Accuracy a) Algorithm Tuning b) Ensembles
  3. Finalize Model a) Predictions on validation dataset b) Create standalone model on entire training dataset c) Save model for later use

https://www.kaggle.com/dennise/coursera-competition-getting-started-eda

例(train: ECサイトの transaction)

準備

1. repare Problem a) Load libraries b) Load dataset

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
color = sns.color_palette()
%matplotlib inline 

from scipy import stats


test=pd.read_csv('test.csv.gz',compression='gzip')
train=pd.read_csv('train.csv.gz',compression='gzip')

# データ初見確認
train.info()
train.describe() # count が index と等しければ、non-null と考えられる
train.head()

2. Summarize Data a) Descriptive statistics b) Data visualizations

train.price.hist()
train.price.value_counts()
train.price.nunique()

# histgram
train.price.apply(lambda x: np.log10(x+2)).hist(figsize=(12,16))

# index 順の plot
train.block_num.plot(figsize=(20,4))

# データ作り込み
train["category"]=train.name.apply(lambda x:x.split()[0])

date_format = '%d.%m.%Y'
train['day'] = pd.to_datetime(train['date'], format=date_format).dt.day
train['month'] = pd.to_datetime(train['date'], format=date_format).dt.month
train['year'] = pd.to_datetime(train['date'], format=date_format).dt.year
train['weekday'] = pd.to_datetime(train['date'], format=date_format).dt.dayofweek

train["revenue"]=train.item_cnt_day * train.price

train.groupby("item_category_id").sum()["revenue"].hist(figsize=(20,4),bins=100)

# 不必要な行の削除
train.drop("date",axis=1,inplace=True)
train.drop("name",axis=1,inplace=True)
train.drop("item_cnt_day",axis=1,inplace=True)

# 売上数の推移
train.groupby("block_num").sum()['revenue'].plot()
train.groupby("block_num").mean()['revenue'].plot()

# 特定商品名に対する価格の散布図
prices_hoge=train[train.category=="hogehoge"]["item_price"]

plt.figure(figsize=(20, 8), dpi=80)
plt.scatter(prices_hoge.index, prices_hoge,s=0.1)

# groupby した結果を pivot table で見る
# upnstack <--> stack (階層化データの変換)
train.groupby(["block_num","item_category_id"]).sum()["revenue"].unstack()
train.pivot_table(index=['shop_id','item_id'], columns='block_num', values='item_cnt_day',aggfunc='sum').fillna(0.0)

train.groupby(["block_num","item_category_id"]).sum()["revenue"].unstack().plot(figsize=(20,20))
train.groupby(["block_num","shop_id"]).sum()["revenue"].unstack().plot(figsize=(20,20))

# データの相関
sns.pairplot(train)

# データの入力ミスの修正
train.loc[train["shop_id"]==0,"shop_id"]=1

# テストデータの内容確認
test_list = test.shop_id.unique()
out_of_test = [i for i in train.shop_id.unique() if i not in test_list]

3. Prepare Data a) Data Cleaning b) Feature Selection c) Data Transforms (Normalize,...)

# a) Data Cleaning
columns_needed = ['item_id', 'shop_id', ...]
df = train[columns_needed]

# b) Feature Selection c) Data Transforms (Normalize,...)

df["price_category"]=np.nan
df["price_category"][(df["price"]>=0)&(df["price"]<=10000)]=0
df["price_category"][(df["price"]>10000)]=1

from sklearn import preprocessing

le = preprocessing.LabelEncoder()
le.fit(df.category)
df["meta_category"] = le.transform(df.category)

scaler = preprocessing.StandardScaler()
le.fit(df.star)
df['star'] = scaler.transform(df.star)

X_train=df.drop("item_cnt_month", axis=1)
# Reason for dropping item_price explained below
y_train=df["item_cnt_month"]

X_train.fillna(0, inplace=True)

分析

1. Evaluate Algorithms a) Split-out validation dataset b) Test options and evaluation metric c) Spot Check Algorithms d) Compare Algorithms

import lightgbm as lgb
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score

# 線形回帰
linmodel=LinearRegression()
linmodel.fit(X_train, y_train)
lin_pred=linmodel.predict(X_train)

print('R-squared is %f' % r2_score(lin_pred, y_train))

# boost model
lgb_params = {
               'feature_fraction': 0.75,
               'metric': 'rmse',
               'nthread':1, 
               'min_data_in_leaf': 2**7, 
               'bagging_fraction': 0.75, 
               'learning_rate': 0.03, 
               'objective': 'mse', 
               'bagging_seed': 2**7, 
               'num_leaves': 2**7,
               'bagging_freq':1,
               'verbose':0 
              }

model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)
pred_lgb = model.predict(X_train)

print('R-squared is %f' % r2_score(pred_lgb, y_train))

http://kidnohr.hatenadiary.com/entry/2018/09/21/012446

2. Improve Accuracy a) Algorithm Tuning b) Ensembles

meta_feature = np.c_[lin_pred, pred_lgb]
meta_lr = LinearRegression()
meta_lr.fit(meta_feature, final_train[33])

meta_pred = meta_lr.predict(meta_feature)

print('R-squared is %f' % r2_score(meta_pred, y_train))

3. Finalize Model a) Predictions on validation dataset b) Create standalone model on entire training dataset c) Save model for later use




    
  

python の future の使い方

import concurrent.futures
import os

score_list = []

def worker(my_random_seed):
    model = CatBoostClassifier(
        iterations=300,
        learning_rate=0.1,
        random_seed=my_random_seed
    )
    model.fit(
        X_train, y_train,
        cat_features=cat_features,
        eval_set=(X_validation, y_validation),
    )
    return model.best_score_['validation_0']['Logloss']

with concurrent.futures.ProcessPoolExecutor(max_workers=os.cpu_count()) as executor:
    futures = {executor.submit(worker, i) for i in range(10)}
    for future in concurrent.futures.as_completed(futures):
        score_list.append(future.result())

機械学習の黄色本 について

機械学習の黄色本

www.amazon.co.jp

https://www.amazon.co.jp/dp/4621061240www.amazon.co.jp

Web PDF

https://www.microsoft.com/en-us/research/uploads/prod/2006/01/Bishop-Pattern-Recognition-and-Machine-Learning-2006.pdf

演習問題 (www) の解答

https://www.microsoft.com/en-us/research/wp-content/uploads/2016/05/prml-web-sol-2009-09-08.pdf

元サイト

www.microsoft.com