janusgraph+cassandra で graph database を構築

Cassandra の yum install

  • /etc/yum.repos.d/cassandra.repo
name=Apache Cassandra
yum -y install cassandra

vim /etc/cassandra/conf/cassandra.yaml

### : /etc/cassandra/conf/cassandra.yaml
<start_rpc: false
>start_rpc: true

service cassandra start
service cassandra enable


wget https://github.com/JanusGraph/janusgraph/releases/download/v0.2.0/janusgraph-0.2.0-hadoop2.zip

unzip janusgraph-0.2.0-hadoop2.zip

cd janusgraph-0.2.0-hadoop2

gremlin> graph = JanusGraphFactory.open('conf/janusgraph-cassandra.properties')
gremlin> GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true)
gremlin> g = graph.traversal()
==>graphtraversalsource[standardjanusgraph[cassandrathrift:[]], standard]

Python での S3 からファイル取得(boto3)

boto3というモジュールが存在して、それを使ってS3 のファイルが取得できる。


In [1]: import boto3

In [7]: import botocore

In [21]: s3 = boto3.resource('s3', aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY,)

In [22]: try:
    ...:     s3.meta.client.head_bucket(Bucket=S3_BUCKET)
    ...: except Exception as e:
    ...:     print(e)

In [23]: for bucket in s3.buckets.all():
    ...:     for key in bucket.objects.all():
    ...:          print(key.key)


In [26]:try:
   ...:     s3.Bucket(S3_BUCKET).download_file(KEY, FILE)
   ...: except botocore.exceptions.ClientError as e:
   ...:     if e.response['Error']['Code'] == "404":
   ...:         raise S3FileNotFound
   ...:     else:
   ...:         raise

Boto 3 Documentation — Boto 3 Docs 1.7.19 documentation


private static DataType parseDataType(Config fieldsConfig) {
  String type = fieldsConfig.getString(FIELD_TYPE_CONFIG);
  switch (type) {
    case "string":
      return DataTypes.StringType;
    case "byte":
      return DataTypes.ByteType;
    case "short":
      return DataTypes.ShortType;
    case "int":
      return DataTypes.IntegerType;
    case "long":
      return DataTypes.LongType;
    case "float":
      return DataTypes.FloatType;
    case "double":
      return DataTypes.DoubleType;
    case "decimal":
      ConfigUtils.assertConfig(fieldsConfig, DECIMAL_SCALE_CONFIG);
      ConfigUtils.assertConfig(fieldsConfig, DECIMAL_PRECISION_CONFIG);
      return DataTypes.createDecimalType(
    case "boolean":
      return DataTypes.BooleanType;
    case "binary":
      return DataTypes.BinaryType;
    case "date":
      return DataTypes.DateType;
    case "timestamp":
      return DataTypes.TimestampType;
    case "array":
    case "map":
    case "struct":
      throw new RuntimeException("Schema check does not currently support complex types");
      throw new RuntimeException("Unknown type: " + type);

LDA(Latent Dirichlet Allocation) でのトピック抽出

以下の形式のsample.csvからデータを取得し、sklean の LDA でトピック抽出する。

id text
1 今日は晴れ。明日は雨
2 今日はカープが優勝した。
... ...
  • text2topic.py
#!/usr/bin/env python
# coding:utf-8

from __future__ import print_function
from time import time

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation

import pickle
import os
import pandas as pd

import MeCab

n_samples = 2000
n_features = 1000
n_components = 10
n_top_words = 20

FETCHED_PAGES_DIR_NAME = 'fetched_pages'
QUERIES = '胃もたれ 虫歯 花粉症対策 鬱 機械 骨折 肩こり 書類'.split(' ')
NB_PKL_FILENAME = 'naive_bayes_classifier.pkl'
MIN_TFIDF = 0.001
TFIDF_RESULT_PKL_FILENAME = 'tfidf_result.pkl'
TFIDF_VECTORIZER_PKL_FILENAME = 'tfidf_vectorizer.pkl'

def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])

def is_bigger_than_min_tfidf(term, terms, tfidfs):
    [term for term in terms if is_bigger_than_min_tfidf(term, terms, tfidfs)]で使う
    if tfidfs[terms.index(term)] > MIN_TFIDF:
        return True
    return False

def tfidf(values):
    # analyzerは文字列を入れると文字列のlistが返る関数
    vectorizer = TfidfVectorizer(analyzer=stems, min_df=1, max_df=50, max_features=n_features)
    corpus = [v for v in values]

    x = vectorizer.fit_transform(corpus)

    return x, vectorizer  # xはtfidf_resultとしてmainで受け取る

def countvec(values):
    # analyzerは文字列を入れると文字列のlistが返る関数
    vectorizer = CountVectorizer(analyzer=stems, min_df=1, max_df=50, max_features=n_features)
    corpus = [v for v in values]

    x = vectorizer.fit_transform(corpus)

    return x, vectorizer  # xはtfidf_resultとしてmainで受け取る

def _split_to_words(text, to_stem=False):
    入力: 'すべて自分のほうへ'
    出力: tuple(['すべて', '自分', 'の', 'ほう', 'へ'])
    tagger = MeCab.Tagger('mecabrc')  # 別のTaggerを使ってもいい
    mecab_result = tagger.parse(text)
    info_of_words = mecab_result.split('\n')
    words = []
    for info in info_of_words:
        # macabで分けると、文の最後に’’が、その手前に'EOS'が来る
        if info == 'EOS' or info == '':
            # info => 'な\t助詞,終助詞,*,*,*,*,な,ナ,ナ'
        info_elems = info.split(',')
        # 6番目に、無活用系の単語が入る。もし6番目が'*'だったら0番目を入れる
        if info_elems[6] == '*':
            # info_elems[0] => 'ヴァンロッサム\t名詞'
        if to_stem:
            # 語幹に変換
        # 語をそのまま
    return words

def words(text):
    words = _split_to_words(text=text, to_stem=False)
    return words

def stems(text):
    stems = _split_to_words(text=text, to_stem=True)
    return stems

if __name__ == '__main__':
    print("Loading dataset...")
    t0 = time()
    df = pd.read_csv('/tmp/sample.csv', header=None)
    y = df.values[:n_samples, 0]
    print("done in %0.3fs." % (time() - t0))

    print("Extracting tf features for tfidf...")
    t0 = time()
    tfidf_result, tfidf_vectorizer = tfidf(df.values[:n_samples, 1])  # tfidf_resultはtfidf関数のx

    pkl_tfidf_result_path = os.path.join('.', TFIDF_RESULT_PKL_FILENAME)
    pkl_tfidf_vectorizer_path = os.path.join('.', TFIDF_VECTORIZER_PKL_FILENAME)

    with open(pkl_tfidf_result_path, 'wb') as f:
        pickle.dump(tfidf_result, f)
    with open(pkl_tfidf_vectorizer_path, 'wb') as f:
        pickle.dump(tfidf_vectorizer, f)
    print("done in %0.3fs." % (time() - t0))

    # Use tf (raw term count) features for LDA.
    t0 = time()
    print("Extracting tf features for countvec...")
    tf_result, tf_vectorizer = countvec(df.values[:n_samples, 1])
    t0 = time()
    print("done in %0.3fs." % (time() - t0))

    # Fit the NMF model
    print("Fitting the NMF model (Frobenius norm) with tf-idf features, "
          "n_samples=%d and n_features=%d..."
          % (n_samples, n_features))
    t0 = time()
    nmf = NMF(n_components=n_components, random_state=1,
              alpha=.1, l1_ratio=.5).fit(tfidf_result)
    print("done in %0.3fs." % (time() - t0))

    print("\nTopics in NMF model (Frobenius norm):")
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    print_top_words(nmf, tfidf_feature_names, n_top_words)

    # Fit the NMF model
    print("Fitting the NMF model (generalized Kullback-Leibler divergence) with "
          "tf-idf features, n_samples=%d and n_features=%d..."
          % (n_samples, n_features))
    t0 = time()
    nmf = NMF(n_components=n_components, random_state=1,
              beta_loss='kullback-leibler', solver='mu', max_iter=1000, alpha=.1,
    print("done in %0.3fs." % (time() - t0))

    print("\nTopics in NMF model (generalized Kullback-Leibler divergence):")
    tfidf_feature_names = tfidf_vectorizer.get_feature_names()
    print_top_words(nmf, tfidf_feature_names, n_top_words)

    print("Fitting LDA models with tf features, "
          "n_samples=%d and n_features=%d..."
          % (n_samples, n_features))
    lda = LatentDirichletAllocation(n_components=n_components, max_iter=5,
    t0 = time()
    print("done in %0.3fs." % (time() - t0))

    print("\nTopics in LDA model:")
    tf_feature_names = tf_vectorizer.get_feature_names()
    print_top_words(lda, tf_feature_names, n_top_words)

Topic extraction with Non-negative Matrix Factorization and Latent Dirichlet Allocation — scikit-learn 0.19.1 documentation




The Term Vector Component | Apache Solr Reference Guide 7.1

#!/usr/bin/env python
# coding:utf-8

import requests
import json
from heapq import heappush, heappop

URL = "http://localhost:8983/solr/project/tvrh"

def get_tvlist(url, _id, start, rows):
    params = {
        'q': 'id:{}'.format(_id),
        'rows': rows,
        'start': start,
        'indent': 'true',
        'tv.tf_idf': 'true',
        'tv.fl': 'includes',
        'fl': 'id',
    r = requests.get(URL, params=params)
    dic = json.loads(r.text)
    tv_list = dic.get('termVectors', [])
    return tv_list

def analisys_dic(tv_list):
    res = {}
    num = len(tv_list) // 2
    for i in range(num):
        _id = tv_list[2 * i]
        term_list = tv_list[2 * i + 1][3]
        term_list_num = len(term_list) // 2
        _heap = []
        for h in range(term_list_num):
            string = term_list[2 * h]
            tfidf = term_list[2 * h + 1][1]
            if tfidf < 0.1:
                heappush(_heap, (- tfidf, string))
        tfidf_list = []
        for n in range(10):
            if _heap == []:
            pop_tuple = heappop(_heap)
            tfidf_list.append((pop_tuple[1], - pop_tuple[0]))

        res.update({_id: tfidf_list})
    return res



AWS にて、 dockerの立ち上げがうまく行かなかった



$ git clone https://github.com/efkbook/blog-sample

$ cd blog-sample/

$ docker-compose up -d
Building go
Step 1/5 : FROM golang:1.8
1.8: Pulling from library/golang
4176fe04cefe: Pull complete
851356ecf618: Pull complete
6115379c7b49: Pull complete
69914558965c: Pull complete
b108f9aa98db: Pull complete
df7abcd2981e: Pull complete
3e60cb3f592b: Pull complete
Digest: sha256:f0b5dab7581eddb49dabd1d1b9aa505ca3edcdf79a66395b5bfa4f3c036b49ef
Status: Downloaded newer image for golang:1.8
 ---> 0d283eb41a92
Step 2/5 : RUN mkdir -p /go/src/github.com/efkbook/blog-sample
 ---> Running in 59dba2dba710
 ---> 2fcb72cc73c5
Removing intermediate container 59dba2dba710
Step 3/5 : WORKDIR /go/src/github.com/efkbook/blog-sample
 ---> 9719d83418fe
Removing intermediate container cdab235d0dd4
Step 4/5 : CMD make app/run
 ---> Running in ae90cd6f44cd
 ---> e6a2ebc9eaa0
Removing intermediate container ae90cd6f44cd
Step 5/5 : EXPOSE 8080
 ---> Running in a3e82ed29f86
 ---> c50bc29f92a4
Removing intermediate container a3e82ed29f86
Successfully built c50bc29f92a4
Successfully tagged blogsample_go:latest
WARNING: Image for service go was built because it did not already exist. To rebuild this image you must use `docker-compose build` or `docker-compose up --build`.
Building fluentd
Step 1/7 : FROM debian:jessie
jessie: Pulling from library/debian
f2b6b4884fc8: Pull complete
Digest: sha256:a69efd2cf1f83d7fe6a374aaf27974ed031d72b2886acb4b730c6a2017db0c3d
Status: Downloaded newer image for debian:jessie
 ---> 5dd74d62fab8
Step 2/7 : ENV DEBIAN_FRONTEND noninteractive
 ---> Running in 6bd988af2ec6
 ---> 3cb72631b1ab
Removing intermediate container 6bd988af2ec6
Step 3/7 : RUN apt-get -qq update && apt-get install --no-install-recommends -y curl ca-certificates sudo build-essential libcurl4-gnutls-dev
 ---> Running in f59911d3c822
Reading package lists...
Building dependency tree...
Reading state information...
The following extra packages will be installed:
  binutils bzip2 cpp cpp-4.9 dpkg-dev g++ g++-4.9 gcc gcc-4.9 libasan1
  libatomic1 libc-dev-bin libc6-dev libcilkrts5 libcloog-isl4 libcurl3
  libcurl3-gnutls libdpkg-perl libffi6 libgcc-4.9-dev libgdbm3 libgmp10
  libgnutls-deb0-28 libgomp1 libgssapi-krb5-2 libhogweed2 libidn11 libisl10
  libitm1 libk5crypto3 libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2
  liblsan0 libmpc3 libmpfr4 libnettle4 libp11-kit0 libquadmath0 librtmp1
  libsasl2-2 libsasl2-modules-db libssh2-1 libssl1.0.0 libstdc++-4.9-dev
  libtasn1-6 libtimedate-perl libtsan0 libubsan0 linux-libc-dev make openssl
  patch perl perl-modules xz-utils
Suggested packages:
  binutils-doc bzip2-doc cpp-doc gcc-4.9-locales debian-keyring g++-multilib
  g++-4.9-multilib gcc-4.9-doc libstdc++6-4.9-dbg gcc-multilib manpages-dev
  autoconf automake libtool flex bison gdb gcc-doc gcc-4.9-multilib
  libgcc1-dbg libgomp1-dbg libitm1-dbg libatomic1-dbg libasan1-dbg
  liblsan0-dbg libtsan0-dbg libubsan0-dbg libcilkrts5-dbg libquadmath0-dbg
  glibc-doc libcurl4-doc libcurl3-dbg libgnutls28-dev libidn11-dev libkrb5-dev
  libldap2-dev librtmp-dev libssh2-1-dev pkg-config zlib1g-dev gnutls-bin
  krb5-doc krb5-user libstdc++-4.9-doc make-doc ed diffutils-doc perl-doc
  libterm-readline-gnu-perl libterm-readline-perl-perl libb-lint-perl
  libcpanplus-dist-build-perl libcpanplus-perl libfile-checktree-perl
  liblog-message-simple-perl liblog-message-perl libobject-accessor-perl
Recommended packages:
  fakeroot libalgorithm-merge-perl libfile-fcntllock-perl krb5-locales
  libsasl2-modules rename libarchive-extract-perl libmodule-pluggable-perl
  libpod-latex-perl libterm-ui-perl libtext-soundex-perl libcgi-pm-perl
  libmodule-build-perl libpackage-constants-perl
The following NEW packages will be installed:
  binutils build-essential bzip2 ca-certificates cpp cpp-4.9 curl dpkg-dev g++
  g++-4.9 gcc gcc-4.9 libasan1 libatomic1 libc-dev-bin libc6-dev libcilkrts5
  libcloog-isl4 libcurl3 libcurl3-gnutls libcurl4-gnutls-dev libdpkg-perl
  libffi6 libgcc-4.9-dev libgdbm3 libgmp10 libgnutls-deb0-28 libgomp1
  libgssapi-krb5-2 libhogweed2 libidn11 libisl10 libitm1 libk5crypto3
  libkeyutils1 libkrb5-3 libkrb5support0 libldap-2.4-2 liblsan0 libmpc3
  libmpfr4 libnettle4 libp11-kit0 libquadmath0 librtmp1 libsasl2-2
  libsasl2-modules-db libssh2-1 libssl1.0.0 libstdc++-4.9-dev libtasn1-6
  libtimedate-perl libtsan0 libubsan0 linux-libc-dev make openssl patch perl
  perl-modules sudo xz-utils
0 upgraded, 62 newly installed, 0 to remove and 1 not upgraded.
Need to get 54.6 MB of archives.
After this operation, 186 MB of additional disk space will be used.
Get:1 http://security.debian.org/ jessie/updates/main libssl1.0.0 amd64 1.0.1t-1+deb8u8 [1044 kB]
Get:2 http://security.debian.org/ jessie/updates/main libasan1 amd64 4.9.2-10+deb8u1 [195 kB]
Get:3 http://security.debian.org/ jessie/updates/main libatomic1 amd64 4.9.2-10+deb8u1 [9014 B]
Get:4 http://security.debian.org/ jessie/updates/main libcilkrts5 amd64 4.9.2-10+deb8u1 [40.1 kB]
Get:5 http://security.debian.org/ jessie/updates/main libcurl3 amd64 7.38.0-4+deb8u10 [260 kB]
Get:6 http://security.debian.org/ jessie/updates/main libcurl3-gnutls amd64 7.38.0-4+deb8u10 [252 kB]
Get:7 http://security.debian.org/ jessie/updates/main libgomp1 amd64 4.9.2-10+deb8u1 [37.8 kB]
Get:8 http://security.debian.org/ jessie/updates/main libitm1 amd64 4.9.2-10+deb8u1 [29.3 kB]
Get:9 http://security.debian.org/ jessie/updates/main liblsan0 amd64 4.9.2-10+deb8u1 [92.7 kB]
Get:10 http://security.debian.org/ jessie/updates/main libquadmath0 amd64 4.9.2-10+deb8u1 [129 kB]
Get:11 http://security.debian.org/ jessie/updates/main libtsan0 amd64 4.9.2-10+deb8u1 [212 kB]
Get:12 http://security.debian.org/ jessie/updates/main libubsan0 amd64 4.9.2-10+deb8u1 [82.4 kB]
Get:13 http://security.debian.org/ jessie/updates/main cpp-4.9 amd64 4.9.2-10+deb8u1 [5002 kB]
Get:14 http://security.debian.org/ jessie/updates/main libgcc-4.9-dev amd64 4.9.2-10+deb8u1 [2066 kB]
Get:15 http://security.debian.org/ jessie/updates/main linux-libc-dev amd64 3.16.51-3+deb8u1 [1083 kB]
Get:16 http://deb.debian.org/debian/ jessie/main libgdbm3 amd64 1.8.3-13.1 [30.0 kB]
Get:17 http://security.debian.org/ jessie/updates/main gcc-4.9 amd64 4.9.2-10+deb8u1 [5184 kB]
Get:18 http://security.debian.org/ jessie/updates/main libstdc++-4.9-dev amd64 4.9.2-10+deb8u1 [1120 kB]
Get:19 http://security.debian.org/ jessie/updates/main g++-4.9 amd64 4.9.2-10+deb8u1 [17.3 MB]
Get:20 http://deb.debian.org/debian/ jessie/main libgmp10 amd64 2:6.0.0+dfsg-6 [253 kB]
Get:21 http://security.debian.org/ jessie/updates/main openssl amd64 1.0.1t-1+deb8u8 [664 kB]
Get:22 http://security.debian.org/ jessie/updates/main curl amd64 7.38.0-4+deb8u10 [201 kB]
Get:23 http://security.debian.org/ jessie/updates/main libcurl4-gnutls-dev amd64 7.38.0-4+deb8u10 [329 kB]
Get:24 http://deb.debian.org/debian/ jessie/main libnettle4 amd64 2.7.1-5+deb8u2 [176 kB]
Get:25 http://deb.debian.org/debian/ jessie/main libhogweed2 amd64 2.7.1-5+deb8u2 [125 kB]
Get:26 http://deb.debian.org/debian/ jessie/main libffi6 amd64 3.1-2+deb8u1 [20.2 kB]
Get:27 http://deb.debian.org/debian/ jessie/main libp11-kit0 amd64 0.20.7-1 [81.2 kB]
Get:28 http://deb.debian.org/debian/ jessie/main libtasn1-6 amd64 4.2-3+deb8u3 [49.2 kB]
Get:29 http://deb.debian.org/debian/ jessie/main libgnutls-deb0-28 amd64 3.3.8-6+deb8u7 [696 kB]
Get:30 http://deb.debian.org/debian/ jessie/main libkeyutils1 amd64 1.5.9-5+b1 [12.0 kB]
Get:31 http://deb.debian.org/debian/ jessie/main libkrb5support0 amd64 1.12.1+dfsg-19+deb8u4 [59.4 kB]
Get:32 http://deb.debian.org/debian/ jessie/main libk5crypto3 amd64 1.12.1+dfsg-19+deb8u4 [116 kB]
Get:33 http://deb.debian.org/debian/ jessie/main libkrb5-3 amd64 1.12.1+dfsg-19+deb8u4 [303 kB]
Get:34 http://deb.debian.org/debian/ jessie/main libgssapi-krb5-2 amd64 1.12.1+dfsg-19+deb8u4 [152 kB]
Get:35 http://deb.debian.org/debian/ jessie/main libidn11 amd64 1.29-1+deb8u2 [136 kB]
Get:36 http://deb.debian.org/debian/ jessie/main libsasl2-modules-db amd64 2.1.26.dfsg1-13+deb8u1 [67.1 kB]
Get:37 http://deb.debian.org/debian/ jessie/main libsasl2-2 amd64 2.1.26.dfsg1-13+deb8u1 [105 kB]
Get:38 http://deb.debian.org/debian/ jessie/main libldap-2.4-2 amd64 2.4.40+dfsg-1+deb8u3 [218 kB]
Get:39 http://deb.debian.org/debian/ jessie/main perl-modules all 5.20.2-3+deb8u9 [2553 kB]
Get:40 http://deb.debian.org/debian/ jessie/main perl amd64 5.20.2-3+deb8u9 [2641 kB]
Get:41 http://deb.debian.org/debian/ jessie/main libisl10 amd64 0.12.2-2 [440 kB]
Get:42 http://deb.debian.org/debian/ jessie/main libcloog-isl4 amd64 0.18.2-1+b2 [61.8 kB]
Get:43 http://deb.debian.org/debian/ jessie/main librtmp1 amd64 2.4+20150115.gita107cef-1+deb8u1 [60.0 kB]
Get:44 http://deb.debian.org/debian/ jessie/main libssh2-1 amd64 1.4.3-4.1+deb8u1 [125 kB]
Get:45 http://deb.debian.org/debian/ jessie/main libmpfr4 amd64 3.1.2-2 [527 kB]
Get:46 http://deb.debian.org/debian/ jessie/main libmpc3 amd64 1.0.2-1 [39.3 kB]
Get:47 http://deb.debian.org/debian/ jessie/main bzip2 amd64 1.0.6-7+b3 [46.9 kB]
Get:48 http://deb.debian.org/debian/ jessie/main patch amd64 2.7.5-1 [109 kB]
Get:49 http://deb.debian.org/debian/ jessie/main xz-utils amd64 5.1.1alpha+20120614-2+b3 [221 kB]
Get:50 http://deb.debian.org/debian/ jessie/main binutils amd64 2.25-5+deb8u1 [3496 kB]
Get:51 http://deb.debian.org/debian/ jessie/main libc-dev-bin amd64 2.19-18+deb8u10 [238 kB]
Get:52 http://deb.debian.org/debian/ jessie/main libc6-dev amd64 2.19-18+deb8u10 [2003 kB]
Get:53 http://deb.debian.org/debian/ jessie/main cpp amd64 4:4.9.2-2 [17.3 kB]
Get:54 http://deb.debian.org/debian/ jessie/main gcc amd64 4:4.9.2-2 [5136 B]
Get:55 http://deb.debian.org/debian/ jessie/main g++ amd64 4:4.9.2-2 [1530 B]
Get:56 http://deb.debian.org/debian/ jessie/main make amd64 4.0-8.1 [349 kB]
Get:57 http://deb.debian.org/debian/ jessie/main libtimedate-perl all 2.3000-2 [42.2 kB]
Get:58 http://deb.debian.org/debian/ jessie/main libdpkg-perl all 1.17.27 [1075 kB]
Get:59 http://deb.debian.org/debian/ jessie/main dpkg-dev all 1.17.27 [1548 kB]
Get:60 http://deb.debian.org/debian/ jessie/main build-essential amd64 11.7 [7114 B]
Get:61 http://deb.debian.org/debian/ jessie/main ca-certificates all 20141019+deb8u3 [207 kB]
Get:62 http://deb.debian.org/debian/ jessie/main sudo amd64 1.8.10p3-1+deb8u5 [854 kB]
debconf: delaying package configuration, since apt-utils is not installed
Fetched 54.6 MB in 12s (4386 kB/s)
Selecting previously unselected package libgdbm3:amd64.
(Reading database ... 7561 files and directories currently installed.)
Preparing to unpack .../libgdbm3_1.8.3-13.1_amd64.deb ...
Unpacking libgdbm3:amd64 (1.8.3-13.1) ...
Selecting previously unselected package libssl1.0.0:amd64.
Preparing to unpack .../libssl1.0.0_1.0.1t-1+deb8u8_amd64.deb ...
Unpacking libssl1.0.0:amd64 (1.0.1t-1+deb8u8) ...
Selecting previously unselected package libgmp10:amd64.
Preparing to unpack .../libgmp10_2%3a6.0.0+dfsg-6_amd64.deb ...
Unpacking libgmp10:amd64 (2:6.0.0+dfsg-6) ...
Selecting previously unselected package libnettle4:amd64.
Preparing to unpack .../libnettle4_2.7.1-5+deb8u2_amd64.deb ...
Unpacking libnettle4:amd64 (2.7.1-5+deb8u2) ...
Selecting previously unselected package libhogweed2:amd64.
Preparing to unpack .../libhogweed2_2.7.1-5+deb8u2_amd64.deb ...
Unpacking libhogweed2:amd64 (2.7.1-5+deb8u2) ...
Selecting previously unselected package libffi6:amd64.
Preparing to unpack .../libffi6_3.1-2+deb8u1_amd64.deb ...
Unpacking libffi6:amd64 (3.1-2+deb8u1) ...
Selecting previously unselected package libp11-kit0:amd64.
Preparing to unpack .../libp11-kit0_0.20.7-1_amd64.deb ...
Unpacking libp11-kit0:amd64 (0.20.7-1) ...
Selecting previously unselected package libtasn1-6:amd64.
Preparing to unpack .../libtasn1-6_4.2-3+deb8u3_amd64.deb ...
Unpacking libtasn1-6:amd64 (4.2-3+deb8u3) ...
Selecting previously unselected package libgnutls-deb0-28:amd64.
Preparing to unpack .../libgnutls-deb0-28_3.3.8-6+deb8u7_amd64.deb ...
Unpacking libgnutls-deb0-28:amd64 (3.3.8-6+deb8u7) ...
Selecting previously unselected package libkeyutils1:amd64.
Preparing to unpack .../libkeyutils1_1.5.9-5+b1_amd64.deb ...
Unpacking libkeyutils1:amd64 (1.5.9-5+b1) ...
Selecting previously unselected package libkrb5support0:amd64.
Preparing to unpack .../libkrb5support0_1.12.1+dfsg-19+deb8u4_amd64.deb ...
Unpacking libkrb5support0:amd64 (1.12.1+dfsg-19+deb8u4) ...
Selecting previously unselected package libk5crypto3:amd64.
Preparing to unpack .../libk5crypto3_1.12.1+dfsg-19+deb8u4_amd64.deb ...
Unpacking libk5crypto3:amd64 (1.12.1+dfsg-19+deb8u4) ...
Selecting previously unselected package libkrb5-3:amd64.
Preparing to unpack .../libkrb5-3_1.12.1+dfsg-19+deb8u4_amd64.deb ...
Unpacking libkrb5-3:amd64 (1.12.1+dfsg-19+deb8u4) ...
Selecting previously unselected package libgssapi-krb5-2:amd64.
Preparing to unpack .../libgssapi-krb5-2_1.12.1+dfsg-19+deb8u4_amd64.deb ...
Unpacking libgssapi-krb5-2:amd64 (1.12.1+dfsg-19+deb8u4) ...
Selecting previously unselected package libidn11:amd64.
Preparing to unpack .../libidn11_1.29-1+deb8u2_amd64.deb ...
Unpacking libidn11:amd64 (1.29-1+deb8u2) ...
Selecting previously unselected package libsasl2-modules-db:amd64.
Preparing to unpack .../libsasl2-modules-db_2.1.26.dfsg1-13+deb8u1_amd64.deb ...
Unpacking libsasl2-modules-db:amd64 (2.1.26.dfsg1-13+deb8u1) ...
Selecting previously unselected package libsasl2-2:amd64.
Preparing to unpack .../libsasl2-2_2.1.26.dfsg1-13+deb8u1_amd64.deb ...
Unpacking libsasl2-2:amd64 (2.1.26.dfsg1-13+deb8u1) ...
Selecting previously unselected package libldap-2.4-2:amd64.
Preparing to unpack .../libldap-2.4-2_2.4.40+dfsg-1+deb8u3_amd64.deb ...
Unpacking libldap-2.4-2:amd64 (2.4.40+dfsg-1+deb8u3) ...
Selecting previously unselected package perl-modules.
Preparing to unpack .../perl-modules_5.20.2-3+deb8u9_all.deb ...
Unpacking perl-modules (5.20.2-3+deb8u9) ...
Selecting previously unselected package perl.
Preparing to unpack .../perl_5.20.2-3+deb8u9_amd64.deb ...
Unpacking perl (5.20.2-3+deb8u9) ...
Selecting previously unselected package libasan1:amd64.
Preparing to unpack .../libasan1_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libasan1:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libatomic1:amd64.
Preparing to unpack .../libatomic1_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libatomic1:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libcilkrts5:amd64.
Preparing to unpack .../libcilkrts5_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libcilkrts5:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libisl10:amd64.
Preparing to unpack .../libisl10_0.12.2-2_amd64.deb ...
Unpacking libisl10:amd64 (0.12.2-2) ...
Selecting previously unselected package libcloog-isl4:amd64.
Preparing to unpack .../libcloog-isl4_0.18.2-1+b2_amd64.deb ...
Unpacking libcloog-isl4:amd64 (0.18.2-1+b2) ...
Selecting previously unselected package librtmp1:amd64.
Preparing to unpack .../librtmp1_2.4+20150115.gita107cef-1+deb8u1_amd64.deb ...
Unpacking librtmp1:amd64 (2.4+20150115.gita107cef-1+deb8u1) ...
Selecting previously unselected package libssh2-1:amd64.
Preparing to unpack .../libssh2-1_1.4.3-4.1+deb8u1_amd64.deb ...
Unpacking libssh2-1:amd64 (1.4.3-4.1+deb8u1) ...
Selecting previously unselected package libcurl3:amd64.
Preparing to unpack .../libcurl3_7.38.0-4+deb8u10_amd64.deb ...
Unpacking libcurl3:amd64 (7.38.0-4+deb8u10) ...
Selecting previously unselected package libcurl3-gnutls:amd64.
Preparing to unpack .../libcurl3-gnutls_7.38.0-4+deb8u10_amd64.deb ...
Unpacking libcurl3-gnutls:amd64 (7.38.0-4+deb8u10) ...
Selecting previously unselected package libgomp1:amd64.
Preparing to unpack .../libgomp1_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libgomp1:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libitm1:amd64.
Preparing to unpack .../libitm1_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libitm1:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package liblsan0:amd64.
Preparing to unpack .../liblsan0_4.9.2-10+deb8u1_amd64.deb ...
Unpacking liblsan0:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libmpfr4:amd64.
Preparing to unpack .../libmpfr4_3.1.2-2_amd64.deb ...
Unpacking libmpfr4:amd64 (3.1.2-2) ...
Selecting previously unselected package libquadmath0:amd64.
Preparing to unpack .../libquadmath0_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libquadmath0:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libtsan0:amd64.
Preparing to unpack .../libtsan0_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libtsan0:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libubsan0:amd64.
Preparing to unpack .../libubsan0_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libubsan0:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package libmpc3:amd64.
Preparing to unpack .../libmpc3_1.0.2-1_amd64.deb ...
Unpacking libmpc3:amd64 (1.0.2-1) ...
Selecting previously unselected package bzip2.
Preparing to unpack .../bzip2_1.0.6-7+b3_amd64.deb ...
Unpacking bzip2 (1.0.6-7+b3) ...
Selecting previously unselected package patch.
Preparing to unpack .../patch_2.7.5-1_amd64.deb ...
Unpacking patch (2.7.5-1) ...
Selecting previously unselected package xz-utils.
Preparing to unpack .../xz-utils_5.1.1alpha+20120614-2+b3_amd64.deb ...
Unpacking xz-utils (5.1.1alpha+20120614-2+b3) ...
Selecting previously unselected package binutils.
Preparing to unpack .../binutils_2.25-5+deb8u1_amd64.deb ...
Unpacking binutils (2.25-5+deb8u1) ...
Selecting previously unselected package libc-dev-bin.
Preparing to unpack .../libc-dev-bin_2.19-18+deb8u10_amd64.deb ...
Unpacking libc-dev-bin (2.19-18+deb8u10) ...
Selecting previously unselected package linux-libc-dev:amd64.
Preparing to unpack .../linux-libc-dev_3.16.51-3+deb8u1_amd64.deb ...
Unpacking linux-libc-dev:amd64 (3.16.51-3+deb8u1) ...
Selecting previously unselected package libc6-dev:amd64.
Preparing to unpack .../libc6-dev_2.19-18+deb8u10_amd64.deb ...
Unpacking libc6-dev:amd64 (2.19-18+deb8u10) ...
Selecting previously unselected package cpp-4.9.
Preparing to unpack .../cpp-4.9_4.9.2-10+deb8u1_amd64.deb ...
Unpacking cpp-4.9 (4.9.2-10+deb8u1) ...
Selecting previously unselected package cpp.
Preparing to unpack .../cpp_4%3a4.9.2-2_amd64.deb ...
Unpacking cpp (4:4.9.2-2) ...
Selecting previously unselected package libgcc-4.9-dev:amd64.
Preparing to unpack .../libgcc-4.9-dev_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libgcc-4.9-dev:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package gcc-4.9.
Preparing to unpack .../gcc-4.9_4.9.2-10+deb8u1_amd64.deb ...
Unpacking gcc-4.9 (4.9.2-10+deb8u1) ...
Selecting previously unselected package gcc.
Preparing to unpack .../gcc_4%3a4.9.2-2_amd64.deb ...
Unpacking gcc (4:4.9.2-2) ...
Selecting previously unselected package libstdc++-4.9-dev:amd64.
Preparing to unpack .../libstdc++-4.9-dev_4.9.2-10+deb8u1_amd64.deb ...
Unpacking libstdc++-4.9-dev:amd64 (4.9.2-10+deb8u1) ...
Selecting previously unselected package g++-4.9.
Preparing to unpack .../g++-4.9_4.9.2-10+deb8u1_amd64.deb ...
Unpacking g++-4.9 (4.9.2-10+deb8u1) ...
Selecting previously unselected package g++.
Preparing to unpack .../g++_4%3a4.9.2-2_amd64.deb ...
Unpacking g++ (4:4.9.2-2) ...
Selecting previously unselected package make.
Preparing to unpack .../make_4.0-8.1_amd64.deb ...
Unpacking make (4.0-8.1) ...
Selecting previously unselected package libtimedate-perl.
Preparing to unpack .../libtimedate-perl_2.3000-2_all.deb ...
Unpacking libtimedate-perl (2.3000-2) ...
Selecting previously unselected package libdpkg-perl.
Preparing to unpack .../libdpkg-perl_1.17.27_all.deb ...
Unpacking libdpkg-perl (1.17.27) ...
Selecting previously unselected package dpkg-dev.
Preparing to unpack .../dpkg-dev_1.17.27_all.deb ...
Unpacking dpkg-dev (1.17.27) ...
Selecting previously unselected package build-essential.
Preparing to unpack .../build-essential_11.7_amd64.deb ...
Unpacking build-essential (11.7) ...
Selecting previously unselected package openssl.
Preparing to unpack .../openssl_1.0.1t-1+deb8u8_amd64.deb ...
Unpacking openssl (1.0.1t-1+deb8u8) ...
Selecting previously unselected package ca-certificates.
Preparing to unpack .../ca-certificates_20141019+deb8u3_all.deb ...
Unpacking ca-certificates (20141019+deb8u3) ...
Selecting previously unselected package curl.
Preparing to unpack .../curl_7.38.0-4+deb8u10_amd64.deb ...
Unpacking curl (7.38.0-4+deb8u10) ...
Selecting previously unselected package libcurl4-gnutls-dev:amd64.
Preparing to unpack .../libcurl4-gnutls-dev_7.38.0-4+deb8u10_amd64.deb ...
Unpacking libcurl4-gnutls-dev:amd64 (7.38.0-4+deb8u10) ...
Selecting previously unselected package sudo.
Preparing to unpack .../sudo_1.8.10p3-1+deb8u5_amd64.deb ...
Unpacking sudo (1.8.10p3-1+deb8u5) ...
Processing triggers for systemd (215-17+deb8u7) ...
Setting up libgdbm3:amd64 (1.8.3-13.1) ...
Setting up libssl1.0.0:amd64 (1.0.1t-1+deb8u8) ...
Setting up libgmp10:amd64 (2:6.0.0+dfsg-6) ...
Setting up libnettle4:amd64 (2.7.1-5+deb8u2) ...
Setting up libhogweed2:amd64 (2.7.1-5+deb8u2) ...
Setting up libffi6:amd64 (3.1-2+deb8u1) ...
Setting up libp11-kit0:amd64 (0.20.7-1) ...
Setting up libtasn1-6:amd64 (4.2-3+deb8u3) ...
Setting up libgnutls-deb0-28:amd64 (3.3.8-6+deb8u7) ...
Setting up libkeyutils1:amd64 (1.5.9-5+b1) ...
Setting up libkrb5support0:amd64 (1.12.1+dfsg-19+deb8u4) ...
Setting up libk5crypto3:amd64 (1.12.1+dfsg-19+deb8u4) ...
Setting up libkrb5-3:amd64 (1.12.1+dfsg-19+deb8u4) ...
Setting up libgssapi-krb5-2:amd64 (1.12.1+dfsg-19+deb8u4) ...
Setting up libidn11:amd64 (1.29-1+deb8u2) ...
Setting up libsasl2-modules-db:amd64 (2.1.26.dfsg1-13+deb8u1) ...
Setting up libsasl2-2:amd64 (2.1.26.dfsg1-13+deb8u1) ...
Setting up libldap-2.4-2:amd64 (2.4.40+dfsg-1+deb8u3) ...
Setting up perl-modules (5.20.2-3+deb8u9) ...
Setting up perl (5.20.2-3+deb8u9) ...
update-alternatives: using /usr/bin/prename to provide /usr/bin/rename (rename) in auto mode
Setting up libasan1:amd64 (4.9.2-10+deb8u1) ...
Setting up libatomic1:amd64 (4.9.2-10+deb8u1) ...
Setting up libcilkrts5:amd64 (4.9.2-10+deb8u1) ...
Setting up libisl10:amd64 (0.12.2-2) ...
Setting up libcloog-isl4:amd64 (0.18.2-1+b2) ...
Setting up librtmp1:amd64 (2.4+20150115.gita107cef-1+deb8u1) ...
Setting up libssh2-1:amd64 (1.4.3-4.1+deb8u1) ...
Setting up libcurl3:amd64 (7.38.0-4+deb8u10) ...
Setting up libcurl3-gnutls:amd64 (7.38.0-4+deb8u10) ...
Setting up libgomp1:amd64 (4.9.2-10+deb8u1) ...
Setting up libitm1:amd64 (4.9.2-10+deb8u1) ...
Setting up liblsan0:amd64 (4.9.2-10+deb8u1) ...
Setting up libmpfr4:amd64 (3.1.2-2) ...
Setting up libquadmath0:amd64 (4.9.2-10+deb8u1) ...
Setting up libtsan0:amd64 (4.9.2-10+deb8u1) ...
Setting up libubsan0:amd64 (4.9.2-10+deb8u1) ...
Setting up libmpc3:amd64 (1.0.2-1) ...
Setting up bzip2 (1.0.6-7+b3) ...
Setting up patch (2.7.5-1) ...
Setting up xz-utils (5.1.1alpha+20120614-2+b3) ...
update-alternatives: using /usr/bin/xz to provide /usr/bin/lzma (lzma) in auto mode
Setting up binutils (2.25-5+deb8u1) ...
Setting up libc-dev-bin (2.19-18+deb8u10) ...
Setting up linux-libc-dev:amd64 (3.16.51-3+deb8u1) ...
Setting up libc6-dev:amd64 (2.19-18+deb8u10) ...
Setting up cpp-4.9 (4.9.2-10+deb8u1) ...
Setting up cpp (4:4.9.2-2) ...
Setting up libgcc-4.9-dev:amd64 (4.9.2-10+deb8u1) ...
Setting up gcc-4.9 (4.9.2-10+deb8u1) ...
Setting up gcc (4:4.9.2-2) ...
Setting up libstdc++-4.9-dev:amd64 (4.9.2-10+deb8u1) ...
Setting up g++-4.9 (4.9.2-10+deb8u1) ...
Setting up g++ (4:4.9.2-2) ...
update-alternatives: using /usr/bin/g++ to provide /usr/bin/c++ (c++) in auto mode
Setting up make (4.0-8.1) ...
Setting up libtimedate-perl (2.3000-2) ...
Setting up libdpkg-perl (1.17.27) ...
Setting up dpkg-dev (1.17.27) ...
Setting up build-essential (11.7) ...
Setting up openssl (1.0.1t-1+deb8u8) ...
Setting up ca-certificates (20141019+deb8u3) ...
Updating certificates in /etc/ssl/certs... 174 added, 0 removed; done.
Setting up curl (7.38.0-4+deb8u10) ...
Setting up libcurl4-gnutls-dev:amd64 (7.38.0-4+deb8u10) ...
Setting up sudo (1.8.10p3-1+deb8u5) ...
Processing triggers for libc-bin (2.19-18+deb8u10) ...
Processing triggers for ca-certificates (20141019+deb8u3) ...
Updating certificates in /etc/ssl/certs... 0 added, 0 removed; done.
Running hooks in /etc/ca-certificates/update.d....done.
Processing triggers for systemd (215-17+deb8u7) ...
 ---> 5c0310ae3ddc
Removing intermediate container f59911d3c822
Step 4/7 : RUN curl -L https://toolbelt.treasuredata.com/sh/install-debian-jessie-td-agent2.sh | sh
 ---> Running in e83c29c33e61
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0==============================
 td-agent Installation Script
This script requires superuser access to install apt packages.
You will be prompted for your password by sudo.
100   738  100   738    0     0   1449      0 --:--:-- --:--:-- --:--:--  1447
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  3157  100  3157    0     0   105k      0 --:--:-- --:--:-- --:--:--  106k
Get:1 http://packages.treasuredata.com jessie InRelease [1842 B]
Get:2 http://packages.treasuredata.com jessie/contrib amd64 Packages [399 B]
Hit http://security.debian.org jessie/updates InRelease
Get:3 http://security.debian.org jessie/updates/main amd64 Packages [644 kB]
Ign http://deb.debian.org jessie InRelease
Hit http://deb.debian.org jessie-updates InRelease
Hit http://deb.debian.org jessie Release.gpg
Hit http://deb.debian.org jessie Release
Get:4 http://deb.debian.org jessie-updates/main amd64 Packages [23.1 kB]
Get:5 http://deb.debian.org jessie/main amd64 Packages [9064 kB]
Fetched 9734 kB in 5s (1824 kB/s)
Reading package lists...
Reading package lists...
Building dependency tree...
Reading state information...
The following NEW packages will be installed:
0 upgraded, 1 newly installed, 0 to remove and 1 not upgraded.
Need to get 55.0 MB of archives.
After this operation, 183 MB of additional disk space will be used.
Get:1 http://packages.treasuredata.com/2/debian/jessie/ jessie/contrib td-agent amd64 2.3.6-0 [55.0 MB]
debconf: delaying package configuration, since apt-utils is not installed
Fetched 55.0 MB in 2s (20.0 MB/s)
Selecting previously unselected package td-agent.
(Reading database ... 12532 files and directories currently installed.)
Preparing to unpack .../td-agent_2.3.6-0_amd64.deb ...
Unpacking td-agent (2.3.6-0) ...
Setting up td-agent (2.3.6-0) ...
Adding system user `td-agent' (UID 104) ...
Adding new group `td-agent' (GID 107) ...
Adding new user `td-agent' (UID 104) with group `td-agent' ...
Not creating home directory `/home/td-agent'.
Installing default conffile /etc/td-agent/td-agent.conf ...
invoke-rc.d: policy-rc.d denied execution of start.
Processing triggers for libc-bin (2.19-18+deb8u10) ...

Installation completed. Happy Logging!

 ---> 6bb3f47cc669
Removing intermediate container e83c29c33e61
Step 5/7 : RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch fluent-plugin-record-reformer
 ---> Running in 4358c3476ca1
Successfully installed serverengine-2.0.6
Building native extensions.  This could take a while...
invalid options: -SHN
(invalid options are ignored)
Successfully installed strptime-0.2.3
Successfully installed dig_rb-1.0.1
Successfully installed fluentd-1.1.3
Successfully installed excon-0.62.0
Successfully installed multipart-post-2.0.0
Successfully installed faraday-0.14.0
Successfully installed elasticsearch-transport-6.0.2
Successfully installed elasticsearch-api-6.0.2
Successfully installed elasticsearch-6.0.2
Successfully installed fluent-plugin-elasticsearch-2.8.6
Parsing documentation for serverengine-2.0.6
Installing ri documentation for serverengine-2.0.6
Parsing documentation for strptime-0.2.3
Installing ri documentation for strptime-0.2.3
Parsing documentation for dig_rb-1.0.1
Installing ri documentation for dig_rb-1.0.1
Parsing documentation for fluentd-1.1.3
Installing ri documentation for fluentd-1.1.3
Parsing documentation for excon-0.62.0
Installing ri documentation for excon-0.62.0
Parsing documentation for multipart-post-2.0.0
Installing ri documentation for multipart-post-2.0.0
Parsing documentation for faraday-0.14.0
Installing ri documentation for faraday-0.14.0
Parsing documentation for elasticsearch-transport-6.0.2
Installing ri documentation for elasticsearch-transport-6.0.2
Parsing documentation for elasticsearch-api-6.0.2
Installing ri documentation for elasticsearch-api-6.0.2
Parsing documentation for elasticsearch-6.0.2
Installing ri documentation for elasticsearch-6.0.2
Parsing documentation for fluent-plugin-elasticsearch-2.8.6
Installing ri documentation for fluent-plugin-elasticsearch-2.8.6
Done installing documentation for serverengine, strptime, dig_rb, fluentd, excon, multipart-post, faraday, elasticsearch-transport, elasticsearch-api, elasticsearch, fluent-plugin-elasticsearch after 11 seconds
Successfully installed fluent-plugin-record-reformer-0.9.1
Parsing documentation for fluent-plugin-record-reformer-0.9.1
Installing ri documentation for fluent-plugin-record-reformer-0.9.1
Done installing documentation for fluent-plugin-record-reformer after 0 seconds
12 gems installed
 ---> 4db79640c552
Removing intermediate container 4358c3476ca1
Step 6/7 : EXPOSE 24224
 ---> Running in e09f02db180e
 ---> 75708fd35464
Removing intermediate container e09f02db180e
Step 7/7 : CMD exec td-agent -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
 ---> Running in 927bdb731fc6
 ---> 9ff77d0acf21
Removing intermediate container 927bdb731fc6
Successfully built 9ff77d0acf21
Successfully tagged blogsample_fluentd:latest
WARNING: Image for service fluentd was built because it did not already exist. To rebuild this image you must use `docker-compose build` or `docker-compose up --build`.
Building elasticsearch
Step 1/2 : FROM docker.elastic.co/elasticsearch/elasticsearch:5.4.3
5.4.3: Pulling from elasticsearch/elasticsearch
d3aeceeb0289: Pull complete
8f71a54ee36c: Pull complete
71f1935b230b: Pull complete
16c3cc6647f0: Pull complete
c23ca9956682: Pull complete
fa98162d672f: Pull complete
34efa21beb0e: Pull complete
139c723cd9ea: Pull complete
03a95a5ee04e: Pull complete
7f59c15ccc56: Pull complete
Digest: sha256:b14e0b6d7819d9a755d380cf32641befadac1c3a96c48a69831005250cfe54fd
Status: Downloaded newer image for docker.elastic.co/elasticsearch/elasticsearch:5.4.3
 ---> 2ae8547160a7
Step 2/2 : RUN cd /usr/share/elasticsearch &&   bin/elasticsearch-plugin install analysis-kuromoji &&   bin/elasticsearch-plugin install analysis-icu
 ---> Running in 79819937416c
-> Downloading analysis-kuromoji from elastic
[=================================================] 100%??
-> Installed analysis-kuromoji
-> Downloading analysis-icu from elastic
[=================================================] 100%??
-> Installed analysis-icu
 ---> 9f70a0c52685
Removing intermediate container 79819937416c
Successfully built 9f70a0c52685
Successfully tagged blogsample_elasticsearch:latest
WARNING: Image for service elasticsearch was built because it did not already exist. To rebuild this image you must use `docker-compose build` or `docker-compose up --build`.
Pulling kibana (docker.elastic.co/kibana/kibana:5.4.3)...
5.4.3: Pulling from kibana/kibana
d3aeceeb0289: Already exists
8579720ccec7: Pull complete
047c80c6b8c9: Pull complete
dd3d39cdec23: Pull complete
360978fe41a9: Pull complete
35aa697b4267: Pull complete
166e29386d64: Pull complete
d7dd1d81bf77: Pull complete
ed50f0e6ccee: Pull complete
Digest: sha256:a14294558c0d4a03d0abf5264b8147fa2a00f9b090e48448a1586b075d14e4b1
Status: Downloaded newer image for docker.elastic.co/kibana/kibana:5.4.3
Pulling nginx (nginx:1.11)...
1.11: Pulling from library/nginx
6d827a3ef358: Already exists
f8f2e0556751: Pull complete
5c9972dca3fd: Pull complete
451b9524cb06: Pull complete
Digest: sha256:e6693c20186f837fc393390135d8a598a96a833917917789d63766cab6c59582
Status: Downloaded newer image for nginx:1.11
Creating blogsample_nginx_1         ...
Creating blogsample_fluentd_1       ... error
Creating blogsample_elasticsearch_1 ...
Creating blogsample_fluentd_1       ...
Creating blogsample_go_1            ...
Creating blogsample_go_1            ... done
ERROR: for blogsample_fluentd_1  Cannot start service fluentd: driver failed programming eCreating blogsample_elasticsearch_1 ... done
Creating blogsample_kibana_1        ... done
Creating blogsample_nginx_1         ... done

ERROR: for fluentd  Cannot start service fluentd: driver failed programming external connectivity on endpoint blogsample_fluentd_1 (59456b19d7383bd270ed7424a55702be74a9b924c77c2db0275d6758d718bf32): Error starting userland proxy: listen tcp bind: address already in use
ERROR: Encountered errors while bringing up the project.


$ free -m
             total       used       free     shared    buffers     cached
Mem:          2001       1938         62          0          0         27
-/+ buffers/cache:       1910         90
Swap:            0          0          0

$ free -m
             total       used       free     shared    buffers     cached
Mem:          2001       1084        917          0         12         81
-/+ buffers/cache:        990       1010
Swap:            0          0          0

## CPU
$ sar -u
12:00:01 AM     CPU     %user     %nice   %system   %iowait    %steal     %idle
05:20:01 PM     all      0.04      0.00      0.03      0.01      0.02     99.90
05:30:01 PM     all      0.04      0.00      0.03      0.01      0.02     99.91
05:40:01 PM     all      0.04      0.00      0.03      0.01      0.02     99.90
05:50:01 PM     all     13.82      0.00      3.01      1.03      0.08     82.06
06:00:36 PM     all     14.26      0.00      5.28     78.84      1.12      0.50
06:10:01 PM     all     34.88      0.00      6.14     56.52      0.89      1.58

## ロードアベレージ
$ sar -q
12:00:01 AM   runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15
05:20:01 PM         0       147      0.00      0.00      0.00
05:30:01 PM         1       147      0.00      0.00      0.00
05:40:01 PM         0       147      0.00      0.00      0.00
05:50:01 PM         1       157      0.94      0.39      0.15
06:00:36 PM         0       277      9.69      7.82      4.20
06:10:01 PM         0       201      1.51      6.54      5.89

## メモリ
$ sar -r
12:00:01 AM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit

05:20:01 PM    909852   1139400     55.60    166756    410508    971816     47.42
05:30:01 PM    909884   1139368     55.60    166756    410508    971816     47.42
05:40:01 PM    909764   1139488     55.61    166756    410524    971816     47.42
05:50:01 PM     74796   1974456     96.35    234604   1085612   1069228     52.18
06:00:36 PM     65172   1984080     96.82       116     25968   3084236    150.51
06:10:01 PM    336440   1712812     83.58      4668     46448   2591224    126.45

## Disk I/O
$ sar -b

12:00:01 AM       tps      rtps      wtps   bread/s   bwrtn/s
05:20:01 PM      0.16      0.00      0.16      0.00      1.90
05:30:01 PM      0.15      0.00      0.15      0.00      1.70
05:40:01 PM      0.16      0.00      0.16      0.00      1.82
05:50:01 PM     89.59      6.19     83.40    330.35  12059.86
06:00:36 PM   1179.01   1157.77     21.24 110175.33   3395.56
06:10:01 PM    791.31    786.20      5.11  81238.48    647.16