Change source.list to yakkety and then apt-get update like these.
# cat /etc/apt/sources.list
#------------------------------------------------------------------------------#
###### Ubuntu Main Repos
deb [arch=armhf] http://ports.ubuntu.com/ yakkety main restricted universe
deb-src [arch=armhf] http://ports.ubuntu.com/ yakkety main
###### Ubuntu Update Repos
deb [arch=armhf] http://ports.ubuntu.com/ yakkety-security main restricted universe
deb [arch=armhf] http://ports.ubuntu.com/ yakkety-updates main restricted universe
###### Ubuntu Main Repos
deb [arch=armhf] http://ports.ubuntu.com/ trusty main restricted universe
deb-src [arch=armhf] http://ports.ubuntu.com/ trusty main
###### Ubuntu Update Repos
deb [arch=armhf] http://ports.ubuntu.com/ trusty-security main restricted universe
deb [arch=armhf] http://ports.ubuntu.com/ trusty-updates main restricted universe
Sometime you must remove ceph and cuda source list.
rm /etc/apt/sources.list.d/ceph.list
rm /etc/apt/sources.list.d/cuda-6-5-local.list
rm /etc/apt/sources.list.d/opencv4tegra-repo.list
If you cluster exists Ceph
$ ceph-deploy purge node-master node-1
$ ceph-deploy purgedata node-master node-1
$ ceph-deploy forgetkeys
and set /etc/hosts for local resolv host inside all cluster node.
SHELL=/bin/bash
HOME=/home/ceph
add ceph user for all node.
# userdel ceph
# rm -rf /home/ceph/
# useradd -d /home/ceph -m ceph && chown -R ceph /home/ceph/ && passwd ceph
$ echo "ceph ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/ceph
$ sudo chmod 0440 /etc/sudoers.d/ceph
login as ceph and then.
$ ssh-keygen
$ ssh-copy-id ceph@node-{1,2,3}
$ sudo iptables -A INPUT -i eth3 -p tcp -s 192.168.5.{201,202,203,204}/24 --dport 6789 -j ACCEPT
$ sudo iptables-save ## For all nodes.
192.168.5.201 node-master
192.168.5.202 node-1
192.168.5.203 node-2
192.168.5.204 node-3
and change your hostname to node-{name} because when you create monitor its will be error if not set exactly hostname.
Then deploy ceph cluster use command following.
$ mkdir my-cluster
$ cd my-cluster
$ ceph-deploy new node-master node-1 node-2 node-3
$ ceph-deploy install node-master node-1 node-2 node-3
=====
make sure you have free space >= 6GB because sometime error occur!!!!
and newest ceph-deploy by apt-get install ceph-deploy
=====
$ ceph-deploy mon create-initial
$ ceph-deploy gatherkeys node-master
Logs
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy new node-master node-1 node-2 node-3
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy new node-master node-1 node-2 node-3
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] func : <function new at 0xb6967c70>
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb689bd78>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] mon : ['node-master', 'node-1', 'node-2', 'node-3']
[ceph_deploy.cli][INFO ] public_network : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster_network : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /bin/ip link show
[node-master][INFO ] Running command: sudo /bin/ip addr show
[node-master][DEBUG ] IP addresses found: ['192.168.5.201']
[ceph_deploy.new][DEBUG ] Resolving host node-master
[ceph_deploy.new][DEBUG ] Monitor node-master at 192.168.5.201
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[node-1][DEBUG ] connected to host: node-master
[node-1][INFO ] Running command: ssh -CT -o BatchMode=yes node-1
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /bin/ip link show
[node-1][INFO ] Running command: sudo /bin/ip addr show
[node-1][DEBUG ] IP addresses found: ['192.168.5.202']
[ceph_deploy.new][DEBUG ] Resolving host node-1
[ceph_deploy.new][DEBUG ] Monitor node-1 at 192.168.5.202
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[node-2][DEBUG ] connected to host: node-master
[node-2][INFO ] Running command: ssh -CT -o BatchMode=yes node-2
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /bin/ip link show
[node-2][INFO ] Running command: sudo /bin/ip addr show
[node-2][DEBUG ] IP addresses found: ['192.168.5.203']
[ceph_deploy.new][DEBUG ] Resolving host node-2
[ceph_deploy.new][DEBUG ] Monitor node-2 at 192.168.5.203
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[node-3][DEBUG ] connected to host: node-master
[node-3][INFO ] Running command: ssh -CT -o BatchMode=yes node-3
[node-3][DEBUG ] connection detected need for sudo
[node-3][DEBUG ] connected to host: node-3
[node-3][DEBUG ] detect platform information from remote host
[node-3][DEBUG ] detect machine type
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo /sbin/initctl version
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo /bin/ip link show
[node-3][INFO ] Running command: sudo /bin/ip addr show
[node-3][DEBUG ] IP addresses found: ['192.168.5.204']
[ceph_deploy.new][DEBUG ] Resolving host node-3
[ceph_deploy.new][DEBUG ] Monitor node-3 at 192.168.5.204
[ceph_deploy.new][DEBUG ] Monitor initial members are ['node-master', 'node-1', 'node-2', 'node-3']
[ceph_deploy.new][DEBUG ] Monitor addrs are ['192.168.5.201', '192.168.5.202', '192.168.5.203', '192.168.5.204']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy install node-master node-1 node-2 node-3
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy install node-master node-1 node-2 node-3
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] testing : None
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb6893210>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] dev_commit : None
[ceph_deploy.cli][INFO ] install_mds : False
[ceph_deploy.cli][INFO ] stable : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] adjust_repos : False
[ceph_deploy.cli][INFO ] func : <function install at 0xb692d730>
[ceph_deploy.cli][INFO ] install_all : False
[ceph_deploy.cli][INFO ] repo : False
[ceph_deploy.cli][INFO ] host : ['node-master', 'node-1', 'node-2', 'node-3']
[ceph_deploy.cli][INFO ] install_rgw : False
[ceph_deploy.cli][INFO ] install_tests : False
[ceph_deploy.cli][INFO ] repo_url : None
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] install_osd : False
[ceph_deploy.cli][INFO ] version_kind : stable
[ceph_deploy.cli][INFO ] install_common : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] dev : master
[ceph_deploy.cli][INFO ] local_mirror : None
[ceph_deploy.cli][INFO ] release : None
[ceph_deploy.cli][INFO ] install_mon : False
[ceph_deploy.cli][INFO ] gpg_url : None
[ceph_deploy.install][DEBUG ] Installing stable version infernalis on cluster ceph hosts node-master node-1 node-2 node-3
[ceph_deploy.install][DEBUG ] Detecting platform for host node-master ...
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[ceph_deploy.install][INFO ] Distro info: Ubuntu 14.04 trusty
[node-master][INFO ] installing Ceph on node-master
[node-master][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https
[node-master][DEBUG ] Reading package lists...
[node-master][DEBUG ] Building dependency tree...
[node-master][DEBUG ] Reading state information...
[node-master][DEBUG ] apt-transport-https is already the newest version (1.2.11).
[node-master][DEBUG ] ca-certificates is already the newest version (20160104ubuntu1).
[node-master][DEBUG ] The following packages were automatically installed and are no longer required:
[node-master][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-master][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-master][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-master][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-master][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-master][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-master][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-master][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-master][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-master][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-master][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-master][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-master][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-master][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-master][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-master][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-master][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-master][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-master][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-master][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-master][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-master][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-master][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-master][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-master][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-master][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-master][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-master][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-master][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-master][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-master][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-master][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-master][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-master][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-master][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-master][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-master][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-master][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-master][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-master][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-master][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-master][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1442 not upgraded.
[node-master][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update
[node-master][DEBUG ] Hit:1 http://ports.ubuntu.com yakkety InRelease
[node-master][DEBUG ] Hit:2 http://ports.ubuntu.com yakkety-security InRelease
[node-master][DEBUG ] Hit:3 http://ports.ubuntu.com yakkety-updates InRelease
[node-master][DEBUG ] Ign:4 http://ports.ubuntu.com trusty InRelease
[node-master][DEBUG ] Hit:5 http://ports.ubuntu.com trusty-security InRelease
[node-master][DEBUG ] Hit:6 http://ports.ubuntu.com trusty-updates InRelease
[node-master][DEBUG ] Hit:7 http://ports.ubuntu.com trusty Release
[node-master][DEBUG ] Reading package lists...
[node-master][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-mds radosgw
[node-master][DEBUG ] Reading package lists...
[node-master][DEBUG ] Building dependency tree...
[node-master][DEBUG ] Reading state information...
[node-master][DEBUG ] ceph is already the newest version (10.2.0-0ubuntu1).
[node-master][DEBUG ] radosgw is already the newest version (10.2.0-0ubuntu1).
[node-master][DEBUG ] ceph-mds is already the newest version (10.2.0-0ubuntu1).
[node-master][DEBUG ] The following packages were automatically installed and are no longer required:
[node-master][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-master][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-master][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-master][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-master][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-master][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-master][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-master][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-master][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-master][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-master][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-master][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-master][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-master][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-master][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-master][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-master][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-master][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-master][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-master][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-master][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-master][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-master][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-master][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-master][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-master][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-master][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-master][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-master][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-master][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-master][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-master][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-master][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-master][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-master][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-master][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-master][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-master][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-master][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-master][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-master][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-master][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1442 not upgraded.
[node-master][INFO ] Running command: sudo ceph --version
[node-master][DEBUG ] ceph version 10.2.0 (3a9fba20ec743699b69bd0181dd6c54dc01c64b9)
[ceph_deploy.install][DEBUG ] Detecting platform for host node-1 ...
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[ceph_deploy.install][INFO ] Distro info: Ubuntu 14.04 trusty
[node-1][INFO ] installing Ceph on node-1
[node-1][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https
[node-1][DEBUG ] Reading package lists...
[node-1][DEBUG ] Building dependency tree...
[node-1][DEBUG ] Reading state information...
[node-1][DEBUG ] apt-transport-https is already the newest version (1.2.11).
[node-1][DEBUG ] ca-certificates is already the newest version (20160104ubuntu1).
[node-1][DEBUG ] The following packages were automatically installed and are no longer required:
[node-1][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-1][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-1][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-1][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-1][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-1][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-1][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-1][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-1][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-1][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-1][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-1][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-1][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-1][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-1][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-1][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-1][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-1][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-1][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-1][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-1][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-1][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-1][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-1][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-1][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-1][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-1][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-1][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-1][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-1][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-1][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-1][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-1][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-1][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-1][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-1][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-1][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-1][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-1][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-1][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-1][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-1][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1443 not upgraded.
[node-1][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update
[node-1][DEBUG ] Hit:1 http://ports.ubuntu.com yakkety InRelease
[node-1][DEBUG ] Hit:2 http://ports.ubuntu.com yakkety-security InRelease
[node-1][DEBUG ] Hit:3 http://ports.ubuntu.com yakkety-updates InRelease
[node-1][DEBUG ] Ign:4 http://ports.ubuntu.com trusty InRelease
[node-1][DEBUG ] Hit:5 http://ports.ubuntu.com trusty-security InRelease
[node-1][DEBUG ] Hit:6 http://ports.ubuntu.com trusty-updates InRelease
[node-1][DEBUG ] Hit:7 http://ports.ubuntu.com trusty Release
[node-1][DEBUG ] Reading package lists...
[node-1][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-mds radosgw
[node-1][DEBUG ] Reading package lists...
[node-1][DEBUG ] Building dependency tree...
[node-1][DEBUG ] Reading state information...
[node-1][DEBUG ] ceph is already the newest version (10.2.0-0ubuntu1).
[node-1][DEBUG ] radosgw is already the newest version (10.2.0-0ubuntu1).
[node-1][DEBUG ] ceph-mds is already the newest version (10.2.0-0ubuntu1).
[node-1][DEBUG ] The following packages were automatically installed and are no longer required:
[node-1][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-1][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-1][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-1][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-1][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-1][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-1][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-1][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-1][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-1][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-1][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-1][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-1][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-1][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-1][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-1][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-1][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-1][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-1][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-1][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-1][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-1][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-1][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-1][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-1][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-1][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-1][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-1][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-1][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-1][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-1][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-1][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-1][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-1][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-1][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-1][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-1][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-1][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-1][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-1][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-1][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-1][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1443 not upgraded.
[node-1][INFO ] Running command: sudo ceph --version
[node-1][DEBUG ] ceph version 10.2.0 (3a9fba20ec743699b69bd0181dd6c54dc01c64b9)
[ceph_deploy.install][DEBUG ] Detecting platform for host node-2 ...
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[ceph_deploy.install][INFO ] Distro info: Ubuntu 14.04 trusty
[node-2][INFO ] installing Ceph on node-2
[node-2][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https
[node-2][DEBUG ] Reading package lists...
[node-2][DEBUG ] Building dependency tree...
[node-2][DEBUG ] Reading state information...
[node-2][DEBUG ] apt-transport-https is already the newest version (1.2.11).
[node-2][DEBUG ] ca-certificates is already the newest version (20160104ubuntu1).
[node-2][DEBUG ] ca-certificates set to manually installed.
[node-2][DEBUG ] The following packages were automatically installed and are no longer required:
[node-2][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-2][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-2][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-2][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-2][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-2][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-2][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-2][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-2][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-2][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-2][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-2][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-2][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-2][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-2][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-2][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-2][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-2][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-2][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-2][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-2][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-2][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-2][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-2][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-2][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-2][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-2][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-2][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-2][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-2][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-2][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-2][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-2][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-2][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-2][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-2][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-2][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-2][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-2][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-2][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-2][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-2][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1442 not upgraded.
[node-2][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update
[node-2][DEBUG ] Get:1 http://ports.ubuntu.com yakkety InRelease [247 kB]
[node-2][DEBUG ] Hit:2 http://ports.ubuntu.com yakkety-security InRelease
[node-2][DEBUG ] Hit:3 http://ports.ubuntu.com yakkety-updates InRelease
[node-2][DEBUG ] Ign:4 http://ports.ubuntu.com trusty InRelease
[node-2][DEBUG ] Hit:5 http://ports.ubuntu.com trusty-security InRelease
[node-2][DEBUG ] Hit:6 http://ports.ubuntu.com trusty-updates InRelease
[node-2][DEBUG ] Get:7 http://ports.ubuntu.com yakkety/universe armhf Packages [7,402 kB]
[node-2][DEBUG ] Get:8 http://ports.ubuntu.com yakkety/universe Translation-en [4,421 kB]
[node-2][DEBUG ] Hit:9 http://ports.ubuntu.com trusty Release
[node-2][DEBUG ] Fetched 12.1 MB in 1min 7s (179 kB/s)
[node-2][DEBUG ] Reading package lists...
[node-2][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-mds radosgw
[node-2][DEBUG ] Reading package lists...
[node-2][DEBUG ] Building dependency tree...
[node-2][DEBUG ] Reading state information...
[node-2][DEBUG ] ceph is already the newest version (10.2.0-0ubuntu1).
[node-2][DEBUG ] radosgw is already the newest version (10.2.0-0ubuntu1).
[node-2][DEBUG ] ceph-mds is already the newest version (10.2.0-0ubuntu1).
[node-2][DEBUG ] The following packages were automatically installed and are no longer required:
[node-2][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-2][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-2][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-2][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-2][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-2][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-2][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-2][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-2][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-2][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-2][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-2][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-2][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-2][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-2][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-2][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-2][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-2][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-2][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-2][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-2][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-2][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-2][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-2][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-2][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-2][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-2][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-2][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-2][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-2][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-2][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-2][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-2][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-2][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-2][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-2][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-2][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-2][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-2][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-2][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-2][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-2][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1442 not upgraded.
[node-2][INFO ] Running command: sudo ceph --version
[node-2][DEBUG ] ceph version 10.2.0 (3a9fba20ec743699b69bd0181dd6c54dc01c64b9)
[ceph_deploy.install][DEBUG ] Detecting platform for host node-3 ...
[node-3][DEBUG ] connection detected need for sudo
[node-3][DEBUG ] connected to host: node-3
[node-3][DEBUG ] detect platform information from remote host
[node-3][DEBUG ] detect machine type
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo /sbin/initctl version
[ceph_deploy.install][INFO ] Distro info: Ubuntu 14.04 trusty
[node-3][INFO ] installing Ceph on node-3
[node-3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install ca-certificates apt-transport-https
[node-3][DEBUG ] Reading package lists...
[node-3][DEBUG ] Building dependency tree...
[node-3][DEBUG ] Reading state information...
[node-3][DEBUG ] The following packages were automatically installed and are no longer required:
[node-3][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-3][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-3][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-3][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-3][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-3][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-3][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-3][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-3][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-3][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-3][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-3][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-3][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-3][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-3][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-3][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-3][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-3][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-3][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-3][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-3][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-3][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-3][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-3][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-3][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-3][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-3][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-3][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-3][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-3][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-3][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-3][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-3][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-3][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-3][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-3][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-3][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-3][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-3][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-3][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-3][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-3][DEBUG ] The following NEW packages will be installed:
[node-3][DEBUG ] apt-transport-https
[node-3][DEBUG ] The following packages will be upgraded:
[node-3][DEBUG ] ca-certificates
[node-3][DEBUG ] 1 upgraded, 1 newly installed, 0 to remove and 1442 not upgraded.
[node-3][DEBUG ] Need to get 214 kB of archives.
[node-3][DEBUG ] After this operation, 219 kB of additional disk space will be used.
[node-3][DEBUG ] Get:1 http://ports.ubuntu.com yakkety/main armhf apt-transport-https armhf 1.2.11 [23.0 kB]
[node-3][DEBUG ] Get:2 http://ports.ubuntu.com yakkety/main armhf ca-certificates all 20160104ubuntu1 [191 kB]
[node-3][WARNIN] Unescaped left brace in regex is deprecated, passed through in regex; marked by <-- HERE in m/^(.*?)(\\)?\${ <-- HERE ([^{}]+)}(.*)$/ at /usr/share/perl5/Debconf/Question.pm line 72.
[node-3][WARNIN] Unescaped left brace in regex is deprecated, passed through in regex; marked by <-- HERE in m/\${ <-- HERE ([^}]+)}/ at /usr/share/perl5/Debconf/Config.pm line 30.
[node-3][DEBUG ] Preconfiguring packages ...
[node-3][DEBUG ] Fetched 214 kB in 0s (556 kB/s)
[node-3][DEBUG ] Selecting previously unselected package apt-transport-https.
(Reading database ... 149206 files and directories currently installed.)
[node-3][DEBUG ] Preparing to unpack .../apt-transport-https_1.2.11_armhf.deb ...
[node-3][DEBUG ] Unpacking apt-transport-https (1.2.11) ...
[node-3][DEBUG ] Preparing to unpack .../ca-certificates_20160104ubuntu1_all.deb ...
[node-3][DEBUG ] Unpacking ca-certificates (20160104ubuntu1) over (20141019ubuntu0.14.04.1) ...
[node-3][DEBUG ] Processing triggers for man-db (2.6.7.1-1ubuntu1) ...
[node-3][DEBUG ] Setting up apt-transport-https (1.2.11) ...
[node-3][DEBUG ] Setting up ca-certificates (20160104ubuntu1) ...
[node-3][DEBUG ] Unescaped left brace in regex is deprecated, passed through in regex; marked by <-- HERE in m/^(.*?)(\\)?\${ <-- HERE ([^{}]+)}(.*)$/ at /usr/share/perl5/Debconf/Question.pm line 72.
[node-3][DEBUG ] Unescaped left brace in regex is deprecated, passed through in regex; marked by <-- HERE in m/\${ <-- HERE ([^}]+)}/ at /usr/share/perl5/Debconf/Config.pm line 30.
[node-3][DEBUG ] Processing triggers for ca-certificates (20160104ubuntu1) ...
[node-3][DEBUG ] Updating certificates in /etc/ssl/certs...
[node-3][DEBUG ] WARNING: Skipping duplicate certificate Go_Daddy_Class_2_CA.pem
[node-3][DEBUG ] WARNING: Skipping duplicate certificate Go_Daddy_Class_2_CA.pem
[node-3][DEBUG ] 19 added, 19 removed; done.
[node-3][DEBUG ] Running hooks in /etc/ca-certificates/update.d...
[node-3][DEBUG ]
[node-3][DEBUG ] Adding debian:CA_WoSign_ECC_Root.pem
[node-3][DEBUG ] Adding debian:Certification_Authority_of_WoSign_G2.pem
[node-3][DEBUG ] Adding debian:Certinomis_-_Root_CA.pem
[node-3][DEBUG ] Adding debian:CFCA_EV_ROOT.pem
[node-3][DEBUG ] Adding debian:COMODO_RSA_Certification_Authority.pem
[node-3][DEBUG ] Adding debian:Entrust_Root_Certification_Authority_-_EC1.pem
[node-3][DEBUG ] Adding debian:Entrust_Root_Certification_Authority_-_G2.pem
[node-3][DEBUG ] Adding debian:GlobalSign_ECC_Root_CA_-_R4.pem
[node-3][DEBUG ] Adding debian:GlobalSign_ECC_Root_CA_-_R5.pem
[node-3][DEBUG ] Adding debian:IdenTrust_Commercial_Root_CA_1.pem
[node-3][DEBUG ] Adding debian:IdenTrust_Public_Sector_Root_CA_1.pem
[node-3][DEBUG ] Adding debian:OISTE_WISeKey_Global_Root_GB_CA.pem
[node-3][DEBUG ] Adding debian:Staat_der_Nederlanden_EV_Root_CA.pem
[node-3][DEBUG ] Adding debian:Staat_der_Nederlanden_Root_CA_-_G3.pem
[node-3][DEBUG ] Adding debian:S-TRUST_Universal_Root_CA.pem
[node-3][DEBUG ] Adding debian:TÜRKTRUST_Elektronik_Sertifika_Hizmet_Sağlayıcısı_H5.pem
[node-3][DEBUG ] Adding debian:TÜRKTRUST_Elektronik_Sertifika_Hizmet_Sağlayıcısı_H6.pem
[node-3][DEBUG ] Adding debian:USERTrust_ECC_Certification_Authority.pem
[node-3][DEBUG ] Adding debian:USERTrust_RSA_Certification_Authority.pem
[node-3][DEBUG ] Removing debian:A-Trust-nQual-03.pem
[node-3][DEBUG ] Removing debian:America_Online_Root_Certification_Authority_1.pem
[node-3][DEBUG ] Removing debian:America_Online_Root_Certification_Authority_2.pem
[node-3][DEBUG ] Removing debian:Buypass_Class_3_CA_1.pem
[node-3][DEBUG ] Removing debian:ComSign_Secured_CA.pem
[node-3][DEBUG ] Removing debian:Digital_Signature_Trust_Co._Global_CA_1.pem
[node-3][DEBUG ] Removing debian:Digital_Signature_Trust_Co._Global_CA_3.pem
[node-3][DEBUG ] Removing debian:E-Guven_Kok_Elektronik_Sertifika_Hizmet_Saglayicisi.pem
[node-3][DEBUG ] Removing debian:GTE_CyberTrust_Global_Root.pem
[node-3][DEBUG ] Removing debian:TC_TrustCenter_Class_2_CA_II.pem
[node-3][DEBUG ] Removing debian:TC_TrustCenter_Universal_CA_I.pem
[node-3][DEBUG ] Removing debian:TURKTRUST_Certificate_Services_Provider_Root_1.pem
[node-3][DEBUG ] Removing debian:TURKTRUST_Certificate_Services_Provider_Root_2.pem
[node-3][DEBUG ] Removing debian:Thawte_Premium_Server_CA.pem
[node-3][DEBUG ] Removing debian:Thawte_Server_CA.pem
[node-3][DEBUG ] Removing debian:UTN_DATACorp_SGC_Root_CA.pem
[node-3][DEBUG ] Removing debian:Verisign_Class_4_Public_Primary_Certification_Authority_-_G3.pem
[node-3][DEBUG ] Removing debian:spi-cacert-2008.pem
[node-3][DEBUG ] Removing debian:SG_TRUST_SERVICES_RACINE.pem
[node-3][DEBUG ] done.
[node-3][DEBUG ] done.
[node-3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q update
[node-3][DEBUG ] Get:1 http://ports.ubuntu.com yakkety InRelease [247 kB]
[node-3][DEBUG ] Hit:2 http://ports.ubuntu.com yakkety-security InRelease
[node-3][DEBUG ] Hit:3 http://ports.ubuntu.com yakkety-updates InRelease
[node-3][DEBUG ] Ign:4 http://ports.ubuntu.com trusty InRelease
[node-3][DEBUG ] Hit:5 http://ports.ubuntu.com trusty-security InRelease
[node-3][DEBUG ] Hit:6 http://ports.ubuntu.com trusty-updates InRelease
[node-3][DEBUG ] Get:7 http://ports.ubuntu.com yakkety/main Sources [869 kB]
[node-3][DEBUG ] Get:8 http://ports.ubuntu.com yakkety/main armhf Packages [1,146 kB]
[node-3][DEBUG ] Get:9 http://ports.ubuntu.com yakkety/universe armhf Packages [7,402 kB]
[node-3][DEBUG ] Get:10 http://ports.ubuntu.com yakkety/universe Translation-en [4,421 kB]
[node-3][DEBUG ] Hit:11 http://ports.ubuntu.com trusty Release
[node-3][DEBUG ] Fetched 14.1 MB in 7s (1,918 kB/s)
[node-3][DEBUG ] Reading package lists...
[node-3][INFO ] Running command: sudo env DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew ceph ceph-mds radosgw
[node-3][DEBUG ] Reading package lists...
[node-3][DEBUG ] Building dependency tree...
[node-3][DEBUG ] Reading state information...
[node-3][DEBUG ] ceph is already the newest version (10.2.0-0ubuntu1).
[node-3][DEBUG ] radosgw is already the newest version (10.2.0-0ubuntu1).
[node-3][DEBUG ] ceph-mds is already the newest version (10.2.0-0ubuntu1).
[node-3][DEBUG ] The following packages were automatically installed and are no longer required:
[node-3][DEBUG ] account-plugin-windows-live ant ant-optional aspectj bsh
[node-3][DEBUG ] ca-certificates-java default-jre default-jre-headless fonts-dejavu-extra fop
[node-3][DEBUG ] gstreamer0.10-nice icedtea-7-jre-jamvm java-common java-wrappers junit
[node-3][DEBUG ] junit4 libaether-java libaopalliance-java libapache-pom-java
[node-3][DEBUG ] libapparmor-perl libasm3-java libaspectj-java libasync-http-client-java
[node-3][DEBUG ] libatinject-jsr330-api-java libatk-wrapper-java libatk-wrapper-java-jni
[node-3][DEBUG ] libavalon-framework-java libbatik-java libbonobo2-0 libbonobo2-common
[node-3][DEBUG ] libbsf-java libbsh-java libcdi-api-java libcglib-java libclassworlds-java
[node-3][DEBUG ] libcommons-beanutils-java libcommons-cli-java libcommons-codec-java
[node-3][DEBUG ] libcommons-collections3-java libcommons-configuration-java
[node-3][DEBUG ] libcommons-digester-java libcommons-httpclient-java libcommons-io-java
[node-3][DEBUG ] libcommons-jexl2-java libcommons-jxpath-java libcommons-lang-java
[node-3][DEBUG ] libcommons-logging-java libcommons-net2-java libcommons-parent-java
[node-3][DEBUG ] libcommons-vfs-java libdom4j-java libdoxia-java libeasymock-java libelfg0
[node-3][DEBUG ] libevdev2 libfarstream-0.1-0 libfop-java libganymed-ssh2-java libgconf2-4
[node-3][DEBUG ] libgeronimo-interceptor-3.0-spec-java libgeronimo-jpa-2.0-spec-java
[node-3][DEBUG ] libgeronimo-osgi-support-java libgif4 libgnome-desktop-3-12 libgnome2-0
[node-3][DEBUG ] libgnome2-bin libgnome2-common libgnomevfs2-0 libgnomevfs2-common
[node-3][DEBUG ] libgssglue1 libguava-java libguice-java libhamcrest-java libhttpclient-java
[node-3][DEBUG ] libhttpcore-java libidl-common libidl0 libinput10 libitext1-java
[node-3][DEBUG ] libjackrabbit-java libjaxen-java libjaxp1.3-java libjcommander-java
[node-3][DEBUG ] libjdom1-java libjetty-java libjline-java libjsch-java libjsoup-java
[node-3][DEBUG ] libjsr305-java liblog4j1.2-java libmaven-parent-java libmaven-scm-java
[node-3][DEBUG ] libmaven2-core-java libmeanwhile1 libmodello-java libnetbeans-cvsclient-java
[node-3][DEBUG ] libnetty-java liborbit-2-0 liborbit2 libosgi-compendium-java
[node-3][DEBUG ] libosgi-core-java libosgi-foundation-ee-java libplexus-ant-factory-java
[node-3][DEBUG ] libplexus-archiver-java libplexus-bsh-factory-java libplexus-build-api-java
[node-3][DEBUG ] libplexus-cipher-java libplexus-classworlds-java libplexus-classworlds2-java
[node-3][DEBUG ] libplexus-cli-java libplexus-container-default-java
[node-3][DEBUG ] libplexus-containers-java libplexus-containers1.5-java libplexus-i18n-java
[node-3][DEBUG ] libplexus-interactivity-api-java libplexus-interpolation-java
[node-3][DEBUG ] libplexus-io-java libplexus-sec-dispatcher-java libplexus-utils-java
[node-3][DEBUG ] libplexus-utils2-java libpurple-bin libqdox-java libregexp-java
[node-3][DEBUG ] librhino-java libsaxon-java libservlet2.5-java libservlet3.0-java
[node-3][DEBUG ] libsisu-guice-java libsisu-ioc-java libslf4j-java libupstart1 libwagon-java
[node-3][DEBUG ] libwagon2-java libxalan2-java libxbean-java libxerces2-java
[node-3][DEBUG ] libxml-commons-external-java libxml-commons-resolver1.1-java
[node-3][DEBUG ] libxmlgraphics-commons-java libxom-java libxpp2-java libxpp3-java
[node-3][DEBUG ] libyaml-snake-java libzephyr4 openjdk-7-jre openjdk-7-jre-headless
[node-3][DEBUG ] overlay-scrollbar-gtk2 pidgin-data python-pycurl rhino testng tzdata-java
[node-3][DEBUG ] Use 'sudo apt autoremove' to remove them.
[node-3][DEBUG ] 0 upgraded, 0 newly installed, 0 to remove and 1442 not upgraded.
[node-3][INFO ] Running command: sudo ceph --version
[node-3][DEBUG ] ceph version 10.2.0 (3a9fba20ec743699b69bd0181dd6c54dc01c64b9)
ceph@node-master:/home/ceph/my-cluster$
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy mon create-initial
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy mon create-initial
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : create-initial
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb68172d8>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function mon at 0xb67fe8b0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] keyrings : None
[ceph_deploy.mon][DEBUG ] Deploying mon, cluster ceph hosts node-master node-1 node-2 node-3
[ceph_deploy.mon][DEBUG ] detecting platform for host node-master ...
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 14.04 trusty
[node-master][DEBUG ] determining if provided host has same hostname in remote
[node-master][DEBUG ] get remote short hostname
[node-master][DEBUG ] deploying mon to node-master
[node-master][DEBUG ] get remote short hostname
[node-master][DEBUG ] remote hostname: node-master
[node-master][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[node-master][DEBUG ] create the mon path if it does not exist
[node-master][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-master/done
[node-master][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-master/done
[node-master][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-master.mon.keyring
[node-master][DEBUG ] create the monitor keyring file
[node-master][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-master --keyring /var/lib/ceph/tmp/ceph-node-master.mon.keyring --setuser 1001 --setgroup 1002
[node-master][DEBUG ] ceph-mon: mon.noname-a 192.168.5.201:6789/0 is local, renaming to mon.node-master
[node-master][DEBUG ] ceph-mon: set fsid to 37defaff-03ea-4952-bd25-d07310aab339
[node-master][DEBUG ] ceph-mon: created monfs at /var/lib/ceph/mon/ceph-node-master for mon.node-master
[node-master][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-master.mon.keyring
[node-master][DEBUG ] create a done file to avoid re-doing the mon deployment
[node-master][DEBUG ] create the init path if it does not exist
[node-master][INFO ] Running command: sudo initctl emit ceph-mon cluster=ceph id=node-master
[node-master][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-master.asok mon_status
[node-master][DEBUG ] ********************************************************************************
[node-master][DEBUG ] status for monitor: mon.node-master
[node-master][DEBUG ] {
[node-master][DEBUG ] "election_epoch": 0,
[node-master][DEBUG ] "extra_probe_peers": [
[node-master][DEBUG ] "192.168.5.202:6789/0",
[node-master][DEBUG ] "192.168.5.203:6789/0",
[node-master][DEBUG ] "192.168.5.204:6789/0"
[node-master][DEBUG ] ],
[node-master][DEBUG ] "monmap": {
[node-master][DEBUG ] "created": "2016-05-11 22:17:04.612828",
[node-master][DEBUG ] "epoch": 0,
[node-master][DEBUG ] "fsid": "37defaff-03ea-4952-bd25-d07310aab339",
[node-master][DEBUG ] "modified": "2016-05-11 22:17:04.612828",
[node-master][DEBUG ] "mons": [
[node-master][DEBUG ] {
[node-master][DEBUG ] "addr": "192.168.5.201:6789/0",
[node-master][DEBUG ] "name": "node-master",
[node-master][DEBUG ] "rank": 0
[node-master][DEBUG ] },
[node-master][DEBUG ] {
[node-master][DEBUG ] "addr": "0.0.0.0:0/1",
[node-master][DEBUG ] "name": "node-1",
[node-master][DEBUG ] "rank": 1
[node-master][DEBUG ] },
[node-master][DEBUG ] {
[node-master][DEBUG ] "addr": "0.0.0.0:0/2",
[node-master][DEBUG ] "name": "node-2",
[node-master][DEBUG ] "rank": 2
[node-master][DEBUG ] },
[node-master][DEBUG ] {
[node-master][DEBUG ] "addr": "0.0.0.0:0/3",
[node-master][DEBUG ] "name": "node-3",
[node-master][DEBUG ] "rank": 3
[node-master][DEBUG ] }
[node-master][DEBUG ] ]
[node-master][DEBUG ] },
[node-master][DEBUG ] "name": "node-master",
[node-master][DEBUG ] "outside_quorum": [
[node-master][DEBUG ] "node-master"
[node-master][DEBUG ] ],
[node-master][DEBUG ] "quorum": [],
[node-master][DEBUG ] "rank": 0,
[node-master][DEBUG ] "state": "probing",
[node-master][DEBUG ] "sync_provider": []
[node-master][DEBUG ] }
[node-master][DEBUG ] ********************************************************************************
[node-master][INFO ] monitor: mon.node-master is running
[node-master][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-master.asok mon_status
[ceph_deploy.mon][DEBUG ] detecting platform for host node-1 ...
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 14.04 trusty
[node-1][DEBUG ] determining if provided host has same hostname in remote
[node-1][DEBUG ] get remote short hostname
[node-1][DEBUG ] deploying mon to node-1
[node-1][DEBUG ] get remote short hostname
[node-1][DEBUG ] remote hostname: node-1
[node-1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[node-1][DEBUG ] create the mon path if it does not exist
[node-1][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-1/done
[node-1][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-1/done
[node-1][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-1.mon.keyring
[node-1][DEBUG ] create the monitor keyring file
[node-1][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-1 --keyring /var/lib/ceph/tmp/ceph-node-1.mon.keyring --setuser 1001 --setgroup 1002
[node-1][DEBUG ] ceph-mon: mon.noname-b 192.168.5.202:6789/0 is local, renaming to mon.node-1
[node-1][DEBUG ] ceph-mon: set fsid to 37defaff-03ea-4952-bd25-d07310aab339
[node-1][DEBUG ] ceph-mon: created monfs at /var/lib/ceph/mon/ceph-node-1 for mon.node-1
[node-1][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-1.mon.keyring
[node-1][DEBUG ] create a done file to avoid re-doing the mon deployment
[node-1][DEBUG ] create the init path if it does not exist
[node-1][INFO ] Running command: sudo initctl emit ceph-mon cluster=ceph id=node-1
[node-1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-1.asok mon_status
[node-1][DEBUG ] ********************************************************************************
[node-1][DEBUG ] status for monitor: mon.node-1
[node-1][DEBUG ] {
[node-1][DEBUG ] "election_epoch": 0,
[node-1][DEBUG ] "extra_probe_peers": [
[node-1][DEBUG ] "192.168.5.201:6789/0",
[node-1][DEBUG ] "192.168.5.203:6789/0",
[node-1][DEBUG ] "192.168.5.204:6789/0"
[node-1][DEBUG ] ],
[node-1][DEBUG ] "monmap": {
[node-1][DEBUG ] "created": "2016-05-11 22:17:10.521770",
[node-1][DEBUG ] "epoch": 0,
[node-1][DEBUG ] "fsid": "37defaff-03ea-4952-bd25-d07310aab339",
[node-1][DEBUG ] "modified": "2016-05-11 22:17:10.521770",
[node-1][DEBUG ] "mons": [
[node-1][DEBUG ] {
[node-1][DEBUG ] "addr": "192.168.5.201:6789/0",
[node-1][DEBUG ] "name": "node-master",
[node-1][DEBUG ] "rank": 0
[node-1][DEBUG ] },
[node-1][DEBUG ] {
[node-1][DEBUG ] "addr": "192.168.5.202:6789/0",
[node-1][DEBUG ] "name": "node-1",
[node-1][DEBUG ] "rank": 1
[node-1][DEBUG ] },
[node-1][DEBUG ] {
[node-1][DEBUG ] "addr": "0.0.0.0:0/2",
[node-1][DEBUG ] "name": "node-2",
[node-1][DEBUG ] "rank": 2
[node-1][DEBUG ] },
[node-1][DEBUG ] {
[node-1][DEBUG ] "addr": "0.0.0.0:0/3",
[node-1][DEBUG ] "name": "node-3",
[node-1][DEBUG ] "rank": 3
[node-1][DEBUG ] }
[node-1][DEBUG ] ]
[node-1][DEBUG ] },
[node-1][DEBUG ] "name": "node-1",
[node-1][DEBUG ] "outside_quorum": [
[node-1][DEBUG ] "node-1",
[node-1][DEBUG ] "node-master"
[node-1][DEBUG ] ],
[node-1][DEBUG ] "quorum": [],
[node-1][DEBUG ] "rank": 1,
[node-1][DEBUG ] "state": "probing",
[node-1][DEBUG ] "sync_provider": []
[node-1][DEBUG ] }
[node-1][DEBUG ] ********************************************************************************
[node-1][INFO ] monitor: mon.node-1 is running
[node-1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-1.asok mon_status
[ceph_deploy.mon][DEBUG ] detecting platform for host node-2 ...
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 14.04 trusty
[node-2][DEBUG ] determining if provided host has same hostname in remote
[node-2][DEBUG ] get remote short hostname
[node-2][DEBUG ] deploying mon to node-2
[node-2][DEBUG ] get remote short hostname
[node-2][DEBUG ] remote hostname: node-2
[node-2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[node-2][DEBUG ] create the mon path if it does not exist
[node-2][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-2/done
[node-2][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-2/done
[node-2][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-2.mon.keyring
[node-2][DEBUG ] create the monitor keyring file
[node-2][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-2 --keyring /var/lib/ceph/tmp/ceph-node-2.mon.keyring --setuser 1001 --setgroup 1002
[node-2][DEBUG ] ceph-mon: mon.noname-c 192.168.5.203:6789/0 is local, renaming to mon.node-2
[node-2][DEBUG ] ceph-mon: set fsid to 37defaff-03ea-4952-bd25-d07310aab339
[node-2][DEBUG ] ceph-mon: created monfs at /var/lib/ceph/mon/ceph-node-2 for mon.node-2
[node-2][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-2.mon.keyring
[node-2][DEBUG ] create a done file to avoid re-doing the mon deployment
[node-2][DEBUG ] create the init path if it does not exist
[node-2][INFO ] Running command: sudo initctl emit ceph-mon cluster=ceph id=node-2
[node-2][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-2.asok mon_status
[node-2][DEBUG ] ********************************************************************************
[node-2][DEBUG ] status for monitor: mon.node-2
[node-2][DEBUG ] {
[node-2][DEBUG ] "election_epoch": 1,
[node-2][DEBUG ] "extra_probe_peers": [
[node-2][DEBUG ] "192.168.5.201:6789/0",
[node-2][DEBUG ] "192.168.5.202:6789/0",
[node-2][DEBUG ] "192.168.5.204:6789/0"
[node-2][DEBUG ] ],
[node-2][DEBUG ] "monmap": {
[node-2][DEBUG ] "created": "2016-05-11 22:17:16.457014",
[node-2][DEBUG ] "epoch": 0,
[node-2][DEBUG ] "fsid": "37defaff-03ea-4952-bd25-d07310aab339",
[node-2][DEBUG ] "modified": "2016-05-11 22:17:16.457014",
[node-2][DEBUG ] "mons": [
[node-2][DEBUG ] {
[node-2][DEBUG ] "addr": "192.168.5.201:6789/0",
[node-2][DEBUG ] "name": "node-master",
[node-2][DEBUG ] "rank": 0
[node-2][DEBUG ] },
[node-2][DEBUG ] {
[node-2][DEBUG ] "addr": "192.168.5.202:6789/0",
[node-2][DEBUG ] "name": "node-1",
[node-2][DEBUG ] "rank": 1
[node-2][DEBUG ] },
[node-2][DEBUG ] {
[node-2][DEBUG ] "addr": "192.168.5.203:6789/0",
[node-2][DEBUG ] "name": "node-2",
[node-2][DEBUG ] "rank": 2
[node-2][DEBUG ] },
[node-2][DEBUG ] {
[node-2][DEBUG ] "addr": "0.0.0.0:0/3",
[node-2][DEBUG ] "name": "node-3",
[node-2][DEBUG ] "rank": 3
[node-2][DEBUG ] }
[node-2][DEBUG ] ]
[node-2][DEBUG ] },
[node-2][DEBUG ] "name": "node-2",
[node-2][DEBUG ] "outside_quorum": [],
[node-2][DEBUG ] "quorum": [],
[node-2][DEBUG ] "rank": 2,
[node-2][DEBUG ] "state": "electing",
[node-2][DEBUG ] "sync_provider": []
[node-2][DEBUG ] }
[node-2][DEBUG ] ********************************************************************************
[node-2][INFO ] monitor: mon.node-2 is running
[node-2][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-2.asok mon_status
[ceph_deploy.mon][DEBUG ] detecting platform for host node-3 ...
[node-3][DEBUG ] connection detected need for sudo
[node-3][DEBUG ] connected to host: node-3
[node-3][DEBUG ] detect platform information from remote host
[node-3][DEBUG ] detect machine type
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo /sbin/initctl version
[node-3][DEBUG ] find the location of an executable
[ceph_deploy.mon][INFO ] distro info: Ubuntu 14.04 trusty
[node-3][DEBUG ] determining if provided host has same hostname in remote
[node-3][DEBUG ] get remote short hostname
[node-3][DEBUG ] deploying mon to node-3
[node-3][DEBUG ] get remote short hostname
[node-3][DEBUG ] remote hostname: node-3
[node-3][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[node-3][DEBUG ] create the mon path if it does not exist
[node-3][DEBUG ] checking for done path: /var/lib/ceph/mon/ceph-node-3/done
[node-3][DEBUG ] done path does not exist: /var/lib/ceph/mon/ceph-node-3/done
[node-3][INFO ] creating keyring file: /var/lib/ceph/tmp/ceph-node-3.mon.keyring
[node-3][DEBUG ] create the monitor keyring file
[node-3][INFO ] Running command: sudo ceph-mon --cluster ceph --mkfs -i node-3 --keyring /var/lib/ceph/tmp/ceph-node-3.mon.keyring --setuser 1001 --setgroup 1002
[node-3][DEBUG ] ceph-mon: mon.noname-d 192.168.5.204:6789/0 is local, renaming to mon.node-3
[node-3][DEBUG ] ceph-mon: set fsid to 37defaff-03ea-4952-bd25-d07310aab339
[node-3][DEBUG ] ceph-mon: created monfs at /var/lib/ceph/mon/ceph-node-3 for mon.node-3
[node-3][INFO ] unlinking keyring file /var/lib/ceph/tmp/ceph-node-3.mon.keyring
[node-3][DEBUG ] create a done file to avoid re-doing the mon deployment
[node-3][DEBUG ] create the init path if it does not exist
[node-3][INFO ] Running command: sudo initctl emit ceph-mon cluster=ceph id=node-3
[node-3][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-3.asok mon_status
[node-3][DEBUG ] ********************************************************************************
[node-3][DEBUG ] status for monitor: mon.node-3
[node-3][DEBUG ] {
[node-3][DEBUG ] "election_epoch": 1,
[node-3][DEBUG ] "extra_probe_peers": [
[node-3][DEBUG ] "192.168.5.201:6789/0",
[node-3][DEBUG ] "192.168.5.202:6789/0",
[node-3][DEBUG ] "192.168.5.203:6789/0"
[node-3][DEBUG ] ],
[node-3][DEBUG ] "monmap": {
[node-3][DEBUG ] "created": "2016-05-11 22:17:22.334718",
[node-3][DEBUG ] "epoch": 0,
[node-3][DEBUG ] "fsid": "37defaff-03ea-4952-bd25-d07310aab339",
[node-3][DEBUG ] "modified": "2016-05-11 22:17:22.334718",
[node-3][DEBUG ] "mons": [
[node-3][DEBUG ] {
[node-3][DEBUG ] "addr": "192.168.5.201:6789/0",
[node-3][DEBUG ] "name": "node-master",
[node-3][DEBUG ] "rank": 0
[node-3][DEBUG ] },
[node-3][DEBUG ] {
[node-3][DEBUG ] "addr": "192.168.5.202:6789/0",
[node-3][DEBUG ] "name": "node-1",
[node-3][DEBUG ] "rank": 1
[node-3][DEBUG ] },
[node-3][DEBUG ] {
[node-3][DEBUG ] "addr": "192.168.5.203:6789/0",
[node-3][DEBUG ] "name": "node-2",
[node-3][DEBUG ] "rank": 2
[node-3][DEBUG ] },
[node-3][DEBUG ] {
[node-3][DEBUG ] "addr": "192.168.5.204:6789/0",
[node-3][DEBUG ] "name": "node-3",
[node-3][DEBUG ] "rank": 3
[node-3][DEBUG ] }
[node-3][DEBUG ] ]
[node-3][DEBUG ] },
[node-3][DEBUG ] "name": "node-3",
[node-3][DEBUG ] "outside_quorum": [],
[node-3][DEBUG ] "quorum": [],
[node-3][DEBUG ] "rank": 3,
[node-3][DEBUG ] "state": "electing",
[node-3][DEBUG ] "sync_provider": []
[node-3][DEBUG ] }
[node-3][DEBUG ] ********************************************************************************
[node-3][INFO ] monitor: mon.node-3 is running
[node-3][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-3.asok mon_status
[ceph_deploy.mon][INFO ] processing monitor mon.node-master
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-master.asok mon_status
[ceph_deploy.mon][WARNIN] mon.node-master monitor is not yet in quorum, tries left: 5
[ceph_deploy.mon][WARNIN] waiting 5 seconds before retrying
[node-master][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-master.asok mon_status
[ceph_deploy.mon][WARNIN] mon.node-master monitor is not yet in quorum, tries left: 4
[ceph_deploy.mon][WARNIN] waiting 10 seconds before retrying
[node-master][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-master.asok mon_status
[ceph_deploy.mon][INFO ] mon.node-master monitor has reached quorum!
[ceph_deploy.mon][INFO ] processing monitor mon.node-1
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-1.asok mon_status
[ceph_deploy.mon][INFO ] mon.node-1 monitor has reached quorum!
[ceph_deploy.mon][INFO ] processing monitor mon.node-2
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-2.asok mon_status
[ceph_deploy.mon][INFO ] mon.node-2 monitor has reached quorum!
[ceph_deploy.mon][INFO ] processing monitor mon.node-3
[node-3][DEBUG ] connection detected need for sudo
[node-3][DEBUG ] connected to host: node-3
[node-3][DEBUG ] detect platform information from remote host
[node-3][DEBUG ] detect machine type
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo /sbin/initctl version
[node-3][DEBUG ] find the location of an executable
[node-3][INFO ] Running command: sudo ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.node-3.asok mon_status
[ceph_deploy.mon][INFO ] mon.node-3 monitor has reached quorum!
[ceph_deploy.mon][INFO ] all initial monitors are running and have formed quorum
[ceph_deploy.mon][INFO ] Running gatherkeys...
[ceph_deploy.gatherkeys][DEBUG ] Checking node-master for /etc/ceph/ceph.client.admin.keyring
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] fetch remote file
[ceph_deploy.gatherkeys][DEBUG ] Got ceph.client.admin.keyring key from node-master.
[ceph_deploy.gatherkeys][DEBUG ] Have ceph.mon.keyring
[ceph_deploy.gatherkeys][DEBUG ] Checking node-master for /var/lib/ceph/bootstrap-osd/ceph.keyring
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] fetch remote file
[ceph_deploy.gatherkeys][DEBUG ] Got ceph.bootstrap-osd.keyring key from node-master.
[ceph_deploy.gatherkeys][DEBUG ] Checking node-master for /var/lib/ceph/bootstrap-mds/ceph.keyring
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] fetch remote file
[ceph_deploy.gatherkeys][DEBUG ] Got ceph.bootstrap-mds.keyring key from node-master.
[ceph_deploy.gatherkeys][DEBUG ] Checking node-master for /var/lib/ceph/bootstrap-rgw/ceph.keyring
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] fetch remote file
[ceph_deploy.gatherkeys][DEBUG ] Got ceph.bootstrap-rgw.keyring key from node-master.
change permission and create folder in all node
root@node-master:~# mkdir /media/data_ssd/ceph_data
prepare OSD
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy osd prepare node-1:/media/data_ssd/ceph_data node-2:/media/data_ssd/ceph_data
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy osd prepare node-1:/media/data_ssd/ceph_data node-2:/media/data_ssd/ceph_data
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] disk : [('node-1', '/media/data_ssd/ceph_data', None), ('node-2', '/media/data_ssd/ceph_data', None)]
[ceph_deploy.cli][INFO ] dmcrypt : False
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] bluestore : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : prepare
[ceph_deploy.cli][INFO ] dmcrypt_key_dir : /etc/ceph/dmcrypt-keys
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb68c65a8>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] fs_type : xfs
[ceph_deploy.cli][INFO ] func : <function osd at 0xb68af330>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] zap_disk : False
[ceph_deploy.osd][DEBUG ] Preparing cluster ceph disks node-1:/media/data_ssd/ceph_data: node-2:/media/data_ssd/ceph_data:
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 14.04 trusty
[ceph_deploy.osd][DEBUG ] Deploying osd to node-1
[node-1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.osd][DEBUG ] Preparing host node-1 disk /media/data_ssd/ceph_data journal None activate False
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /usr/sbin/ceph-disk -v prepare --cluster ceph --fs-type xfs -- /media/data_ssd/ceph_data
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=fsid
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --check-allows-journal -i 0 --cluster ceph
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --check-wants-journal -i 0 --cluster ceph
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --check-needs-journal -i 0 --cluster ceph
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=osd_journal_size
[node-1][WARNIN] populate_data_path: Preparing osd data dir /media/data_ssd/ceph_data
[node-1][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/ceph_fsid.10107.tmp
[node-1][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/fsid.10107.tmp
[node-1][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/magic.10107.tmp
[node-1][INFO ] checking OSD status...
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /usr/bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host node-1 is now ready for osd use.
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 14.04 trusty
[ceph_deploy.osd][DEBUG ] Deploying osd to node-2
[node-2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.osd][DEBUG ] Preparing host node-2 disk /media/data_ssd/ceph_data journal None activate False
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /usr/sbin/ceph-disk -v prepare --cluster ceph --fs-type xfs -- /media/data_ssd/ceph_data
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=fsid
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --check-allows-journal -i 0 --cluster ceph
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --check-wants-journal -i 0 --cluster ceph
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --check-needs-journal -i 0 --cluster ceph
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=osd_journal_size
[node-2][WARNIN] populate_data_path: Preparing osd data dir /media/data_ssd/ceph_data
[node-2][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/ceph_fsid.15136.tmp
[node-2][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/fsid.15136.tmp
[node-2][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/magic.15136.tmp
[node-2][INFO ] checking OSD status...
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /usr/bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host node-2 is now ready for osd use.
Sometimes you must change permisstion (Because I run with root fixed with root@node-1:~# chmod 777 -R /media/data_ssd/ceph_data)
And activate OSD(s).
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy osd activate node-1:/media/data_ssd/ceph_data node-2:/media/data_ssd/ceph_data
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy osd activate node-1:/media/data_ssd/ceph_data node-2:/media/data_ssd/ceph_data
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] subcommand : activate
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb680e5a8>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] func : <function osd at 0xb67f7330>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] disk : [('node-1', '/media/data_ssd/ceph_data', None), ('node-2', '/media/data_ssd/ceph_data', None)]
[ceph_deploy.osd][DEBUG ] Activating cluster ceph disks node-1:/media/data_ssd/ceph_data: node-2:/media/data_ssd/ceph_data:
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 14.04 trusty
[ceph_deploy.osd][DEBUG ] activating host node-1 disk /media/data_ssd/ceph_data
[ceph_deploy.osd][DEBUG ] will use init type: upstart
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /usr/sbin/ceph-disk -v activate --mark-init upstart --mount /media/data_ssd/ceph_data
[node-1][WARNIN] main_activate: path = /media/data_ssd/ceph_data
[node-1][WARNIN] activate: Cluster uuid is 37defaff-03ea-4952-bd25-d07310aab339
[node-1][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=fsid
[node-1][WARNIN] activate: Cluster name is ceph
[node-1][WARNIN] activate: OSD uuid is 4bf56255-fa11-4302-9721-d3da846ffd5b
[node-1][WARNIN] allocate_osd_id: Allocating OSD id...
[node-1][WARNIN] command: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd create --concise 4bf56255-fa11-4302-9721-d3da846ffd5b
[node-1][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/whoami.10286.tmp
[node-1][WARNIN] activate: OSD id is 0
[node-1][WARNIN] activate: Initializing OSD...
[node-1][WARNIN] command_check_call: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /media/data_ssd/ceph_data/activate.monmap
[node-1][WARNIN] got monmap epoch 2
[node-1][WARNIN] command_check_call: Running command: /usr/bin/ceph-osd --cluster ceph --mkfs --mkkey -i 0 --monmap /media/data_ssd/ceph_data/activate.monmap --osd-data /media/data_ssd/ceph_data --osd-journal /media/data_ssd/ceph_data/journal --osd-uuid 4bf56255-fa11-4302-9721-d3da846ffd5b --keyring /media/data_ssd/ceph_data/keyring --setuser ceph --setgroup ceph
[node-1][WARNIN] 2016-05-11 22:25:51.728453 b656c000 -1 filestore(/media/data_ssd/ceph_data) WARNING: max attr value size (1024) is smaller than osd_max_object_name_len (2048). Your backend filesystem appears to not support attrs large enough to handle the configured max rados name size. You may get unexpected ENAMETOOLONG errors on rados operations or buggy behavior
[node-1][WARNIN] 2016-05-11 22:25:51.754338 b656c000 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
[node-1][WARNIN] 2016-05-11 22:25:51.851178 b656c000 -1 filestore(/media/data_ssd/ceph_data) WARNING: max attr value size (1024) is smaller than osd_max_object_name_len (2048). Your backend filesystem appears to not support attrs large enough to handle the configured max rados name size. You may get unexpected ENAMETOOLONG errors on rados operations or buggy behavior
[node-1][WARNIN] 2016-05-11 22:25:51.979179 b656c000 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
[node-1][WARNIN] 2016-05-11 22:25:51.991966 b656c000 -1 filestore(/media/data_ssd/ceph_data) could not find #-1:7b3f43c4:::osd_superblock:0# in index: (2) No such file or directory
[node-1][WARNIN] 2016-05-11 22:25:52.188778 b656c000 -1 created object store /media/data_ssd/ceph_data for osd.0 fsid 37defaff-03ea-4952-bd25-d07310aab339
[node-1][WARNIN] 2016-05-11 22:25:52.188976 b656c000 -1 auth: error reading file: /media/data_ssd/ceph_data/keyring: can't open /media/data_ssd/ceph_data/keyring: (2) No such file or directory
[node-1][WARNIN] 2016-05-11 22:25:52.189542 b656c000 -1 created new key in keyring /media/data_ssd/ceph_data/keyring
[node-1][WARNIN] activate: Marking with init system upstart
[node-1][WARNIN] activate: Authorizing OSD key...
[node-1][WARNIN] command_check_call: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring auth add osd.0 -i /media/data_ssd/ceph_data/keyring osd allow * mon allow profile osd
[node-1][WARNIN] added key for osd.0
[node-1][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/active.10286.tmp
[node-1][WARNIN] activate: ceph osd.0 data dir is ready at /media/data_ssd/ceph_data
[node-1][WARNIN] activate_dir: Creating symlink /var/lib/ceph/osd/ceph-0 -> /media/data_ssd/ceph_data
[node-1][WARNIN] start_daemon: Starting ceph osd.0...
[node-1][WARNIN] command_check_call: Running command: /sbin/initctl emit --no-wait -- ceph-osd cluster=ceph id=0
[node-1][INFO ] checking OSD status...
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /usr/bin/ceph --cluster=ceph osd stat --format=json
[node-1][WARNIN] there is 1 OSD down
[node-1][WARNIN] there is 1 OSD out
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO ] Distro info: Ubuntu 14.04 trusty
[ceph_deploy.osd][DEBUG ] activating host node-2 disk /media/data_ssd/ceph_data
[ceph_deploy.osd][DEBUG ] will use init type: upstart
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /usr/sbin/ceph-disk -v activate --mark-init upstart --mount /media/data_ssd/ceph_data
[node-2][WARNIN] main_activate: path = /media/data_ssd/ceph_data
[node-2][WARNIN] activate: Cluster uuid is 37defaff-03ea-4952-bd25-d07310aab339
[node-2][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=fsid
[node-2][WARNIN] activate: Cluster name is ceph
[node-2][WARNIN] activate: OSD uuid is a40c21a2-f826-4adf-a3b7-93527ac9507f
[node-2][WARNIN] allocate_osd_id: Allocating OSD id...
[node-2][WARNIN] command: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd create --concise a40c21a2-f826-4adf-a3b7-93527ac9507f
[node-2][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/whoami.15315.tmp
[node-2][WARNIN] activate: OSD id is 1
[node-2][WARNIN] activate: Initializing OSD...
[node-2][WARNIN] command_check_call: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /media/data_ssd/ceph_data/activate.monmap
[node-2][WARNIN] got monmap epoch 2
[node-2][WARNIN] command_check_call: Running command: /usr/bin/ceph-osd --cluster ceph --mkfs --mkkey -i 1 --monmap /media/data_ssd/ceph_data/activate.monmap --osd-data /media/data_ssd/ceph_data --osd-journal /media/data_ssd/ceph_data/journal --osd-uuid a40c21a2-f826-4adf-a3b7-93527ac9507f --keyring /media/data_ssd/ceph_data/keyring --setuser ceph --setgroup ceph
[node-2][WARNIN] 2016-05-11 22:26:03.892106 b65da000 -1 filestore(/media/data_ssd/ceph_data) WARNING: max attr value size (1024) is smaller than osd_max_object_name_len (2048). Your backend filesystem appears to not support attrs large enough to handle the configured max rados name size. You may get unexpected ENAMETOOLONG errors on rados operations or buggy behavior
[node-2][WARNIN] 2016-05-11 22:26:03.932268 b65da000 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
[node-2][WARNIN] 2016-05-11 22:26:04.010871 b65da000 -1 filestore(/media/data_ssd/ceph_data) WARNING: max attr value size (1024) is smaller than osd_max_object_name_len (2048). Your backend filesystem appears to not support attrs large enough to handle the configured max rados name size. You may get unexpected ENAMETOOLONG errors on rados operations or buggy behavior
[node-2][WARNIN] 2016-05-11 22:26:04.130923 b65da000 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway
[node-2][WARNIN] 2016-05-11 22:26:04.144328 b65da000 -1 filestore(/media/data_ssd/ceph_data) could not find #-1:7b3f43c4:::osd_superblock:0# in index: (2) No such file or directory
[node-2][WARNIN] 2016-05-11 22:26:04.327988 b65da000 -1 created object store /media/data_ssd/ceph_data for osd.1 fsid 37defaff-03ea-4952-bd25-d07310aab339
[node-2][WARNIN] 2016-05-11 22:26:04.328166 b65da000 -1 auth: error reading file: /media/data_ssd/ceph_data/keyring: can't open /media/data_ssd/ceph_data/keyring: (2) No such file or directory
[node-2][WARNIN] 2016-05-11 22:26:04.328597 b65da000 -1 created new key in keyring /media/data_ssd/ceph_data/keyring
[node-2][WARNIN] activate: Marking with init system upstart
[node-2][WARNIN] activate: Authorizing OSD key...
[node-2][WARNIN] command_check_call: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring auth add osd.1 -i /media/data_ssd/ceph_data/keyring osd allow * mon allow profile osd
[node-2][WARNIN] added key for osd.1
[node-2][WARNIN] command: Running command: /bin/chown -R ceph:ceph /media/data_ssd/ceph_data/active.15315.tmp
[node-2][WARNIN] activate: ceph osd.1 data dir is ready at /media/data_ssd/ceph_data
[node-2][WARNIN] activate_dir: Creating symlink /var/lib/ceph/osd/ceph-1 -> /media/data_ssd/ceph_data
[node-2][WARNIN] start_daemon: Starting ceph osd.1...
[node-2][WARNIN] command_check_call: Running command: /sbin/initctl emit --no-wait -- ceph-osd cluster=ceph id=1
[node-2][INFO ] checking OSD status...
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /usr/bin/ceph --cluster=ceph osd stat --format=json
[node-2][WARNIN] there are 2 OSDs down
[node-2][WARNIN] there are 2 OSDs out
Spread admin
ceph@node-master:/home/ceph/my-cluster$ ceph-deploy admin node-master node-1 node-2
[ceph_deploy.conf][DEBUG ] found configuration file at: /var/lib/ceph/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (1.5.32): /usr/bin/ceph-deploy admin node-master node-1 node-2
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf instance at 0xb68097b0>
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] client : ['node-master', 'node-1', 'node-2']
[ceph_deploy.cli][INFO ] func : <function admin at 0xb688dcf0>
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-master
[node-master][DEBUG ] connection detected need for sudo
[node-master][DEBUG ] connected to host: node-master
[node-master][DEBUG ] detect platform information from remote host
[node-master][DEBUG ] detect machine type
[node-master][DEBUG ] find the location of an executable
[node-master][INFO ] Running command: sudo /sbin/initctl version
[node-master][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-1
[node-1][DEBUG ] connection detected need for sudo
[node-1][DEBUG ] connected to host: node-1
[node-1][DEBUG ] detect platform information from remote host
[node-1][DEBUG ] detect machine type
[node-1][DEBUG ] find the location of an executable
[node-1][INFO ] Running command: sudo /sbin/initctl version
[node-1][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to node-2
[node-2][DEBUG ] connection detected need for sudo
[node-2][DEBUG ] connected to host: node-2
[node-2][DEBUG ] detect platform information from remote host
[node-2][DEBUG ] detect machine type
[node-2][DEBUG ] find the location of an executable
[node-2][INFO ] Running command: sudo /sbin/initctl version
[node-2][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
Change permission.
root@node-master:~# chmod +r /etc/ceph/ceph.client.admin.keyring
Check status.
root@node-master:/media/data_ssd/my-cluster# ceph -s
cluster 0498163e-bfdb-41c8-adf9-f14799e0ad9b
health HEALTH_ERR
64 pgs are stuck inactive for more than 300 seconds
64 pgs stuck inactive
mon.node-1 low disk space
monmap e1: 2 mons at {node-1=192.168.5.224:6789/0,node-master=192.168.5.201:6789/0}
election epoch 4, quorum 0,1 node-master,node-1
osdmap e3: 1 osds: 0 up, 0 in
flags sortbitwise
pgmap v4: 64 pgs, 1 pools, 0 bytes data, 0 objects
0 kB used, 0 kB / 0 kB avail
64 creating
Mark OSD(s) in cluster
$ root@node-master:/media/data_ssd/my-cluster# ceph osd in 0
apt-get install rbd-fuse
ปัญหาเกิดจาก File System จริงๆ ด้วยครับ ขนาดชื่อไฟล์ยาวเกินได้
ผมไปเพิ่ม
osd_max_object_name_len = 256
osd_max_object_namespace_len = 64
ในไฟล์ ceph.conf