Build Ceph FS for arm.

Build_CephFS_for_arm.md

first, clone git repository frome ceph upstream or inside my account (https://github.com/Aorjoa/ceph).

./install-deps.sh
./autogen.sh
./configure --without-tcmalloc

remove option -momit-leaf-frame-pointer in this file (because it's not support yet). vi /media/data_ssd/docker_engine_arm/ceph/src/rocksdb/Makefile and then run make command.

while i build it (and stuck because some bugs). i got a new solution

root@node-master:/# cat /etc/apt/sources.list.d/ceph.list 
deb [arch=armhf] http://download.ceph.com/debian-dumpling/ wheezy main
root@node-master:/# wget --no-check-certificate -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add -
#For QA and Dev only
root@node-master:/# wget --no-check-certificate -q -O- 'https://download.ceph.com/keys/autobuild.asc' | apt-key add -

(note that : sometimes we use deb http://ceph.com/debian-giant/ trusty main in ceph.list because more than packages found there.) and sources.list here.

# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.

deb http://ports.ubuntu.com/ubuntu-ports/ wily main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily main restricted

## Major bug fix updates produced after the final release of the
## distribution.
deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates main restricted

## Uncomment the following two lines to add software from the 'universe'
## repository.
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb http://ports.ubuntu.com/ubuntu-ports/ wily universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily universe
deb http://ports.ubuntu.com/ubuntu-ports/ wily-updates universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-updates universe

## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
# deb http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted
# deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-backports main restricted

deb http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ wily-security universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security universe
# deb http://ports.ubuntu.com/ubuntu-ports/ wily-security multiverse
# deb-src http://ports.ubuntu.com/ubuntu-ports/ wily-security multiverse

to avoid wget bug for check certificate.

root@node-master:/# cat /root/.wgetrc
check_certificate = off

then install ceph-deploy

ln -s /usr/lib/insserv/insserv /sbin/insserv
root@node-master:/# apt-get update
root@node-master:/# apt-get upgrade
root@node-master:~/my-cluster# apt-get install openssh-server
root@node-master:~/my-cluster# apt-get install ntp
root@node-master:~/my-cluster# DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -q -o Dpkg::Options::=--force-confnew --no-install-recommends --assume-yes install -- ceph ceph-mds ceph-common ceph-fs-common gdisk

make sure target node NOT contain armhf AND arm it's bug in apt-get use only armhf once.

root@node-2:~# cat /var/lib/dpkg/arch 
armhf

set /etc/hosts for resolv hostname

192.168.5.201 node-master
192.168.5.202 node-1
192.168.5.203 node-2
192.168.5.204 node-3

initiate Ceph FS

mkdir my-cluster
cd my-cluster
root@node-master:~/my-cluster# mkdir -p /etc/ceph/
root@node-master:~/my-cluster# ceph-deploy new node-master node-1 node-2 node-3
root@node-master:~/my-cluster# ceph-deploy install node-master node-1 node-2 node-3
===== (make sure you have free space >= 6GB because some case error occur) =====
root@node-master:~/my-cluster# ceph-deploy mon create node-master node-1 node-2 node-3

and run command for ensure it's form cluster ceph-deploy mon create-initial

# ceph-deploy gatherkeys  node-master

change permission and create folder in all node

root@node-master:~# mkdir /media/data_ssd/ceph_data
root@node-master:~# sudo chmod +r  /etc/ceph/ceph.client.admin.keyring

prepare osd

root@0d24bd587071:/data/aiyara-ceph-cluster# ceph-deploy osd prepare node-1:/media/data_ssd/ceph_data

activate osd

root@0d24bd587071:/data/aiyara-ceph-cluster# ceph-deploy osd activate node-1:/media/data_ssd/ceph_data

check status

root@node-master:~# ceph -s
    cluster 21050870-a4e0-44ce-a8e5-a5284a88ae30
     health HEALTH_WARN clock skew detected on mon.node-1, mon.node-2, mon.node-3 mon.node-master low disk space; Monitor clock skew detected 
     monmap e1: 4 mons at {node-1=192.168.5.202:6789/0,node-2=192.168.5.203:6789/0,node-3=192.168.5.204:6789/0,node-master=192.168.5.201:6789/0}, election epoch 30, quorum 0,1,2,3 node-master,node-1,node-2,node-3
     mdsmap e15: 1/1/1 up {0=node-master=up:active}
     osdmap e45: 4 osds: 4 up, 4 in
      pgmap v149: 192 pgs, 3 pools, 2023 bytes data, 22 objects
            43844 MB used, 792 GB / 879 GB avail
                 192 active+clean
  client io 2763 B/s rd, 2 op/s

mount point

apt-get install rbd-fuse
root@node-master:~# rbd-fuse /media/data_ssd/aiyara_data_mountpoint/
mkfs.ext4 /media/data_ssd/aiyara_data_mountpoint/aiyara_repo
root@node-master:~# mount /media/data_ssd/aiyara_data_mountpoint/aiyara_repo /test/

describe disk

root@node-master:~# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/root        14G   12G  1.9G  86% /
devtmpfs        937M  4.0K  937M   1% /dev
none            4.0K     0  4.0K   0% /sys/fs/cgroup
none            189M  708K  189M   1% /run
none            5.0M     0  5.0M   0% /run/lock
none            945M  144K  945M   1% /run/shm
none            100M   28K  100M   1% /run/user
/dev/sda1       220G   28G  182G  14% /media/data_ssd
rbd-fuse        2.0G  2.0G     0 100% /media/data_ssd/aiyara_data_mountpoint
/dev/loop0      2.0G  3.0M  1.8G   1% /test

/=========/
some time GPG error because our behide firewall and we fixed with specific protocol and port 80.

apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 40976EAF437D05B5

If you have small free storage it's may be can't start.