Mesos Troubleshooting

1mesos-troubleshooting.md

Master

# Environment Files:
/opt/mesosphere/etc/cfn_signal_metadata
/opt/mesosphere/environment

# Run
/opt/mesosphere/bin/cfn-signal



cat /opt/mesosphere/bin/exhibitor_wait.py
cat /opt/mesosphere/etc/master_count
curl  http://127.0.0.1:8181/exhibitor/v1/cluster/status
cat /var/lib/dcos/cluster-id

journalctl --identifier=coreos-cloudinit

Exhibitor Config:

cat /etc/mesosphere/setup-packages/dcos-provider-aws--setup/etc/dns_config
cat /etc/mesosphere/setup-packages/dcos-provider-aws--setup/etc/exhibitor


vi  /opt/mesosphere/packages/exhibitor--acde090d6148629034335be3d53a0343c44fefa1/usr/exhibitor/start_exhibitor.py
vi /opt/mesosphere/etc/master_count
cat /opt/mesosphere/etc/mesos-master-provider
cat /opt/mesosphere/etc/mesos-master


systemctl stop dcos-exhibitor.service
systemctl start dcos-exhibitor.service
systemctl status dcos-exhibitor.service

# Browse to 
# http://internal-mesos-mas-internal-1o27dple0oo08-1940258802.us-east-1.elb.amazonaws.com:8181/exhibitor/v1/ui/index.html


cat /opt/mesosphere/etc/exhibitor
cat /opt/mesosphere/etc/dns_config
cat /run/dcos_exhibitor/exhibitor_defaults.conf

cat /opt/mesosphere/active/exhibitor/usr/zookeeper/bin/../conf/zoo.cfg
sudo journalctl -u dcos-exhibitor.service

AWS CLI:

sudo su -
cat >> Dockerfile << _EOF 
FROM python:3.4
RUN pip install awscli
VOLUME /root/.aws
WORKDIR /work
ENTRYPOINT ["aws"]
_EOF

docker build -t awscli .
alias aws="docker run --rm -it -v $(pwd):/work awscli "
aws s3 ls
echo test > test
aws s3 cp test s3://mesos-infra-exhibitors3bucket-b5k56gq1ahr3/

# Alternate
alias aws="docker run --rm -it -v $(pwd):/work -v ~/.aws:/root/.aws:ro awscli "
alias aws="docker run --rm -it -e AWS_ACCESS_KEY_ID=<key> -e AWS_SECRET_ACCESS_KEY=<secret> -e AWS_DEFAULT_REGION=us-east-1 -v $(pwd):/work  awscli "

DDT Service:

curl http://10.120.170.226:5050/state
curl -v  http://$(hostname -i):5050/state

Signal Service:

curl  http://127.0.0.1:/system/health/v1/report

Code Snippet :

get_private_ip_from_metaserver()
{
    curl -fsSL http://169.254.169.254/latest/meta-data/local-ipv4
}

echo ${COREOS_PRIVATE_IPV4:-$(get_private_ip_from_metaserver)}

2marathon-api.md

API

Python Code snippet to extract port details for a given service

import requests
import json

headers = {'Accept': 'application/json'}
url = "https://marathon.myco.com/v2/tasks"
r = requests.get(url,headers=headers)

for task in r.json()['tasks']:
   print task['id'],
   print task['host']
   print task['appId'],
   print task['ports'][0],
   print task['servicePorts'][0]


i = [ task['ports'][0] for task in r.json()['tasks'] if task['appId'] == u'/delta-service' ]
print i

Curl:

curl https://marathon.myco.com/v2/apps/delta-service/tasks -H "Accept: application/json" | python -m 'json.tool'