Skip to content

Instantly share code, notes, and snippets.

@hantuzun

hantuzun/add-spark-agents.patch

Created December 13, 2017 09:22
Show Gist options
  • Save hantuzun/6e72be94883f37d26cedeb770aa58fb9 to your computer and use it in GitHub Desktop.
Save hantuzun/6e72be94883f37d26cedeb770aa58fb9 to your computer and use it in GitHub Desktop.
Download ZIP
add-spark-agents.patch
Raw
add-spark-agents.patch
diff --git a/dcos_installer/action_lib.py b/dcos_installer/action_lib.py
index d47821d8..d6690c68 100644
--- a/dcos_installer/action_lib.py
+++ b/dcos_installer/action_lib.py
@@ -54,7 +54,8 @@ class ExecuteException(Exception):
def nodes_count_by_type(config):
total_agents_count = len(config.hacky_default_get('agent_list', [])) + \
- len(config.hacky_default_get('public_agent_list', []))
+ len(config.hacky_default_get('public_agent_list', [])) + \
+ len(config.hacky_default_get('spark_agent_list', []))
return {
'total_masters': len(config['master_list']),
'total_agents': total_agents_count
@@ -69,7 +70,8 @@ def get_full_nodes_list(config):
node_role_map = {
'master_list': 'master',
'agent_list': 'agent',
- 'public_agent_list': 'public_agent'
+ 'public_agent_list': 'public_agent',
+ 'spark_agent_list': 'spark_agent'
}
full_target_list = []
for config_field, role in node_role_map.items():
@@ -225,6 +227,10 @@ def install_dcos(
'public_agent': {
'tags': {'role': 'public_agent', 'dcos_install_param': 'slave_public'},
'hosts': config.hacky_default_get('public_agent_list', [])
+ },
+ 'spark_agent': {
+ 'tags': {'role': 'spark_agent', 'dcos_install_param': 'slave_spark'},
+ 'hosts': config.hacky_default_get('spark_agent_list', [])
}
}
diff --git a/dcos_installer/upgrade.py b/dcos_installer/upgrade.py
index 5dad08ce..d28eb8ae 100644
--- a/dcos_installer/upgrade.py
+++ b/dcos_installer/upgrade.py
@@ -92,10 +92,10 @@ fi
# Determine this node's role.
ROLE_DIR=/etc/mesosphere/roles
-num_roles=$( (ls --format=single-column $ROLE_DIR/{master,slave,slave_public} || true) 2>/dev/null | wc -l)
+num_roles=$( (ls --format=single-column $ROLE_DIR/{master,slave,slave_public,slave_spark} || true) 2>/dev/null | wc -l)
if [ "$num_roles" -ne "1" ]; then
echo "ERROR: Can't determine this node's role." \
- "One of master, slave, or slave_public must be present under $ROLE_DIR."
+ "One of master, slave, slave_public or slave_spark must be present under $ROLE_DIR."
exit 1
fi
@@ -108,6 +108,9 @@ elif [ -f $ROLE_DIR/slave ]; then
elif [ -f $ROLE_DIR/slave_public ]; then
role="slave_public"
role_name="public agent"
+elif [ -f $ROLE_DIR/slave_spark ]; then
+ role="slave_spark"
+ role_name="spark agent"
fi
echo "Upgrading DC/OS $role_name {{ installed_cluster_version }} -> {{ installer_version }}"
diff --git a/docs/styleguides/python.md b/docs/styleguides/python.md
index 6fd02b2c..3241d617 100644
--- a/docs/styleguides/python.md
+++ b/docs/styleguides/python.md
@@ -165,6 +165,7 @@ from test_util.helpers import (CI_CREDENTIALS,
public_masters: List[str],
slaves: List[str],
public_slaves: List[str],
+ spark_slaves: List[str],
default_os_user: str) -> DcosApiSession:
```
@@ -220,6 +221,7 @@ class DcosApiSession(ARNodeApiClientMixin, ApiClientSession):
public_masters: List[str],
slaves: List[str],
public_slaves: List[str],
+ spark_slaves: List[str],
default_os_user: str,
auth_user: Optional[DcosUser]) -> None:
diff --git a/gen/__init__.py b/gen/__init__.py
index a958292b..efc59c30 100644
--- a/gen/__init__.py
+++ b/gen/__init__.py
@@ -32,7 +32,7 @@ from pkgpanda import PackageId
from pkgpanda.util import hash_checkout, json_prettyprint, load_string, make_tar, split_by_token, write_json, write_yaml
# List of all roles all templates should have.
-role_names = {"master", "slave", "slave_public"}
+role_names = {"master", "slave", "slave_public", "slave_spark"}
role_template = '/etc/mesosphere/roles/{}'
diff --git a/gen/build_deploy/azure.py b/gen/build_deploy/azure.py
index fdb57f88..bfecd935 100644
--- a/gen/build_deploy/azure.py
+++ b/gen/build_deploy/azure.py
@@ -35,6 +35,10 @@ INSTANCE_GROUPS = {
'slave_public': {
'report_name': 'PublicSlaveServerGroup',
'roles': ['slave_public']
+ },
+ 'slave_spark': {
+ 'report_name': 'SparkSlaveServerGroup',
+ 'roles': ['slave_spark']
}
}
@@ -58,7 +62,8 @@ azure_base_source = Source(entry={
'exhibitor_storage_backend': 'azure',
'master_cloud_config': '{{ master_cloud_config }}',
'slave_cloud_config': '{{ slave_cloud_config }}',
- 'slave_public_cloud_config': '{{ slave_public_cloud_config }}'
+ 'slave_public_cloud_config': '{{ slave_public_cloud_config }}',
+ 'slave_spark_cloud_config': '{{ slave_spark_cloud_config }}'
},
'conditional': {
'oauth_available': {
@@ -117,12 +122,14 @@ def render_arm(
arm_template,
master_cloudconfig_yaml_str,
slave_cloudconfig_yaml_str,
- slave_public_cloudconfig_yaml_str):
+ slave_public_cloudconfig_yaml_str,
+ slave_spark_cloudconfig_yaml_str):
template_str = gen.template.parse_str(arm_template).render({
'master_cloud_config': transform(master_cloudconfig_yaml_str),
'slave_cloud_config': transform(slave_cloudconfig_yaml_str),
- 'slave_public_cloud_config': transform(slave_public_cloudconfig_yaml_str)
+ 'slave_public_cloud_config': transform(slave_public_cloudconfig_yaml_str),
+ 'slave_spark_cloud_config': transform(slave_spark_cloudconfig_yaml_str)
})
# Add in some metadata to help support engineers
@@ -151,7 +158,7 @@ def gen_templates(gen_arguments, arm_template, extra_sources):
# Add general services
cloud_config = results.utils.add_services(cloud_config, 'canonical')
- # Specialize for master, slave, slave_public
+ # Specialize for master, slave, slave_public, slave_spark
variant_cloudconfig = {}
for variant, params in INSTANCE_GROUPS.items():
cc_variant = deepcopy(cloud_config)
@@ -169,7 +176,8 @@ def gen_templates(gen_arguments, arm_template, extra_sources):
results.templates[arm_template + '.json'],
variant_cloudconfig['master'],
variant_cloudconfig['slave'],
- variant_cloudconfig['slave_public'])
+ variant_cloudconfig['slave_public'],
+ variant_cloudconfig['slave_spark'])
return (arm, results)
diff --git a/gen/build_deploy/bash.py b/gen/build_deploy/bash.py
index 57984f9c..77a21246 100644
--- a/gen/build_deploy/bash.py
+++ b/gen/build_deploy/bash.py
@@ -287,7 +287,7 @@ function check_all() {
echo -e "${BOLD}Running preflight checks${NORMAL}"
AGENT_ONLY=0
for ROLE in $ROLES; do
- if [[ $ROLE = "slave" || $ROLE = "slave_public" ]]; then
+ if [[ $ROLE = "slave" || $ROLE = "slave_public" || $ROLE = "slave_spark" ]]; then
AGENT_ONLY=1
break
fi
@@ -401,7 +401,7 @@ function check_all() {
check_service $service
done
else
- # agent / public agent node service checks
+ # agent / public agent / spark agent node service checks
for service in \
"53 spartan" \
"5051 mesos-agent" \
@@ -423,8 +423,8 @@ function check_all() {
for role in "$ROLES"
do
- if [ "$role" != "master" -a "$role" != "slave" -a "$role" != "slave_public" -a "$role" != "minuteman" ]; then
- echo -e "${RED}FAIL Invalid role $role. Role must be one of {master,slave,slave_public}${NORMAL}"
+ if [ "$role" != "master" -a "$role" != "slave" -a "$role" != "slave_public" -a "$role" != "slave_spark" -a "$role" != "minuteman" ]; then
+ echo -e "${RED}FAIL Invalid role $role. Role must be one of {master,slave,slave_public,slave_spark}${NORMAL}"
(( OVERALL_RC += 1 ))
fi
done
diff --git a/gen/dcos-config.yaml b/gen/dcos-config.yaml
index d13c0014..cf2852ec 100644
--- a/gen/dcos-config.yaml
+++ b/gen/dcos-config.yaml
@@ -64,6 +64,16 @@ package:
"bind_ip_blacklist": {{ dns_bind_ip_blacklist }},
"forward_zones": {{ dns_forward_zones }}
}
+ - path: /etc_slave_spark/spartan.json
+ content: |
+ {
+ "upstream_resolvers": {{ resolvers }},
+ "bind_interface": "spartan",
+ "udp_port": 53,
+ "tcp_port": 53,
+ "bind_ip_blacklist": {{ dns_bind_ip_blacklist }},
+ "forward_zones": {{ dns_forward_zones }}
+ }
- path: /etc_master/spartan.json
content: |
{
@@ -468,6 +478,9 @@ package:
MESOS_RESOURCES=[{"name":"ports","type":"RANGES","ranges": {"range": [{"begin": 1, "end": 21},{"begin": 23, "end": 5050},{"begin": 5052, "end": 32000}]}}]
MESOS_DEFAULT_ROLE=slave_public
MESOS_ATTRIBUTES=public_ip:true
+ - path: /etc/mesos-slave-spark
+ content: |
+ MESOS_RESOURCES=[{"name":"ports","type":"RANGES","ranges": {"range": [{"begin": 1025, "end": 2180},{"begin": 2182, "end": 3887},{"begin": 3889, "end": 5049},{"begin": 5052, "end": 8079},{"begin": 8082, "end": 8180},{"begin": 8182, "end": 32000}]}}]
- path: /etc/mesos-executor-environment.json
content: |
{
@@ -495,6 +508,11 @@ package:
{
"role": "agent_public"
}
+ - path: /etc_slave_spark/dcos-log-config.json
+ content: |
+ {
+ "role": "agent_spark"
+ }
- path: /etc/dns_search_config
content: |
{{ dcos_gen_resolvconf_search_str }}
@@ -549,6 +567,13 @@ package:
"endpoint-config": "/opt/mesosphere/etc/dcos-diagnostics-endpoint-config.json",
"agent-port": 61001
}
+ - path: /etc_slave_spark/dcos-diagnostics-config.json
+ content: |
+ {
+ "role": "agent_spark",
+ "endpoint-config": "/opt/mesosphere/etc/dcos-diagnostics-endpoint-config.json",
+ "agent-port": 61001
+ }
- path: /etc/dcos-diagnostics-endpoint-config.json
content: |
{
@@ -696,27 +721,27 @@ package:
{
"Port": 5051,
"Uri": "/__processes__",
- "Role":["agent", "agent_public"]
+ "Role":["agent", "agent_public", "agent_spark"]
},
{
"Port": 5051,
"Uri": "/metrics/snapshot",
- "Role":["agent", "agent_public"]
+ "Role":["agent", "agent_public", "agent_spark"]
},
{
"Port": 5051,
"Uri": "/slave/flags",
- "Role":["agent", "agent_public"]
+ "Role":["agent", "agent_public", "agent_spark"]
},
{
"Port": 5051,
"Uri": "/slave/state",
- "Role":["agent", "agent_public"]
+ "Role":["agent", "agent_public", "agent_spark"]
},
{
"Port": 5051,
"Uri": "/system/stats.json",
- "Role":["agent", "agent_public"]
+ "Role":["agent", "agent_public", "agent_spark"]
},
{
"Port": 8123,
diff --git a/notes.md b/notes.md
new file mode 100644
index 00000000..b0c7f776
--- /dev/null
+++ b/notes.md
@@ -0,0 +1,7 @@
+Search in project:
+
+slave|agent
+
+exclude:
+
+-*.md, -/Users/han/repos/dcos/packages/dcos-integration-test/*, -/Users/han/repos/dcos/packages/adminrouter/extra/src/test-harness/*, -/Users/han/repos/dcos/test_util/*, -/Users/han/repos/dcos/gen/aws/*, -/Users/han/repos/dcos/gen/build_deploy/aws.py, -/Users/han/repos/dcos/dcos_installer/test_async_server.py, -/Users/han/repos/dcos/gen/azure/templates/*, -/Users/han/repos/dcos/pkgpanda/test_setup.py, -/Users/han/repos/dcos/pkgpanda/test_resources/*, -/Users/han/repos/dcos/gen/tests/*
diff --git a/packages/adminrouter/build b/packages/adminrouter/build
index ff2117fe..da31e297 100755
--- a/packages/adminrouter/build
+++ b/packages/adminrouter/build
@@ -29,7 +29,7 @@ tmp="$PKG_PATH/dcos.target.wants_master/dcos-adminrouter.service"
mkdir -vp "$(dirname "$tmp")"
envsubst '$PKG_PATH' < "/pkg/extra/systemd/dcos-adminrouter.service" > "$tmp"
-for at in slave slave_public; do
+for at in slave slave_public slave_spark; do
tmp="$PKG_PATH/dcos.target.wants_${at}/dcos-adminrouter-agent.service"
mkdir -vp "$(dirname "$tmp")"
envsubst '$PKG_PATH' < "/pkg/extra/systemd/dcos-adminrouter-agent.service" > "$tmp"
diff --git a/packages/bootstrap/extra/dcos_internal_utils/cli.py b/packages/bootstrap/extra/dcos_internal_utils/cli.py
index 2871bdfc..1970c70e 100644
--- a/packages/bootstrap/extra/dcos_internal_utils/cli.py
+++ b/packages/bootstrap/extra/dcos_internal_utils/cli.py
@@ -67,6 +67,7 @@ bootstrappers = {
'dcos-mesos-master': noop,
'dcos-mesos-slave': noop,
'dcos-mesos-slave-public': noop,
+ 'dcos-mesos-slave-spark': noop,
'dcos-cosmos': noop,
'dcos-metronome': noop,
'dcos-history': noop,
@@ -126,7 +127,7 @@ def get_zookeeper_address():
if 'master' in roles:
return '127.0.0.1:2181'
- if 'slave' in roles or 'slave_public' in roles:
+ if 'slave' in roles or 'slave_public' in roles or 'slave_spark' in roles:
return get_zookeeper_address_agent()
raise Exception("Can't get zookeeper address. Unknown role: {}".format(roles))
diff --git a/packages/dcos-diagnostics/build b/packages/dcos-diagnostics/build
index 88b03491..7319c3de 100755
--- a/packages/dcos-diagnostics/build
+++ b/packages/dcos-diagnostics/build
@@ -11,18 +11,23 @@ cp -r /pkg/bin/ "$PKG_PATH"
master_service=${PKG_PATH}/dcos.target.wants_master/dcos-diagnostics.service
slave_service=${PKG_PATH}/dcos.target.wants_slave/dcos-diagnostics.service
slave_public_service=${PKG_PATH}/dcos.target.wants_slave_public/dcos-diagnostics.service
+slave_spark_service=${PKG_PATH}/dcos.target.wants_slave_spark/dcos-diagnostics.service
master_socket_service=${PKG_PATH}/dcos.target.wants_master/dcos-diagnostics.socket
slave_socket_service=${PKG_PATH}/dcos.target.wants_slave/dcos-diagnostics.socket
slave_public_socket_service=${PKG_PATH}/dcos.target.wants_slave_public/dcos-diagnostics.socket
+slave_spark_socket_service=${PKG_PATH}/dcos.target.wants_slave_spark/dcos-diagnostics.socket
mkdir -p $(dirname $master_service)
mkdir -p $(dirname $slave_service)
mkdir -p $(dirname $slave_public_service)
+mkdir -p $(dirname $slave_spark_service)
cp /pkg/extra/dcos-diagnostics-master.service "$master_service"
cp /pkg/extra/dcos-diagnostics-agent.service "$slave_service"
cp /pkg/extra/dcos-diagnostics-agent.service "$slave_public_service"
+cp /pkg/extra/dcos-diagnostics-agent.service "$slave_spark_service"
cp /pkg/extra/dcos-diagnostics.socket "$master_socket_service"
cp /pkg/extra/dcos-diagnostics.socket "$slave_socket_service"
cp /pkg/extra/dcos-diagnostics.socket "$slave_public_socket_service"
+cp /pkg/extra/dcos-diagnostics.socket "$slave_spark_socket_service"
diff --git a/packages/dcos-log/build b/packages/dcos-log/build
index c56a2130..0eb0d016 100755
--- a/packages/dcos-log/build
+++ b/packages/dcos-log/build
@@ -11,18 +11,23 @@ cp -r /pkg/bin/ "$PKG_PATH"
master_service=${PKG_PATH}/dcos.target.wants_master/dcos-log-master.service
slave_service=${PKG_PATH}/dcos.target.wants_slave/dcos-log-agent.service
slave_public_service=${PKG_PATH}/dcos.target.wants_slave_public/dcos-log-agent.service
+slave_spark_service=${PKG_PATH}/dcos.target.wants_slave_spark/dcos-log-agent.service
master_socket_service=${PKG_PATH}/dcos.target.wants_master/dcos-log-master.socket
slave_socket_service=${PKG_PATH}/dcos.target.wants_slave/dcos-log-agent.socket
slave_public_socket_service=${PKG_PATH}/dcos.target.wants_slave_public/dcos-log-agent.socket
+slave_spark_socket_service=${PKG_PATH}/dcos.target.wants_slave_spark/dcos-log-agent.socket
mkdir -p $(dirname $master_service)
mkdir -p $(dirname $slave_service)
mkdir -p $(dirname $slave_public_service)
+mkdir -p $(dirname $slave_spark_service)
cp /pkg/extra/dcos-log-master.service "$master_service"
cp /pkg/extra/dcos-log-agent.service "$slave_service"
cp /pkg/extra/dcos-log-agent.service "$slave_public_service"
+cp /pkg/extra/dcos-log-agent.service "$slave_spark_service"
cp /pkg/extra/dcos-log.socket "$master_socket_service"
cp /pkg/extra/dcos-log.socket "$slave_socket_service"
cp /pkg/extra/dcos-log.socket "$slave_public_socket_service"
+cp /pkg/extra/dcos-log.socket "$slave_spark_socket_service"
diff --git a/packages/dcos-metrics/build b/packages/dcos-metrics/build
index 3c3f7450..e44984d7 100644
--- a/packages/dcos-metrics/build
+++ b/packages/dcos-metrics/build
@@ -53,6 +53,10 @@ agent_public_service="$PKG_PATH/dcos.target.wants_slave_public/dcos-metrics-agen
mkdir -p "$(dirname "$agent_public_service")"
cp /pkg/extra/dcos-metrics-agent.service "$agent_public_service"
+agent_public_service="$PKG_PATH/dcos.target.wants_slave_spark/dcos-metrics-agent.service"
+mkdir -p "$(dirname "$agent_public_service")"
+cp /pkg/extra/dcos-metrics-agent.service "$agent_public_service"
+
master_service="$PKG_PATH/dcos.target.wants_master/dcos-metrics-master.service"
mkdir -p "$(dirname "$master_service")"
cp /pkg/extra/dcos-metrics-master.service "$master_service"
@@ -66,6 +70,10 @@ agent_public_socket="$PKG_PATH/dcos.target.wants_slave_public/dcos-metrics-agent
mkdir -p "$(dirname "$agent_public_socket")"
cp /pkg/extra/dcos-metrics-agent.socket "$agent_public_socket"
+agent_public_socket="$PKG_PATH/dcos.target.wants_slave_spark/dcos-metrics-agent.socket"
+mkdir -p "$(dirname "$agent_public_socket")"
+cp /pkg/extra/dcos-metrics-agent.socket "$agent_public_socket"
+
master_socket="$PKG_PATH/dcos.target.wants_master/dcos-metrics-master.socket"
mkdir -p "$(dirname "$master_socket")"
cp /pkg/extra/dcos-metrics-master.socket "$master_socket"
diff --git a/packages/docker-gc/build b/packages/docker-gc/build
index d28517db..0bce5aa1 100644
--- a/packages/docker-gc/build
+++ b/packages/docker-gc/build
@@ -10,6 +10,10 @@ service="$PKG_PATH/dcos.target.wants_slave_public/dcos-docker-gc.service"
mkdir -p $(dirname "$service")
envsubst '$PKG_PATH' < /pkg/extra/dcos-docker-gc.service > "$service"
+service="$PKG_PATH/dcos.target.wants_slave_spark/dcos-docker-gc.service"
+mkdir -p $(dirname "$service")
+envsubst '$PKG_PATH' < /pkg/extra/dcos-docker-gc.service > "$service"
+
timer="$PKG_PATH/dcos.target.wants_slave/dcos-docker-gc.timer"
mkdir -p $(dirname "$timer")
envsubst '$PKG_PATH' < /pkg/extra/dcos-docker-gc.timer > "$timer"
@@ -18,4 +22,6 @@ timer="$PKG_PATH/dcos.target.wants_slave_public/dcos-docker-gc.timer"
mkdir -p $(dirname "$timer")
envsubst '$PKG_PATH' < /pkg/extra/dcos-docker-gc.timer > "$timer"
-
+timer="$PKG_PATH/dcos.target.wants_slave_spark/dcos-docker-gc.timer"
+mkdir -p $(dirname "$timer")
+envsubst '$PKG_PATH' < /pkg/extra/dcos-docker-gc.timer > "$timer"
diff --git a/packages/logrotate/build b/packages/logrotate/build
index 51a6dc29..876396f9 100755
--- a/packages/logrotate/build
+++ b/packages/logrotate/build
@@ -22,11 +22,16 @@ systemd_agent_public="$PKG_PATH"/dcos.target.wants_slave_public/dcos-logrotate-a
mkdir -p "$(dirname "$systemd_agent_public")"
envsubst '$PKG_PATH' < /pkg/extra/dcos-logrotate-agent.service > "$systemd_agent_public"
+systemd_agent_spark="$PKG_PATH"/dcos.target.wants_slave_spark/dcos-logrotate-agent.service
+mkdir -p "$(dirname "$systemd_agent_spark")"
+envsubst '$PKG_PATH' < /pkg/extra/dcos-logrotate-agent.service > "$systemd_agent_spark"
+
logrotate_timer="/pkg/extra/dcos-logrotate.timer"
cp "$logrotate_timer" "$PKG_PATH/dcos.target.wants_master/dcos-logrotate-master.timer"
cp "$logrotate_timer" "$PKG_PATH/dcos.target.wants_slave/dcos-logrotate-agent.timer"
cp "$logrotate_timer" "$PKG_PATH/dcos.target.wants_slave_public/dcos-logrotate-agent.timer"
+cp "$logrotate_timer" "$PKG_PATH/dcos.target.wants_slave_spark/dcos-logrotate-agent.timer"
postrotate_script="$PKG_PATH/bin/delete-oldest-unmanaged-files.py"
diff --git a/packages/mesos/build b/packages/mesos/build
index a0beba3e..000d7ab3 100755
--- a/packages/mesos/build
+++ b/packages/mesos/build
@@ -55,6 +55,10 @@ systemd_slave_public="$PKG_PATH"/dcos.target.wants_slave_public/dcos-mesos-slave
mkdir -p "$(dirname "$systemd_slave_public")"
envsubst '$PKG_PATH' < /pkg/extra/dcos-mesos-slave-public.service > "$systemd_slave_public"
+systemd_slave_spark="$PKG_PATH"/dcos.target.wants_slave_spark/dcos-mesos-slave-spark.service
+mkdir -p "$(dirname "$systemd_slave_spark")"
+envsubst '$PKG_PATH' < /pkg/extra/dcos-mesos-slave-spark.service > "$systemd_slave_spark"
+
# setup additonal volume service which discovers /dcos/volumeN mounts and creates an optional
# EnvironmentFile that contains a MESOS_RESOURCES env variable. This MESOS_RESOURCES adds
# Mount resources to another existing MESOS_RESOURCES variable, which means the EnvironmentFile
diff --git a/packages/mesos/buildinfo.json b/packages/mesos/buildinfo.json
index 907832d5..79d29266 100644
--- a/packages/mesos/buildinfo.json
+++ b/packages/mesos/buildinfo.json
@@ -19,6 +19,10 @@
"dcos-mesos-slave-public": {
"vm.max_map_count": 262144,
"vm.swappiness": 1
+ },
+ "dcos-mesos-slave-spark": {
+ "vm.max_map_count": 262144,
+ "vm.swappiness": 1
}
}
}
diff --git a/packages/mesos/extra/dcos-mesos-slave-spark.service b/packages/mesos/extra/dcos-mesos-slave-spark.service
new file mode 100644
index 00000000..77fd9dcc
--- /dev/null
+++ b/packages/mesos/extra/dcos-mesos-slave-spark.service
@@ -0,0 +1,26 @@
+[Unit]
+Description=Mesos Agent Spark: distributed systems kernel spark agent
+
+[Service]
+Restart=always
+StartLimitInterval=0
+RestartSec=5
+KillMode=control-group
+Delegate=true
+LimitNOFILE=infinity
+TasksMax=infinity
+SyslogIdentifier=mesos-agent
+EnvironmentFile=/opt/mesosphere/environment
+EnvironmentFile=/opt/mesosphere/etc/mesos-slave-common
+EnvironmentFile=/opt/mesosphere/etc/mesos-slave-spark
+EnvironmentFile=/opt/mesosphere/etc/proxy.env
+EnvironmentFile=-/opt/mesosphere/etc/mesos-slave-common-extras
+EnvironmentFile=-/var/lib/dcos/mesos-slave-common
+EnvironmentFile=-/var/lib/dcos/mesos-resources
+EnvironmentFile=-/run/dcos/etc/mesos-slave-spark
+ExecStartPre=/bin/ping -c1 ready.spartan
+ExecStartPre=/bin/ping -c1 leader.mesos
+ExecStartPre=/opt/mesosphere/bin/bootstrap dcos-mesos-slave-spark
+ExecStartPre=/opt/mesosphere/bin/make_disk_resources.py /var/lib/dcos/mesos-resources
+ExecStartPre=/bin/bash -c 'for i in /proc/sys/net/ipv4/conf/*/rp_filter; do echo 2 > $i; echo -n "$i: "; cat $i; done'
+ExecStart=$PKG_PATH/bin/mesos-agent
diff --git a/packages/pkgpanda-role/build b/packages/pkgpanda-role/build
index 1b084d8f..f36c3a4b 100644
--- a/packages/pkgpanda-role/build
+++ b/packages/pkgpanda-role/build
@@ -4,3 +4,4 @@
mkdir -p $PKG_PATH/etc_master/roles/master
mkdir -p $PKG_PATH/etc_slave/roles/slave
mkdir -p $PKG_PATH/etc_slave_public/roles/slave_public
+mkdir -p $PKG_PATH/etc_slave_spark/roles/slave_spark
diff --git a/packages/rexray/build b/packages/rexray/build
index 1966748e..314b3753 100644
--- a/packages/rexray/build
+++ b/packages/rexray/build
@@ -9,6 +9,10 @@ systemd_slave_public=$PKG_PATH/dcos.target.wants_slave_public/dcos-rexray.servic
mkdir -p $(dirname $systemd_slave_public)
cp "$systemd_slave" "$systemd_slave_public"
+systemd_slave_spark=$PKG_PATH/dcos.target.wants_slave_spark/dcos-rexray.service
+mkdir -p $(dirname $systemd_slave_spark)
+cp "$systemd_slave" "$systemd_slave_spark"
+
srcdir=$GOPATH/src/github.com/codedellemc/rexray
mkdir -p $(dirname $srcdir)
ln -s /pkg/src/rexray $srcdir
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment