Created
June 20, 2011 07:59
-
-
Save cloudartisan/1035272 to your computer and use it in GitHub Desktop.
Pallet + EC2 == too hard :-(
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (ns staging.core | |
| (:require | |
| [pallet.core :as core] | |
| [pallet.compute :as compute] | |
| [pallet.phase :as phase] | |
| [pallet.crate.automated-admin-user :as automated-admin-user] | |
| [staging.nodes :as staging-nodes] | |
| [staging.crates.mongodb :as mongodb])) | |
| (def ec2-service | |
| (compute/compute-service-from-config-file :ec2)) | |
| ; FIXME | |
| ; | |
| ; Does not work. Need a way to specify existing SSH keypair before launch, | |
| ; preferably at the cluster-spec level. | |
| ; | |
| ; http://code.google.com/p/jclouds/issues/detail?id=602 | |
| ; | |
| ;(def ubuntu-admin-user | |
| ; (core/admin-user "ubuntu" | |
| ; :private-key-path "~/.ssh/assistly/staging.pem")) | |
| ; FIXME | |
| ; | |
| ; I want to ensure the "staging" security group is created before any | |
| ; instance is launched and have all instances "inherit" that security group. | |
| ; | |
| ; If all instances are destroyed I want to destroy the "staging" security | |
| ; group. | |
| ; | |
| ; I want to automatically create security groups at the server-spec level | |
| ; and have all instances "inherit" those security groups at the group-spec | |
| ; level. | |
| ; | |
| ; Eg, if a group-spec extends "with-redis" and "with-mongodb" I want to make | |
| ; sure any instance launched from that group-spec is in both the "mongodb" | |
| ; and "redis" security groups. If all instances in the "redis" security | |
| ; group are destroyed I want to destroy the "redis" security group. | |
| ; | |
| ; I also want to be able to specify security groups as members of other | |
| ; security groups. Eg, so that only instances in the "web" security group | |
| ; are able to talk to instances in the "database" security group. | |
| ; | |
| ; I have no idea where to call these... | |
| ; | |
| ; (.createSecurityGroupInRegion ec2-security-group-client "us-east-1" "sg" "desc") | |
| ; (.deleteSecurityGroupInRegion ec2-security-group-client "us-east-1" "sg") | |
| ; | |
| (def ec2-security-group-client | |
| (.. | |
| (clojure.contrib.reflect/get-field | |
| org.jclouds.ec2.compute.EC2ComputeService "ec2Client" (.compute ec2-service)) | |
| getSecurityGroupServices)) | |
| ; With an automatically-created admin user account for the current user | |
| ; | |
| ; FIXME | |
| ; | |
| ; Get rid of this as soon as this bug is fixed: | |
| ; http://code.google.com/p/jclouds/issues/detail?id=602 | |
| ; | |
| (def with-automated-admin-user | |
| (core/server-spec | |
| :phases {:bootstrap (phase/phase-fn (automated-admin-user/automated-admin-user))})) | |
| ; With MongoDB from the 10gen repository | |
| ; | |
| ; We use the 10gen-provided MongoDB to get the most recent packaged version. | |
| ; | |
| (def with-mongodb-10gen | |
| (core/server-spec | |
| :phases {:configure (phase/phase-fn (mongodb/install)) | |
| :start-mongodb (phase/phase-fn (mongodb/start)) | |
| :stop-mongodb (phase/phase-fn (mongodb/stop)) | |
| :restart-mongodb (phase/phase-fn (mongodb/restart))})) | |
| ; Recommended EC2 configuration: | |
| ; http://www.mongodb.org/display/DOCS/Amazon+EC2 | |
| ; "use a 64 bit instance as this is required for all MongoDB databases of | |
| ; significant size" | |
| ; | |
| (def mongodb-us-east-1a | |
| (core/group-spec | |
| "mongodb" | |
| :extends [with-automated-admin-user | |
| with-mongodb-10gen] | |
| :node-spec staging-nodes/ubuntu-1104-m1-large-us-east-1a)) | |
| ; Recommended EC2 configuration: | |
| ; http://www.mongodb.org/display/DOCS/Amazon+EC2 | |
| ; "use a 64 bit instance as this is required for all MongoDB databases of | |
| ; significant size" | |
| ; | |
| (def mongodb-us-east-1b | |
| (core/group-spec | |
| "mongodb" | |
| :extends [with-automated-admin-user | |
| with-mongodb-10gen] | |
| :node-spec staging-nodes/ubuntu-1104-m1-large-us-east-1b)) | |
| (def mongodb-cluster | |
| (core/cluster-spec | |
| "mongodb" | |
| :groups [mongodb-us-east-1a | |
| mongodb-us-east-1b])) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (ns staging.crate.mongodb | |
| (:require | |
| [pallet.action.exec-script :as exec-script] | |
| [pallet.action.directory :as directory] | |
| [pallet.action.file :as file] | |
| [pallet.action.package :as package] | |
| [pallet.action.service :as service] | |
| [pallet.script :as script])) | |
| (def default-configuration | |
| {:dbpath "/var/lib/mongodb" | |
| :logpath "/var/log/mongodb/mongodb.log" | |
| :logappend true | |
| :port 27017 | |
| :cpu false | |
| :noauth true | |
| :auth false | |
| :verbose false | |
| :objcheck false | |
| :quota false | |
| :diaglog 0 | |
| :nocursors false | |
| :nohints false | |
| :nohttpinterface true | |
| :noscripting false | |
| :notablescan false | |
| :noprealloc false | |
| :master false | |
| :slave false | |
| :journal true}) | |
| (defn install | |
| "Install MongoDB server (from the 10gen repository)" | |
| [session] | |
| (-> session | |
| (package/package-source | |
| "10gen" | |
| :aptitude {:source-type "deb" | |
| :url "http://downloads-distro.mongodb.org/repo/ubuntu-upstart" | |
| :release "dist" | |
| :scopes ["10gen"] | |
| :key-id "7F0CEB10"} ) | |
| (package/package-manager :update) | |
| (package/package "mongodb-10gen" :action :install) | |
| (service/service "mongodb" :action :restart))) | |
| (defn start | |
| "Start MongoDB service" | |
| [session] | |
| (-> session | |
| (service/service "mongodb" :action :start))) | |
| (defn stop | |
| "Stop MongoDB service" | |
| [session] | |
| (-> session | |
| (service/service "mongodb" :action :stop))) | |
| (defn restart | |
| "Restart MongoDB service" | |
| [session] | |
| (-> session | |
| (service/service "mongodb" :action :restart))) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ lein pallet converge staging.core/mongodb-cluster 1 | |
| WARN implementation - pallet.compute.vmfest provider failed to load: java.io.FileNotFoundException: Could not locate vmfest/virtualbox/virtualbox__init.class or vmfest/virtualbox/virtualbox.clj on classpath: (vmfest.clj:1) | |
| INFO core - retrieving nodes | |
| ERROR main - Exception | |
| java.lang.RuntimeException: java.lang.RuntimeException: java.lang.NullPointerException | |
| at clojure.lang.LazySeq.sval(LazySeq.java:47) | |
| at clojure.lang.LazySeq.seq(LazySeq.java:56) | |
| at clojure.lang.RT.seq(RT.java:450) | |
| at clojure.core$seq.invoke(core.clj:122) | |
| at clojure.core$r.invoke(core.clj:793) | |
| at pallet.core$nodes_in_set.invoke(core.clj:841) | |
| at pallet.core$session_with_groups.invoke(core.clj:915) | |
| at pallet.core$converge_STAR_.invoke(core.clj:980) | |
| at pallet.core$converge.doInvoke(core.clj:1148) | |
| at clojure.lang.RestFn.applyTo(RestFn.java:139) | |
| at clojure.core$apply.invoke(core.clj:540) | |
| at pallet.task.converge$converge.doInvoke(converge.clj:40) | |
| at clojure.lang.RestFn.invoke(RestFn.java:439) | |
| at clojure.lang.Var.invoke(Var.java:373) | |
| at clojure.lang.AFn.applyToHelper(AFn.java:167) | |
| at clojure.lang.Var.applyTo(Var.java:482) | |
| at clojure.core$apply.invoke(core.clj:542) | |
| at pallet.main_invoker$invoke.invoke(main_invoker.clj:102) | |
| at clojure.lang.Var.invoke(Var.java:373) | |
| at pallet.main$pallet_task.invoke(main.clj:138) | |
| at pallet.main$_main.doInvoke(main.clj:152) | |
| at clojure.lang.RestFn.invoke(RestFn.java:482) | |
| at clojure.lang.Var.invoke(Var.java:381) | |
| at user$eval368.invoke(NO_SOURCE_FILE:1) | |
| at clojure.lang.Compiler.eval(Compiler.java:5424) | |
| at clojure.lang.Compiler.eval(Compiler.java:5415) | |
| at clojure.lang.Compiler.eval(Compiler.java:5415) | |
| at clojure.lang.Compiler.eval(Compiler.java:5391) | |
| at clojure.core$eval.invoke(core.clj:2382) | |
| at clojure.main$eval_opt.invoke(main.clj:235) | |
| at clojure.main$initialize.invoke(main.clj:254) | |
| at clojure.main$null_opt.invoke(main.clj:279) | |
| at clojure.main$main.doInvoke(main.clj:354) | |
| at clojure.lang.RestFn.invoke(RestFn.java:421) | |
| at clojure.lang.Var.invoke(Var.java:369) | |
| at clojure.lang.AFn.applyToHelper(AFn.java:163) | |
| at clojure.lang.Var.applyTo(Var.java:482) | |
| at clojure.main.main(main.java:37) | |
| Caused by: java.lang.RuntimeException: java.lang.NullPointerException | |
| at clojure.lang.LazySeq.sval(LazySeq.java:47) | |
| at clojure.lang.LazySeq.seq(LazySeq.java:56) | |
| at clojure.lang.RT.seq(RT.java:450) | |
| at clojure.core$seq.invoke(core.clj:122) | |
| at clojure.core$zipmap.invoke(core.clj:2301) | |
| at pallet.core$node_map_with_prefix.invoke(core.clj:766) | |
| at pallet.core$nodes_in_set.invoke(core.clj:835) | |
| at pallet.core$nodes_in_set$fn__6162.invoke(core.clj:841) | |
| at clojure.core$map$fn__3699.invoke(core.clj:2096) | |
| at clojure.lang.LazySeq.sval(LazySeq.java:42) | |
| ... 37 more | |
| Caused by: java.lang.NullPointerException | |
| at clojure.core$name.invoke(core.clj:1245) | |
| at pallet.core$group_with_prefix$fn__6129.invoke(core.clj:761) | |
| at clojure.lang.AFn.applyToHelper(AFn.java:161) | |
| at clojure.lang.AFn.applyTo(AFn.java:151) | |
| at clojure.core$apply.invoke(core.clj:542) | |
| at clojure.core$update_in.doInvoke(core.clj:4969) | |
| at clojure.lang.RestFn.invoke(RestFn.java:445) | |
| at pallet.core$group_with_prefix.invoke(core.clj:760) | |
| at pallet.core$node_map_with_prefix$fn__6133.invoke(core.clj:765) | |
| at clojure.core$map$fn__3699.invoke(core.clj:2096) | |
| at clojure.lang.LazySeq.sval(LazySeq.java:42) | |
| ... 46 more | |
| java.lang.RuntimeException: java.lang.NullPointerException |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| $ lein pallet converge staging.core/mongodb-us-east-1a 1 | |
| WARN implementation - pallet.compute.vmfest provider failed to load: java.io.FileNotFoundException: Could not locate vmfest/virtualbox/virtualbox__init.class or vmfest/virtualbox/virtualbox.clj on classpath: (vmfest.clj:1) | |
| INFO core - retrieving nodes | |
| INFO core - converging nodes | |
| INFO core - adjust-node-count :mongodb 1 | |
| INFO core - Starting 1 nodes for :mongodb os-family :ubuntu | |
| INFO jclouds - building node template for :mongodb | |
| INFO jclouds - authorizing /Users/david/.ssh/id_rsa.pub | |
| INFO jclouds - options {:image-id "us-east-1/ami-1aad5273", :os-64-bit true, :os-description-matches "11.04", :os-family :ubuntu, :hardware-id "m1.large", :location-id "us-east-1a", :inbound-ports [22]} | |
| WARN compute - to avoid creating temporary keys in aws-ec2, use templateOption overrideLoginCredentialWith(id_rsa) | |
| WARN compute - to avoid creating temporary keys in aws-ec2, use templateOption overrideLoginCredentialWith(id_rsa) | |
| ERROR futures - Adjust node count exception: requested location us-east-1a, which is not in the configured locations: {ap-northeast-1=https://ec2.ap-northeast-1.amazonaws.com, eu-west-1=https://ec2.eu-west-1.amazonaws.com, us-east-1=https://ec2.us-east-1.amazonaws.com, us-west-1=https://ec2.us-west-1.amazonaws.com, ap-southeast-1=https://ec2.ap-southeast-1.amazonaws.com} | |
| java.lang.IllegalArgumentException: requested location us-east-1a, which is not in the configured locations: {ap-northeast-1=https://ec2.ap-northeast-1.amazonaws.com, eu-west-1=https://ec2.eu-west-1.amazonaws.com, us-east-1=https://ec2.us-east-1.amazonaws.com, us-west-1=https://ec2.us-west-1.amazonaws.com, ap-southeast-1=https://ec2.ap-southeast-1.amazonaws.com} | |
| at com.google.common.base.Preconditions.checkArgument(Preconditions.java:115) | |
| at org.jclouds.location.functions.RegionToEndpointOrProviderIfNull.apply(RegionToEndpointOrProviderIfNull.java:63) | |
| at org.jclouds.location.functions.RegionToEndpointOrProviderIfNull.apply(RegionToEndpointOrProviderIfNull.java:43) | |
| at org.jclouds.rest.internal.RestAnnotationProcessor.getEndpointInParametersOrNull(RestAnnotationProcessor.java:670) | |
| at org.jclouds.rest.internal.RestAnnotationProcessor.getEndpointFor(RestAnnotationProcessor.java:702) | |
| at org.jclouds.rest.internal.RestAnnotationProcessor.createRequest(RestAnnotationProcessor.java:372) | |
| at org.jclouds.rest.internal.AsyncRestClientProxy.createListenableFuture(AsyncRestClientProxy.java:145) | |
| at org.jclouds.rest.internal.AsyncRestClientProxy.invoke(AsyncRestClientProxy.java:126) | |
| at $Proxy71.runInstancesInRegion(Unknown Source) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | |
| at java.lang.reflect.Method.invoke(Method.java:597) | |
| at org.jclouds.concurrent.internal.SyncProxy.invoke(SyncProxy.java:130) | |
| at $Proxy72.runInstancesInRegion(Unknown Source) | |
| at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.createNodesInRegionAndZone(EC2CreateNodesInGroupThenAddToSet.java:166) | |
| at org.jclouds.aws.ec2.compute.strategy.AWSEC2CreateNodesInGroupThenAddToSet.createNodesInRegionAndZone(AWSEC2CreateNodesInGroupThenAddToSet.java:99) | |
| at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.createKeyPairAndSecurityGroupsAsNeededThenRunInstances(EC2CreateNodesInGroupThenAddToSet.java:152) | |
| at org.jclouds.ec2.compute.strategy.EC2CreateNodesInGroupThenAddToSet.execute(EC2CreateNodesInGroupThenAddToSet.java:116) | |
| at org.jclouds.compute.internal.BaseComputeService.createNodesInGroup(BaseComputeService.java:229) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) | |
| at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) | |
| at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) | |
| at java.lang.reflect.Method.invoke(Method.java:597) | |
| at clojure.lang.Reflector.invokeMatchingMethod(Reflector.java:90) | |
| at clojure.lang.Reflector.invokeInstanceMethod(Reflector.java:28) | |
| at org.jclouds.compute$create_nodes.invoke(compute.clj:213) | |
| at clojure.lang.AFn.applyToHelper(AFn.java:172) | |
| at clojure.lang.AFn.applyTo(AFn.java:151) | |
| at clojure.core$apply.invoke(core.clj:540) | |
| at org.jclouds.compute$run_nodes.doInvoke(compute.clj:215) | |
| at clojure.lang.RestFn.invoke(RestFn.java:457) | |
| at pallet.compute.jclouds.JcloudsService.run_nodes(jclouds.clj:404) | |
| at pallet.core$create_nodes.invoke(core.clj:408) | |
| at pallet.core$adjust_node_count.invoke(core.clj:450) | |
| at pallet.core$parallel_adjust_node_counts$p_a_n_c_future__5973$fn__5974.invoke(core.clj:473) | |
| at clojure.lang.AFn.call(AFn.java:18) | |
| at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:138) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908) | |
| at java.lang.Thread.run(Thread.java:680) | |
| INFO core - lift-nodes phases [:settings :configure], groups [:mongodb] | |
| INFO core - apply-phase :pallet.phase/pre-settings for :mongodb with 0 nodes | |
| INFO core - apply-phase :settings for :mongodb with 0 nodes | |
| INFO core - apply-phase :pallet.phase/post-settings for :mongodb with 0 nodes | |
| INFO core - apply-phase :pallet.phase/pre-configure for :mongodb with 0 nodes | |
| INFO core - apply-phase :configure for :mongodb with 0 nodes | |
| INFO core - apply-phase :pallet.phase/post-configure for :mongodb with 0 nodes |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| (ns staging.nodes | |
| (:require | |
| [pallet.core :as core])) | |
| ; Defines an EC2 m1.large instance in us-east-1a using the Ubuntu | |
| ; 11.04 AMI | |
| ; | |
| (def ubuntu-1104-m1-large-us-east-1a | |
| (core/node-spec | |
| :image {:os-family :ubuntu | |
| :os-description-matches "11.04" | |
| :os-64-bit true | |
| :image-id "us-east-1/ami-1aad5273"} | |
| :hardware {:hardware-id "m1.large"} | |
| :network {:inbound-ports [22] | |
| ; FIXME | |
| ; Does not work. | |
| ; "requested location us-east-1a, which is not in the configured locations" | |
| :location-id "us-east-1b" | |
| ; FIXME | |
| ; Need to make sure these security groups are pre-created | |
| ;:security-groups ["staging"] | |
| ; FIXME | |
| ; Looks like this is broken... | |
| ; http://code.google.com/p/jclouds/issues/detail?id=602 | |
| ;:key-pair "us-east-1a_staging" | |
| })) | |
| ; Defines an EC2 m1.large instance in us-east-1b using the Ubuntu | |
| ; 11.04 AMI | |
| ; | |
| (def ubuntu-1104-m1-large-us-east-1b | |
| (core/node-spec | |
| :image {:os-family :ubuntu | |
| :os-description-matches "11.04" | |
| :os-64-bit true | |
| :image-id "us-east-1/ami-1aad5273"} | |
| :hardware {:hardware-id "m1.large"} | |
| :network {:inbound-ports [22] | |
| ; FIXME | |
| ; Does not work. | |
| ; "requested location us-east-1a, which is not in the configured locations" | |
| :location-id "us-east-1b" | |
| ; FIXME | |
| ; Need to make sure these security groups are pre-created | |
| ;:security-groups ["staging"] | |
| ; FIXME | |
| ; Looks like this is broken... | |
| ; http://code.google.com/p/jclouds/issues/detail?id=602 | |
| ;:key-pair "us-east-1a_staging" | |
| })) |
I spoke too soon - reproduced using the clojure api
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Availability zone issue is logged here:
http://code.google.com/p/jclouds/issues/detail?id=608
Adrian has been unable to reproduce the issue, so needs help to get this resolved for you