Skip to content

Instantly share code, notes, and snippets.

@mstump

mstump/ruby-test.clj

Created May 15, 2012 03:51
Show Gist options
  • Save mstump/2698992 to your computer and use it in GitHub Desktop.
Save mstump/2698992 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
ruby-test.clj
(ns courier.test.storm.ruby
(:use [backtype.storm clojure]
[backtype.storm bootstrap testing]
clojure.test
midje.sweet)
(:require [courier.db :as db]
[courier.util :as util]
[courier.lib.ruby.core :as ruby]
[courier.storm.ruby :as sruby]
[courier.models.package :as pkg]
[courier.models.package-version :as pv]
[cheshire.core :as json]))
(bootstrap)
(defn package []
(util/convert-hash-keys-to-keywords (json/parse-string (slurp "dev-resources/ruby/_.json"))))
(defn versions []
(map util/convert-hash-keys-to-keywords
(json/parse-string (slurp "dev-resources/ruby/_.versions.json"))))
(fact
"check that hashes are built - always passes"
(db/with-test-keyspace (concat pkg/column-family-defs pv/column-family-defs)
(with-simulated-time-local-cluster [cluster]
(let [keyspace_name (db/keyspace-name)
results (complete-topology cluster
(sruby/mk-redis-topology keyspace_name "" -1)
:mock-sources {"input" [["_"]]}
:storm-conf {TOPOLOGY-DEBUG true})]
(let [tuples (read-tuples results "build-hashes")]
(count tuples) => 1))))
(against-background
(courier.lib.ruby.core/get-gem "_") => (package)
(courier.lib.ruby.core/get-versions "_") => (versions)))
(fact
"end-to-end cassandra bolt test - fails if previous test is commented out"
(db/with-test-keyspace (concat pkg/column-family-defs pv/column-family-defs)
(with-simulated-time-local-cluster [cluster]
(let [keyspace_name (db/keyspace-name)
results (complete-topology cluster
(sruby/mk-redis-topology keyspace_name "" -1)
:mock-sources {"input" [["_"]]}
:storm-conf {TOPOLOGY-DEBUG true})]
(let [p (ruby/build-package-hash (ruby/get-gem "_"))]
(pkg/fetch-one (db/keyspace) (:id p)) => (contains (dissoc p :tags))
(:tags (pkg/fetch-one (db/keyspace) (:id p))) => (contains (:tags p) :in-any-order)
(doseq [pv (map (partial ruby/build-package-version-hash p) (ruby/get-versions "_"))]
(dissoc (pv/fetch-one (db/keyspace) (:id pv)) :summary) => (dissoc pv :summary))))))
(against-background
(courier.lib.ruby.core/get-gem "_") => (package)
(courier.lib.ruby.core/get-versions "_") => (versions)))
Raw
ruby.clj
(ns courier.storm.ruby
(:use [backtype.storm clojure config])
(:require [cheshire.core :as json]
[courier.lib.git.core :as git]
[courier.lib.ruby.core :as ruby]
[courier.storm.core :as core]
[courier.storm.package :as pkg]
[courier.storm.package-version :as pv]
[courier.storm.sqs :as sqs]
[courier.util :as util])
(:import [yieldbot.storm.spout RedisPubSubSpout]
[java.lang Exception])
(:gen-class))
(defbolt build-package-and-package-version-hashes-bolt ["package" "package_versions"]
[tuple collector]
(try
(let [name (:message tuple)
package (ruby/build-package-hash (ruby/get-gem name))
versions (vec (git/populate-summaries (map (partial ruby/build-package-version-hash package) (ruby/get-versions name)) (:git package)))]
(emit-bolt! collector [package versions] :anchor tuple))
(ack! collector tuple)
(catch Exception e
(core/log-exception e tuple)
(fail! collector tuple))))
(defn- base-topology
[keyspace_name]
{"build-hashes" (bolt-spec {"input" :shuffle}
build-package-and-package-version-hashes-bolt
:p 4)
"write-package-to-db" (bolt-spec {"build-hashes" :shuffle}
(pkg/put-bolt keyspace_name))
"write-package-versions-to-db" (bolt-spec {"build-hashes" :shuffle}
(pv/put-many-bolt keyspace_name))})
(defn mk-redis-topology
[keyspace_name host port]
(topology
{"input" (spout-spec (RedisPubSubSpout. host port ruby/queue))}
(base-topology keyspace_name)))
(defn mk-sqs-topology
[keyspace_name awsid secret queue]
(topology
{"input" (spout-spec (sqs/sqs-spout awsid secret queue true))}
(base-topology keyspace_name)))
Raw
stdout.txt
2658 [Thread-9] INFO backtype.storm.daemon.task - Opening spout input:9
2658 [Thread-9] INFO backtype.storm.daemon.task - Opened spout input:9
2660 [Thread-33] INFO backtype.storm.daemon.task - Activating spout input:9
2663 [Thread-9] INFO backtype.storm.daemon.task - Finished loading task input:9
2663 [Thread-33] INFO backtype.storm.daemon.task - Emitting: input default ["_"]
2665 [Thread-9] INFO backtype.storm.daemon.worker - Launching receive-thread for eee1b650-2a86-4adf-afa9-29c321851116:1
2673 [Thread-33] INFO backtype.storm.daemon.task - Emitting: input __ack_init [-8398043255983614990 1426071082436781173 9]
2673 [Thread-30] INFO backtype.storm.daemon.task - Emitting: a3142dd9-afef-465d-b45e-7f3272795251 __ack_ack [-8398043255983614990 -7202334585926724812]
2680 [Thread-9] INFO backtype.storm.daemon.worker - Worker has topology config {"storm.id" "topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0", "dev.zookeeper.path" "/tmp/dev-storm-zookeeper", "topology.fall.back.on.java.serialization" true, "zmq.linger.millis" 0, "topology.skip.missing.kryo.registrations" true, "ui.childopts" "-Xmx768m", "storm.zookeeper.session.timeout" 20000, "nimbus.reassign" true, "nimbus.monitor.freq.secs" 10, "java.library.path" "/usr/local/lib:/opt/local/lib:/usr/lib", "storm.local.dir" "/var/folders/kg/c_cdzn2j2bv0zqrh6yz4w_780000gn/T//f6342dc7-3477-44de-8652-df657ddeb2a9", "supervisor.worker.start.timeout.secs" 240, "nimbus.cleanup.inbox.freq.secs" 600, "nimbus.inbox.jar.expiration.secs" 3600, "nimbus.host" "localhost", "storm.zookeeper.port" 2000, "transactional.zookeeper.port" nil, "transactional.zookeeper.servers" nil, "storm.zookeeper.root" "/storm", "supervisor.enable" true, "storm.zookeeper.servers" ["localhost"], "transactional.zookeeper.root" "/transactional", "topology.name" "topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7", "topology.worker.childopts" nil, "worker.childopts" "-Xmx768m", "supervisor.heartbeat.frequency.secs" 5, "drpc.port" 3772, "supervisor.monitor.frequency.secs" 3, "task.heartbeat.frequency.secs" 3, "topology.max.spout.pending" nil, "storm.zookeeper.retry.interval" 1000, "supervisor.slots.ports" (1 2 3), "topology.debug" true, "nimbus.task.launch.secs" 120, "nimbus.supervisor.timeout.secs" 60, "topology.kryo.register" nil, "topology.message.timeout.secs" 30, "task.refresh.poll.secs" 10, "topology.workers" 1, "supervisor.childopts" "-Xmx1024m", "nimbus.thrift.port" 6627, "topology.stats.sample.rate" 0.05, "worker.heartbeat.frequency.secs" 1, "nimbus.task.timeout.secs" 30, "drpc.invocations.port" 3773, "zmq.threads" 1, "storm.zookeeper.retry.times" 5, "topology.state.synchronization.timeout.secs" 60, "supervisor.worker.timeout.secs" 30, "nimbus.file.copy.expiration.secs" 600, "drpc.request.timeout.secs" 600, "storm.local.mode.zmq" false, "ui.port" 8080, "nimbus.childopts" "-Xmx1024m", "topology.ackers" 1, "storm.cluster.mode" "local", "topology.optimize" true, "topology.max.task.parallelism" nil}
2680 [Thread-9] INFO backtype.storm.daemon.worker - Worker ea550b4e-fdf1-419c-85fd-10037e0fa895 for storm topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0 on eee1b650-2a86-4adf-afa9-29c321851116:1 has finished loading
3998 [Thread-33] INFO backtype.storm.daemon.task - Failing message ff9fd283-89ec-49d0-b9eb-fa47c5cdad0b: {:stream "default", :values ["_"]}
4081 [main] INFO backtype.storm.daemon.nimbus - Delaying event :remove for 30 secs for topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0
4086 [main] INFO backtype.storm.daemon.nimbus - Updated topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0 with status {:type :killed, :kill-time-secs 30}
4130 [Thread-33] INFO backtype.storm.daemon.task - Deactivating spout input:9
4693 [Thread-5] INFO backtype.storm.daemon.nimbus - Killing topology: topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0
4724 [Thread-11] INFO backtype.storm.daemon.supervisor - Removing code for storm id topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0
4724 [Thread-8] INFO backtype.storm.daemon.supervisor - Removing code for storm id topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0
4752 [Thread-9] INFO backtype.storm.daemon.supervisor - Shutting down and clearing state for id ea550b4e-fdf1-419c-85fd-10037e0fa895. State: :disallowed, Heartbeat: #backtype.storm.daemon.common.WorkerHeartbeat{:time-secs 80, :storm-id "topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0", :task-ids #{1 2 3 4 5 6 7 8 9}, :port 1}
4753 [Thread-9] INFO backtype.storm.daemon.supervisor - Shutting down eee1b650-2a86-4adf-afa9-29c321851116:ea550b4e-fdf1-419c-85fd-10037e0fa895
4753 [Thread-9] INFO backtype.storm.process-simulator - Killing process 5f80b29b-3d81-40c2-9973-fa0de40d167d
4753 [Thread-9] INFO backtype.storm.daemon.worker - Shutting down worker topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0 eee1b650-2a86-4adf-afa9-29c321851116 1
4753 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:1
4759 [Thread-14] INFO backtype.storm.util - Async loop interrupted!
4763 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:1
4763 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:2
4763 [Thread-16] INFO backtype.storm.util - Async loop interrupted!
4766 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:2
4766 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:3
4767 [Thread-19] INFO backtype.storm.util - Async loop interrupted!
4769 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:3
4769 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:4
4984 [Thread-22] INFO backtype.storm.daemon.task - Emitting: build-hashes default [{:name "_", :urls {"git" "https://github.com/mame/_.git", "RubyGems Project" "http://rubygems.org/gems/_", "Gem Location" "http://rubygems.org/gems/_-1.2.gem", "Homepage" "http://github.com/mame/_/tree/master"}, :meta_source_description "RubyGems", :git "https://github.com/mame/_.git", :source {}, :summary "_ allows you to write Ruby script by using only _.", :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :meta_source_type "gem", :tags [], :aliases []} [{:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.2", :downloads "1498", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :summary "* * first commit.\n", :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:8114b9dabe64741f5700c676da07d4182530a754", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []} {:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.1", :downloads "1089", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :summary "", :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:4491f8b5ecc09d9de308ea37e6a23ad2dd66c256", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []} {:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.0", :downloads "1069", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :summary "", :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:e8dc057d3346e56aed7cf252185dbe1fa6454411", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []}]]
4985 [Thread-22] INFO backtype.storm.daemon.task - Emitting: build-hashes __ack_ack [-8398043255983614990 4106391416840464094]
4985 [Thread-30] INFO backtype.storm.daemon.task - Emitting: a3142dd9-afef-465d-b45e-7f3272795251 __ack_ack [-8398043255983614990 -6407591154751858535]
4986 [Thread-21] INFO backtype.storm.util - Async loop interrupted!
4988 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:4
4988 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:5
4989 [Thread-23] INFO backtype.storm.util - Async loop interrupted!
4991 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:5
4991 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:6
4992 [Thread-25] INFO backtype.storm.util - Async loop interrupted!
4995 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:6
4995 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:7
4995 [Thread-27] INFO backtype.storm.util - Async loop interrupted!
4998 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:7
4998 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:8
4999 [Thread-29] INFO backtype.storm.util - Async loop interrupted!
5001 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:8
5001 [Thread-9] INFO backtype.storm.daemon.task - Shutting down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:9
5001 [Thread-33] INFO backtype.storm.util - Async loop interrupted!
5002 [Thread-31] INFO backtype.storm.util - Async loop interrupted!
5003 [Thread-9] INFO backtype.storm.daemon.task - Shut down task topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0:9
5004 [Thread-9] INFO backtype.storm.messaging.loader - Shutting down receiving-thread: [topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0, 1]
5004 [Thread-9] INFO backtype.storm.messaging.loader - Waiting for receiving-thread:[topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0, 1] to die
5004 [Thread-36] INFO backtype.storm.messaging.loader - Receiving-thread:[topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0, 1] received shutdown notice
5004 [Thread-9] INFO backtype.storm.messaging.loader - Shutdown receiving-thread: [topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0, 1]
5004 [Thread-9] INFO backtype.storm.daemon.worker - Terminating zmq context
5004 [Thread-9] INFO backtype.storm.daemon.worker - Waiting for threads to die
5004 [Thread-35] INFO backtype.storm.util - Async loop interrupted!
5006 [Thread-9] INFO backtype.storm.daemon.worker - Disconnecting from storm cluster state context
5008 [Thread-9] INFO backtype.storm.daemon.worker - Shut down worker topologytest-827a24bc-44e1-4298-b847-d1d08e15ffc7-1-0 eee1b650-2a86-4adf-afa9-29c321851116 1
5010 [Thread-9] INFO backtype.storm.daemon.supervisor - Shut down eee1b650-2a86-4adf-afa9-29c321851116:ea550b4e-fdf1-419c-85fd-10037e0fa895
FAIL at (ruby.clj:47)
Actual result did not agree with the checking function.
Actual result: nil
Checking function: (contains (dissoc p :tags))
The checker said this about the reason:
You can't compare nil () to {:name "_", :urls {"git" "https://github.com/mame/_.git", "RubyGems Project" "http://rubygems.org/gems/_", "Gem Location" "http://rubygems.org/gems/_-1.2.gem", "Homepage" "http://github.com/mame/_/tree/master"}, :meta_source_description "RubyGems", :git "https://github.com/mame/_.git", :source {}, :summary "_ allows you to write Ruby script by using only _.", :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :meta_source_type "gem", :aliases []} (class clojure.lang.PersistentHashMap).
FAIL at (ruby.clj:48)
Actual result did not agree with the checking function.
Actual result: nil
Checking function: (contains (:tags p) :in-any-order)
The checker said this about the reason:
You can't compare nil () to [] (class clojure.lang.PersistentVector).
FAIL at (ruby.clj:51)
Expected: {:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.2", :downloads "1498", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:8114b9dabe64741f5700c676da07d4182530a754", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []}
Actual: nil
FAIL at (ruby.clj:51)
Expected: {:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.1", :downloads "1089", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:4491f8b5ecc09d9de308ea37e6a23ad2dd66c256", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []}
Actual: nil
FAIL at (ruby.clj:51)
Expected: {:cves_fixed [], :advisories {}, :advisories_vuln [], :package_id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787", :package_name "_", :urls {}, :version "1.0", :downloads "1069", :meta_source_description "RubyGems", :copyright_bodies [], :cves_vuln [], :advisories_fixed [], :license_names [], :released_at "2009-11-10T18:00:00Z", :source {}, :id "gem:53a0acfad59379b3e050338bf9f23cfc172ee787:1257876000000:e8dc057d3346e56aed7cf252185dbe1fa6454411", :meta_source_type "gem", :license_bodies [], :platform "ruby", :tags [], :aliases []}
Actual: nil
5093 [main] INFO backtype.storm.daemon.nimbus - Shutting down master
5094 [main] INFO backtype.storm.daemon.nimbus - Shut down master
5107 [main] INFO backtype.storm.daemon.supervisor - Shutting down supervisor eee1b650-2a86-4adf-afa9-29c321851116
5107 [Thread-8] INFO backtype.storm.event - Event manager interrupted
5108 [Thread-9] INFO backtype.storm.event - Event manager interrupted
5110 [main] INFO backtype.storm.daemon.supervisor - Shutting down supervisor 0d5798cb-8fea-4f1e-910e-325d458ca0e1
5111 [Thread-11] INFO backtype.storm.event - Event manager interrupted
5111 [Thread-12] INFO backtype.storm.event - Event manager interrupted
5112 [main] INFO backtype.storm.testing - Shutting down in process zookeeper
5117 [main] INFO backtype.storm.testing - Done shutting down in process zookeeper
5117 [main] INFO backtype.storm.testing - Deleting temporary path /var/folders/kg/c_cdzn2j2bv0zqrh6yz4w_780000gn/T//f30917d1-cded-4170-b10e-f9dffb9d0e1d
5118 [main] INFO backtype.storm.testing - Deleting temporary path /var/folders/kg/c_cdzn2j2bv0zqrh6yz4w_780000gn/T//66560aed-f076-4ec8-a6c3-83ed1c009ae8
5119 [main] INFO backtype.storm.testing - Deleting temporary path /var/folders/kg/c_cdzn2j2bv0zqrh6yz4w_780000gn/T//f6342dc7-3477-44de-8652-df657ddeb2a9
5122 [main] INFO backtype.storm.testing - Deleting temporary path /var/folders/kg/c_cdzn2j2bv0zqrh6yz4w_780000gn/T//53e6b325-bd19-4b7d-9878-76545b1f2c78
FAILURE: 5 facts were not confirmed.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment