[debug] [2016-03-15T18:57:24.958Z] [on-taskgraph] [Job.Catalog.GenerateSku] [29a5c6] SKU Noop OBM settings for VirtualBox nodes matched 100%
-> /node_modules/on-tasks/lib/jobs/generate-sku.js:90
sku: 56e85a7e64960c2e4e29a5c5
[info] [2016-03-15T18:57:24.959Z] [on-taskgraph] [Job.Catalog.GenerateSku] [29a5c6] Assigning SKU Noop OBM settings for VirtualBox nodes
-> /node_modules/on-tasks/lib/jobs/generate-sku.js:110
sku: 56e85a7e64960c2e4e29a5c5
[info] [2016-03-15T18:57:24.965Z] [on-taskgraph] [TaskGraph.TaskRunner] [Server] Running task
-> /lib/task-runner.js:340
taskRunnerId: 821385c1-e261-4451-bf92-49022d52b16d
taskId: 241f0d7d-a1e2-41c8-acf6-f9702ef85eb2
taskName: Task.Catalog.GenerateEnclosure
[info] [2016-03-15T18:57:24.966Z] [on-taskgraph] [Task.Task] [Server] Running task job.
-> /node_modules/on-tasks/lib/task.js:283
taskId: 241f0d7d-a1e2-41c8-acf6-f9702ef85eb2
name: Generate Enclosure
job: Job.Catalog.GenerateEnclosure
[debug] [2016-03-15T18:57:24.966Z] [on-taskgraph] [Job.Catalog.GenerateEnclosure] [29a5c6] Running job.
-> /node_modules/on-tasks/lib/jobs/base-job.js:112
module: Job.Catalog.GenerateEnclosure
name: GenerateEnclosureJob
options:
nodeId: 56e85aca64960c2e4e29a5c6
taskId: 241f0d7d-a1e2-41c8-acf6-f9702ef85eb2
target: none
[debug] [2016-03-15T18:57:24.988Z] [on-taskgraph] [Job.Catalog.GenerateEnclosure] [29a5c6] No matched enclosure, create a new one
-> /node_modules/on-tasks/lib/jobs/generate-enclosure.js:94
enclosure: 56e85b144202794c4efa9418
Tasks work until we hit this one.
[error] [2016-03-15T18:57:25.072Z] [on-taskgraph] [Task.Task] [Server] Task failed
-> /node_modules/on-tasks/lib/task.js:261
error: AssertionError: undefined (uuid) is required
taskId: 38c66ef4-ad15-428e-bd9d-77d5bd2bd16b
graphId: cfd1109b-8c8e-4bea-9cb6-c04fef508034
injectableName: Task.Graph.Run.SkuSpecific
[info] [2016-03-15T18:57:25.076Z] [on-taskgraph] [TaskGraph.TaskRunner] [Server] Task finished
-> /lib/task-runner.js:296
instanceId: 38c66ef4-ad15-428e-bd9d-77d5bd2bd16b
state: failed
taskRunnerId: 821385c1-e261-4451-bf92-49022d52b16d
[debug] [2016-03-15T18:57:25.079Z] [on-taskgraph] [TaskGraph.TaskScheduler] [Server] Listener received task finished event, triggering evaluation
-> /lib/task-scheduler.js:525
data:
taskId: 38c66ef4-ad15-428e-bd9d-77d5bd2bd16b
graphId: cfd1109b-8c8e-4bea-9cb6-c04fef508034
state: failed
error:
"""
AssertionError: undefined (uuid) is required
at AssertService.(anonymous function) [as uuid] (/var/renasar/on-taskgraph/node_modules/on-core/lib/common/assert.js:30:28)
at TaskGraphRunnerProtocol.runTaskGraph (/var/renasar/on-taskgraph/node_modules/on-core/lib/protocol/task-graph-runner.js:232:16)
at /var/renasar/on-taskgraph/node_modules/on-tasks/lib/jobs/run-sku-graph.js:87:42
at tryCatcher (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/util.js:26:23)
at Promise._settlePromiseFromHandler (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:503:31)
at Promise._settlePromiseAt (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:577:18)
at Promise._settlePromises (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:693:14)
at Async._drainQueue (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:123:16)
at Async._drainQueues (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:133:10)
at Async.drainQueues (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:15:14)
at process._tickDomainCallback (node.js:459:13)
"""
context:
graphId: cfd1109b-8c8e-4bea-9cb6-c04fef508034
terminalOnStates:
- succeeded
- timeout
- cancelled
- failed
schedulerId: 9be614d9-1b58-43c3-909e-13c06864d703
and here
debug] [2016-03-15T18:57:25.088Z] [on-taskgraph] [TaskGraph.TaskScheduler] [Server] cancel/failing pending tasks
-> /lib/task-scheduler.js:378
data:
- 10c9758e-6bd7-4087-ae71-41eaf799075e
- 241f0d7d-a1e2-41c8-acf6-f9702ef85eb2
- 38c66ef4-ad15-428e-bd9d-77d5bd2bd16b
- 40bdf3f0-6d55-4f41-955f-4d072804112c
- ad5e9114-11bf-48c6-9438-26f6ef91e432
- c56fdc35-7528-4269-95e9-3ce54bad33fe
[debug] [2016-03-15T18:57:25.108Z] [on-taskgraph] [TaskGraph.TaskScheduler] [Server] Graph finished
-> /lib/task-scheduler.js:471
createdAt: Tue Mar 15 2016 18:56:10 GMT+0000 (UTC)
name: SKU Discovery
context:
graphId: cfd1109b-8c8e-4bea-9cb6-c04fef508034
updatedAt: Tue Mar 15 2016 18:57:25 GMT+0000 (UTC)
_status: failed
schedulerId: 9be614d9-1b58-43c3-909e-13c06864d703
serviceGraph: null
injectableName: Graph.SKU.Discovery
tasks:
38c66ef4-ad15-428e-bd9d-77d5bd2bd16b:
error:
"""
AssertionError: undefined (uuid) is required
at AssertService.(anonymous function) [as uuid] (/var/renasar/on-taskgraph/node_modules/on-core/lib/common/assert.js:30:28)
at TaskGraphRunnerProtocol.runTaskGraph (/var/renasar/on-taskgraph/node_modules/on-core/lib/protocol/task-graph-runner.js:232:16)
at /var/renasar/on-taskgraph/node_modules/on-tasks/lib/jobs/run-sku-graph.js:87:42
at tryCatcher (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/util.js:26:23)
at Promise._settlePromiseFromHandler (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:503:31)
at Promise._settlePromiseAt (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:577:18)
at Promise._settlePromises (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/promise.js:693:14)
at Async._drainQueue (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:123:16)
at Async._drainQueues (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:133:10)
at Async.drainQueues (/var/renasar/on-taskgraph/node_modules/on-core/node_modules/waterline/node_modules/bluebird/js/main/async.js:15:14)
at process._tickDomainCallback (node.js:459:13)
"""
friendlyName: Run SKU-specific graph
ignoreFailure: false
implementsTask: Task.Base.Graph.RunSku
injectableName: Task.Graph.Run.SkuSpecific
instanceId: 38c66ef4-ad15-428e-bd9d-77d5bd2bd16b
name: Task.Graph.Run.SkuSpecific
options:
nodeId: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Graph.RunSku
state: failed
terminalOnStates:
- succeeded
- timeout
- cancelled
- failed
waitingOn:
10c9758e-6bd7-4087-ae71-41eaf799075e: succeeded
10c9758e-6bd7-4087-ae71-41eaf799075e:
friendlyName: Generate SKU
ignoreFailure: false
implementsTask: Task.Base.Catalog.GenerateSku
injectableName: Task.Catalog.GenerateSku
instanceId: 10c9758e-6bd7-4087-ae71-41eaf799075e
name: Task.Catalog.GenerateSku
options:
nodeId: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Catalog.GenerateSku
state: succeeded
terminalOnStates:
- timeout
- cancelled
- failed
waitingOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
ad5e9114-11bf-48c6-9438-26f6ef91e432:
friendlyName: Create Default Pollers
ignoreFailure: false
implementsTask: Task.Base.Pollers.CreateDefault
injectableName: Task.Inline.Pollers.CreateDefault
instanceId: ad5e9114-11bf-48c6-9438-26f6ef91e432
name: Task.Inline.Pollers.CreateDefault
options:
nodeId: 56e85aca64960c2e4e29a5c6
pollers:
-
type: ipmi
pollInterval: 60000
config:
command: sdr
-
type: ipmi
pollInterval: 60000
config:
command: selInformation
-
type: ipmi
pollInterval: 60000
config:
command: sel
-
type: ipmi
pollInterval: 15000
config:
command: chassis
-
type: ipmi
pollInterval: 30000
config:
command: driveHealth
properties:
runJob: Job.Pollers.CreateDefault
state: succeeded
terminalOnStates:
- succeeded
- timeout
- cancelled
- failed
waitingOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
241f0d7d-a1e2-41c8-acf6-f9702ef85eb2:
friendlyName: Generate Enclosure
ignoreFailure: true
implementsTask: Task.Base.Catalog.GenerateEnclosure
injectableName: Task.Catalog.GenerateEnclosure
instanceId: 241f0d7d-a1e2-41c8-acf6-f9702ef85eb2
name: Task.Catalog.GenerateEnclosure
options:
nodeId: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Catalog.GenerateEnclosure
state: succeeded
terminalOnStates:
- succeeded
- timeout
- cancelled
- failed
waitingOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
40bdf3f0-6d55-4f41-955f-4d072804112c:
friendlyName: Run Discovery Graph
ignoreFailure: false
implementsTask: Task.Base.Graph.Run
injectableName: Task.Graph.Run.Discovery
instanceId: 40bdf3f0-6d55-4f41-955f-4d072804112c
name: Task.Graph.Run.Discovery
options:
graphName: Graph.Discovery
graphOptions:
target: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Graph.Run
state: succeeded
terminalOnStates:
- timeout
- cancelled
- failed
waitingOn:
c56fdc35-7528-4269-95e9-3ce54bad33fe:
friendlyName: Generate Tag
ignoreFailure: false
implementsTask: Task.Base.Catalog.GenerateTag
injectableName: Task.Catalog.GenerateTag
instanceId: c56fdc35-7528-4269-95e9-3ce54bad33fe
name: Task.Catalog.GenerateTag
options:
nodeId: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Catalog.GenerateTag
state: succeeded
terminalOnStates:
- timeout
- cancelled
- failed
waitingOn:
c56fdc35-7528-4269-95e9-3ce54bad33fe:
friendlyName: Generate Tag
ignoreFailure: false
implementsTask: Task.Base.Catalog.GenerateTag
injectableName: Task.Catalog.GenerateTag
instanceId: c56fdc35-7528-4269-95e9-3ce54bad33fe
name: Task.Catalog.GenerateTag
options:
nodeId: 56e85aca64960c2e4e29a5c6
properties:
runJob: Job.Catalog.GenerateTag
state: succeeded
terminalOnStates:
- succeeded
- timeout
- cancelled
- failed
waitingOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
domain: default
node: null
definition:
friendlyName: SKU Discovery
injectableName: Graph.SKU.Discovery
options:
defaults:
graphOptions:
target: 56e85aca64960c2e4e29a5c6
nodeId: 56e85aca64960c2e4e29a5c6
tasks:
-
label: discovery-graph
taskDefinition:
friendlyName: Run Discovery Graph
injectableName: Task.Graph.Run.Discovery
implementsTask: Task.Base.Graph.Run
options:
graphName: Graph.Discovery
graphOptions:
properties:
-
label: generate-sku
waitOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
taskName: Task.Catalog.GenerateSku
-
label: generate-tag
waitOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
taskName: Task.Catalog.GenerateTag
-
label: generate-enclosure
waitOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
taskName: Task.Catalog.GenerateEnclosure
ignoreFailure: true
-
label: create-default-pollers
taskDefinition:
friendlyName: Create Default Pollers
injectableName: Task.Inline.Pollers.CreateDefault
implementsTask: Task.Base.Pollers.CreateDefault
properties:
options:
nodeId: null
pollers:
-
type: ipmi
pollInterval: 60000
config:
command: sdr
-
type: ipmi
pollInterval: 60000
config:
command: selInformation
-
type: ipmi
pollInterval: 60000
config:
command: sel
-
type: ipmi
pollInterval: 15000
config:
command: chassis
-
type: ipmi
pollInterval: 30000
config:
command: driveHealth
waitOn:
40bdf3f0-6d55-4f41-955f-4d072804112c: succeeded
-
label: run-sku-graph
taskDefinition:
friendlyName: Run SKU-specific graph
injectableName: Task.Graph.Run.SkuSpecific
implementsTask: Task.Base.Graph.RunSku
options:
properties:
waitOn:
10c9758e-6bd7-4087-ae71-41eaf799075e: succeeded
instanceId: cfd1109b-8c8e-4bea-9cb6-c04fef508034
logContext:
graphInstance: cfd1109b-8c8e-4bea-9cb6-c04fef508034
graphName: SKU Discovery
_id:
_bsontype: ObjectID
id: VèZÊpý¥<88><93>Ö¥Å
[debug] [2016-03-15T18:58:03.122Z] [on-taskgraph] [Services.ErrorPublisher] [Server] Event Loop Blocked
-> /node_modules/on-core/lib/services/error-publisher.js:85
host: rackhd01
name: on-taskgraph
severity: debug
ms: 13
Then we are at Event Loop Blocked for the next long time.