gistfile1.txt

{"level":"info","ts":1568969687.854493,"logger":"cmd","msg":"Go Version: go1.12.9"}
{"level":"info","ts":1568969687.8545113,"logger":"cmd","msg":"Go OS/Arch: linux/amd64"}
{"level":"info","ts":1568969687.854514,"logger":"cmd","msg":"Version of operator-sdk: v0.8.1"}
{"level":"info","ts":1568969687.8546445,"logger":"leader","msg":"Trying to become the leader."}
{"level":"info","ts":1568969687.896242,"logger":"leader","msg":"No pre-existing lock was found."}
{"level":"info","ts":1568969687.8996854,"logger":"leader","msg":"Became the leader."}
{"level":"info","ts":1568969687.9197435,"logger":"cmd","msg":"Registering Components."}
{"level":"info","ts":1568969687.9200983,"logger":"utils","msg":"detecting platform version..."}
{"level":"info","ts":1568969687.9223013,"logger":"utils","msg":"PlatformInfo [Name: Kubernetes, K8SVersion: 1.14, OS: linux/amd64]"}
{"level":"info","ts":1568969687.9226077,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"wildflyserver-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1568969687.9228241,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"wildflyserver-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1568969687.923078,"logger":"kubebuilder.controller","msg":"Starting EventSource","controller":"wildflyserver-controller","source":"kind source: /, Kind="}
{"level":"info","ts":1568969687.9232504,"logger":"utils","msg":"detecting platform version..."}
{"level":"info","ts":1568969687.9248903,"logger":"utils","msg":"PlatformInfo [Name: Kubernetes, K8SVersion: 1.14, OS: linux/amd64]"}
{"level":"info","ts":1568969687.9645166,"logger":"metrics","msg":"Metrics Service object created","Service.Name":"wildfly-operator","Service.Namespace":"default"}
{"level":"info","ts":1568969687.9649255,"logger":"cmd","msg":"Starting the Cmd."}
{"level":"info","ts":1568969688.0654593,"logger":"kubebuilder.controller","msg":"Starting Controller","controller":"wildflyserver-controller"}
{"level":"info","ts":1568969688.165744,"logger":"kubebuilder.controller","msg":"Starting workers","controller":"wildflyserver-controller","worker count":1}
{"level":"info","ts":1568969688.7577076,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969688.757805,"logger":"controller_wildflyserver","msg":"Creating a new StatefulSet.","Request.Namespace":"default","Request.Name":"quickstart","StatefulSet.Namespace":"default","StatefulSet.Name":"quickstart"}
{"level":"info","ts":1568969688.8046794,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969688.8101578,"logger":"controller_wildflyserver","msg":"Adding Finalizer for the WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart","Finalizer Name":"finalizer.wildfly.org"}
{"level":"error","ts":1568969688.9330847,"logger":"controller_wildflyserver","msg":"Failed to Update StatefulSet.","StatefulSet.Namespace":"default","StatefulSet.Name":"quickstart","error":"Operation cannot be fulfilled on statefulsets.apps \"quickstart\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).checkStatefulSet\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:473\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).Reconcile\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:252\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:215\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"error","ts":1568969688.9332583,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Operation cannot be fulfilled on statefulsets.apps \"quickstart\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969689.9346135,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969689.9411464,"logger":"controller_wildflyserver","msg":"Updating StatefulSet to be up to date with the WildFlyServer Spec","StatefulSet.Namespace":"default","StatefulSet.Name":"quickstart"}
{"level":"info","ts":1568969689.9418623,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969689.9421935,"logger":"controller_wildflyserver","msg":"Creating a new LoadBalancer.","Request.Namespace":"default","Request.Name":"quickstart","LoadBalancer.Namespace":"default","LoadBalancer.Name":"quickstart-loadbalancer"}
{"level":"info","ts":1568969689.95723,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969689.95764,"logger":"controller_wildflyserver","msg":"Creating a new Headless Service.","Request.Namespace":"default","Request.Name":"quickstart","HeadlessService.Namespace":"default","HeadlessServie.Name":"quickstart-headless"}
{"level":"info","ts":1568969689.9636595,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969689.9650993,"logger":"controller_wildflyserver","msg":"Updating the pod status with new status","Request.Namespace":"default","Request.Name":"quickstart","Pod statuses":[{"name":"quickstart-0","podIP":"","state":"ACTIVE"},{"name":"quickstart-1","podIP":"","state":"ACTIVE"}]}
{"level":"info","ts":1568969689.9740279,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969690.049153,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969690.6899667,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969691.9714193,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969694.532269,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969699.6528916,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969709.8934903,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969730.3740113,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969743.1176796,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969743.117955,"logger":"controller_wildflyserver","msg":"Updating the pod status with new status","Request.Namespace":"default","Request.Name":"quickstart","Pod statuses":[{"name":"quickstart-0","podIP":"172.17.0.6","state":"ACTIVE"},{"name":"quickstart-1","podIP":"","state":"ACTIVE"}]}
{"level":"info","ts":1568969743.1573157,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969745.1071322,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969745.108255,"logger":"controller_wildflyserver","msg":"Updating the pod status with new status","Request.Namespace":"default","Request.Name":"quickstart","Pod statuses":[{"name":"quickstart-0","podIP":"172.17.0.6","state":"ACTIVE"},{"name":"quickstart-1","podIP":"172.17.0.7","state":"ACTIVE"}]}
{"level":"info","ts":1568969745.1184168,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969771.334985,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.8778365,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.878114,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.8975797,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.897742,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.899812,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969801.8998535,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969801.923645,"logger":"controller_wildflyserver","msg":"Enabling recovery listener for processing scaledown at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969801.9264023,"logger":"controller_wildflyserver","msg":"Enabling recovery listener for processing scaledown at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969811.8491266,"logger":"controller_wildflyserver","msg":"Reloading WildFly for recovery listener being activated at pod quickstart-1","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969811.8493066,"logger":"controller_wildflyserver","msg":"Reloading WildFly for recovery listener being activated at pod quickstart-0","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969822.100277,"logger":"controller_wildflyserver","msg":"Query to find the transaction recovery port to force scan at pod quickstart-1","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969822.3918269,"logger":"controller_wildflyserver","msg":"Query to find the transaction recovery port to force scan at pod quickstart-0","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"error","ts":1568969866.630489,"logger":"controller_wildflyserver","msg":"Error on reading transaction recovery port with management command","Request.Namespace":"default","Request.Name":"quickstart","Pod name":"quickstart-1","error":"Error on management operation to read transaction recovery socket binding with command /subsystem=transactions:read-attribute(name=socket-binding), error: Cannot execute JBoss CLI command /opt/jboss/wildfly/bin/jboss-cli.sh --output-json -c --command='/subsystem=transactions:read-attribute(name=socket-binding)' at pod quickstart-1. Cause: cannot execute '/opt/jboss/wildfly/bin/jboss-cli.sh --output-json -c --command='/subsystem=transactions:read-attribute(name=socket-binding)'': command terminated with exit code 1","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).checkRecovery\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:1006\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).processTransactionRecoveryScaleDown.func1\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:899"}
{"level":"info","ts":1568969866.6878176,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969873.2104297,"logger":"controller_wildflyserver","msg":"Error on reading transaction recovery port with management command","Request.Namespace":"default","Request.Name":"quickstart","Pod name":"quickstart-0","error":"Error on management operation to read recovery port with command /socket-binding-group=standard-sockets/socket-binding=txn-recovery-environment:read-attribute(name=port), error: Cannot execute JBoss CLI command /opt/jboss/wildfly/bin/jboss-cli.sh --output-json -c --command='/socket-binding-group=standard-sockets/socket-binding=txn-recovery-environment:read-attribute(name=port)' at pod quickstart-0. Cause: cannot execute '/opt/jboss/wildfly/bin/jboss-cli.sh --output-json -c --command='/socket-binding-group=standard-sockets/socket-binding=txn-recovery-environment:read-attribute(name=port)'': command terminated with exit code 137","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).checkRecovery\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:1006\ngithub.com/wildfly/wildfly-operator/pkg/controller/wildflyserver.(*ReconcileWildFlyServer).processTransactionRecoveryScaleDown.func1\n\t/Users/jmesnil/Developer/wildfly-operator/pkg/controller/wildflyserver/wildflyserver_controller.go:899"}
{"level":"info","ts":1568969873.21882,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969873.2233562,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969874.227326,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969874.233197,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969874.2335694,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969874.2350016,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969874.2417471,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969874.2491992,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969874.2512996,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969875.2520194,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969875.2522588,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969875.2525048,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969875.2529943,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969875.2655532,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969875.2659225,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969875.2662818,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969876.2690437,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969876.2693944,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969876.2696114,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969876.2701879,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969876.2861826,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969876.2876515,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969876.2932887,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969877.2942612,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969877.2946396,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969877.2960186,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969877.2963502,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969877.3162963,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568969877.330437,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969877.3306978,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969878.3311634,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969878.3312461,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969878.3314805,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969878.3319817,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969878.3443568,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969878.347262,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969878.348304,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969879.3512163,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969879.3512979,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969879.351411,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969879.35173,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969879.3614955,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969879.3668122,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969879.3692775,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969880.370215,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969880.370288,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969880.370372,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969880.3715932,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969880.3835564,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568969880.3836787,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969880.3903697,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969881.671024,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969881.6712964,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969881.6715457,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969881.672484,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969881.6881673,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568969881.698173,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969881.6985002,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969884.259127,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969884.2593303,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969884.2595484,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969884.2612743,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969884.2702122,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969884.2807765,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969884.281866,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969889.402268,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969889.4046636,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969889.4048676,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969889.4055896,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969889.4135585,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969889.41915,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969889.4213023,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969899.6622293,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969899.6627686,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969899.6633792,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969899.664461,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969899.6741796,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568969899.6774352,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969899.6777468,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969920.158085,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969920.158143,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969920.158241,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969920.1587963,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969920.166504,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568969920.1674857,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568969920.1676219,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568969961.1283891,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969961.129187,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568969961.1296291,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969961.130253,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568969961.1407099,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568969961.1407096,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568969961.1413205,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568970043.0661502,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568970043.0662477,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568970043.0670927,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568970043.0679607,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568970043.0811014,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"info","ts":1568970043.0815203,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"error","ts":1568970043.0820854,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}
{"level":"info","ts":1568970206.9226878,"logger":"controller_wildflyserver","msg":"Reconciling WildFlyServer","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568970206.9228246,"logger":"controller_wildflyserver","msg":"WildflyServer is marked for deletion. Waiting for finalizers to clean the workspace","Request.Namespace":"default","Request.Name":"quickstart"}
{"level":"info","ts":1568970206.9232879,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-0","IP Address":"172.17.0.6","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568970206.9241493,"logger":"controller_wildflyserver","msg":"Transaction recovery scaledown processing","Request.Namespace":"default","Request.Name":"quickstart","Pod Name":"quickstart-1","IP Address":"172.17.0.7","Pod State":"SCALING_DOWN_RECOVERY_INVESTIGATION","Pod Phase":"Running"}
{"level":"info","ts":1568970206.9415684,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-1","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.7","Recovery port":4712}
{"level":"info","ts":1568970206.9469821,"logger":"controller_wildflyserver","msg":"Executing recovery scan at quickstart-0","Request.Namespace":"default","Request.Name":"quickstart","Pod IP":"172.17.0.6","Recovery port":4712}
{"level":"error","ts":1568970206.947268,"logger":"kubebuilder.controller","msg":"Reconciler error","controller":"wildflyserver-controller","request":"default/quickstart","error":"Finalizer processing: failed transaction recovery for WildflyServer default:quickstart name Error: Found 2 errors:\n [[Failed to run transaction recovery scan for scaling down pod quickstart-1. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.7:4712, error: dial tcp 172.17.0.7:4712: connect: connection refused]], [[Failed to run transaction recovery scan for scaling down pod quickstart-0. Please, verify the pod log file. Error: Cannot process TCP connection to 172.17.0.6:4712, error: dial tcp 172.17.0.6:4712: connect: connection refused]],","stacktrace":"github.com/go-logr/zapr.(*zapLogger).Error\n\t/Users/jmesnil/go/pkg/mod/github.com/go-logr/[email protected]/zapr.go:128\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:217\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1\n\t/Users/jmesnil/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:158\nk8s.io/apimachinery/pkg/util/wait.JitterUntil.func1\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:134\nk8s.io/apimachinery/pkg/util/wait.Until\n\t/Users/jmesnil/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:88"}