heavy string allocation in iptables code on 3.7 node

gistfile1.txt

### Not that busy node

Showing nodes accounting for 51472970, 64.09% of 80315852 total
Dropped 1872 nodes (cum <= 401579)
Showing top 30 nodes out of 277
      flat  flat%   sum%        cum   cum%
         0     0%     0%   80185072 99.84%  runtime.goexit /usr/lib/golang/src/runtime/asm_amd64.s
  39878766 49.65% 49.65%   39878766 49.65%  runtime.rawstringtmp /usr/lib/golang/src/runtime/string.go
         0     0% 49.65%   35614612 44.34%  runtime.slicebytetostring /usr/lib/golang/src/runtime/string.go
         0     0% 49.65%   22676400 28.23%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).syncProxyRules /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go
         0     0% 49.65%   22583009 28.12%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).(github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables.syncProxyRules)-fm /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go
         0     0% 49.65%   22583009 28.12%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).SyncLoop /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go
         0     0% 49.65%   22583009 28.12%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/async.(*BoundedFrequencyRunner).Loop /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/async/bounded_frequency_runner.go
         0     0% 49.65%   22583009 28.12%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/async.(*BoundedFrequencyRunner).tryRun /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/async/bounded_frequency_runner.go
         0     0% 49.65%   17408847 21.68%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
         0     0% 49.65%   17370617 21.63%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).handleStream /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
     45877 0.057% 49.71%   17370617 21.63%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).processUnaryRPC /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
         0     0% 49.71%   16253472 20.24%  encoding/json.(*decodeState).unmarshal /usr/lib/golang/src/encoding/json/decode.go
         0     0% 49.71%   16253472 20.24%  encoding/json.(*decodeState).value /usr/lib/golang/src/encoding/json/decode.go
         0     0% 49.71%   16181708 20.15%  encoding/json.(*decodeState).object /usr/lib/golang/src/encoding/json/decode.go
         0     0% 49.71%   16144031 20.10%  encoding/json.(*Decoder).Decode /usr/lib/golang/src/encoding/json/stream.go
         0     0% 49.71%   15932392 19.84%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/iptables.GetChainLines /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/iptables/save_restore.go
         0     0% 49.71%   15618163 19.45%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/iptables.ReadLine /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/util/iptables/save_restore.go
         0     0% 49.71%   15057558 18.75%  encoding/json.(*decodeState).array /usr/lib/golang/src/encoding/json/decode.go
         0     0% 49.71%   13494267 16.80%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 49.71%   13494267 16.80%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 49.71%   13491550 16.80%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.Until /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 49.71%   12242514 15.24%  github.com/openshift/origin/pkg/cmd/util/docker.(*KubeDocker).ListContainers <autogenerated>
         0     0% 49.71%   12242514 15.24%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.(*instrumentedInterface).ListContainers <autogenerated>
         0     0% 49.71%   12242514 15.24%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.(*kubeDockerClient).ListContainers /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker/kube_docker_client.go
         0     0% 49.71%   12242514 15.24%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.instrumentedInterface.ListContainers /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker/instrumented_client.go
      4682 0.0058% 49.72%   12226130 15.22%  github.com/openshift/origin/vendor/github.com/docker/engine-api/client.(*Client).ContainerList /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/github.com/docker/engine-api/client/container_list.go
  11543645 14.37% 64.09%   11543645 14.37%  runtime.convT2E /usr/lib/golang/src/runtime/iface.go
         0     0% 64.09%    9864707 12.28%  github.com/openshift/origin/pkg/network/node.(*podManager).processCNIRequests /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/pkg/network/node/pod.go
         0     0% 64.09%    9864707 12.28%  github.com/openshift/origin/pkg/network/node.(*podManager).processRequest /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/pkg/network/node/pod.go
         0     0% 64.09%    9255655 11.52%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport.(*hostportSyncer).SyncHostports /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_syncer.go
         
### Busier node

(pprof) top30
Showing nodes accounting for 1650055675, 61.02% of 2704251106 total
Dropped 2655 nodes (cum <= 13521255)
Showing top 30 nodes out of 266
      flat  flat%   sum%        cum   cum%
         0     0%     0% 2699002519 99.81%  runtime.goexit /usr/lib/golang/src/runtime/asm_amd64.s
         0     0%     0%  917232485 33.92%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
    776629 0.029% 0.029%  916424180 33.89%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).processUnaryRPC /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
         0     0% 0.029%  916407796 33.89%  github.com/openshift/origin/vendor/google.golang.org/grpc.(*Server).handleStream /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/google.golang.org/grpc/server.go
 913903831 33.80% 33.82%  913903831 33.80%  runtime.rawstringtmp /usr/lib/golang/src/runtime/string.go
         0     0% 33.82%  864369896 31.96%  encoding/json.(*decodeState).unmarshal /usr/lib/golang/src/encoding/json/decode.go
         0     0% 33.82%  864369896 31.96%  encoding/json.(*decodeState).value /usr/lib/golang/src/encoding/json/decode.go
         0     0% 33.82%  861238942 31.85%  encoding/json.(*decodeState).object /usr/lib/golang/src/encoding/json/decode.go
         0     0% 33.82%  859455315 31.78%  encoding/json.(*Decoder).Decode /usr/lib/golang/src/encoding/json/stream.go
         0     0% 33.82%  827439125 30.60%  encoding/json.(*decodeState).array /usr/lib/golang/src/encoding/json/decode.go
         0     0% 33.82%  757336583 28.01%  runtime.slicebytetostring /usr/lib/golang/src/runtime/string.go
         0     0% 33.82%  658116641 24.34%  github.com/openshift/origin/pkg/cmd/util/docker.(*KubeDocker).ListContainers <autogenerated>
         0     0% 33.82%  658116641 24.34%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.(*instrumentedInterface).ListContainers <autogenerated>
         0     0% 33.82%  658116641 24.34%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.(*kubeDockerClient).ListContainers /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker/kube_docker_client.go
         0     0% 33.82%  658116641 24.34%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker.instrumentedInterface.ListContainers /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker/instrumented_client.go
    406133 0.015% 33.84%  657466713 24.31%  github.com/openshift/origin/vendor/github.com/docker/engine-api/client.(*Client).ContainerList /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/github.com/docker/engine-api/client/container_list.go
         0     0% 33.84%  542817262 20.07%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 33.84%  542817262 20.07%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 33.84%  541935779 20.04%  github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait.Until /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go
         0     0% 33.84%  450344941 16.65%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime._RuntimeService_ListPodSandbox_Handler /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime/api.pb.go
     32769 0.0012% 33.84%  450257557 16.65%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/remote.(*dockerService).ListPodSandbox /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/remote/docker_service.go
   2491860 0.092% 33.93%  450224788 16.65%  github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim.(*dockerService).ListPodSandbox /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/pkg/kubelet/dockershim/docker_sandbox.go
 370072447 13.68% 47.62%  370072447 13.68%  runtime.convT2E /usr/lib/golang/src/runtime/iface.go
 361016148 13.35% 60.97%  361016148 13.35%  reflect.unsafe_New /usr/lib/golang/src/runtime/malloc.go
         0     0% 60.97%  340737048 12.60%  github.com/openshift/origin/vendor/github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func2 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/github.com/prometheus/client_golang/prometheus/registry.go
         0     0% 60.97%  337107375 12.47%  github.com/openshift/origin/vendor/github.com/google/cadvisor/metrics.(*PrometheusCollector).Collect /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/github.com/google/cadvisor/metrics/prometheus.go
   1355858  0.05% 61.02%  337026201 12.46%  github.com/openshift/origin/vendor/github.com/google/cadvisor/metrics.(*PrometheusCollector).collectContainersInfo /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/github.com/google/cadvisor/metrics/prometheus.go
         0     0% 61.02%  279877639 10.35%  reflect.Value.Convert /usr/lib/golang/src/reflect/value.go
         0     0% 61.02%  279877639 10.35%  reflect.cvtBytesString /usr/lib/golang/src/reflect/value.go
         0     0% 61.02%  278219275 10.29%  github.com/openshift/origin/vendor/k8s.io/kubernetes/cmd/kubelet/app.startKubelet.func1 /builddir/build/BUILD/atomic-openshift-git-0.ee33a8f/_output/local/go/src/github.com/openshift/origin/vendor/k8s.io/kubernetes/cmd/kubelet/app/server.go