kubeflow-tensorflow-gpu-serving.yaml

apiVersion: v1
kind: Service
metadata:
  labels:
    app: mnist
  name: mnist-service
  namespace: kubeflow
spec:
  ports:
  - name: grpc-tf-serving
    port: 9000
    targetPort: 9000
  - name: http-tf-serving
    port: 8500
    targetPort: 8500
  selector:
    app: mnist
  # changed the type from ClusterIP to LoadBalancer
  type: LoadBalancer
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: mnist
  name: mnist-v1
  namespace: kubeflow
spec:
  # added replicas: 1
  replicas: 1
  # added selector.
  selector:
    matchLabels:
      app: mnist
  template:
    metadata:
      annotations:
        sidecar.istio.io/inject: "true"
      labels:
        app: mnist
        version: v1
    spec:
      containers:
      - args:
        - --port=9000
        - --rest_api_port=8500
        - --model_name=mnist
        # changed the path to exported Tensorflow model in Google Cloud Storage
        - --model_base_path=gs://<BUCKET_NAME>
        command:
        - /usr/bin/tensorflow_model_server
        env:
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: /secret/gcp-credentials/user-gcp-sa.json
        image: tensorflow/serving:latest-gpu
        imagePullPolicy: IfNotPresent
        livenessProbe:
          initialDelaySeconds: 30
          periodSeconds: 30
          tcpSocket:
            port: 9000
        name: mnist
        ports:
        - containerPort: 9000
        - containerPort: 8500
        resources:
          limits:
            cpu: "4"
            memory: 16Gi
            nvidia.com/gpu: 1
          requests:
            cpu: "1"
            memory: 4Gi
        volumeMounts:
        # - mountPath: /var/config/
        #   name: config-volume
        - mountPath: /secret/gcp-credentials
          name: gcp-credentials
      volumes:
      # - configMap:
      #     name: mnist-v1-config
      #   name: config-volume
      - name: gcp-credentials
        secret:
          secretName: user-gcp-sa
---
apiVersion: networking.istio.io/v1alpha3
kind: DestinationRule
metadata:
  labels:
  name: mnist-service
  namespace: kubeflow
spec:
  host: mnist-service
  subsets:
  - labels:
      version: v1
    name: v1
---
apiVersion: networking.istio.io/v1alpha3
kind: VirtualService
metadata:
  labels:
  name: mnist-service
  namespace: kubeflow
spec:
  gateways:
  - kubeflow-gateway
  hosts:
  - '*'
  http:
  - match:
    - method:
        exact: POST
      uri:
        prefix: /tfserving/models/mnist
    rewrite:
      uri: /v1/models/mnist:predict
    route:
    - destination:
        host: mnist-service
        port:
          number: 8500
        subset: v1
      weight: 100