dan-zheng

struct Model {
  var w: Float

  struct Parameters {
    var w: Float
  }

  var allParameters: Parameters {
    get {
      return Parameters(w: w)

dan-zheng

// RUN: %target-swift-frontend -Xllvm -tf-dump-intermediates -O -emit-sil -verify %s | %FileCheck %s

import TensorFlow

// This test is intended to verify that all of the operations end up in the
// graph: that there are no host/accelerator copies generated.  This tests a
// combination of the partitioning pass being able to recognize various forms,
// but also checks that certain ops implementations are promotable as well.

// Please keep it so no errors or warnings are generated by functions in this

dan-zheng

import TensorFlow
func test() {
  for _ in 0...10 {
    let x = Tensor<Float>(ones: [2, 2])
    _ = x.reshaped(toShape: Tensor<Int32>([4, Int32(1 * 1)]))

    // Alternative 1: literal `1` avoids send/receive.
    // _ = x.reshaped(toShape: Tensor<Int32>([4, 1]))

    // Alternative 2: avoiding `Int32` avoids send/receive.

dan-zheng

%24 = graph_op "MaxPoolV2,i,i,i"(%4 : $TensorHandle<Float>, %14 : $TensorHandle<Int32>, %22 : $TensorHandle<Int32>) {T: $Float, padding: "SAME", data_format: "NHWC", __device: "/device:CPU:0"} : $TensorHandle<Float>

As you can see, kernel size and strides are SSA operands of `graph_op`, not constant attributes.
This is because of how MaxPool is defined:

REGISTER_OP("MaxPoolV2")
    .Attr(
        "T: {half, bfloat16, float, double, int32, int64, uint8, int16, int8, "
        "uint16, qint8} = DT_FLOAT")
    .Attr(GetPaddingAttrString())

dan-zheng

import TensorFlow

func train(iterationCount: Int) {
  let images = Tensor<Float>(ones: [1000, 784])
  let batchSize = Float(images.shape[0])

  print("Begin training for \(iterationCount) iterations.")

  for _ in 0...iterationCount {
    let bound = Int32(batchSize)/25

dan-zheng

struct MNISTParameters : ParameterAggregate {
  var w1 = Tensor<Float>(randomNormal: [784, 30])
  var w2 = Tensor<Float>(randomNormal: [30, 10])
  
  // Compiler-synthesized:
  // static var allKeyPaths: [WritableKeyPath<MNISTParameters, Tensor<Float>>] {
  //   return [\MNISTParameters.w1, \MNISTParameters.w2]
  // }
  // Learn more about key paths here: https://github.com/apple/swift-evolution/blob/master/proposals/0161-key-paths.md
}

dan-zheng

This gist is an in-depth exploration of the Swift parameter update design, focusing on dynamic parameters and how to synthesize allKeyPaths for them.

dan-zheng

import TensorFlow
struct Model: Parameterized, Differentiable {
  @TFParameter var w: Tensor<Float>
  func tangentVector(from cotangent: Parameters) -> Parameters {
    return cotangent
  }
}

let model = Model(w: Tensor<Float>(zeros: [5]))
let pb = pullback(at: model) { m in m.w }

dan-zheng

## Building Swift

./swift/utils/update-checkout --skip-repository swift --clone --scheme tensorflow
./swift/utils/build-script -x -R --debug-swift --skip-build-benchmarks --stdlib-deployment-targets=macosx-x86_64 2>&1 | tee log.txt
./swift/utils/build-script -R --skip-build-benchmarks --stdlib-deployment-targets=macosx-x86_64 --enable-tensorflow --no-swift-stdlib-assertions --reconfigure

dan-zheng

import TensorFlow

extension SGD {
  // Take `Model` metatype as an argument, so it doesn't need to be written out
  // explicitly as a generic parameter.
  // IMPORTANT: This API change might not be desirable, because usage involves
  // getting dynamic type via `type(of:)`.
  convenience init(_ modelType: Model.Type, learningRate: Scalar) {
    self.init(learningRate: learningRate) 
  }