rgov · August 17, 2022 14:36
diff --git a/ModelZoo-Status.md b/ModelZoo-Status.md
diff --git a/Z-0001-Test-TensorFlow-Lite-conversion.patch b/Z-0001-Test-TensorFlow-Lite-conversion.patch
 From 47829edabf1d91f253132a314d08c55db7e0f432 Mon Sep 17 00:00:00 2001
 From: Ryan Govostes <[email protected]>
 Date: Wed, 17 Aug 2022 10:20:37 -0400
 Subject: [PATCH] Test TensorFlow Lite conversion

 ---
 test/test_modelzoo.py | 81 +++++++++++++++++++++++++++++++------------
 1 file changed, 58 insertions(+), 23 deletions(-)

 diff --git a/test/test_modelzoo.py b/test/test_modelzoo.py
 index 0bf526e..3c21908 100644
 --- a/test/test_modelzoo.py
 +++ b/test/test_modelzoo.py
 @@ -120,8 +120,8 @@ def _get_model_and_test_data():
         test_data_file = os.path.join(root, file_name)
         test_data_dir = os.path.join(root, file_name.split('.')[0])
         new_test_data_file = os.path.join(test_data_dir, file_name)
 -        os.mkdir(test_data_dir)
 -        os.rename(test_data_file, new_test_data_file)
 +        os.makedirs(test_data_dir, exist_ok=True)
 +        os.replace(test_data_file, new_test_data_file)
         test_data_set.append(test_data_dir)
   return onnx_model, test_data_set
 
 @@ -268,6 +268,28 @@ def _report_convert_model(model):
     return '{}: {}'.format(type(ex).__name__, stack_trace[0].strip())
 
 
 +def _report_convert_model_tflite(model, select_ops=False):
 +  """Test conversion and returns a report string."""
 +  try:
 +    converter = tf.lite.TFLiteConverter.from_saved_model(_CFG['output_directory'])
 +    converter.target_spec.supported_ops = [
 +      tf.lite.OpsSet.TFLITE_BUILTINS, # enable TensorFlow Lite ops
 +    ]
 +    if select_ops:
 +      converter.target_spec.supported_ops.append(tf.lite.OpsSet.SELECT_TF_OPS)
 +    tf_lite_model = converter.convert()
 +    return ''
 +  except Exception as ex:
 +    # don't delete, as we are going to try again shortly
 +    #_del_location(_CFG['untar_directory'])
 +    #_del_location(_CFG['output_directory'])
 +    stack_trace = str(ex).strip().split('\n')
 +    if len(stack_trace) > 1:
 +      err_msg = stack_trace[-1].strip()
 +      return err_msg
 +    return '{}: {}'.format(type(ex).__name__, stack_trace[0].strip())
 +
 +
 def _get_inputs_outputs_pb(tf_rep, data_dir):
   """Get the input and reference output tensors"""
   inputs = {}
 @@ -366,44 +388,50 @@ def _report_model(file_path, results=Results(), onnx_model_count=1):
     results.skip_count += 1
   else:
     if _CFG['verbose']:
 -      print('Testing', file_path)
 +      print('Testing', file_path, 'at', datetime.datetime.now())
     model = onnx.load(onnx_model)
     ir_version = model.ir_version
     opset_version = model.opset_import[0].version
     check_err = _report_check_model(model)
     convert_err = '' if check_err else _report_convert_model(model)
 -    run_err = '' if convert_err or len(
 -        test_data_set) == 0 else _report_run_model(model, test_data_set)
 +    tflite_err = '' if convert_err else _report_convert_model_tflite(model)
 +    tflite2_err = '' if convert_err or not tflite_err else _report_convert_model_tflite(model, True)
 +
 +    # We are not "running" the model which normally cleans this stuff up, so 
 +    # do it here.
 +    _del_location(_CFG['untar_directory'])
 +    _del_location(_CFG['output_directory'])
 
 -    if (not check_err and not convert_err and not run_err and
 -        len(test_data_set) > 0):
 +    if not check_err and not convert_err and not tflite_err:
       # https://github-emoji-list.herokuapp.com/
       # ran successfully
       emoji_validated = ':ok:'
       emoji_converted = ':ok:'
 -      emoji_ran = ':ok:'
 +      emoji_tflite = ':ok:'
 +      emoji_tflite2 = ':ok:'
       emoji_overall = ':heavy_check_mark:'
       results.pass_count += 1
 -    elif (not check_err and not convert_err and not run_err and
 -          len(test_data_set) == 0):
 -      # validation & conversion passed but no test data available
 +    elif not check_err and not convert_err and not tflite2_err:
       emoji_validated = ':ok:'
       emoji_converted = ':ok:'
 -      emoji_ran = 'No test data provided in model zoo'
 -      emoji_overall = ':warning:'
 -      results.warn_count += 1
 +      emoji_tflite = tflite_err
 +      emoji_tflite2 = ':ok:'
 +      emoji_overall = ':heavy_check_mark:'
 +      results.pass_count += 1
     elif not check_err and not convert_err:
 -      # validation & conversion passed but failed to run
 +      # conversion to tf passed, but to tflite did not
       emoji_validated = ':ok:'
       emoji_converted = ':ok:'
 -      emoji_ran = run_err
 +      emoji_tflite = tflite_err
 +      emoji_tflite2 = tflite2_err
       emoji_overall = ':x:'
       results.fail_count += 1
     elif not check_err:
       # validation pass, but conversion did not
       emoji_validated = ':ok:'
       emoji_converted = convert_err
 -      emoji_ran = ':heavy_minus_sign:'
 +      emoji_tflite = ':heavy_minus_sign:'
 +      emoji_tflite2 = ':heavy_minus_sign:'
       if ('BackendIsNotSupposedToImplementIt' in convert_err or
           'OpUnsupportedException' in convert_err):
         # known limitations
 @@ -419,15 +447,22 @@ def _report_model(file_path, results=Results(), onnx_model_count=1):
       # validation failed
       emoji_validated = check_err
       emoji_converted = ':heavy_minus_sign:'
 -      emoji_ran = ':heavy_minus_sign:'
 +      emoji_tflite = ':heavy_minus_sign:'
 +      emoji_tflite2 = ':heavy_minus_sign:'
       emoji_overall = ':x:'
       results.fail_count += 1
 -
 -  results.append_detail('{} | {}. {} | {} | {} | {} | {} | {} | {}'.format(
 +  
 +  print(f'overall             -> {emoji_overall}')
 +  print(f'validated           -> {emoji_validated}')
 +  print(f'converted           -> {emoji_converted}')
 +  print(f'tflite (base ops)   -> {emoji_tflite}')
 +  print(f'tflite (select ops) -> {emoji_tflite2}')
 +
 +  results.append_detail('{} | {}. {} | {} | {} | {} | {} | {} | {} | {}'.format(
       emoji_overall, onnx_model_count,
       file_path[file_path.rindex('/') + 1:file_path.index('.')],
       _size_with_units(size_pulled[0]), ir_version, opset_version,
 -      emoji_validated, emoji_converted, emoji_ran))
 +      emoji_validated, emoji_converted, emoji_tflite, emoji_tflite2))
 
   if len(test_data_set) == 0:
     _del_location(_CFG['output_directory'])
 @@ -539,10 +574,10 @@ def modelzoo_report(models_dir='models',
         results.append_detail('')
         results.append_detail(
             'Status | Model | Size | IR | Opset | ONNX Checker | '
 -            'ONNX-TF Converted | ONNX-TF Ran')
 +            'ONNX-TF Converted | TF-TFLite Converted | TF-TFLite Converted w/ Select Ops')
         results.append_detail(
             '------ | ----- | ---- | -- | ----- | ------------ | '
 -            '----------------- | -----------')
 +            '----------------- | ------------------- | -----------')
     onnx_model_count = 0
     file_path = ''
     for item in sorted(files):
 -- 
 2.32.1 (Apple Git-133)
Package	Version
Platform	macOS-12.5-arm64-arm-64bit
Python	3.10.6 (main, Aug 11 2022, 13:36:31) [Clang 13.1.6 (clang-1316.0.21.2.5)]
onnx	1.12.0
onnx-tf	1.10.0
tensorflow	2.9.0
Value	Count
Models	39
Total	168
✔️ Passed	123
⚠️ Limitation	9
❌ Failed	36
➖ Skipped	0
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. bertsquad-10	385M	5	10	🆗	🆗	tf.RealDiv(tensor, tensor) -> (tensor) : {device = ""}	🆗
❌	2. bertsquad-12-int8	101M	7	12	🆗	ValueError: 'onnx_tf_prefix_bert/embeddings/MatMul:0_output_quantized_cast' is not a valid root scope name. A root scope name has to match the following pattern: ^[A-Za-z0-9.][A-Za-z0-9_.\/>-]*$	➖	➖
✔️	3. bertsquad-12	384M	7	12	🆗	🆗	tf.RealDiv(tensor, tensor) -> (tensor) : {device = ""}	🆗
✔️	4. bertsquad-8	385M	5	8	🆗	🆗	tf.RealDiv(tensor, tensor) -> (tensor) : {device = ""}	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. gpt2-10	442M	6	10	🆗	🆗	tf.RealDiv(tensor, tensor) -> (tensor) : {device = ""}	🆗
✔️	2. gpt2-lm-head-10	578M	6	10	🆗	🆗	tf.RealDiv(tensor, tensor) -> (tensor) : {device = ""}	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. roberta-base-11	278M	6	11	🆗	🆗	tf.Erf(tensor<?x?x3072xf32>) -> (tensor<?x?x3072xf32>) : {device = ""}	🆗
✔️	2. roberta-sequence-classification-9	411M	6	9	🆗	🆗	tf.Cast(tensor<1xi64>) -> (tensor<1xf64>) : {Truncate = false, device = ""}	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. t5-decoder-with-lm-head-12	275M	6	12	🆗	🆗	tf.Range(tensor, tensor, tensor) -> (tensor<?xi64>) : {device = ""}	🆗
✔️	2. t5-encoder-12	186M	6	12	🆗	🆗	tf.Range(tensor, tensor, tensor) -> (tensor<?xi64>) : {device = ""}	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. age_googlenet	23M	6	11	🆗	🆗	🆗	🆗
✔️	2. gender_googlenet	23M	6	11	🆗	🆗	🆗	🆗
✔️	3. vgg_ilsvrc_16_age_chalearn_iccv2015	514M	6	11	🆗	🆗	🆗	🆗
✔️	4. vgg_ilsvrc_16_age_imdb_wiki	514M	6	11	🆗	🆗	🆗	🆗
✔️	5. vgg_ilsvrc_16_gender_imdb_wiki	512M	6	11	🆗	🆗	🆗	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. emotion-ferplus-2	31M	3	2	🆗	🆗	🆗	🆗
✔️	2. emotion-ferplus-7	31M	3	7	🆗	🆗	🆗	🆗
✔️	3. emotion-ferplus-8	31M	3	8	🆗	🆗	🆗	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
✔️	1. version-RFB-320	1M	4	9	🆗	🆗	🆗	🆗
✔️	2. version-RFB-640	2M	4	9	🆗	🆗	🆗	🆗
Status	Model	Size	IR	Opset	ONNX Checker	ONNX-TF Converted	TF-TFLite Converted	TF-TFLite Converted w/ Select Ops
❌	1. bvlcalexnet-12-int8	39M	7	12	🆗	ValueError: slice index 3 of dimension 0 out of bounds. for '{{node strided_slice_13}} = StridedSlice[Index=DT_INT32, T=DT_INT8, begin_mask=0, ellipsis_mask=0, end_mask=0, new_axis_mask=0, shrink_axis_mask=1](strided_slice_12, strided_slice_13/stack, strided_slice_13/stack_1, strided_slice_13/stack_2)' with input shapes: [3,11,11], [1], [1], [1] and with computed input tensors: input[1] = <3>, input[2] = <4>, input[3] = <1>.	➖	➖
✔️	2. bvlcalexnet-12	216M	7	12	🆗	🆗	🆗	🆗
✔️	3. bvlcalexnet-3	219M	3	3	🆗	🆗	🆗	🆗
✔️	4. bvlcalexnet-6	219M	3	6	🆗	🆗	🆗	🆗
✔️	5. bvlcalexnet-7	216M	3	7	🆗	🆗	🆗	🆗
✔️	6. bvlcalexnet-8	216M	3	8	🆗	🆗	🆗	🆗
✔️	7. bvlcalexnet-9	216M	3	9	🆗	🆗	🆗	🆗