bigquery-schema

[
 {
  "name": "subscription_name",
  "type": "STRING",
  "mode": "NULLABLE"
 },
 {
  "name": "messageid",
  "type": "STRING",
  "mode": "NULLABLE"
 },
 {
  "name": "domain",
  "type": "STRING",
  "mode": "NULLABLE"
 },
 {
  "name": "publish_time",
  "type": "TIMESTAMP",
  "mode": "REQUIRED"
 },
 {
  "name": "attributes",
  "type": "STRING",
  "mode": "NULLABLE"
 },
 {
  "name": "data",
  "type": "RECORD",
  "mode": "NULLABLE",
  "fields": [
   {
    "name": "domain",
    "type": "STRING",
    "mode": "NULLABLE"
   },
   {
    "name": "grade",
    "type": "STRING",
    "mode": "NULLABLE"
   },
   {
    "name": "host",
    "type": "STRING",
    "mode": "NULLABLE"
   },
   {
    "name": "likelihood_indicator",
    "type": "STRING",
    "mode": "NULLABLE"
   },
   {
    "name": "response_headers",
    "type": "RECORD",
    "mode": "REPEATED",
    "fields": [
     {
      "name": "header",
      "type": "STRING",
      "mode": "NULLABLE"
     },
     {
      "name": "value",
      "type": "STRING",
      "mode": "NULLABLE"
     }
    ]
   }
  ]
 }
]

commands

go install github.com/GoogleCloudPlatform/protoc-gen-bq-schema@latest
sed -e '/STRIP/d' httpobs-schema.proto  > /tmp/l.protoc
gcloud pubsub schemas create check-me.httpobs --type=protocol-buffer --definition-file=/tmp/l.protoc
gcloud pubsub topics create check-me.httpobs --schema check-me.httpobs  --message-encoding=json

protoc --bq-schema_out=. httpobs-schema.proto --proto_path=.
bq rm --project_id MYDOMAIN --table MYDOMAIN:checkme.httpobs
bq mk --project_id MYDOMAIN --table --schema  httpobs/httpobs.schema MYDOMAIN:checkme.httpobs

httpobs-schema.proto

syntax = "proto3";
package httpobs;
import "bq_table.proto";  //STRIP
import "bq_field.proto";  //STRIP

message Httpobs {
  option (gen_bq_schema.bigquery_opts).table_name = "httpobs";  //STRIP

  string subscription_name = 1;
  string messageid = 2;
  string domain = 3;
  uint64 publish_time = 4
  [ (gen_bq_schema.bigquery) = { require: true, type_override: 'TIMESTAMP' } ] //STRIP
  ;
  string attributes = 5;

  message HeaderValue {
    string header = 1;
    string value = 2;
  }

  message Data {
    string domain = 1;
    string grade = 2;
    string host = 3;
    string likelihood_indicator = 4;
    repeated HeaderValue response_headers = 5;
  }

  Data data = 6;

}

pubsub schema

syntax = "proto3";
package httpobs;

message Httpobs {

  string subscription_name = 1;
  string messageid = 2;
  string domain = 3;
  uint64 publish_time = 4
  ;
  string attributes = 5;

  message HeaderValue {
    string header = 1;
    string value = 2;
  }

  message Data {
    string domain = 1;
    string grade = 2;
    string host = 3;
    string likelihood_indicator = 4;
    repeated HeaderValue response_headers = 5;
  }

  Data data = 6;

}

PUT

https://pubsub.googleapis.com/v1/projects/MYDOMAIN/subscriptions/check-me.httpobs?alt=json
{
  "bigqueryConfig": {
    "dropUnknownFields": true,
    "table": "MYDOMAIN:checkme.httpobs",
    "useTopicSchema": true,
    "writeMetadata": true
  },
  "name": "projects/MYDOMAIN/subscriptions/check-me.httpobs",
  "topic": "projects/MYDOMAIN/topics/check-me.httpobs"
}

results

gcloud  pubsub subscriptions create check-me.httpobs --topic=check-me.httpobs --bigquery-table=MYDOMAIN:checkme.httpobs --write-metadata  --use-topic-schema --log-http --verbosity debug
DEBUG: Running [gcloud.pubsub.subscriptions.create] with arguments: [--bigquery-table: "MYDOMAIN:checkme.httpobs", --log-http: "true", --topic: "check-me.httpobs", --use-topic-schema: "True", --verbosity: "debug", --write-metadata: "True", SUBSCRIPTION:1: "['check-me.httpobs']"]
INFO: Display format: "none"
=======================
==== request start ====
uri: https://pubsub.googleapis.com/v1/projects/MYDOMAIN/subscriptions/check-me.httpobs?alt=json
method: PUT
== headers start ==
b'accept': b'application/json'
b'accept-encoding': b'gzip, deflate'
b'authorization': --- Token Redacted ---
b'content-length': b'218'
b'content-type': b'application/json'
b'user-agent': b'google-cloud-sdk gcloud/402.0.0 command/gcloud.pubsub.subscriptions.create invocation-id/2d67bb84d974422f9373c1145fd162e1 environment/None environment-version/None interactive/True from-script/False python/3.10.6 term/alacritty (Linux 5.19.5-051905-generic)'
== headers end ==
== body start ==
{"bigqueryConfig": {"table": "MYDOMAIN:checkme.httpobs", "useTopicSchema": true, "writeMetadata": true}, "name": "projects/MYDOMAIN/subscriptions/check-me.httpobs", "topic": "projects/MYDOMAIN/topics/check-me.httpobs"}
== body end ==
==== request end ====
DEBUG: Starting new HTTPS connection (1): pubsub.googleapis.com:443
DEBUG: https://pubsub.googleapis.com:443 "PUT /v1/projects/MYDOMAIN/subscriptions/check-me.httpobs?alt=json HTTP/1.1" 400 None
---- response start ----
status: 400
-- headers start --
Alt-Svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000,h3-Q050=":443"; ma=2592000,h3-Q046=":443"; ma=2592000,h3-Q043=":443"; ma=2592000,quic=":443"; ma=2592000; v="46,43"
Cache-Control: private
Content-Encoding: gzip
Content-Type: application/json; charset=UTF-8
Date: Fri, 30 Sep 2022 18:38:34 GMT
Server: ESF
Transfer-Encoding: chunked
Vary: Origin, X-Origin, Referer
X-Content-Type-Options: nosniff
X-Frame-Options: SAMEORIGIN
X-XSS-Protection: 0
-- headers end --
-- body start --
{
  "error": {
    "code": 400,
    "message": "Request contains an invalid argument.",
    "status": "INVALID_ARGUMENT"
  }
}

-- body end --
total round trip time (request+response): 0.945 secs
---- response end ----
----------------------
ERROR: Failed to create subscription [projects/MYDOMAIN/subscriptions/check-me.httpobs]: Request contains an invalid argument.
DEBUG: (gcloud.pubsub.subscriptions.create) Failed to create the following: [check-me.httpobs].
Traceback (most recent call last):
  File "/home/don/google-cloud-sdk/lib/googlecloudsdk/calliope/cli.py", line 987, in Execute
    resources = calliope_command.Run(cli=self, args=args)
  File "/home/don/google-cloud-sdk/lib/googlecloudsdk/calliope/backend.py", line 811, in Run
    display_info=self.ai.display_info).Display()
  File "/home/don/google-cloud-sdk/lib/googlecloudsdk/calliope/display.py", line 483, in Display
    self._printer.Print(self._resources)
  File "/home/don/google-cloud-sdk/lib/googlecloudsdk/core/resource/resource_printer_base.py", line 298, in Print
    for resource in resources:
  File "/home/don/google-cloud-sdk/lib/surface/pubsub/subscriptions/create.py", line 119, in _Run
    raise util.RequestsFailedError(failed, 'create')
googlecloudsdk.command_lib.pubsub.util.RequestsFailedError: Failed to create the following: [check-me.httpobs].
ERROR: (gcloud.pubsub.subscriptions.create) Failed to create the following: [check-me.httpobs].