Last active
July 29, 2024 19:55
-
-
Save dommmel/4af2742f5e5a351291c48ef03ecfa81c to your computer and use it in GitHub Desktop.
minimal dataflow beam test
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "context" | |
| "flag" | |
| "log" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam/io/pubsubio" | |
| beamLog "github.com/apache/beam/sdks/v2/go/pkg/beam/log" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam/register" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam/x/beamx" | |
| ) | |
| func init() { | |
| register.Function3x0(logAndEmit) | |
| } | |
| var projectId string = "staging-shopifysync" | |
| func main() { | |
| flag.Set("runner", "dataflow") | |
| flag.Set("project", projectId) | |
| flag.Set("staging_location", "gs://flexify-dataflow-staging-temp/binaries") | |
| flag.Parse() | |
| beam.Init() | |
| beamPipeline, scope := beam.NewPipelineWithRoot() | |
| readInput := func(scope beam.Scope) beam.PCollection { | |
| return pubsubio.Read(scope, projectId, "shopify-service", &pubsubio.ReadOptions{IDAttribute: "X-Shopify-Webhook-Id"}) | |
| } | |
| pipeline(scope, readInput) | |
| ctx := context.Background() | |
| err := beamx.Run(ctx, beamPipeline) | |
| if err != nil { | |
| log.Fatalf("Failed to run beam pipeline: %v", err) | |
| } | |
| } | |
| func pipeline(scope beam.Scope, readInput func(scope beam.Scope) beam.PCollection) beam.PCollection { | |
| messages := readInput(scope) | |
| return beam.ParDo(scope, logAndEmit, messages) | |
| } | |
| func logAndEmit(ctx context.Context, msg []byte, emit func(string)) { | |
| beamLog.Infoln(ctx, "test") | |
| beamLog.Infoln(ctx, "Received message:", string(msg)) | |
| emit(string(msg)) | |
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| package main | |
| import ( | |
| "flag" | |
| "testing" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/passert" | |
| "github.com/apache/beam/sdks/v2/go/pkg/beam/testing/ptest" | |
| ) | |
| var productDataTestString string = `{"admin_graphql_api_id":"gid:\/\/shopify\/Product\/4520903934091","body_html":"\u003cp\u003eCHANGES are coming NOW NOW\u003c\/p\u003e\n\u003c!----\u003e","created_at":"2020-02-07T06:13:36-05:00","handle":"5-pocket-jean","id":4520903934091,"product_type":"Bird Cage Food \u0026 Water Dishes","published_at":"2021-10-28T09:13:08-04:00","template_suffix":"","title":"5 Pocket Jean","updated_at":"2024-07-29T13:43:03-04:00","vendor":"Nigel Cabourn","status":"active","published_scope":"web","tags":"03\/03, blue, bounty, denim, indigo, jeans, man, newman, nigel cabourn, SALE, Shot 3\/5, Shot 4\/3, ss15, test, visible, WEEK10","variants":[{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005332619","barcode":"","compare_at_price":"102.99","created_at":"2024-07-18T04:44:38-04:00","id":42621005332619,"inventory_policy":"continue","position":1,"price":"64.60","product_id":4520903934091,"sku":"","taxable":true,"title":"30 \/ Black \/ ver small","updated_at":"2024-07-18T04:44:39-04:00","option1":"30","option2":"Black","option3":"ver small","image_id":null,"inventory_item_id":44719351595147,"inventory_quantity":0,"old_inventory_quantity":0},{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005365387","barcode":"","compare_at_price":"102.99","created_at":"2024-07-18T04:44:38-04:00","id":42621005365387,"inventory_policy":"continue","position":2,"price":"64.60","product_id":4520903934091,"sku":"","taxable":true,"title":"32 \/ Black \/ ver small","updated_at":"2024-07-18T04:44:39-04:00","option1":"32","option2":"Black","option3":"ver small","image_id":null,"inventory_item_id":44719351627915,"inventory_quantity":0,"old_inventory_quantity":0},{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005398155","barcode":"","compare_at_price":"102.99","created_at":"2024-07-18T04:44:38-04:00","id":42621005398155,"inventory_policy":"continue","position":3,"price":"64.60","product_id":4520903934091,"sku":"","taxable":true,"title":"34 \/ Black \/ ver small","updated_at":"2024-07-18T04:44:39-04:00","option1":"34","option2":"Black","option3":"ver small","image_id":null,"inventory_item_id":44719351660683,"inventory_quantity":0,"old_inventory_quantity":0}],"options":[{"name":"MYSIZE","id":5878937976971,"product_id":4520903934091,"position":1,"values":["30","32","34"]},{"name":"Color","id":9635800613003,"product_id":4520903934091,"position":2,"values":["Black"]},{"name":"yoursize","id":8524904628363,"product_id":4520903934091,"position":3,"values":["ver small"]}],"images":[{"id":14173842964619,"product_id":4520903934091,"position":1,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:35:00-04:00","alt":null,"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_08_32020_18028.jpg?v=1713780202","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173842964619"},{"id":14173842997387,"product_id":4520903934091,"position":2,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:36:11-04:00","alt":null,"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_09_30295_18046.jpg?v=1713774970","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173842997387"},{"id":14173843030155,"product_id":4520903934091,"position":3,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:36:28-04:00","alt":"fb-feed","width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_08_32020_18021.jpg?v=1713774988","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173843030155"},{"id":14173843062923,"product_id":4520903934091,"position":4,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":"fb-feed","width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_09_30295_18035.jpg?v=1581074016","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173843062923"},{"id":14173843095691,"product_id":4520903934091,"position":5,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":null,"width":1463,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-03-06_Addis_Look_01_30185_9758.jpg?v=1581074016","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173843095691"},{"id":14173843128459,"product_id":4520903934091,"position":6,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":null,"width":1463,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-03-06_Addis_Look_01_30185_9788.jpg?v=1581074016","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173843128459"}],"image":{"id":14173842964619,"product_id":4520903934091,"position":1,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:35:00-04:00","alt":null,"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_08_32020_18028.jpg?v=1713780202","variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/ProductImage\/14173842964619"},"media":[{"id":6346770776203,"product_id":4520903934091,"position":1,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:35:00-04:00","alt":null,"status":"READY","media_content_type":"IMAGE","preview_image":{"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_08_32020_18028.jpg?v=1713780202","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770776203"},{"id":6346770808971,"product_id":4520903934091,"position":2,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:36:11-04:00","alt":null,"status":"READY","media_content_type":"IMAGE","preview_image":{"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_09_30295_18046.jpg?v=1713774970","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770808971"},{"id":6346770841739,"product_id":4520903934091,"position":3,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2024-04-22T04:36:28-04:00","alt":"fb-feed","status":"READY","media_content_type":"IMAGE","preview_image":{"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_08_32020_18021.jpg?v=1713774988","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770841739"},{"id":6346770874507,"product_id":4520903934091,"position":4,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":"fb-feed","status":"READY","media_content_type":"IMAGE","preview_image":{"width":1462,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-04-03_Jake_Look_09_30295_18035.jpg?v=1581074016","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770874507"},{"id":6346770907275,"product_id":4520903934091,"position":5,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":null,"status":"READY","media_content_type":"IMAGE","preview_image":{"width":1463,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-03-06_Addis_Look_01_30185_9758.jpg?v=1581074016","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770907275"},{"id":6346770940043,"product_id":4520903934091,"position":6,"created_at":"2020-02-07T06:13:36-05:00","updated_at":"2020-02-07T06:13:36-05:00","alt":null,"status":"READY","media_content_type":"IMAGE","preview_image":{"width":1463,"height":2048,"src":"https:\/\/cdn.shopify.com\/s\/files\/1\/0328\/8314\/6891\/products\/2015-03-06_Addis_Look_01_30185_9788.jpg?v=1581074016","status":"READY"},"variant_ids":[],"admin_graphql_api_id":"gid:\/\/shopify\/MediaImage\/6346770940043"}],"variant_gids":[{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005332619","updated_at":"2024-07-18T08:44:39.000Z"},{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005365387","updated_at":"2024-07-18T08:44:39.000Z"},{"admin_graphql_api_id":"gid:\/\/shopify\/ProductVariant\/42621005398155","updated_at":"2024-07-18T08:44:39.000Z"}]}` | |
| func TestPipeline(t *testing.T) { | |
| // setup | |
| flag.Set("runner", "direct") | |
| beam.Init() | |
| beamPipeline, scope := beam.NewPipelineWithRoot() | |
| readInput := readInputFn(productDataTestString) | |
| // when | |
| elements := pipeline(scope, readInput) | |
| passert.Equals(scope, elements, productDataTestString, productDataTestString) | |
| ptest.RunAndValidate(t, beamPipeline) | |
| } | |
| func readInputFn(productDataString string) func(beam.Scope) beam.PCollection { | |
| return func(scope beam.Scope) beam.PCollection { | |
| duplicateWebookSimulatorBytes := make([][]byte, 2) | |
| for i := 0; i < 2; i++ { | |
| duplicateWebookSimulatorBytes[i] = []byte(productDataString) | |
| } | |
| return beam.CreateList(scope, duplicateWebookSimulatorBytes) | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Comment outflag.Set("runner", "dataflow")to make test run with DirectRunnerUpdated the files; now
go run main.gouses the gcloud Dataflow runnergo testuses the local DirectRunner