kshitijcode

{"Emp_id":"10030","Emp_name":"Linden, Mathew","Emp_location":"Chicago","Emp_dept":"IT","Emp_designation":"System Engineer","Last_promotion_date":"NULL","Hired_date":"6/15/2017"}
{"Emp_id":"10025","Emp_name":"Beatrice, Courtney","Emp_location":"Seattle","Emp_dept":"IT","Emp_designation":"Sr System Engineer","Last_promotion_date":"3/1/2020","Hired_date":"3/10/2018"}
{"Emp_id":"10029","Emp_name":"Heitzman, Anthony","Emp_location":"Seattle","Emp_dept":"HR","Emp_designation":"Recruiter","Last_promotion_date":"NULL","Hired_date":"6/2/2018"}
{"Emp_id":"10028","Emp_name":"Eaton, Marianne","Emp_location":"Dallas","Emp_dept":"Finance","Emp_designation":"Finance Director","Last_promotion_date":"3/1/2021","Hired_date":"1/28/2019"}
{"Emp_id":"10027","Emp_name":"Forrest, Alex","Emp_location":"Chicago","Emp_dept":"Finance","Emp_designation":"Sr Financial Analyst","Last_promotion_date":"3/1/2021","Hired_date":"10/1/2019"}
{"Emp_id":"10024","Emp_name":"Barbara, Thomas","Emp_location":"Seattle","Emp_dept":"HR","Emp_designation

kshitijcode

assert pipeline_run_result == constants.PIPELINE_SUCCESS_STATUS

# Check for Data in Raw Zone
parquet_dataframe = adls.read_parquet_file_from_ADLS(
adls_connection_client, container_name, target_path)
num_of_rows = len(parquet_dataframe.index)

#Assert

kshitijcode

We couldn’t find that file to show.

kshitijcode

def test_source_to_curated_workflow(azure_credential,
                        synapse_endpoint: str,
                        pipeline_name: str,
                        storage_account_name: str,
                        container_name: str,
                        base_path: str,
                        input_sample_file_name: str,
                        adls_connection_client,
                        sql_connection_client):

kshitijcode

.
├── README.md
├── data
│   └── employee_sample_integration_test.parquet
├── dataconnectors
│   └── adls.py
├── requirements.txt
├── tests
│   ├── conftest.py
│   └── test_master_pipeline_e2e.py

kshitijcode

.
├── data
├── dataconnectors
│   └── __pycache__
├── tests
└── utils

kshitijcode

"SourceTableSchemaName":"dbo",
"SourceTableName":table_raw,
"SinkTableSchemaName":"dbo",
"SinkTableName": table_curated,
"HasIncrementalData":"false"
"sourceDatabase": database,

kshitijcode

## Base Path of File 
"basePath": base_path,
"filePath": "employee_*.parquet",
## Target Table
"targetTable": target_table,
# Bad table Data
"badDataTable": "bad_employees",
# ADLS Container Name
"containerName": container_name,
# Storage Account Name

kshitijcode

{
  "Logging": {
    "LogLevel": {
      "Default": "Information",
      "Microsoft": "Warning",
      "Microsoft.Hosting.Lifetime": "Information"
    }
  },
  "AllowedHosts": "*",
  "DAPR_HTTP_ENDPOINT": "http://localhost:5380",

kshitijcode

        public void ConfigureServices(IServiceCollection services)
        {

            services.AddControllers().AddDapr(builder => builder.UseHttpEndpoint(Configuration.GetValue<string>("DAPR_HTTP_ENDPOINT"))
                                               .UseGrpcEndpoint(Configuration.GetValue<string>("DAPR_GRPC_ENDPOINT")));
            services.AddControllers();
            services.AddSwaggerGen(c =>
            {
                c.SwaggerDoc("v1", new OpenApiInfo { Title = "DaprAdventures.SaveStateService", Version = "v1" });
            });