nsivabalan

/**
 * UberInterceptor interface is synonymous to {@link UberClientCall}. This class extends {@link
 * UberClientCall} and lends way for developers for easy interceptor development. Every new grpc
 * call by the client/application, will result in creating a new chain of interceptors and so each
 * instance of an interceptor deals with only one grpc call in its lifetime. Any synchronization
 * across different calls needs to be abstracted out to a different class and all interceptor copies
 * will be accessing the a single instance of such class. For eg: Failover module.
 *
 * @param <ReqT> request msg of generic type RequestT.
 * @param <RespT> response msg of generic type RequestT.

nsivabalan

/**
 * An instance of a call to a remote method. A call will send zero or more request messages to the
 * server and receive zero or more response messages back. This exactly mimics grpc's {@link
 * ClientCall} with Uber's specific entities. This will also act as a wrapper for grpc's {@link
 * ClientCall} in Adaptor/ grpc translations layers.
 *
 * <p>Instances are created by a {@link UberChannel} and used by stubs to invoke their remote
 * behavior.
 *
 * <p>More advanced usages may consume this interface directly as opposed to using a stub. Common

nsivabalan

package helloworld;

// The greeting service definition.
service Greeter {
    // Sends a greeting
    rpc SayHello (HelloRequest) returns (HelloReply) {}
}

// The request message containing the user's name.
message HelloRequest {

nsivabalan

11240 [Executor task launch worker for task 60] ERROR org.apache.spark.executor.Executor  - Exception in task 1.0 in stage 57.0 (TID 60)
org.apache.hudi.exception.HoodieIOException: Could not read commit details from /var/folders/2k/v2f22b650rbf2s5zkg3_4cc40000gn/T/junit3188025014584302338/.hoodie/003.deltacommit.inflight
	at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.readDataFromPath(HoodieActiveTimeline.java:465)
	at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.getInstantDetails(HoodieActiveTimeline.java:189)
	at org.apache.hudi.common.table.timeline.HoodieDefaultTimeline.getInstantDetails(HoodieDefaultTimeline.java:296)
	at org.apache.hudi.table.action.rollback.RollbackUtils.lambda$generateRollbackRequestsUsingFileListingMOR$10f8a1d7$1(RollbackUtils.java:179)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1.apply(JavaRDDLike.scala:125)
	at org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1.apply(JavaRDDLike.scala:125)
	at scala.collection.Iterator$$anon$12.nextCur(

nsivabalan

commit e54108864f385c4d2a89d4f1b889082bd947904c
Author: Sivabalan Narayanan <[email protected]>
Date:   Wed Jul 22 11:17:34 2020 -0400

    Adding tests for HoodieInternalRow and HoodieInternalWriteStatus

diff --git a/hudi-client/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java b/hudi-client/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
index 7e54e542..87a117bb 100644
--- a/hudi-client/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java
+++ b/hudi-client/src/main/java/org/apache/hudi/client/HoodieInternalWriteStatus.java

nsivabalan

import io.grpc.CallOptions;
import io.grpc.Channel;
import io.grpc.ClientCall;
import io.grpc.ManagedChannel;
import io.grpc.Metadata;
import io.grpc.MethodDescriptor;
import io.grpc.Status;

public abstract class SimpleClientInterceptor<ReqT, RespT> extends InterceptorChannel {

nsivabalan

package org.apache.hudi.utilities.transform;

import org.apache.hudi.common.config.TypedProperties;

import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.functions;
import org.apache.spark.sql.types.DataTypes;

nsivabalan

spark-submit \
>   --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer $HUDI_UTILITIES_BUNDLE \
>   --table-type COPY_ON_WRITE \
>   --source-class org.apache.hudi.utilities.sources.JsonKafkaSource \
>   --source-ordering-field ts \
>   --target-base-path /user/hive/warehouse/stock_ticks_cow \
>   --target-table stock_ticks_cow \
>   --transformer-class org.apache.hudi.utilities.transform.DeleteTransformer \
>   --props /var/demo/config/kafka-source.properties \
>   --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider

nsivabalan

16020 [main] WARN  org.apache.spark.util.Utils  - Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
16020 [main] WARN  org.apache.spark.util.Utils  - Service 'SparkUI' could not bind on port 4041. Attempting port 4042.
16834 [main] WARN  org.apache.spark.sql.SparkSession$Builder  - Using an existing SparkSession; some configuration may not take effect.
16841 [main] WARN  org.apache.spark.sql.SparkSession$Builder  - Using an existing SparkSession; some configuration may not take effect.
18296 [dispatcher-event-loop-0] WARN  org.apache.spark.scheduler.TaskSetManager  - Stage 0 contains a task of very large size (196 KB). The maximum recommended task size is 100 KB.
20747 [dispatcher-event-loop-1] WARN  org.apache.spark.scheduler.TaskSetManager  - Stage 1 contains a task of very large size (196 KB). The maximum recommended task size is 100 KB.
21805 [dispatcher-event-loop-0] WARN  org.apache.spark.scheduler.TaskSetManager  - Stage 2 contains a task of very large size (196 KB). The maximum reco

nsivabalan

public interface UberCallHeaders {

String getHeaderValue(String key);

void setHeader(String key, String value);

Set<String> getAllHeaderKeys();

Map<String, String> getHeadersAsMap();