BryanCutler

# Addition of `Span` with Series of `SpanDtype` produces another Series
df["span"].iloc[1]  + df["span"].iloc[3:5]

# 3    [0, 16): 'Monty Python and'
# 4    [0, 20): 'Monty Python and the'
# Name: span, dtype: SpanDtype

BryanCutler

@pytest.fixture
def dtype():
    """ Return dtype of your extension array."""
    return TensorDtype()
 
@pytest.fixture
def data(dtype):
    """ Return an extension array as data for the tests."""
    return pd.array(np.array([[i] for i in range(100)]), dtype=dtype)
  

BryanCutler

# Reduce the `SpanArray` to a single `Span` covering tokens 2 up to 5.
df["span"].iloc[2:5].sum()

# [6, 20): 'Python and the'

BryanCutler

class SpanOpMixin:
  def __add__(self, other) -> Union["Span", "SpanArray"]:
 
    if isinstance(other, (ABCDataFrame, ABCSeries, ABCIndexClass)):
      # Rely on pandas to unbox and dispatch to us.
      return NotImplemented

    if isinstance(self, Span) and isinstance(other, Span):
      # Span + *Span = Span
      return Span(self.target_text,

BryanCutler

BryanCutler

BryanCutler

BryanCutler

BryanCutler

def make_remote_dataset(endpoint):
  """Make a TensorFlow Arrow Dataset that reads from a remote Arrow stream."""

  # Create the Arrow Dataset from a remote host serving a stream
  ds = arrow_io.ArrowStreamDataset(
      [endpoint],
      columns=(0, 1, 2),
      output_types=(tf.int64, tf.float64, tf.float64),
      output_shapes=(tf.TensorShape([]), tf.TensorShape([]), tf.TensorShape([])),
      batch_mode='auto')

BryanCutler

def serve_csv_data(ip_addr, port_num, directory):
  """
  Create a socket and serve Arrow record batches as a stream read from the
  given directory containing CVS files.
  """

  # Create the socket
  sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  sock.bind((ip_addr, port_num))
  sock.listen(1)