fsspec_data.interchange

Attributes

_rust

DEFAULT_REGISTRY

Classes

DataFormat

str(object='') -> str

SchemaPolicy

str(object='') -> str

CodecCapabilities

DecodedBatches

DecodedBatchStream

PlannedBatchStream

Codec

CodecRegistry

FieldMapping

InterchangeRequest

InterchangePlan

Functions

plan_schema(→ InterchangePlan)

_ensure_pyarrow(→ pyarrow.Schema)

Module Contents

fsspec_data.interchange._rust
class fsspec_data.interchange.DataFormat[source]

Bases: str, enum.Enum

str(object=’’) -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to ‘strict’.

Initialize self. See help(type(self)) for accurate signature.

ARROW = 'arrow'
PARQUET = 'parquet'
CSV = 'csv'
JSONL = 'jsonl'
class fsspec_data.interchange.SchemaPolicy[source]

Bases: str, enum.Enum

str(object=’’) -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to ‘strict’.

Initialize self. See help(type(self)) for accurate signature.

EXACT = 'exact'
PROJECTION = 'projection'
COMPATIBLE = 'compatible'
COERCE = 'coerce'
class fsspec_data.interchange.CodecCapabilities[source]
encode: bool
decode: bool
streaming: bool
class fsspec_data.interchange.DecodedBatches[source]
schema: pyarrow.Schema
batches: tuple[pyarrow.RecordBatch, Ellipsis]
class fsspec_data.interchange.DecodedBatchStream(schema: pyarrow.Schema, native: Any)[source]
schema
_native
__iter__() → DecodedBatchStream[source]
__next__() → pyarrow.RecordBatch[source]
cancel() → None[source]
collect() → DecodedBatches[source]
class fsspec_data.interchange.PlannedBatchStream(plan: InterchangePlan, source: DecodedBatchStream)[source]
schema
_plan
_source
__iter__() → PlannedBatchStream[source]
__next__() → pyarrow.RecordBatch[source]
cancel() → None[source]
class fsspec_data.interchange.Codec[source]
format: DataFormat
capabilities: CodecCapabilities
encode_batches(batches: list[pyarrow.RecordBatch] | tuple[pyarrow.RecordBatch, Ellipsis], *, schema: pyarrow.Schema | None = None) → bytes[source]
encode_batches_to(batches: collections.abc.Iterable[pyarrow.RecordBatch], output: BinaryIO, *, schema: pyarrow.Schema | None = None) → None[source]
decode_batches(data: bytes | bytearray | memoryview | BinaryIO, *, schema: pyarrow.Schema | None = None, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) → DecodedBatches[source]
iter_batches(data: bytes | bytearray | memoryview | BinaryIO, *, schema: pyarrow.Schema | None = None, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) → DecodedBatchStream[source]
class fsspec_data.interchange.CodecRegistry[source]
get(format: DataFormat | str) → Codec[source]
fsspec_data.interchange.DEFAULT_REGISTRY
class fsspec_data.interchange.FieldMapping[source]
source_index: int
target_index: int
cast: str | None = None
check_nulls: bool = False
class fsspec_data.interchange.InterchangeRequest[source]
provided_format: DataFormat
requested_format: DataFormat
provided_schema: pyarrow.Schema
requested_schema: pyarrow.Schema
policy: SchemaPolicy
plan() → InterchangePlan[source]
class fsspec_data.interchange.InterchangePlan[source]
provided_schema: pyarrow.Schema
requested_schema: pyarrow.Schema
policy: SchemaPolicy
mappings: tuple[FieldMapping, Ellipsis]
provided_format: DataFormat
requested_format: DataFormat
apply_table(table: pyarrow.Table) → pyarrow.Table[source]
apply_batch(batch: pyarrow.RecordBatch) → pyarrow.RecordBatch[source]
_apply_mapping(array: pyarrow.Array | pyarrow.ChunkedArray, mapping: FieldMapping)[source]
_validate_input_schema(schema: pyarrow.Schema) → None[source]
iter_batches(data: bytes | bytearray | memoryview | BinaryIO, *, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) → PlannedBatchStream[source]
convert(data: bytes | bytearray | memoryview | BinaryIO, *, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) → bytes[source]
fsspec_data.interchange.plan_schema(provided_schema: pyarrow.Schema, requested_schema: pyarrow.Schema, policy: SchemaPolicy | str) → InterchangePlan[source]
fsspec_data.interchange._ensure_pyarrow(value: Any) → pyarrow.Schema[source]