fsspec_data.schema

Attributes

SchemaInput
SchemaDecoder
SchemaProvider
DEFAULT_SCHEMA_RESOLVERS

Classes

SchemaFormat	str(object='') -> str
SchemaRef
SchemaProvenance
ResolvedSchema
SchemaDocument
SchemaResolverRegistry

Functions

normalize_schema(→ pyarrow.Schema | SchemaRef | None)
resolve_schema(→ ResolvedSchema | None)
schema_from_options(→ pyarrow.Schema)
_field_from_options(→ pyarrow.Field)
_format_from_url(→ SchemaFormat)
_decode_arrow_json(→ pyarrow.Schema)
_decode_arrow_ipc(→ pyarrow.Schema)
_decode_json_schema(→ pyarrow.Schema)
_resolve_fsspec_document(→ SchemaDocument)
_resolve_confluent_document(→ SchemaDocument)

Module Contents

class fsspec_data.schema.SchemaFormat

Bases: str, enum.Enum

str(object=’’) -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to ‘strict’.

Initialize self. See help(type(self)) for accurate signature.

ARROW_IPC = 'arrow-ipc-schema'

ARROW_JSON = 'arrow-json'

JSON_SCHEMA = 'json-schema'

class fsspec_data.schema.SchemaRef

url: str

format: SchemaFormat | str | None = None

storage_options: collections.abc.Mapping[str, Any]

__post_init__() → None

class fsspec_data.schema.SchemaProvenance

source: str

format: str

provider: str

identifier: str | None = None

version: str | None = None

class fsspec_data.schema.ResolvedSchema

schema: pyarrow.Schema

provenance: SchemaProvenance

class fsspec_data.schema.SchemaDocument

data: bytes

format: SchemaFormat

provenance: SchemaProvenance

type fsspec_data.schema.SchemaInput = pa.Schema | SchemaRef | str | Mapping[str, Any] | Sequence[Mapping[str, Any]] | None

type fsspec_data.schema.SchemaDecoder = Callable[[bytes], pa.Schema]

type fsspec_data.schema.SchemaProvider = Callable[[SchemaRef], SchemaDocument]

class fsspec_data.schema.SchemaResolverRegistry

_decoders: dict[SchemaFormat, SchemaDecoder]

_providers: dict[str, SchemaProvider]

register(format: SchemaFormat | str, decoder: SchemaDecoder) → None

register_provider(protocol: str, provider: SchemaProvider) → None

resolve(reference: SchemaRef) → ResolvedSchema

fsspec_data.schema.normalize_schema(value: SchemaInput) → pyarrow.Schema | SchemaRef | None

fsspec_data.schema.resolve_schema(value: pyarrow.Schema | SchemaRef | None, registry: SchemaResolverRegistry | None = None) → ResolvedSchema | None

fsspec_data.schema.schema_from_options(value: collections.abc.Mapping[str, Any] | collections.abc.Sequence[collections.abc.Mapping[str, Any]]) → pyarrow.Schema

fsspec_data.schema._field_from_options(value: collections.abc.Mapping[str, Any]) → pyarrow.Field

fsspec_data.schema._format_from_url(url: str) → SchemaFormat

fsspec_data.schema._decode_arrow_json(data: bytes) → pyarrow.Schema

fsspec_data.schema._decode_arrow_ipc(data: bytes) → pyarrow.Schema

fsspec_data.schema._decode_json_schema(data: bytes) → pyarrow.Schema

fsspec_data.schema._resolve_fsspec_document(reference: SchemaRef) → SchemaDocument

fsspec_data.schema._resolve_confluent_document(reference: SchemaRef) → SchemaDocument

fsspec_data.schema.DEFAULT_SCHEMA_RESOLVERS