fsspec_data
===========

.. py:module:: fsspec_data


Submodules
----------

.. toctree::
   :maxdepth: 1

   /autoapi/fsspec_data/_version/index
   /autoapi/fsspec_data/backends/index
   /autoapi/fsspec_data/confluent/index
   /autoapi/fsspec_data/converters/index
   /autoapi/fsspec_data/filesystem/index
   /autoapi/fsspec_data/interchange/index
   /autoapi/fsspec_data/json_schema/index
   /autoapi/fsspec_data/schema/index
   /autoapi/fsspec_data/tests/index
   /autoapi/fsspec_data/xarray_zarr/index


Attributes
----------

.. autoapisummary::

   fsspec_data.CONVERTER_ENTRY_POINT_GROUP
   fsspec_data.DEFAULT_CONVERTERS
   fsspec_data.DEFAULT_REGISTRY
   fsspec_data.DEFAULT_SCHEMA_RESOLVERS
   fsspec_data.__version__


Classes
-------

.. autoapisummary::

   fsspec_data.Converter
   fsspec_data.ConverterRegistry
   fsspec_data.DataFileSystem
   fsspec_data.Codec
   fsspec_data.CodecCapabilities
   fsspec_data.CodecRegistry
   fsspec_data.DataFormat
   fsspec_data.DecodedBatches
   fsspec_data.DecodedBatchStream
   fsspec_data.FieldMapping
   fsspec_data.InterchangePlan
   fsspec_data.InterchangeRequest
   fsspec_data.PlannedBatchStream
   fsspec_data.SchemaPolicy
   fsspec_data.ResolvedSchema
   fsspec_data.SchemaDocument
   fsspec_data.SchemaFormat
   fsspec_data.SchemaProvenance
   fsspec_data.SchemaRef
   fsspec_data.SchemaResolverRegistry


Functions
---------

.. autoapisummary::

   fsspec_data.plan_schema
   fsspec_data.json_schema_to_arrow
   fsspec_data.resolve_schema


Package Contents
----------------

.. py:data:: CONVERTER_ENTRY_POINT_GROUP
   :value: 'fsspec_data.converters'


.. py:data:: DEFAULT_CONVERTERS

.. py:class:: Converter

   .. py:attribute:: source_type
      :type:  str


   .. py:attribute:: target_type
      :type:  str


   .. py:attribute:: handler
      :type:  ConverterHandler


   .. py:method:: __post_init__() -> None


   .. py:method:: convert(source: Any, target: Any | None = None, *, source_options: collections.abc.Mapping[str, Any] | None = None, target_options: collections.abc.Mapping[str, Any] | None = None, conversion_options: collections.abc.Mapping[str, Any] | None = None) -> Any


.. py:class:: ConverterRegistry(*, entry_point_group: str = CONVERTER_ENTRY_POINT_GROUP, discover_entry_points: bool = True)

   .. py:attribute:: entry_point_group
      :value: 'fsspec_data.converters'



   .. py:attribute:: discover_entry_points
      :value: True



   .. py:attribute:: _converters
      :type:  dict[tuple[str, str], Converter]


   .. py:attribute:: _entry_points_loaded
      :value: False



   .. py:method:: register(converter: Converter, *, replace: bool = False) -> None


   .. py:method:: load_entry_points() -> None


   .. py:method:: get(source_type: str, target_type: str) -> Converter


   .. py:method:: convert(source_type: str, target_type: str, source: Any, target: Any | None = None, *, source_options: collections.abc.Mapping[str, Any] | None = None, target_options: collections.abc.Mapping[str, Any] | None = None, conversion_options: collections.abc.Mapping[str, Any] | None = None) -> Any


.. py:class:: DataFileSystem(fo: str, target_protocol: str | None = None, target_options: dict[str, Any] | None = None, fs: fsspec.AbstractFileSystem | None = None, provided_format: fsspec_data.interchange.DataFormat | str | None = None, requested_format: fsspec_data.interchange.DataFormat | str | None = None, provided_schema: fsspec_data.schema.SchemaInput = None, requested_schema: fsspec_data.schema.SchemaInput = None, schema_policy: fsspec_data.interchange.SchemaPolicy | str = SchemaPolicy.EXACT, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None, spool_max_size: int = 8 * 1024 * 1024, **kwargs: Any)

   Bases: :py:obj:`fsspec.implementations.chained.ChainedFileSystem`


   Read-only format and schema conversion layered over another filesystem.

   Create and configure file-system instance

   Instances may be cachable, so if similar enough arguments are seen
   a new instance is not required. The token attribute exists to allow
   implementations to cache instances if they wish.

   A reasonable default should be provided if there are no arguments.

   Subclasses should call this method.

   :param use_listings_cache: passed to ``DirCache``, if the implementation supports
                              directory listing caching. Pass use_listings_cache=False
                              to disable such caching.
   :param listings_expiry_time: passed to ``DirCache``, if the implementation supports
                                directory listing caching. Pass use_listings_cache=False
                                to disable such caching.
   :param max_paths: passed to ``DirCache``, if the implementation supports
                     directory listing caching. Pass use_listings_cache=False
                     to disable such caching.
   :param skip_instance_cache: If this is a cachable implementation, pass True here to force
                               creating a new instance even if a matching instance exists, and prevent
                               storing this instance.
   :type skip_instance_cache: bool
   :param asynchronous:
   :type asynchronous: bool
   :param loop:
   :type loop: asyncio-compatible IOLoop or None


   .. py:attribute:: protocol
      :value: 'fsspec-data'



   .. py:attribute:: fs
      :value: None



   .. py:attribute:: fo


   .. py:attribute:: provided_format


   .. py:attribute:: requested_format


   .. py:attribute:: provided_schema
      :value: None



   .. py:attribute:: requested_schema
      :value: None



   .. py:attribute:: provided_schema_provenance
      :type:  fsspec_data.schema.SchemaProvenance | None
      :value: None



   .. py:attribute:: requested_schema_provenance
      :type:  fsspec_data.schema.SchemaProvenance | None
      :value: None



   .. py:attribute:: schema_policy


   .. py:attribute:: batch_size
      :value: 1024



   .. py:attribute:: row_limit
      :value: None



   .. py:attribute:: byte_limit
      :value: None



   .. py:attribute:: spool_max_size
      :value: 8388608



   .. py:attribute:: _sizes
      :type:  dict[str, int]


   .. py:method:: _open(path: str, mode: str = 'rb', block_size: int | None = None, autocommit: bool = True, cache_options: dict[str, Any] | None = None, **kwargs: Any)

      Return raw bytes-mode file-like from the file-system



   .. py:method:: info(path: str, **kwargs: Any) -> dict[str, Any]

      Give details of entry at path

      Returns a single dictionary, with exactly the same information as ``ls``
      would with ``detail=True``.

      The default implementation calls ls and could be overridden by a
      shortcut. kwargs are passed on to ```ls()``.

      Some file systems might not be able to measure the file's size, in
      which case, the returned dict will include ``'size': None``.

      :returns: * **dict with keys** (*name (full path in the FS), size (in bytes), type (file,*)
                * *directory, or something else) and other FS-specific keys.*



   .. py:method:: ls(path: str, detail: bool = True, **kwargs: Any)

      List objects at path.

      This should include subdirectories and files at that location. The
      difference between a file and a directory must be clear when details
      are requested.

      The specific keys, or perhaps a FileInfo class, or similar, is TBD,
      but must be consistent across implementations.
      Must include:

      - full path to the entry (without protocol)
      - size of the entry, in bytes. If the value cannot be determined, will
        be ``None``.
      - type of entry, "file", "directory" or other

      Additional information
      may be present, appropriate to the file-system, e.g., generation,
      checksum, etc.

      May use refresh=True|False to allow use of self._ls_from_cache to
      check for a saved listing and avoid calling the backend. This would be
      common where listing may be expensive.

      :param path:
      :type path: str
      :param detail: if True, gives a list of dictionaries, where each is the same as
                     the result of ``info(path)``. If False, gives a list of paths
                     (str).
      :type detail: bool
      :param kwargs: information
      :type kwargs: may have additional backend-specific options, such as version

      :returns: * *List of strings if detail is False, or list of directory information*
                * *dicts if detail is True.*



   .. py:method:: _convert(path: str, output) -> None


.. py:data:: DEFAULT_REGISTRY

.. py:class:: Codec

   .. py:attribute:: format
      :type:  DataFormat


   .. py:attribute:: capabilities
      :type:  CodecCapabilities


   .. py:method:: encode_batches(batches: list[pyarrow.RecordBatch] | tuple[pyarrow.RecordBatch, Ellipsis], *, schema: pyarrow.Schema | None = None) -> bytes


   .. py:method:: encode_batches_to(batches: collections.abc.Iterable[pyarrow.RecordBatch], output: BinaryIO, *, schema: pyarrow.Schema | None = None) -> None


   .. py:method:: decode_batches(data: bytes | bytearray | memoryview | BinaryIO, *, schema: pyarrow.Schema | None = None, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) -> DecodedBatches


   .. py:method:: iter_batches(data: bytes | bytearray | memoryview | BinaryIO, *, schema: pyarrow.Schema | None = None, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) -> DecodedBatchStream


.. py:class:: CodecCapabilities

   .. py:attribute:: encode
      :type:  bool


   .. py:attribute:: decode
      :type:  bool


   .. py:attribute:: streaming
      :type:  bool


.. py:class:: CodecRegistry

   .. py:method:: get(format: DataFormat | str) -> Codec


.. py:class:: DataFormat

   Bases: :py:obj:`str`, :py:obj:`enum.Enum`


   str(object='') -> str
   str(bytes_or_buffer[, encoding[, errors]]) -> str

   Create a new string object from the given object. If encoding or
   errors is specified, then the object must expose a data buffer
   that will be decoded using the given encoding and error handler.
   Otherwise, returns the result of object.__str__() (if defined)
   or repr(object).
   encoding defaults to sys.getdefaultencoding().
   errors defaults to 'strict'.

   Initialize self.  See help(type(self)) for accurate signature.


   .. py:attribute:: ARROW
      :value: 'arrow'



   .. py:attribute:: PARQUET
      :value: 'parquet'



   .. py:attribute:: CSV
      :value: 'csv'



   .. py:attribute:: JSONL
      :value: 'jsonl'



.. py:class:: DecodedBatches

   .. py:attribute:: schema
      :type:  pyarrow.Schema


   .. py:attribute:: batches
      :type:  tuple[pyarrow.RecordBatch, Ellipsis]


.. py:class:: DecodedBatchStream(schema: pyarrow.Schema, native: Any)

   .. py:attribute:: schema


   .. py:attribute:: _native


   .. py:method:: __iter__() -> DecodedBatchStream


   .. py:method:: __next__() -> pyarrow.RecordBatch


   .. py:method:: cancel() -> None


   .. py:method:: collect() -> DecodedBatches


.. py:class:: FieldMapping

   .. py:attribute:: source_index
      :type:  int


   .. py:attribute:: target_index
      :type:  int


   .. py:attribute:: cast
      :type:  str | None
      :value: None



   .. py:attribute:: check_nulls
      :type:  bool
      :value: False



.. py:class:: InterchangePlan

   .. py:attribute:: provided_schema
      :type:  pyarrow.Schema


   .. py:attribute:: requested_schema
      :type:  pyarrow.Schema


   .. py:attribute:: policy
      :type:  SchemaPolicy


   .. py:attribute:: mappings
      :type:  tuple[FieldMapping, Ellipsis]


   .. py:attribute:: provided_format
      :type:  DataFormat


   .. py:attribute:: requested_format
      :type:  DataFormat


   .. py:method:: apply_table(table: pyarrow.Table) -> pyarrow.Table


   .. py:method:: apply_batch(batch: pyarrow.RecordBatch) -> pyarrow.RecordBatch


   .. py:method:: _apply_mapping(array: pyarrow.Array | pyarrow.ChunkedArray, mapping: FieldMapping)


   .. py:method:: _validate_input_schema(schema: pyarrow.Schema) -> None


   .. py:method:: iter_batches(data: bytes | bytearray | memoryview | BinaryIO, *, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) -> PlannedBatchStream


   .. py:method:: convert(data: bytes | bytearray | memoryview | BinaryIO, *, batch_size: int = 1024, row_limit: int | None = None, byte_limit: int | None = None) -> bytes


.. py:class:: InterchangeRequest

   .. py:attribute:: provided_format
      :type:  DataFormat


   .. py:attribute:: requested_format
      :type:  DataFormat


   .. py:attribute:: provided_schema
      :type:  pyarrow.Schema


   .. py:attribute:: requested_schema
      :type:  pyarrow.Schema


   .. py:attribute:: policy
      :type:  SchemaPolicy


   .. py:method:: plan() -> InterchangePlan


.. py:class:: PlannedBatchStream(plan: InterchangePlan, source: DecodedBatchStream)

   .. py:attribute:: schema


   .. py:attribute:: _plan


   .. py:attribute:: _source


   .. py:method:: __iter__() -> PlannedBatchStream


   .. py:method:: __next__() -> pyarrow.RecordBatch


   .. py:method:: cancel() -> None


.. py:class:: SchemaPolicy

   Bases: :py:obj:`str`, :py:obj:`enum.Enum`


   str(object='') -> str
   str(bytes_or_buffer[, encoding[, errors]]) -> str

   Create a new string object from the given object. If encoding or
   errors is specified, then the object must expose a data buffer
   that will be decoded using the given encoding and error handler.
   Otherwise, returns the result of object.__str__() (if defined)
   or repr(object).
   encoding defaults to sys.getdefaultencoding().
   errors defaults to 'strict'.

   Initialize self.  See help(type(self)) for accurate signature.


   .. py:attribute:: EXACT
      :value: 'exact'



   .. py:attribute:: PROJECTION
      :value: 'projection'



   .. py:attribute:: COMPATIBLE
      :value: 'compatible'



   .. py:attribute:: COERCE
      :value: 'coerce'



.. py:function:: plan_schema(provided_schema: pyarrow.Schema, requested_schema: pyarrow.Schema, policy: SchemaPolicy | str) -> InterchangePlan

.. py:function:: json_schema_to_arrow(value: collections.abc.Mapping[str, Any]) -> pyarrow.Schema

.. py:data:: DEFAULT_SCHEMA_RESOLVERS

.. py:class:: ResolvedSchema

   .. py:attribute:: schema
      :type:  pyarrow.Schema


   .. py:attribute:: provenance
      :type:  SchemaProvenance


.. py:class:: SchemaDocument

   .. py:attribute:: data
      :type:  bytes


   .. py:attribute:: format
      :type:  SchemaFormat


   .. py:attribute:: provenance
      :type:  SchemaProvenance


.. py:class:: SchemaFormat

   Bases: :py:obj:`str`, :py:obj:`enum.Enum`


   str(object='') -> str
   str(bytes_or_buffer[, encoding[, errors]]) -> str

   Create a new string object from the given object. If encoding or
   errors is specified, then the object must expose a data buffer
   that will be decoded using the given encoding and error handler.
   Otherwise, returns the result of object.__str__() (if defined)
   or repr(object).
   encoding defaults to sys.getdefaultencoding().
   errors defaults to 'strict'.

   Initialize self.  See help(type(self)) for accurate signature.


   .. py:attribute:: ARROW_IPC
      :value: 'arrow-ipc-schema'



   .. py:attribute:: ARROW_JSON
      :value: 'arrow-json'



   .. py:attribute:: JSON_SCHEMA
      :value: 'json-schema'



.. py:class:: SchemaProvenance

   .. py:attribute:: source
      :type:  str


   .. py:attribute:: format
      :type:  str


   .. py:attribute:: provider
      :type:  str


   .. py:attribute:: identifier
      :type:  str | None
      :value: None



   .. py:attribute:: version
      :type:  str | None
      :value: None



.. py:class:: SchemaRef

   .. py:attribute:: url
      :type:  str


   .. py:attribute:: format
      :type:  SchemaFormat | str | None
      :value: None



   .. py:attribute:: storage_options
      :type:  collections.abc.Mapping[str, Any]


   .. py:method:: __post_init__() -> None


.. py:class:: SchemaResolverRegistry

   .. py:attribute:: _decoders
      :type:  dict[SchemaFormat, SchemaDecoder]


   .. py:attribute:: _providers
      :type:  dict[str, SchemaProvider]


   .. py:method:: register(format: SchemaFormat | str, decoder: SchemaDecoder) -> None


   .. py:method:: register_provider(protocol: str, provider: SchemaProvider) -> None


   .. py:method:: resolve(reference: SchemaRef) -> ResolvedSchema


.. py:function:: resolve_schema(value: pyarrow.Schema | SchemaRef | None, registry: SchemaResolverRegistry | None = None) -> ResolvedSchema | None

.. py:data:: __version__
   :value: '0.2.3'