Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .copier-answers.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Changes here will be overwritten by Copier
_commit: 7d58f0e
_commit: '4372075'
_src_path: https://github.com/python-project-templates/base.git
add_docs: false
add_extension: python
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/build.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ on:
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

permissions:
Expand Down
12 changes: 6 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,16 @@ lint-py: ## lint python with ruff
python -m ruff format --check fsspec_python

lint-docs: ## lint docs with mdformat and codespell
python -m mdformat --check README.md
python -m codespell_lib README.md
python -m mdformat --check README.md $(wildcard docs/src)
python -m codespell_lib README.md $(wildcard docs/src)

fix-py: ## autoformat python code with ruff
python -m ruff check --fix fsspec_python
python -m ruff format fsspec_python

fix-docs: ## autoformat docs with mdformat and codespell
python -m mdformat README.md
python -m codespell_lib --write README.md
python -m mdformat README.md $(wildcard docs/src)
python -m codespell_lib --write README.md $(wildcard docs/src)

lint: lint-py lint-docs ## run all linters
lints: lint
Expand All @@ -50,9 +50,9 @@ check-dist: ## check python sdist and wheel with check-dist
check-dist -v

check-types: ## check python types with ty
ty check --python $$(which python)
ty check --python "$$(python -c 'import sys; print(sys.executable)')"

checks: check-dist
checks: check-dist check-types

# Alias
check: checks
Expand Down
7 changes: 4 additions & 3 deletions fsspec_python/fs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import inspect
from typing import ClassVar

from fsspec import filesystem, register_implementation
from fsspec.implementations.chained import ChainedFileSystem
Expand All @@ -11,7 +12,7 @@
class PythonFileSystem(ChainedFileSystem):
"""Python import filesystem"""

protocol: str = "python"
protocol: ClassVar[str] = "python"
root: str = "/"

def __init__(self, target_protocol=None, target_options=None, fs=None, install: bool = True, **kwargs):
Expand All @@ -29,11 +30,11 @@ def __init__(self, target_protocol=None, target_options=None, fs=None, install:
raise ValueError("Both filesystems (fs) and target_protocol may not be both given.")

target_options = target_options or {}
self.fs = fs if fs is not None else filesystem(target_protocol, **target_options)
self.target_protocol = (
target_protocol if isinstance(target_protocol, str) else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
target_protocol if isinstance(target_protocol, str) else (self.fs.protocol if isinstance(self.fs.protocol, str) else self.fs.protocol[0])
)

self.fs = fs if fs is not None else filesystem(target_protocol, **target_options)
if isinstance(self.fs, ChainedFileSystem):
self.root = "/"
kwargs.pop("fo", None)
Expand Down
11 changes: 5 additions & 6 deletions fsspec_python/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from importlib.machinery import SOURCE_SUFFIXES, ModuleSpec
from os.path import join
from types import ModuleType
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any

from fsspec import url_to_fs
from fsspec.implementations.local import AbstractFileSystem
Expand All @@ -28,7 +28,7 @@
class FSSpecImportFinder(MetaPathFinder):
def __init__(self, fs: PythonFileSystem) -> None:
self.fs: PythonFileSystem = fs
self.remote_modules: dict[str, str] = {}
self.remote_modules: dict[str, ModuleSpec] = {}

def find_spec(self, fullname: str, path: Sequence[str | bytes] | None, target: ModuleType | None = None) -> ModuleSpec | None:
for suffix in SOURCE_SUFFIXES:
Expand Down Expand Up @@ -76,7 +76,7 @@ def get_data(self, path: str | bytes) -> bytes:
# source = source_bytes.decode("utf-8")


def install_importer(fs: str | AbstractFileSystem, **kwargs: str) -> FSSpecImportFinder:
def install_importer(fs: str | AbstractFileSystem, **kwargs: Any) -> PythonFileSystem:
"""Install the fsspec importer."""
if isinstance(fs, AbstractFileSystem):
fsspec_str = normalize_fsspec(fs=fs, **kwargs)
Expand Down Expand Up @@ -106,9 +106,8 @@ def uninstall_importer(fs: str | AbstractFileSystem = "") -> None:
fsspec_str = normalize_fsspec(fs=fs) if isinstance(fs, AbstractFileSystem) else fs

if fsspec_str in _finders:
finder = _finders.pop(fsspec_str, None)
if finder:
finder.unload()
finder = _finders.pop(fsspec_str)
finder.unload()
if finder in sys.meta_path:
sys.meta_path.remove(finder)
return
Expand Down
4 changes: 2 additions & 2 deletions fsspec_python/open.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
_original_open = builtins.open

# Patch fsspec's local open to avoid recursion
fsspec.implementations.local.open = _original_open
vars(fsspec.implementations.local)["open"] = _original_open


def install_open_hook(fsspec: str, **fsspec_args):
Expand All @@ -24,7 +24,7 @@ def open_from_fsspec(file, mode="r", buffering=-1, encoding=None, errors=None, n
return fsspec_fs.open(filename, mode=mode, encoding=encoding, errors=errors, newline=newline)
return _original_open(file, mode, buffering, encoding, errors, newline, closefd, opener)

builtins.open = open_from_fsspec
vars(builtins)["open"] = open_from_fsspec
globals()["open"] = open_from_fsspec


Expand Down
2 changes: 1 addition & 1 deletion fsspec_python/tests/test_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

class TestFs:
def test_fs_import(self, fs_importer):
import my_local_file2
import my_local_file2 # ty: ignore[unresolved-import] # Module provided by the filesystem import fixture.

assert my_local_file2.baz() == "This is a local file."

Expand Down
8 changes: 4 additions & 4 deletions fsspec_python/tests/test_remote_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,21 @@
class TestRemoteImport:
@pytest.mark.skipif(not os.environ.get("FSSPEC_S3_ENDPOINT_URL"), reason="S3 not configured")
def test_importer_s3(self, s3_importer):
import my_remote_file
import my_remote_file # ty: ignore[unresolved-import] # Module provided by the filesystem import fixture.

assert my_remote_file.foo() == "This is a remote file."

def test_importer_local(self, local_importer):
import my_local_file
import my_local_file # ty: ignore[unresolved-import] # Module provided by the filesystem import fixture.

assert my_local_file.bar() == "This is a local file."

def test_importer_local_multi(self, local_importer_multi):
import fsspec_python.importer

assert len(fsspec_python.importer._finders) == 2
import my_local_file
import my_local_file2
import my_local_file # ty: ignore[unresolved-import] # Module provided by the filesystem import fixture.
import my_local_file2 # ty: ignore[unresolved-import] # Module provided by the filesystem import fixture.

assert my_local_file.bar() == "This is a local file."
assert my_local_file2.baz() == "This is a local file."
2 changes: 1 addition & 1 deletion fsspec_python/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


def normalize_fsspec(
fs: AbstractFileSystem = None,
fs: AbstractFileSystem | None = None,
target_protocol: str | None = None,
target_options: dict | None = None,
fo: str = "",
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ develop = [
"codespell",
"hatchling",
"mdformat",
"mdformat-frontmatter",
"mdformat-tables>=1",
"pytest",
"pytest-cov",
Expand Down
Loading