fixup! [wip][feature] Add support for fsspec backends

.github/workflows/tests.yml

@@ -35,7 +35,8 @@ jobs:
     - name: Lint With Codespell
       run: |
         python3 -m pip install codespell
-        codespell --ignore-words-list fo,Nd,unx $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
+        # fsspec uses cachable instead of cacheable ...
+        codespell --ignore-words-list fo,Nd,unx,cachable $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
 
     - name: Lint With Flake8
       run: |
@@ -230,6 +231,7 @@ jobs:
         python3 -m pip install -r tests/requirements-tests.txt
         # Explicitly install pygit2 even on Python 3.13+ because we have set up libgit2 manually.
         python3 -m pip install pygit2
+        python3 -c 'import pygit2'
 
     - name: Unit Tests
       if: false

core/pyproject.toml

@@ -85,6 +85,7 @@ full = [
     # pygit2 1.15 introduced many breaking changes!
     # https://github.com/libgit2/pygit2/issues/1316
     # https://github.com/fsspec/filesystem_spec/pull/1703
+    # build error in Python 3.13 because it requires libgit2 1.8.1 and there are no wheels
     "pygit2<1.15",
     "fsspec",
     "s3fs",
@@ -106,7 +107,8 @@ fsspec = [
     # pygit2 1.15 introduced many breaking changes!
     # https://github.com/libgit2/pygit2/issues/1316
     # https://github.com/fsspec/filesystem_spec/pull/1703
-    "pygit2<1.15",  # build error in Python 3.13 because it requires libgit2 1.8.1
+    # build error in Python 3.13 because it requires libgit2 1.8.1 and there are no wheels
+    "pygit2<1.15",
     "fsspec",
     "s3fs",
     "gcsfs",

core/ratarmountcore/factory.py

@@ -34,7 +34,7 @@
 try:
     from sshfs import SSHFileSystem
 except ImportError:
-    SSHFileSystem= None  # type: ignore
+    SSHFileSystem = None  # type: ignore
 
 
 def _openRarMountSource(fileOrPath: Union[str, IO[bytes]], **options) -> Optional[MountSource]:
@@ -144,12 +144,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[
             print("[Info] Try to open with fsspec")
 
         # Suppress warning about (default!) encoding not being support for Python<3.9 -.-.
-        if sys.version_info < (3,9) and protocol == 'ftp':
+        if sys.version_info < (3, 9) and protocol == 'ftp':
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore")
                 openFile = fsspec.open(url)
         elif protocol in FixedSSHFileSystem.protocols:
-            fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url))
+            fs = FixedSSHFileSystem(**FixedSSHFileSystem._get_kwargs_from_urls(url))  # pytype: disable=attribute-error
 
             # Remove one leading / in order to add support for relative paths. E.g.:
             #   ssh://127.0.0.1/relative/path
@@ -185,10 +185,12 @@ def openFsspec(url, options, printDebug: int) -> Optional[Union[MountSource, IO[
 
         # Avoid resource leaks, e.g., when the seek check fails.
         oldDel = getattr(result, '__del__', None)
+
         def newDel():
             if callable(oldDel):
                 oldDel()
             result.close()
+
         result.__del__ = newDel
 
         # Check that seeking works. May fail when, e.g., the HTTP server does not support range requests.
@@ -224,7 +226,7 @@ def openMountSource(fileOrPath: Union[str, IO[bytes]], **options) -> MountSource
         result = openFsspec(fileOrPath, options, printDebug=printDebug)
         if isinstance(result, MountSource):
             return result
-        if isinstance(result, str) or not result is None:
+        if isinstance(result, str) or result is not None:
             fileOrPath = result
             if not isinstance(fileOrPath, str) and printDebug >= 3:
                 print("[Info] Opened remote file with fsspec.")

tests/.pylintrc

@@ -5,7 +5,7 @@ init-hook='import sys; sys.path.append("./core")'
 # run arbitrary code.
 extension-pkg-whitelist=indexed_gzip,indexed_bzip2,indexed_zstd,libarchive,libarchive.ffi,lzmaffi,rapidgzip,isal,
                         PySquashfsImage,PySquashfsImage.compressor,zstandard,lz4,deflate,pyminizip,fast_zip_decryption,
-                        asyncssh,sshfs
+                        asyncssh,sshfs,fsspec
 
 # Specify a score threshold to be exceeded before program exits with error.
 fail-under=10.0

tests/runtests.sh

@@ -1846,7 +1846,7 @@ checkURLProtocolFTP()
 killRogueSSH()
 {
     local pid
-    for pid in $( ps au | grep -e '(start-asyncssh-server|ssh://)' | grep -v grep | awk '{ print $2; }' ); do
+    for pid in $( pgrep start-asyncssh-server ) $( pgrep -f ssh:// ); do
         kill "$pid"
         sleep 1
         kill -9 "$pid"
@@ -2148,7 +2148,8 @@ if [[ -z "$CI" ]]; then
     while read -r file; do
         filesToSpellCheck+=( "$file" )
     done < <( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)' )
-    codespell --ignore-words-list fo,Nd,unx "${filesToSpellCheck[@]}"
+    # fsspec uses cachable instead of cacheable ...
+    codespell --ignore-words-list fo,Nd,unx,cachable "${filesToSpellCheck[@]}"
 
     flake8 --config tests/.flake8 "${files[@]}" "${testFiles[@]}" || returnError "$LINENO" 'Flake8 failed!'
 

tests/start-asyncssh-server.py

@@ -6,6 +6,7 @@
 # for pid in $( ps aux | grep start-asyncssh-server | grep -v grep | awk '{ print $2; }' ); do
 #     kill "$pid"; sleep 0.1; kill -9 "$pid"; done
 
+
 async def start_server():
     await asyncssh.listen(
         "127.0.0.1",