From c730b30d08424c5a2181b497492cf9cca0ac8930 Mon Sep 17 00:00:00 2001 From: mxmlnkn Date: Sat, 12 Oct 2024 13:07:32 +0200 Subject: [PATCH] fixup! [wip] Add tests and documentation for IPFS, TODO: SMB, DropBox, Hf --- core/ratarmountcore/FSSpecMountSource.py | 26 ++++++-- core/ratarmountcore/factory.py | 38 +++++++++++ tests/requirements-tests.txt | 2 + tests/runtests.sh | 83 +++++++++++++++++++++++- 4 files changed, 140 insertions(+), 9 deletions(-) diff --git a/core/ratarmountcore/FSSpecMountSource.py b/core/ratarmountcore/FSSpecMountSource.py index f5b62736..3fa7be0f 100644 --- a/core/ratarmountcore/FSSpecMountSource.py +++ b/core/ratarmountcore/FSSpecMountSource.py @@ -17,6 +17,11 @@ except ImportError: fsspec = None # type: ignore +try: + from webdav4.fsspec import WebdavFileSystem +except ImportError: + WebdavFileSystem = None # type: ignore + class FSSpecMountSource(MountSource): """ @@ -71,12 +76,16 @@ def __init__(self, urlOrFS, prefix: Optional[str] = None, **options) -> None: # The fsspec filesystems are not uniform! http:// expects the arguments to isdir with prefixed # protocol while other filesystem implementations are fine with only the path. - # https://github.com/ray-project/ray/issues/26423#issuecomment-1179561181 - self._isHTTP = isinstance(self.fileSystem, fsspec.implementations.http.HTTPFileSystem) + # - https://github.com/ray-project/ray/issues/26423#issuecomment-1179561181 + # - https://github.com/fsspec/filesystem_spec/issues/1713 + # - https://github.com/skshetry/webdav4/issues/198 + self._pathsRequireQuoting = isinstance( + self.fileSystem, (fsspec.implementations.http.HTTPFileSystem, WebdavFileSystem) + ) self.prefix = prefix.rstrip("/") if prefix and prefix.strip("/") and self.fileSystem.isdir(prefix) else "" def _getPath(self, path: str) -> str: - if self._isHTTP: + if self._pathsRequireQuoting: path = urllib.parse.quote(path) if self.prefix: if not path or path == "/": @@ -110,9 +119,10 @@ def _convertToFileInfo(entry, path) -> FileInfo: # They kinda work only like hardlinks. # https://github.com/fsspec/filesystem_spec/issues/1679 # https://github.com/fsspec/filesystem_spec/issues/1680 + size = entry.get('size', 0) return FileInfo( # fmt: off - size = entry.get('size', 0), + size = size if size else 0, mtime = FSSpecMountSource._getModificationTime(entry), mode = FSSpecMountSource._getMode(entry), linkname = "", @@ -181,10 +191,14 @@ def _listDir(self, path: str, onlyMode: bool) -> Optional[Union[Iterable[str], D ) for entry in result } - if self._isHTTP: + + # For HTTPFileSystem, we need to filter out the entries for sorting. + # For WebDAV we do not even need to unquote! We get unquoted file names with ls! + if isinstance(self.fileSystem, fsspec.implementations.http.HTTPFileSystem): return { urllib.parse.unquote(name): info for name, info in result.items() if not name.startswith(('?', '#')) } + return result @overrides(MountSource) @@ -214,7 +228,7 @@ def _getFileInfoHTTP(self, path: str) -> Optional[FileInfo]: @overrides(MountSource) def getFileInfo(self, path: str, fileVersion: int = 0) -> Optional[FileInfo]: - if self._isHTTP: + if isinstance(self.fileSystem, fsspec.implementations.http.HTTPFileSystem): return self._getFileInfoHTTP(path) path = self._getPath(path) diff --git a/core/ratarmountcore/factory.py b/core/ratarmountcore/factory.py index 5e44bab9..0fd75a11 100644 --- a/core/ratarmountcore/factory.py +++ b/core/ratarmountcore/factory.py @@ -4,7 +4,9 @@ # pylint: disable=no-member,abstract-method # Disable pylint errors. See https://github.com/fsspec/filesystem_spec/issues/1678 +import http import os +import re import stat import sys import traceback @@ -255,6 +257,42 @@ def tryOpenURL(url, printDebug: int) -> Union[MountSource, IO[bytes], str]: with warnings.catch_warnings(): warnings.simplefilter("ignore") fileSystem, path = fsspec.url_to_fs(url) + elif protocol == 'webdav': + # This needs special handling because we need to decide between HTTP and HTTPS and because of: + # https://github.com/skshetry/webdav4/issues/197 + try: + from webdav4.fsspec import WebdavFileSystem + except ImportError: + raise RatarmountError("Install the webdav4 Python package to mount webdav://.") + + matchedURI = re.match("(?:([^:/]*):([^@/]*)@)?([^/]*)(.*)", splitURI[1]) + if not matchedURI: + raise RatarmountError( + "Failed to match WebDAV URI of the format webdav://[user:password@]host[:port]/path\n" + "If your user name or password contains special characters such as ':/@', then use the environment " + "variables WEBDAV_USER and WEBDAV_PASSWORD to specify them." + ) + username, password, baseURL, path = matchedURI.groups() + if path is None: + path = "" + if username is None and 'WEBDAV_USER' in os.environ: + username = os.environ.get('WEBDAV_USER') + if password is None and 'WEBDAV_PASSWORD' in os.environ: + password = os.environ.get('WEBDAV_PASSWORD') + auth = None if username is None or password is None else (username, password) + + def checkForHTTPS(url): + try: + connection = http.client.HTTPSConnection(url, timeout=2) + connection.request("HEAD", "/") + return bool(connection.getresponse()) + except Exception as exception: + if printDebug >= 3: + print("[Info] Determined WebDAV URL to not use HTTP instead HTTPS because of:", exception) + return False + + transportProtocol = "https" if checkForHTTPS(baseURL) else "http" + fileSystem = WebdavFileSystem(f"{transportProtocol}://{baseURL}", auth=auth) else: fileSystem, path = fsspec.url_to_fs(url) diff --git a/tests/requirements-tests.txt b/tests/requirements-tests.txt index ecff4da6..dca3718a 100644 --- a/tests/requirements-tests.txt +++ b/tests/requirements-tests.txt @@ -26,3 +26,5 @@ pyminizip pyopenssl>=23 rangehttpserver boto3 +wsgidav +cheroot diff --git a/tests/runtests.sh b/tests/runtests.sh index 4ced2a02..2a0e086d 100755 --- a/tests/runtests.sh +++ b/tests/runtests.sh @@ -2115,7 +2115,7 @@ checkURLProtocolS3() weedFolder=$( mktemp -d --suffix .test.ratarmount ) TMP_FILES_TO_CLEANUP+=( "$weedFolder" ) - ./weed server -dir="$weedFolder" -s3 -s3.port "$port" -idleTimeout=30 -ip 127.0.0.1 2>weed.log & + ./weed server -dir="$weedFolder" -s3 -s3.port "$port" -idleTimeout=30 -ip 127.0.0.1 & pid=$! # Wait for port to open @@ -2209,6 +2209,13 @@ checkURLProtocolSamba() checkURLProtocolIPFS() { + # TODO ipfsspec still fails to import with Python 3.14 + # https://github.com/eigenein/protobuf/issues/177 + python3MinorVersion=$( python3 -c 'import sys; print(sys.version_info.minor)' ) + if [[ -n "$python3MinorVersion" && "$python3MinorVersion" -ge 14 ]]; then + return 0 + fi + # Using impacket/examples/smbserver.py does not work for a multidude of reasons. # Therefore set up a server with tests/install-smbd.sh from outside and check for its existence here. local ipfs @@ -2222,9 +2229,11 @@ checkURLProtocolIPFS() ipfs=kubo/ipfs fi + local pid= $ipfs init --profile server if ! pgrep ipfs; then $ipfs daemon & + pid=$! sleep 5 fi @@ -2240,6 +2249,75 @@ checkURLProtocolIPFS() d3b07384d113edec49eaa6238ad5ff00 || returnError "$LINENO" 'Failed to read from IPFS' checkFileInTARForeground "ipfs://QmZwm9gKZaayGWqYtMgj6cd4JaNK1Yp2ChYZhXrERGq4Gi" single-file.tar \ 1a28538854d1884e4415cb9bfb7a2ad8 || returnError "$LINENO" 'Failed to read from IPFS' + + if [[ -n "$pid" ]]; then kill "$pid"; fi +} + + +checkURLProtocolWebDAV() +{ + if ! pip show wsgidav &>/dev/null; then + echoerr "Skipping WebDAV test because wsigdav package is not installed." + return 0 + fi + + local port=8047 + # BEWARE OF LOOP MOUNTS when testing locally! + # It will time out, when trying to expose PWD via WebDAV while mounting into PWD/mounted. + wsgidav --host=127.0.0.1 --port=8047 --root="$PWD" --auth=anonymous & + local pid=$! + sleep 5 + + checkFileInTARForeground "webdav://127.0.0.1:$port/single-file.tar" bar \ + d3b07384d113edec49eaa6238ad5ff00 || returnError "$LINENO" 'Failed to read from WebDAV server' + checkFileInTARForeground "webdav://127.0.0.1:$port" single-file.tar \ + 1a28538854d1884e4415cb9bfb7a2ad8 || returnError "$LINENO" 'Failed to read from WebDAV server' + + kill "$pid" + + local user password + user='pqvfumqbqp' + password='ioweb123GUIweb' + +cat < wsgidav-config.yaml +http_authenticator: + domain_controller: null # Same as wsgidav.dc.simple_dc.SimpleDomainController + accept_basic: true # Pass false to prevent sending clear text passwords + accept_digest: true + default_to_digest: true + +simple_dc: + user_mapping: + "*": + "$user": + password: "$password" +EOF + + wsgidav --host=127.0.0.1 --port=8047 --root="$PWD" --config=wsgidav-config.yaml & + pid=$! + sleep 5 + + checkFileInTARForeground "webdav://$user:$password@127.0.0.1:$port/single-file.tar" bar \ + d3b07384d113edec49eaa6238ad5ff00 || returnError "$LINENO" 'Failed to read from WebDAV server' + checkFileInTARForeground "webdav://$user:$password@127.0.0.1:$port" single-file.tar \ + 1a28538854d1884e4415cb9bfb7a2ad8 || returnError "$LINENO" 'Failed to read from WebDAV server' + + export WEBDAV_USER=$user + export WEBDAV_PASSWORD=$password + checkFileInTARForeground "webdav://127.0.0.1:$port/single-file.tar" bar \ + d3b07384d113edec49eaa6238ad5ff00 || returnError "$LINENO" 'Failed to read from WebDAV server' + checkFileInTARForeground "webdav://127.0.0.1:$port" single-file.tar \ + 1a28538854d1884e4415cb9bfb7a2ad8 || returnError "$LINENO" 'Failed to read from WebDAV server' + unset WEBDAV_USER + unset WEBDAV_PASSWORD + + # This server using SSL also works, but do not overload it with regular tests. + # ratarmount 'webdav://www.dlp-test.com\WebDAV:WebDAV@www.dlp-test.com/webdav' mounted + # checkFileInTARForeground "webdav://www.dlp-test.com\WebDAV:WebDAV@www.dlp-test.com/webdav" \ + # mounted/WebDAV_README.txt 87d13914fe24e486be943cb6b1f4e224 || + # returnError "$LINENO" 'Failed to read from WebDAV server' + + kill "$pid" } @@ -2259,8 +2337,7 @@ checkRemoteSupport() checkURLProtocolSSH || returnError 'Failed ssh:// check' checkURLProtocolSamba || returnError 'Failed smb:// check' - # TODO Add and test IPFS - # TODO look for other fsspec implementations in an automated manner + checkURLProtocolWebDAV || returnError 'Failed webdav:// check' }