From bd6caf835dfa7470f6926a96790f92cb27f8099e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Sep 2024 01:30:49 +0200 Subject: [PATCH] add support for rclone:// repositories (via borgstore) --- docs/quickstart.rst | 1 + docs/usage/general/file-systems.rst.inc | 2 +- docs/usage/general/repository-urls.rst.inc | 9 +++++++-- pyproject.toml | 2 +- src/borg/archiver/_common.py | 2 +- src/borg/helpers/parseformat.py | 15 ++++++++++++++- src/borg/testsuite/helpers.py | 8 ++++++++ 7 files changed, 33 insertions(+), 6 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 918f059cbd..47c493fc08 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -411,6 +411,7 @@ Due to using the `borgstore` project, borg now also supports other kinds of - sftp: the borg client will directly talk to an sftp server. This does not require borg being installed on the sftp server. +- rclone: the borg client will talk via rclone to cloud storage. - Others may come in the future, adding backends to `borgstore` is rather simple. Restoring a backup diff --git a/docs/usage/general/file-systems.rst.inc b/docs/usage/general/file-systems.rst.inc index d53eb96bf9..342c1ee53a 100644 --- a/docs/usage/general/file-systems.rst.inc +++ b/docs/usage/general/file-systems.rst.inc @@ -24,7 +24,7 @@ Pros: and re-write segment files to free space. - In future, easier to adapt to other kinds of storage: borgstore's backends are quite simple to implement. - A ``sftp:`` backend already exists, cloud storage might be easy to add. + ``sftp:`` and ``rclone:`` backends already exists, others might be easy to add. - Parallel repository access with less locking is easier to implement. Cons: diff --git a/docs/usage/general/repository-urls.rst.inc b/docs/usage/general/repository-urls.rst.inc index db2a1e19a3..058047d6b4 100644 --- a/docs/usage/general/repository-urls.rst.inc +++ b/docs/usage/general/repository-urls.rst.inc @@ -14,7 +14,7 @@ Note: you may also prepend a ``file://`` to a filesystem path to get URL style. **Remote repositories** accessed via ssh user@host: -``ssh://user@host:port/path/to/repo`` - absolute path` +``ssh://user@host:port/path/to/repo`` - absolute path ``ssh://user@host:port/./path/to/repo`` - path relative to current directory @@ -22,10 +22,15 @@ Note: you may also prepend a ``file://`` to a filesystem path to get URL style. **Remote repositories** accessed via sftp: -``sftp://user@host:port/path/to/repo`` - absolute path` +``sftp://user@host:port/path/to/repo`` - absolute path For ssh and sftp URLs, the ``user@`` and ``:port`` parts are optional. +**Remote repositories** accessed via rclone: + +``rclone://remote:path`` + + If you frequently need the same repo URL, it is a good idea to set the ``BORG_REPO`` environment variable to set a default for the repo URL: diff --git a/pyproject.toml b/pyproject.toml index 2f6f146974..fcadcc01f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ classifiers = [ ] license = {text="BSD"} dependencies = [ - "borgstore ~= 0.0.1", + "borgstore ~= 0.0.4", "msgpack >=1.0.3, <=1.1.0", "packaging", "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0, diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index f8195b6218..3b9453b126 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -46,7 +46,7 @@ def get_repository( args=args, ) - elif location.proto in ("sftp", "file") and not v1_or_v2: # stuff directly supported by borgstore + elif location.proto in ("sftp", "file", "rclone") and not v1_or_v2: # stuff directly supported by borgstore repository = Repository( location, create=create, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index ee30d0eb83..171e1462dc 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -467,6 +467,14 @@ class Location: re.VERBOSE, ) # path + rclone_re = re.compile( + r""" + (?Prclone):// # rclone:// + (?P(.*)) + """, + re.VERBOSE, + ) # path + socket_re = re.compile( r""" (?Psocket):// # socket:// @@ -546,6 +554,11 @@ def normpath_special(p): self.port = m.group("port") and int(m.group("port")) or None self.path = normpath_special(m.group("path")) return True + m = self.rclone_re.match(text) + if m: + self.proto = m.group("proto") + self.path = m.group("path") + return True m = self.file_re.match(text) if m: self.proto = m.group("proto") @@ -575,7 +588,7 @@ def __str__(self): def to_key_filename(self): name = re.sub(r"[^\w]", "_", self.path).strip("_") - if self.proto not in ("file", "socket"): + if self.proto not in ("file", "socket", "rclone"): name = re.sub(r"[^\w]", "_", self.host) + "__" + name if len(name) > 100: # Limit file names to some reasonable length. Most file systems diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 4f29ce7481..e7ce9eea5d 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -187,6 +187,14 @@ def test_ssh(self, monkeypatch, keys_dir): "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='/some/path')" ) + def test_rclone(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("rclone://remote:path")) + == "Location(proto='rclone', user=None, host=None, port=None, path='remote:path')" + ) + assert Location("rclone://remote:path").to_key_filename() == keys_dir + "remote_path" + def test_sftp(self, monkeypatch, keys_dir): monkeypatch.delenv("BORG_REPO", raising=False) assert (