Merge tag 'v1.125.0'

Release 1.125.0

.gitignore

@@ -18,6 +18,9 @@ python/.eggs
 __pycache__
 python/src/deltachat/capi*.so
 python/.venv/
+python/venv/
+venv/
+env/
 
 python/liveconfig*
 

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## [1.125.0] - 2023-10-14
+
+### API-Changes
+
+- [**breaking**] deltachat-rpc-client: Replace `asyncio` with threads.
+- Validate boolean values passed to `set_config`. Attempts to set values other than `0` and `1` will result in an error.
+
+### CI
+
+- Reduce required Python version for deltachat-rpc-client from 3.8 to 3.7.
+
+### Features / Changes
+
+- Add developer option to disable IDLE.
+
+### Fixes
+
+- `deltachat-rpc-client`: Run `deltachat-rpc-server` in its own process group. This prevents reception of `SIGINT` by the server when the bot is terminated with `^C`.
+- python: Don't automatically set the displayname to "bot" when setting log level.
+- Don't update `timestamp`, `timestamp_rcvd`, `state` when replacing partially downloaded message ([#4700](https://github.com/deltachat/deltachat-core-rust/pull/4700)).
+- Assign encrypted partially downloaded group messages to 1:1 chat ([#4757](https://github.com/deltachat/deltachat-core-rust/pull/4757)).
+- Return all contacts from `Contact::get_all` for bots ([#4811](https://github.com/deltachat/deltachat-core-rust/pull/4811)).
+- Set connectivity status to "connected" during fake idle.
+- Return verifier contacts regardless of their origin.
+- Don't try to send more MDNs if there's a temporary SMTP error ([#4534](https://github.com/deltachat/deltachat-core-rust/pull/4534)).
+
+### Refactor
+
+- deltachat-rpc-client: Close stdin instead of sending `SIGTERM`.
+- deltachat-rpc-client: Remove print() calls. Standard `logging` package is for logging instead.
+
+### Tests
+
+- deltachat-rpc-client: Enable logs in pytest.
+
 ## [1.124.1] - 2023-10-05
 
 ### Fixes
@@ -2879,3 +2914,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0
 [1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0
 [1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1
+[1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.124.1"
+version = "1.125.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.67"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.124.1"
+version = "1.125.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -492,6 +492,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
+ *                    0=use IMAP IDLE if the server supports it.
+ *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
  *                    For larger messages, only the header is downloaded and a placeholder is shown.
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
@@ -500,6 +503,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
+ * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
+ *                    seconds. 2 days by default.
+ *                    This is not supposed to be changed by UIs and only used for testing.
  * - `verified_one_on_one_chats` = Feature flag for verified 1:1 chats; the UI should set it
  *                    to 1 if it supports verified 1:1 chats.
  *                    Regardless of this setting, `dc_chat_is_protected()` returns true while the key is verified,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.124.1"
+version = "1.125.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/typescript/package.json

@@ -55,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.124.1"
+  "version": "1.125.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.124.1"
+version = "1.125.0"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-rpc-client/pyproject.toml

@@ -7,7 +7,6 @@ name = "deltachat-rpc-client"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
-    "Framework :: AsyncIO",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
     "Operating System :: POSIX :: Linux",
@@ -72,3 +71,6 @@ line-length = 120
 
 [tool.isort]
 profile = "black"
+
+[tool.pytest.ini_options]
+log_cli = true

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.124.1"
+version = "1.125.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.124.1"
+  "version": "1.125.0"
 }

python/src/deltachat/account.py

@@ -617,18 +617,18 @@ def set_location(self, latitude: float = 0.0, longitude: float = 0.0, accuracy:
     # meta API for start/stop and event based processing
     #
 
-    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
-        from .events import FFIEventLogger
-
+    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False, displayname=None):
         """get the account running, configure it if necessary. add plugins if provided.
 
         :param addr: the email address of the account
         :param password: the password of the account
         :param account_plugins: a list of plugins to add
         :param show_ffi: show low level ffi events
+        :param displayname: the display name of the account
         """
+        from .events import FFIEventLogger
+
         if show_ffi:
-            self.set_config("displayname", "bot")
             log = FFIEventLogger(self)
             self.add_account_plugin(log)
 
@@ -644,6 +644,8 @@ def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=F
             configtracker = self.configure()
             configtracker.wait_finish()
 
+        if displayname:
+            self.set_config("displayname", displayname)
         # start IO threads and configure if necessary
         self.start_io()
 

python/tests/test_0_complex_or_slow.py

@@ -542,8 +542,6 @@ def test_see_new_verified_member_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.text == msg_out.text
     assert msg_in.get_sender_contact().addr == ac2_addr
 
-    ac1.set_config("bcc_self", "0")
-
 
 def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     """Another test for the bug #3836:
@@ -592,4 +590,67 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
     assert msg_in.text == msg_out.text
 
-    ac2.set_config("bcc_self", "0")
+
+def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
+    """Test for the issue #4346:
+    - User is added to a verified group.
+    - First device of the user downloads "member added" from the group.
+    - First device removes "member added" from the server.
+    - Some new messages are sent to the group.
+    - Second device comes online, receives these new messages. The result is a verified group with unverified members.
+    - First device re-gossips Autocrypt keys to the group.
+    - Now the seconds device has all members verified.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2)
+    for ac in [ac2, ac2_offl]:
+        ac.set_config("bcc_self", "1")
+    ac2.set_config("delete_server_after", "1")
+    ac2.set_config("gossip_period", "0")  # Re-gossip in every message
+    acfactory.bring_accounts_online()
+    dir = tmp_path / "exportdir"
+    dir.mkdir()
+    ac2.export_self_keys(str(dir))
+    ac2_offl.import_self_keys(str(dir))
+    ac2_offl.stop_io()
+
+    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
+    chat1 = ac1.create_group_chat("hello", verified=True)
+    assert chat1.is_protected()
+    qr = chat1.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    chat2 = ac2.qr_join_chat(qr)
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+    # Wait for "Member Me (<addr>) added by <addr>." message.
+    msg_in = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in.is_system_message()
+
+    lp.sec("ac2: waiting for 'member added' to be deleted on the server")
+    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+
+    lp.sec("ac1: sending 'hi' to the group")
+    ac2.set_config("delete_server_after", "0")
+    chat1.send_text("hi")
+
+    lp.sec("ac2_offl: going online, checking the 'hi' message")
+    ac2_offl.start_io()
+    msg_in = ac2_offl._evtracker.wait_next_incoming_message()
+    assert not msg_in.is_system_message()
+    assert msg_in.text.startswith("[Sender of this message is not verified:")
+    ac2_offl_ac1_contact = msg_in.get_sender_contact()
+    assert ac2_offl_ac1_contact.addr == ac1.get_config("addr")
+    assert not ac2_offl_ac1_contact.is_verified()
+    chat2_offl = msg_in.chat
+    assert chat2_offl.is_protected()
+
+    lp.sec("ac2: sending message re-gossiping Autocrypt keys")
+    chat2.send_text("hi2")
+
+    lp.sec("ac2_offl: receiving message")
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    msg_in = ac2_offl.get_message_by_id(ev.data2)
+    assert not msg_in.is_system_message()
+    assert msg_in.text == "hi2"
+    assert msg_in.chat == chat2_offl
+    assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
+    assert ac2_offl_ac1_contact.is_verified()

python/tests/test_1_online.py

@@ -1718,12 +1718,10 @@ def test_qr_new_group_unblocked(acfactory, lp):
 
     ac1_new_chat = ac1.create_group_chat("Another group")
     ac1_new_chat.add_contact(ac2)
-    ac1_new_chat.send_text("Hello!")
-
     # Receive "Member added" message.
     ac2._evtracker.wait_next_incoming_message()
 
-    # Receive "Hello!" message.
+    ac1_new_chat.send_text("Hello!")
     ac2_msg = ac2._evtracker.wait_next_incoming_message()
     assert ac2_msg.text == "Hello!"
     assert ac2_msg.chat.is_contact_request()
@@ -1946,13 +1944,15 @@ def test_set_get_group_image(acfactory, data, lp):
     lp.sec("ac1: add ac2 to promoted group chat")
     chat.add_contact(ac2)  # sends one message
 
+    lp.sec("ac2: wait for receiving member added message from ac1")
+    msg1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg1.is_system_message()  # Member added
+
     lp.sec("ac1: send a first message to ac2")
     chat.send_text("hi")  # sends another message
     assert chat.is_promoted()
 
     lp.sec("ac2: wait for receiving message from ac1")
-    msg1 = ac2._evtracker.wait_next_incoming_message()
-    assert msg1.is_system_message()  # Member added
     msg2 = ac2._evtracker.wait_next_incoming_message()
     assert msg2.text == "hi"
     assert msg1.chat.id == msg2.chat.id

release-date.in

@@ -1 +1 @@
-2023-10-05
+2023-10-14

scripts/README.md

@@ -18,7 +18,7 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `make-python-testenv.sh` creates or updates local python test development environment.
+- `make-python-testenv.sh` creates local python test development environment.
   Reusing the same environment is faster than running `run-python-test.sh` which always
   recreates environment from scratch and runs additional lints.
 

scripts/make-python-testenv.sh

@@ -4,18 +4,14 @@
 # It rebuilds the core and bindings as needed.
 #
 # After running the script, you can either
-# run `pytest` directly with `env/bin/pytest python/`
-# or activate the environment with `. env/bin/activacte`
+# run `pytest` directly with `venv/bin/pytest python/`
+# or activate the environment with `. venv/bin/activate`
 # and run `pytest` from there.
 set -euo pipefail
 
 export DCC_RS_TARGET=debug
 export DCC_RS_DEV="$PWD"
 cargo build -p deltachat_ffi --features jsonrpc
 
-if test -d env; then
-	env/bin/pip install -e python --force-reinstall
-else
-	tox -e py --devenv env
-	env/bin/pip install --upgrade pip
-fi
+tox -c python -e py --devenv venv
+env/bin/pip install --upgrade pip