Install aiodns and brotli in Docker image to speed up aiohttp

[skip ci]

.github/workflows/python-lint.yml

@@ -6,16 +6,17 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: "3.13"
     - uses: isort/isort-action@master
       with:
         sortPaths: "./maubot"
     - uses: psf/black@stable
       with:
         src: "./maubot"
+        version: "24.10.0"
     - name: pre-commit
       run: |
         pip install pre-commit

.gitignore

@@ -13,6 +13,7 @@ __pycache__
 !example-config.yaml
 !.pre-commit-config.yaml
 
+/start
 logs/
 plugins/
 trash/

.gitlab-ci.yml

@@ -10,7 +10,7 @@ default:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
 build frontend:
-  image: node:18-alpine
+  image: node:24-alpine
   stage: build frontend
   before_script: []
   variables:

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         exclude_types: [markdown]
@@ -8,13 +8,13 @@ repos:
       - id: check-yaml
       - id: check-added-large-files
   - repo: https://github.com/psf/black
-    rev: 23.1.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
         files: ^maubot/.*\.pyi?$
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         files: ^maubot/.*\.pyi?$

CHANGELOG.md

@@ -1,3 +1,52 @@
+# v0.5.2 (2025-05-05)
+
+* Improved tombstone handling to ensure that the tombstone sender has
+  permissions to invite users to the target room.
+* Fixed autojoin and online flags not being applied if set during client
+  creation (thanks to [@bnsh] in [#258]).
+* Fixed plugin web apps not being cleared properly when unloading plugins.
+
+[@bnsh]: https://github.com/bnsh
+[#258]: https://github.com/maubot/maubot/pull/258
+
+# v0.5.1 (2025-01-03)
+
+* Updated Docker image to Alpine 3.21.
+* Updated media upload/download endpoints in management frontend
+  (thanks to [@domrim] in [#253]).
+* Fixed plugin web app base path not including a trailing slash
+  (thanks to [@jkhsjdhjs] in [#240]).
+* Changed markdown parsing to cut off plaintext body if necessary to allow
+  longer formatted messages.
+* Updated dependencies to fix Python 3.13 compatibility.
+
+[@domrim]: https://github.com/domrim
+[@jkhsjdhjs]: https://github.com/jkhsjdhjs
+[#253]: https://github.com/maubot/maubot/pull/253
+[#240]: https://github.com/maubot/maubot/pull/240
+
+# v0.5.0 (2024-08-24)
+
+* Dropped Python 3.9 support.
+* Updated Docker image to Alpine 3.20.
+* Updated mautrix-python to 0.20.6 to support authenticated media.
+* Removed hard dependency on SQLAlchemy.
+* Fixed `main_class` to default to being loaded from the last module instead of
+  the first if a module name is not explicitly specified.
+  * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html),
+    and loading from the first module doesn't make sense due to import order.
+* Added simple scheduler utility for running background tasks periodically or
+  after a certain delay.
+* Added testing framework for plugins (thanks to [@abompard] in [#225]).
+* Changed `mbc build` to ignore directories declared in `modules` that are
+  missing an `__init__.py` file.
+  * Importing the modules at runtime would fail and break the plugin.
+    To include non-code resources outside modules in the mbp archive,
+    use `extra_files` instead.
+
+[#225]: https://github.com/maubot/maubot/issues/225
+[@abompard]: https://github.com/abompard
+
 # v0.4.2 (2023-09-20)
 
 * Updated Pillow to 10.0.1.

Dockerfile

@@ -1,9 +1,9 @@
-FROM node:18 AS frontend-builder
+FROM node:24 AS frontend-builder
 
 COPY ./maubot/management/frontend /frontend
 RUN cd /frontend && yarn --prod && yarn build
 
-FROM alpine:3.18
+FROM alpine:3.22
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -11,7 +11,8 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
+        py3-aiodns \
+        py3-brotli \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -34,20 +35,19 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
         py3-dateutil \
         py3-lxml \
-        py3-semver \
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
+        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
 
 COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery tzlocal \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

Dockerfile.ci

@@ -1,4 +1,4 @@
-FROM alpine:3.18
+FROM alpine:3.22
 
 RUN apk add --no-cache \
         python3 py3-pip py3-setuptools py3-wheel \
@@ -6,7 +6,6 @@ RUN apk add --no-cache \
         su-exec \
         yq \
         py3-aiohttp \
-        py3-sqlalchemy \
         py3-attrs \
         py3-bcrypt \
         py3-cffi \
@@ -30,11 +29,10 @@ RUN apk add --no-cache \
         py3-unpaddedbase64 \
         py3-future \
         # plugin deps
-        #py3-pillow \
+        py3-pillow \
         py3-magic \
         py3-feedparser \
-        py3-lxml \
-  && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community
+        py3-lxml
 #        py3-gitlab
 #        py3-semver
 # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies
@@ -43,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt
 COPY optional-requirements.txt /opt/maubot/optional-requirements.txt
 WORKDIR /opt/maubot
 RUN apk add --virtual .build-deps python3-dev build-base git \
-    && pip3 install -r requirements.txt -r optional-requirements.txt \
+    && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
         dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \
     && apk del .build-deps
 # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies

README.md

@@ -22,7 +22,7 @@ All setup and usage instructions are located on
 Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net)
 
 ## Plugins
-A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/).
+A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/).
 
 To add your plugin to the list, send a pull request to <https://github.com/maubot/plugins.maubot.xyz>.
 

dev-requirements.txt

@@ -1,3 +1,3 @@
 pre-commit>=2.10.1,<3
 isort>=5.10.1,<6
-black>=23,<24
+black>=24,<25

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.4.2"
+__version__ = "0.5.2"

maubot/cli/commands/build.py

@@ -93,10 +93,16 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None:
             if os.path.isfile(f"{module}.py"):
                 zip.write(f"{module}.py")
             elif module is not None and os.path.isdir(module):
-                zipdir(zip, module)
+                if os.path.isfile(f"{module}/__init__.py"):
+                    zipdir(zip, module)
+                else:
+                    print(
+                        Fore.YELLOW
+                        + f"Module {module} is missing __init__.py, skipping"
+                        + Fore.RESET
+                    )
             else:
                 print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET)
-
         for pattern in meta.extra_files:
             for file in glob.iglob(pattern):
                 zip.write(file)

maubot/client.py

@@ -350,11 +350,41 @@ class Client(DBClient):
         }
 
     async def _handle_tombstone(self, evt: StateEvent) -> None:
+        if evt.state_key != "":
+            return
         if not evt.content.replacement_room:
             self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring")
             return
+        is_joined = await self.client.state_store.is_joined(
+            evt.content.replacement_room,
+            self.client.mxid,
+        )
+        if is_joined:
+            self.log.debug(
+                f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} "
+                f"sent by {evt.sender}: already joined to replacement room"
+            )
+            return
+        self.log.debug(
+            f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} "
+            f"sent by {evt.sender}"
+        )
         _, server = self.client.parse_user_id(evt.sender)
-        await self.client.join_room(evt.content.replacement_room, servers=[server])
+        room_id = await self.client.join_room(evt.content.replacement_room, servers=[server])
+        power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS)
+        create_event = await self.client.get_state_event(
+            room_id, EventType.ROOM_CREATE, format="event"
+        )
+        if power_levels.get_user_level(evt.sender, create_event) < power_levels.invite:
+            self.log.warning(
+                f"{evt.room_id} was tombstoned into {room_id} by {evt.sender},"
+                " but the sender doesn't have invite power levels, leaving..."
+            )
+            await self.client.leave_room(
+                room_id,
+                f"Followed tombstone from {evt.room_id} by {evt.sender},"
+                " but sender doesn't have sufficient power level for invites",
+            )
 
     async def _handle_invite(self, evt: StrippedStateEvent) -> None:
         if evt.state_key == self.id and evt.content.membership == Membership.INVITE:
@@ -474,8 +504,14 @@ class Client(DBClient):
         self.start_sync()
 
     async def _update_remote_profile(self) -> None:
-        profile = await self.client.get_profile(self.id)
-        self.remote_displayname, self.remote_avatar_url = profile.displayname, profile.avatar_url
+        try:
+            profile = await self.client.get_profile(self.id)
+            self.remote_displayname, self.remote_avatar_url = (
+                profile.displayname,
+                profile.avatar_url,
+            )
+        except Exception:
+            self.log.warning("Failed to update own profile from server", exc_info=True)
 
     async def delete(self) -> None:
         try:

maubot/example-config.yaml

@@ -1,5 +1,4 @@
 # The full URI to the database. SQLite and Postgres are fully supported.
-# Other DBMSes supported by SQLAlchemy may or may not work.
 # Format examples:
 #   SQLite:   sqlite:filename.db
 #   Postgres: postgresql://username:password@hostname/dbname
@@ -55,7 +54,8 @@ server:
     port: 29316
     # Public base URL where the server is visible.
     public_url: https://example.com
-    # The base path for the UI.
+    # The base path for the UI. Note that this does not change the API path.
+    # Add a path prefix to public_url if you want everything on a subpath.
     ui_base_path: /_matrix/maubot
     # The base path for plugin endpoints. The instance ID will be appended directly.
     plugin_base_path: /_matrix/maubot/plugin/
@@ -79,8 +79,9 @@ homeservers:
         # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will.
         secret: null
 
-# List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password
-# to prevent normal login. Root is a special user that can't have a password and will always exist.
+# List of administrator users. Each key is a username and the value is the password.
+# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login.
+# Root is a special user that can't have a password and will always exist.
 admins:
     root: ""
 

maubot/instance.py

@@ -25,7 +25,6 @@ import os.path
 
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
-import sqlalchemy as sql
 
 from mautrix.types import UserID
 from mautrix.util import background_task
@@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger
 
 from .client import Client
 from .db import DatabaseEngine, Instance as DBInstance
+from .lib.optionalalchemy import Engine, MetaData, create_engine
 from .lib.plugin_db import ProxyPostgresDatabase
 from .loader import DatabaseType, PluginLoader, ZippedPluginLoader
 from .plugin_base import Plugin
@@ -128,7 +128,7 @@ class PluginInstance(DBInstance):
         }
 
     def _introspect_sqlalchemy(self) -> dict:
-        metadata = sql.MetaData()
+        metadata = MetaData()
         metadata.reflect(self.inst_db)
         return {
             table.name: {
@@ -214,7 +214,7 @@ class PluginInstance(DBInstance):
 
     async def get_db_tables(self) -> dict:
         if self.inst_db_tables is None:
-            if isinstance(self.inst_db, sql.engine.Engine):
+            if isinstance(self.inst_db, Engine):
                 self.inst_db_tables = self._introspect_sqlalchemy()
             elif self.inst_db.scheme == Scheme.SQLITE:
                 self.inst_db_tables = await self._introspect_sqlite()
@@ -294,7 +294,7 @@ class PluginInstance(DBInstance):
                     "Instance database engine is marked as Postgres, but plugin uses legacy "
                     "database interface, which doesn't support postgres."
                 )
-            self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}")
+            self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}")
         elif self.loader.meta.database_type == DatabaseType.ASYNCPG:
             if self.database_engine is None:
                 if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db:
@@ -329,7 +329,7 @@ class PluginInstance(DBInstance):
     async def stop_database(self) -> None:
         if isinstance(self.inst_db, Database):
             await self.inst_db.stop()
-        elif isinstance(self.inst_db, sql.engine.Engine):
+        elif isinstance(self.inst_db, Engine):
             self.inst_db.dispose()
         else:
             raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}")

maubot/lib/optionalalchemy.py

@@ -0,0 +1,19 @@
+try:
+    from sqlalchemy import MetaData, asc, create_engine, desc
+    from sqlalchemy.engine import Engine
+    from sqlalchemy.exc import IntegrityError, OperationalError
+except ImportError:
+
+    class FakeError(Exception):
+        pass
+
+    class FakeType:
+        def __init__(self, *args, **kwargs):
+            raise Exception("SQLAlchemy is not installed")
+
+    def create_engine(*args, **kwargs):
+        raise Exception("SQLAlchemy is not installed")
+
+    MetaData = Engine = FakeType
+    IntegrityError = OperationalError = FakeError
+    asc = desc = lambda a: a

maubot/loader/zip.py

@@ -31,7 +31,7 @@ from ..config import Config
 from ..lib.zipimport import ZipImportError, zipimporter
 from ..plugin_base import Plugin
 from .abc import IDConflictError, PluginClass, PluginLoader
-from .meta import PluginMeta
+from .meta import DatabaseType, PluginMeta
 
 current_version = Version(__version__)
 yaml = YAML()
@@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader):
         return file, meta
 
     @classmethod
-    def verify_meta(cls, source) -> tuple[str, Version]:
+    def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]:
         _, meta = cls._read_meta(source)
-        return meta.id, meta.version
+        return meta.id, meta.version, meta.database_type if meta.database else None
 
     def _load_meta(self) -> None:
         file, meta = self._read_meta(self.path)
@@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader):
         if "/" in meta.main_class:
             self.main_module, self.main_class = meta.main_class.split("/")[:2]
         else:
-            self.main_module = meta.modules[0]
+            self.main_module = meta.modules[-1]
             self.main_class = meta.main_class
         self._file = file
 

maubot/management/api/client.py

@@ -78,8 +78,8 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response:
     )
     client.enabled = data.get("enabled", True)
     client.sync = data.get("sync", True)
-    client.autojoin = data.get("autojoin", True)
-    client.online = data.get("online", True)
+    await client.update_autojoin(data.get("autojoin", True), save=False)
+    await client.update_online(data.get("online", True), save=False)
     client.displayname = data.get("displayname", "disable")
     client.avatar_url = data.get("avatar_url", "disable")
     await client.update()

maubot/management/api/instance_database.py

@@ -19,12 +19,12 @@ from datetime import datetime
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy import asc, desc, engine, exc
 import aiosqlite
 
 from mautrix.util.async_db import Database
 
 from ...instance import PluginInstance
+from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc
 from .base import routes
 from .responses import resp
 
@@ -56,15 +56,17 @@ async def get_table(request: web.Request) -> web.Response:
     try:
         order = [tuple(order.split(":")) for order in request.query.getall("order")]
         order = [
-            (asc if sort.lower() == "asc" else desc)(table.columns[column])
-            if sort
-            else table.columns[column]
+            (
+                (asc if sort.lower() == "asc" else desc)(table.columns[column])
+                if sort
+                else table.columns[column]
+            )
             for column, sort in order
         ]
     except KeyError:
         order = []
     limit = int(request.query.get("limit", "100"))
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit))
 
 
@@ -82,7 +84,7 @@ async def query(request: web.Request) -> web.Response:
     except KeyError:
         return resp.query_missing
     rows_as_dict = data.get("rows_as_dict", False)
-    if isinstance(instance.inst_db, engine.Engine):
+    if isinstance(instance.inst_db, Engine):
         return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict)
     elif isinstance(instance.inst_db, Database):
         try:
@@ -131,12 +133,12 @@ async def _execute_query_asyncpg(
 def _execute_query_sqlalchemy(
     instance: PluginInstance, sql_query: str, rows_as_dict: bool = False
 ) -> web.Response:
-    assert isinstance(instance.inst_db, engine.Engine)
+    assert isinstance(instance.inst_db, Engine)
     try:
         res = instance.inst_db.execute(sql_query)
-    except exc.IntegrityError as e:
+    except IntegrityError as e:
         return resp.sql_integrity_error(e, sql_query)
-    except exc.OperationalError as e:
+    except OperationalError as e:
         return resp.sql_operational_error(e, sql_query)
     data = {
         "ok": True,

maubot/management/api/plugin_upload.py

@@ -23,10 +23,17 @@ import traceback
 from aiohttp import web
 from packaging.version import Version
 
-from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader
+from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader
 from .base import get_config, routes
 from .responses import resp
 
+try:
+    import sqlalchemy
+
+    has_alchemy = True
+except ImportError:
+    has_alchemy = False
+
 log = logging.getLogger("maubot.server.upload")
 
 
@@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     if pid != plugin_id:
         return resp.pid_mismatch
     plugin = PluginLoader.id_cache.get(plugin_id, None)
@@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response:
     content = await request.read()
     file = BytesIO(content)
     try:
-        pid, version = ZippedPluginLoader.verify_meta(file)
+        pid, version, db_type = ZippedPluginLoader.verify_meta(file)
     except MaubotZipImportError as e:
         return resp.plugin_import_error(str(e), traceback.format_exc())
+    if db_type == DatabaseType.SQLALCHEMY and not has_alchemy:
+        return resp.sqlalchemy_not_installed
     plugin = PluginLoader.id_cache.get(pid, None)
     if not plugin:
         return await upload_new_plugin(content, pid, version)

maubot/management/api/responses.py

@@ -15,13 +15,16 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
 from http import HTTPStatus
 
 from aiohttp import web
 from asyncpg import PostgresError
-from sqlalchemy.exc import IntegrityError, OperationalError
 import aiosqlite
 
+if TYPE_CHECKING:
+    from sqlalchemy.exc import IntegrityError, OperationalError
+
 
 class _Response:
     @property
@@ -324,6 +327,16 @@ class _Response:
             }
         )
 
+    @property
+    def sqlalchemy_not_installed(self) -> web.Response:
+        return web.json_response(
+            {
+                "error": "This plugin requires a legacy database, but SQLAlchemy is not installed",
+                "errcode": "unsupported_plugin_database",
+            },
+            status=HTTPStatus.NOT_IMPLEMENTED,
+        )
+
     @property
     def table_not_found(self) -> web.Response:
         return web.json_response(

maubot/management/frontend/src/api.js

@@ -205,7 +205,7 @@ export const getClients = () => defaultGet("/clients")
 export const getClient = id => defaultGet(`/clients/${id}`)
 
 export async function uploadAvatar(id, data, mime) {
-    const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/r0/upload`, {
+    const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, {
         headers: getHeaders(mime),
         body: data,
         method: "POST",
@@ -217,8 +217,9 @@ export function getAvatarURL({ id, avatar_url }) {
     if (!avatar_url?.startsWith("mxc://")) {
         return null
     }
-    avatar_url = avatar_url.substr("mxc://".length)
-    return `${BASE_PATH}/proxy/${id}/_matrix/media/r0/download/${avatar_url}?access_token=${
+    avatar_url = avatar_url.substring("mxc://".length)
+    // Note: the maubot backend will replace the query param with an authorization header
+    return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${
         localStorage.accessToken}`
 }
 

maubot/management/frontend/src/pages/dashboard/Client.js

@@ -70,9 +70,9 @@ class Client extends BaseMainView {
     get initialState() {
         return {
             id: "",
-            displayname: "",
+            displayname: "disable",
             homeserver: "",
-            avatar_url: "",
+            avatar_url: "disable",
             access_token: "",
             device_id: "",
             fingerprint: null,

maubot/matrix.py

@@ -15,7 +15,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from typing import Awaitable
+from typing import Any, Awaitable
 from html import escape
 import asyncio
 
@@ -62,7 +62,10 @@ async def parse_formatted(
         html = message
     else:
         return message, escape(message)
-    return (await MaubotHTMLParser().parse(html)).text, html
+    text = (await MaubotHTMLParser().parse(html)).text
+    if len(text) > 100 and len(text) + len(html) > 40000:
+        text = text[:100] + "[long message cut off]"
+    return text, html
 
 
 class MaubotMessageEvent(MessageEvent):
@@ -85,6 +88,7 @@ class MaubotMessageEvent(MessageEvent):
         reply: bool | str = False,
         in_thread: bool | None = None,
         edits: EventID | MessageEvent | None = None,
+        extra_content: dict[str, Any] | None = None,
     ) -> EventID:
         """
         Respond to the message.
@@ -104,6 +108,7 @@ class MaubotMessageEvent(MessageEvent):
                 the root if necessary.
             edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters
                 are ignored, as edits can't change the reply or thread status.
+            extra_content: Extra content to add to the event.
 
         Returns:
             The ID of the response event.
@@ -140,6 +145,9 @@ class MaubotMessageEvent(MessageEvent):
                 )
             else:
                 content.set_reply(self)
+        if extra_content:
+            for k, v in extra_content.items():
+                content[k] = v
         return await self.client.send_message_event(self.room_id, event_type, content)
 
     def reply(
@@ -149,6 +157,7 @@ class MaubotMessageEvent(MessageEvent):
         markdown: bool = True,
         allow_html: bool = False,
         in_thread: bool | None = None,
+        extra_content: dict[str, Any] | None = None,
     ) -> Awaitable[EventID]:
         """
         Reply to the message. The parameters are the same as :meth:`respond`,
@@ -166,6 +175,7 @@ class MaubotMessageEvent(MessageEvent):
                 thread. If set to ``False``, the response will never be in a thread. If set to
                 ``True``, the response will always be in a thread, creating one with this event as
                 the root if necessary.
+            extra_content: Extra content to add to the event.
 
         Returns:
             The ID of the response event.
@@ -177,6 +187,7 @@ class MaubotMessageEvent(MessageEvent):
             reply=True,
             in_thread=in_thread,
             allow_html=allow_html,
+            extra_content=extra_content,
         )
 
     def mark_read(self) -> Awaitable[None]:
@@ -253,14 +264,18 @@ class MaubotMatrixClient(MatrixClient):
         markdown: str,
         *,
         allow_html: bool = False,
+        render_markdown: bool = True,
         msgtype: MessageType = MessageType.TEXT,
         edits: EventID | MessageEvent | None = None,
         relates_to: RelatesTo | None = None,
+        extra_content: dict[str, Any] = None,
         **kwargs,
     ) -> EventID:
         content = TextMessageEventContent(msgtype=msgtype, format=Format.HTML)
         content.body, content.formatted_body = await parse_formatted(
-            markdown, allow_html=allow_html
+            markdown,
+            allow_html=allow_html,
+            render_markdown=render_markdown,
         )
         if relates_to:
             if edits:
@@ -268,6 +283,9 @@ class MaubotMatrixClient(MatrixClient):
             content.relates_to = relates_to
         elif edits:
             content.set_edit(edits)
+        if extra_content:
+            for k, v in extra_content.items():
+                content[k] = v
         return await self.send_message(room_id, content, **kwargs)
 
     def dispatch_event(self, event: Event, source: SyncStream) -> list[asyncio.Task]:

maubot/plugin_base.py

@@ -20,14 +20,17 @@ from abc import ABC
 from asyncio import AbstractEventLoop
 
 from aiohttp import ClientSession
-from sqlalchemy.engine.base import Engine
 from yarl import URL
 
 from mautrix.util.async_db import Database, UpgradeTable
 from mautrix.util.config import BaseProxyConfig
 from mautrix.util.logging import TraceLogger
 
+from .scheduler import BasicScheduler
+
 if TYPE_CHECKING:
+    from sqlalchemy.engine.base import Engine
+
     from .client import MaubotMatrixClient
     from .loader import BasePluginLoader
     from .plugin_server import PluginWebApp
@@ -40,6 +43,7 @@ class Plugin(ABC):
     log: TraceLogger
     loop: AbstractEventLoop
     loader: BasePluginLoader
+    sched: BasicScheduler
     config: BaseProxyConfig | None
     database: Engine | Database | None
     webapp: PluginWebApp | None
@@ -53,11 +57,12 @@ class Plugin(ABC):
         instance_id: str,
         log: TraceLogger,
         config: BaseProxyConfig | None,
-        database: Engine | None,
+        database: Engine | Database | None,
         webapp: PluginWebApp | None,
         webapp_url: str | None,
         loader: BasePluginLoader,
     ) -> None:
+        self.sched = BasicScheduler(log=log.getChild("scheduler"))
         self.client = client
         self.loop = loop
         self.http = http
@@ -117,6 +122,7 @@ class Plugin(ABC):
             self.client.remove_event_handler(event_type, func)
         if self.webapp is not None:
             self.webapp.clear()
+        self.sched.stop()
         await self.stop()
 
     async def stop(self) -> None:

maubot/plugin_server.py

@@ -40,6 +40,8 @@ class PluginWebApp(web.UrlDispatcher):
         self._resources = []
         self._named_resources = {}
         self._middleware = []
+        self._resource_index = {}
+        self._matched_sub_app_resources = []
 
     async def handle(self, request: web.Request) -> web.StreamResponse:
         match_info = await self.resolve(request)

maubot/scheduler.py

@@ -0,0 +1,159 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2024 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+from typing import Awaitable, Callable
+import asyncio
+import logging
+
+
+class BasicScheduler:
+    background_loop: asyncio.Task | None
+    tasks: set[asyncio.Task]
+    log: logging.Logger
+
+    def __init__(self, log: logging.Logger) -> None:
+        self.log = log
+        self.tasks = set()
+
+    def _find_caller(self) -> str:
+        try:
+            file_name, line_number, function_name, _ = self.log.findCaller()
+            return f"{function_name} at {file_name}:{line_number}"
+        except ValueError:
+            return "unknown function"
+
+    def run_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        run_task_in_background: bool = False,
+        catch_errors: bool = True,
+    ) -> asyncio.Task:
+        """
+        Run a function periodically in the background.
+
+        Args:
+            period: The period in seconds between each call to the function.
+            func: The function to run. No parameters will be provided,
+                use :meth:`functools.partial` if you need to pass parameters.
+            run_task_in_background: If ``True``, the function will be run in a background task.
+                If ``False`` (the default), the loop will wait for the task to return before
+                sleeping for the next period.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors. This parameter has no effect if ``run_task_in_background``
+                is ``True``.
+
+        Returns:
+            The asyncio task object representing the background loop.
+        """
+        task = asyncio.create_task(
+            self._call_periodically(
+                period,
+                func,
+                caller=self._find_caller(),
+                catch_errors=catch_errors,
+                run_task_in_background=run_task_in_background,
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def run_later(
+        self, delay: float | int, coro: Awaitable, catch_errors: bool = True
+    ) -> asyncio.Task:
+        """
+        Run a coroutine after a delay.
+
+        Examples:
+            >>> self.sched.run_later(5, self.async_task(meow=True))
+
+        Args:
+            delay: The delay in seconds to await the coroutine after.
+            coro: The coroutine to await.
+            catch_errors: Whether the scheduler should catch and log any errors.
+                If ``False``, errors will be raised, and the caller must await the returned task
+                to find errors.
+
+        Returns:
+            The asyncio task object representing the scheduled task.
+        """
+        task = asyncio.create_task(
+            self._call_with_delay(
+                delay, coro, caller=self._find_caller(), catch_errors=catch_errors
+            )
+        )
+        self._register_task(task)
+        return task
+
+    def _register_task(self, task: asyncio.Task) -> None:
+        self.tasks.add(task)
+        task.add_done_callback(self.tasks.discard)
+
+    async def _call_periodically(
+        self,
+        period: float | int,
+        func: Callable[[], Awaitable],
+        caller: str,
+        catch_errors: bool,
+        run_task_in_background: bool,
+    ) -> None:
+        while True:
+            try:
+                await asyncio.sleep(period)
+                if run_task_in_background:
+                    self._register_task(
+                        asyncio.create_task(self._call_periodically_background(func(), caller))
+                    )
+                else:
+                    await func()
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                if catch_errors:
+                    self.log.exception(f"Uncaught error in background loop (created in {caller})")
+                else:
+                    raise
+
+    async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None:
+        try:
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            self.log.exception(f"Uncaught error in background loop subtask (created in {caller})")
+
+    async def _call_with_delay(
+        self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool
+    ) -> None:
+        try:
+            await asyncio.sleep(delay)
+            await coro
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            if catch_errors:
+                self.log.exception(f"Uncaught error in scheduled task (created in {caller})")
+            else:
+                raise
+
+    def stop(self) -> None:
+        """
+        Stop all scheduled tasks and background loops.
+        """
+        for task in self.tasks:
+            task.cancel(msg="Scheduler stopped")

maubot/server.py

@@ -64,14 +64,14 @@ class MaubotServer:
             if request.path.startswith(path):
                 request = request.clone(
                     rel_url=request.rel_url.with_path(
-                        request.rel_url.path[len(path) :]
+                        request.rel_url.path[len(path) - 1 :]
                     ).with_query(request.query_string)
                 )
                 return await app.handle(request)
         return web.Response(status=404)
 
     def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]:
-        subpath = self.config["server.plugin_base_path"] + instance_id
+        subpath = self.config["server.plugin_base_path"] + instance_id + "/"
         url = self.config["server.public_url"] + subpath
         try:
             return self.plugin_routes[subpath], url
@@ -82,7 +82,7 @@ class MaubotServer:
 
     def remove_instance_webapp(self, instance_id: str) -> None:
         try:
-            subpath = self.config["server.plugin_base_path"] + instance_id
+            subpath = self.config["server.plugin_base_path"] + instance_id + "/"
             self.plugin_routes.pop(subpath).clear()
         except KeyError:
             return

maubot/standalone/Dockerfile

@@ -1,9 +1,8 @@
-FROM docker.io/alpine:3.18
+FROM docker.io/alpine:3.21
 
 RUN apk add --no-cache \
       python3 py3-pip py3-setuptools py3-wheel \
       py3-aiohttp \
-      py3-sqlalchemy \
       py3-attrs \
       py3-bcrypt \
       py3-cffi \
@@ -26,8 +25,8 @@ RUN cd /opt/maubot \
       python3-dev \
       libffi-dev \
       build-base \
-  && pip3 install -r requirements.txt -r optional-requirements.txt \
+  && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \
   && apk del .build-deps
 
 COPY . /opt/maubot
-RUN cd /opt/maubot && pip3 install .
+RUN cd /opt/maubot && pip3 install --break-system-packages .

maubot/standalone/__main__.py

@@ -115,7 +115,7 @@ with open(args.meta, "r") as meta_file:
 if "/" in meta.main_class:
     module, main_class = meta.main_class.split("/", 1)
 else:
-    module = meta.modules[0]
+    module = meta.modules[-1]
     main_class = meta.main_class
 
 if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "":

maubot/testing/__init__.py

@@ -0,0 +1,17 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .bot import TestBot, make_message  # noqa: F401
+from .fixtures import *  # noqa: F401,F403

maubot/testing/bot.py

@@ -0,0 +1,100 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import asyncio
+import time
+
+from attr import dataclass
+
+from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent
+from mautrix.api import HTTPAPI
+from mautrix.types import (
+    EventContent,
+    EventType,
+    MessageEvent,
+    MessageType,
+    RoomID,
+    TextMessageEventContent,
+)
+
+
+@dataclass
+class MatrixEvent:
+    room_id: RoomID
+    event_type: EventType
+    content: EventContent
+    kwargs: dict
+
+
+class TestBot:
+    """A mocked bot used for testing purposes.
+
+    Send messages to the mock Matrix server with the ``send()`` method.
+    Look into the ``responded`` list to get what server has replied.
+    """
+
+    def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"):
+        api = HTTPAPI(base_url=mxurl)
+        self.client = MaubotMatrixClient(api=api)
+        self.responded = []
+        self.client.mxid = mxid
+        self.client.send_message_event = self._mock_send_message_event
+
+    async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs):
+        self.responded.append(
+            MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs)
+        )
+
+    async def dispatch(self, event_type: EventType, event):
+        tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True)
+        return await asyncio.gather(*tasks)
+
+    async def send(
+        self,
+        content,
+        html=None,
+        room_id="testroom",
+        msg_type=MessageType.TEXT,
+        sender="@dummy:example.com",
+        timestamp=None,
+    ):
+        event = make_message(
+            content,
+            html=html,
+            room_id=room_id,
+            msg_type=msg_type,
+            sender=sender,
+            timestamp=timestamp,
+        )
+        await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client))
+
+
+def make_message(
+    content,
+    html=None,
+    room_id="testroom",
+    msg_type=MessageType.TEXT,
+    sender="@dummy:example.com",
+    timestamp=None,
+):
+    """Make a Matrix message event."""
+    return MessageEvent(
+        type=EventType.ROOM_MESSAGE,
+        room_id=room_id,
+        event_id="test",
+        sender=sender,
+        timestamp=timestamp or int(time.time() * 1000),
+        content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html),
+    )

maubot/testing/fixtures.py

@@ -0,0 +1,135 @@
+# maubot - A plugin-based Matrix bot system.
+# Copyright (C) 2023 Aurélien Bompard
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from pathlib import Path
+import asyncio
+import logging
+
+from ruamel.yaml import YAML
+import aiohttp
+import pytest
+import pytest_asyncio
+
+from maubot import Plugin
+from maubot.loader import PluginMeta
+from maubot.standalone.loader import FileSystemLoader
+from mautrix.util.async_db import Database
+from mautrix.util.config import BaseProxyConfig, RecursiveDict
+from mautrix.util.logging import TraceLogger
+
+from .bot import TestBot
+
+
+@pytest_asyncio.fixture
+async def maubot_test_bot():
+    return TestBot()
+
+
+@pytest.fixture
+def maubot_upgrade_table():
+    return None
+
+
+@pytest.fixture
+def maubot_plugin_path():
+    return Path(".")
+
+
+@pytest.fixture
+def maubot_plugin_meta(maubot_plugin_path):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh:
+        plugin_meta = PluginMeta.deserialize(yaml.load(fh.read()))
+    return plugin_meta
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table):
+    if not maubot_plugin_meta.get("database", False):
+        return
+    db_path = tmp_path.joinpath("maubot-tests.db").as_posix()
+    db = Database.create(
+        f"sqlite:{db_path}",
+        upgrade_table=maubot_upgrade_table,
+        log=logging.getLogger("db"),
+    )
+    await db.start()
+    yield db
+    await db.stop()
+
+
+@pytest.fixture
+def maubot_plugin_class():
+    return Plugin
+
+
+@pytest.fixture
+def maubot_plugin_config_class():
+    return BaseProxyConfig
+
+
+@pytest.fixture
+def maubot_plugin_config_dict():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config_overrides():
+    return {}
+
+
+@pytest.fixture
+def maubot_plugin_config(
+    maubot_plugin_path,
+    maubot_plugin_config_class,
+    maubot_plugin_config_dict,
+    maubot_plugin_config_overrides,
+):
+    yaml = YAML()
+    with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh:
+        base_config = RecursiveDict(yaml.load(fh))
+    maubot_plugin_config_dict.update(maubot_plugin_config_overrides)
+    return maubot_plugin_config_class(
+        load=lambda: maubot_plugin_config_dict,
+        load_base=lambda: base_config,
+        save=lambda c: None,
+    )
+
+
+@pytest_asyncio.fixture
+async def maubot_plugin(
+    maubot_test_bot,
+    maubot_plugin_db,
+    maubot_plugin_class,
+    maubot_plugin_path,
+    maubot_plugin_config,
+    maubot_plugin_meta,
+):
+    loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta)
+    async with aiohttp.ClientSession() as http:
+        instance = maubot_plugin_class(
+            client=maubot_test_bot.client,
+            loop=asyncio.get_running_loop(),
+            http=http,
+            instance_id="tests",
+            log=TraceLogger("test"),
+            config=maubot_plugin_config,
+            database=maubot_plugin_db,
+            webapp=None,
+            webapp_url=None,
+            loader=loader,
+        )
+        await instance.internal_start()
+        yield instance

optional-requirements.txt

@@ -5,3 +5,10 @@
 python-olm>=3,<4
 pycryptodome>=3,<4
 unpaddedbase64>=1,<3
+
+#/testing
+pytest
+pytest-asyncio
+
+#/legacydb
+SQLAlchemy>1,<1.4

pyproject.toml

@@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"]
 
 [tool.black]
 line-length = 99
-target-version = ["py38"]
+target-version = ["py310"]
 force-exclude = "maubot/management/frontend"

requirements.txt

@@ -1,16 +1,16 @@
-mautrix>=0.20.2,<0.21
+mautrix>=0.20.9rc3,<0.21
 aiohttp>=3,<4
 yarl>=1,<2
-SQLAlchemy>=1,<1.4
-asyncpg>=0.20,<0.29
-aiosqlite>=0.16,<0.19
+asyncpg>=0.20,<1
+aiosqlite>=0.16,<1
 commonmark>=0.9,<1
-ruamel.yaml>=0.15.35,<0.18
+ruamel.yaml>=0.15.35,<0.19
 attrs>=18.1.0
 bcrypt>=3,<5
 packaging>=10
 
 click>=7,<9
 colorama>=0.4,<0.5
-questionary>=1,<2
+questionary>=1,<3
 jinja2>=2,<4
+setuptools

setup.py

@@ -41,7 +41,7 @@ setuptools.setup(
 
     install_requires=install_requires,
     extras_require=extras_require,
-    python_requires="~=3.9",
+    python_requires="~=3.10",
 
     classifiers=[
         "Development Status :: 4 - Beta",
@@ -50,13 +50,16 @@ setuptools.setup(
         "Framework :: AsyncIO",
         "Programming Language :: Python",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ],
     entry_points="""
         [console_scripts]
         mbc=maubot.cli:app
+        [pytest11]
+        maubot=maubot.testing
     """,
     data_files=[
         (".", ["maubot/example-config.yaml"]),