diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index d88b8a8..28d6df2 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -6,16 +6,17 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.13" - uses: isort/isort-action@master with: sortPaths: "./maubot" - uses: psf/black@stable with: src: "./maubot" + version: "24.10.0" - name: pre-commit run: | pip install pre-commit diff --git a/.gitignore b/.gitignore index c57bb3c..9fd28ef 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __pycache__ !example-config.yaml !.pre-commit-config.yaml +/start logs/ plugins/ trash/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0c5196e..8c8a44e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,7 @@ default: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY build frontend: - image: node:18-alpine + image: node:24-alpine stage: build frontend before_script: [] variables: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0de079e..4a6328e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - id: trailing-whitespace exclude_types: [markdown] @@ -8,13 +8,13 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 23.1.0 + rev: 24.10.0 hooks: - id: black language_version: python3 files: ^maubot/.*\.pyi?$ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort files: ^maubot/.*\.pyi?$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a3e691..d9de2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,52 @@ +# v0.5.2 (2025-05-05) + +* Improved tombstone handling to ensure that the tombstone sender has + permissions to invite users to the target room. +* Fixed autojoin and online flags not being applied if set during client + creation (thanks to [@bnsh] in [#258]). +* Fixed plugin web apps not being cleared properly when unloading plugins. + +[@bnsh]: https://github.com/bnsh +[#258]: https://github.com/maubot/maubot/pull/258 + +# v0.5.1 (2025-01-03) + +* Updated Docker image to Alpine 3.21. +* Updated media upload/download endpoints in management frontend + (thanks to [@domrim] in [#253]). +* Fixed plugin web app base path not including a trailing slash + (thanks to [@jkhsjdhjs] in [#240]). +* Changed markdown parsing to cut off plaintext body if necessary to allow + longer formatted messages. +* Updated dependencies to fix Python 3.13 compatibility. + +[@domrim]: https://github.com/domrim +[@jkhsjdhjs]: https://github.com/jkhsjdhjs +[#253]: https://github.com/maubot/maubot/pull/253 +[#240]: https://github.com/maubot/maubot/pull/240 + +# v0.5.0 (2024-08-24) + +* Dropped Python 3.9 support. +* Updated Docker image to Alpine 3.20. +* Updated mautrix-python to 0.20.6 to support authenticated media. +* Removed hard dependency on SQLAlchemy. +* Fixed `main_class` to default to being loaded from the last module instead of + the first if a module name is not explicitly specified. + * This was already the [documented behavior](https://docs.mau.fi/maubot/dev/reference/plugin-metadata.html), + and loading from the first module doesn't make sense due to import order. +* Added simple scheduler utility for running background tasks periodically or + after a certain delay. +* Added testing framework for plugins (thanks to [@abompard] in [#225]). +* Changed `mbc build` to ignore directories declared in `modules` that are + missing an `__init__.py` file. + * Importing the modules at runtime would fail and break the plugin. + To include non-code resources outside modules in the mbp archive, + use `extra_files` instead. + +[#225]: https://github.com/maubot/maubot/issues/225 +[@abompard]: https://github.com/abompard + # v0.4.2 (2023-09-20) * Updated Pillow to 10.0.1. diff --git a/Dockerfile b/Dockerfile index f4bd0df..204a78a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM node:18 AS frontend-builder +FROM node:24 AS frontend-builder COPY ./maubot/management/frontend /frontend RUN cd /frontend && yarn --prod && yarn build -FROM alpine:3.18 +FROM alpine:3.22 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -11,7 +11,8 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ - py3-sqlalchemy \ + py3-aiodns \ + py3-brotli \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -34,20 +35,19 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - #py3-pillow \ + py3-pillow \ py3-magic \ py3-feedparser \ py3-dateutil \ py3-lxml \ - py3-semver \ - && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community + py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ dateparser langdetect python-gitlab pyquery tzlocal \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/Dockerfile.ci b/Dockerfile.ci index 7a957f8..fb76f50 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:3.18 +FROM alpine:3.22 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ @@ -6,7 +6,6 @@ RUN apk add --no-cache \ su-exec \ yq \ py3-aiohttp \ - py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -30,11 +29,10 @@ RUN apk add --no-cache \ py3-unpaddedbase64 \ py3-future \ # plugin deps - #py3-pillow \ + py3-pillow \ py3-magic \ py3-feedparser \ - py3-lxml \ - && apk add --no-cache py3-pillow --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community + py3-lxml # py3-gitlab # py3-semver # TODO remove pillow, magic, feedparser, lxml, gitlab and semver when maubot supports installing dependencies @@ -43,7 +41,7 @@ COPY requirements.txt /opt/maubot/requirements.txt COPY optional-requirements.txt /opt/maubot/optional-requirements.txt WORKDIR /opt/maubot RUN apk add --virtual .build-deps python3-dev build-base git \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ dateparser langdetect python-gitlab pyquery semver tzlocal cssselect \ && apk del .build-deps # TODO also remove dateparser, langdetect and pyquery when maubot supports installing dependencies diff --git a/README.md b/README.md index a2eda36..02a4b6f 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ All setup and usage instructions are located on Matrix room: [#maubot:maunium.net](https://matrix.to/#/#maubot:maunium.net) ## Plugins -A list of plugins can be found at [plugins.maubot.xyz](https://plugins.maubot.xyz/). +A list of plugins can be found at [plugins.mau.bot](https://plugins.mau.bot/). To add your plugin to the list, send a pull request to . diff --git a/dev-requirements.txt b/dev-requirements.txt index 5cd14c2..bb8c2a0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ pre-commit>=2.10.1,<3 isort>=5.10.1,<6 -black>=23,<24 +black>=24,<25 diff --git a/maubot/__meta__.py b/maubot/__meta__.py index df12433..7225152 100644 --- a/maubot/__meta__.py +++ b/maubot/__meta__.py @@ -1 +1 @@ -__version__ = "0.4.2" +__version__ = "0.5.2" diff --git a/maubot/cli/commands/build.py b/maubot/cli/commands/build.py index ec3ac26..39eca53 100644 --- a/maubot/cli/commands/build.py +++ b/maubot/cli/commands/build.py @@ -93,10 +93,16 @@ def write_plugin(meta: PluginMeta, output: str | IO) -> None: if os.path.isfile(f"{module}.py"): zip.write(f"{module}.py") elif module is not None and os.path.isdir(module): - zipdir(zip, module) + if os.path.isfile(f"{module}/__init__.py"): + zipdir(zip, module) + else: + print( + Fore.YELLOW + + f"Module {module} is missing __init__.py, skipping" + + Fore.RESET + ) else: print(Fore.YELLOW + f"Module {module} not found, skipping" + Fore.RESET) - for pattern in meta.extra_files: for file in glob.iglob(pattern): zip.write(file) diff --git a/maubot/client.py b/maubot/client.py index bdb76fc..972f0de 100644 --- a/maubot/client.py +++ b/maubot/client.py @@ -350,11 +350,41 @@ class Client(DBClient): } async def _handle_tombstone(self, evt: StateEvent) -> None: + if evt.state_key != "": + return if not evt.content.replacement_room: self.log.info(f"{evt.room_id} tombstoned with no replacement, ignoring") return + is_joined = await self.client.state_store.is_joined( + evt.content.replacement_room, + self.client.mxid, + ) + if is_joined: + self.log.debug( + f"Ignoring tombstone from {evt.room_id} to {evt.content.replacement_room} " + f"sent by {evt.sender}: already joined to replacement room" + ) + return + self.log.debug( + f"Following tombstone from {evt.room_id} to {evt.content.replacement_room} " + f"sent by {evt.sender}" + ) _, server = self.client.parse_user_id(evt.sender) - await self.client.join_room(evt.content.replacement_room, servers=[server]) + room_id = await self.client.join_room(evt.content.replacement_room, servers=[server]) + power_levels = await self.client.get_state_event(room_id, EventType.ROOM_POWER_LEVELS) + create_event = await self.client.get_state_event( + room_id, EventType.ROOM_CREATE, format="event" + ) + if power_levels.get_user_level(evt.sender, create_event) < power_levels.invite: + self.log.warning( + f"{evt.room_id} was tombstoned into {room_id} by {evt.sender}," + " but the sender doesn't have invite power levels, leaving..." + ) + await self.client.leave_room( + room_id, + f"Followed tombstone from {evt.room_id} by {evt.sender}," + " but sender doesn't have sufficient power level for invites", + ) async def _handle_invite(self, evt: StrippedStateEvent) -> None: if evt.state_key == self.id and evt.content.membership == Membership.INVITE: @@ -474,8 +504,14 @@ class Client(DBClient): self.start_sync() async def _update_remote_profile(self) -> None: - profile = await self.client.get_profile(self.id) - self.remote_displayname, self.remote_avatar_url = profile.displayname, profile.avatar_url + try: + profile = await self.client.get_profile(self.id) + self.remote_displayname, self.remote_avatar_url = ( + profile.displayname, + profile.avatar_url, + ) + except Exception: + self.log.warning("Failed to update own profile from server", exc_info=True) async def delete(self) -> None: try: diff --git a/maubot/example-config.yaml b/maubot/example-config.yaml index a16ea19..7991461 100644 --- a/maubot/example-config.yaml +++ b/maubot/example-config.yaml @@ -1,5 +1,4 @@ # The full URI to the database. SQLite and Postgres are fully supported. -# Other DBMSes supported by SQLAlchemy may or may not work. # Format examples: # SQLite: sqlite:filename.db # Postgres: postgresql://username:password@hostname/dbname @@ -55,7 +54,8 @@ server: port: 29316 # Public base URL where the server is visible. public_url: https://example.com - # The base path for the UI. + # The base path for the UI. Note that this does not change the API path. + # Add a path prefix to public_url if you want everything on a subpath. ui_base_path: /_matrix/maubot # The base path for plugin endpoints. The instance ID will be appended directly. plugin_base_path: /_matrix/maubot/plugin/ @@ -79,8 +79,9 @@ homeservers: # When this is empty, `mbc auth --register` won't work, but `mbc auth` (login) will. secret: null -# List of administrator users. Plaintext passwords will be bcrypted on startup. Set empty password -# to prevent normal login. Root is a special user that can't have a password and will always exist. +# List of administrator users. Each key is a username and the value is the password. +# Plaintext passwords will be bcrypted on startup. Set empty password to prevent normal login. +# Root is a special user that can't have a password and will always exist. admins: root: "" diff --git a/maubot/instance.py b/maubot/instance.py index 2905d12..8427e3c 100644 --- a/maubot/instance.py +++ b/maubot/instance.py @@ -25,7 +25,6 @@ import os.path from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedMap -import sqlalchemy as sql from mautrix.types import UserID from mautrix.util import background_task @@ -36,6 +35,7 @@ from mautrix.util.logging import TraceLogger from .client import Client from .db import DatabaseEngine, Instance as DBInstance +from .lib.optionalalchemy import Engine, MetaData, create_engine from .lib.plugin_db import ProxyPostgresDatabase from .loader import DatabaseType, PluginLoader, ZippedPluginLoader from .plugin_base import Plugin @@ -128,7 +128,7 @@ class PluginInstance(DBInstance): } def _introspect_sqlalchemy(self) -> dict: - metadata = sql.MetaData() + metadata = MetaData() metadata.reflect(self.inst_db) return { table.name: { @@ -214,7 +214,7 @@ class PluginInstance(DBInstance): async def get_db_tables(self) -> dict: if self.inst_db_tables is None: - if isinstance(self.inst_db, sql.engine.Engine): + if isinstance(self.inst_db, Engine): self.inst_db_tables = self._introspect_sqlalchemy() elif self.inst_db.scheme == Scheme.SQLITE: self.inst_db_tables = await self._introspect_sqlite() @@ -294,7 +294,7 @@ class PluginInstance(DBInstance): "Instance database engine is marked as Postgres, but plugin uses legacy " "database interface, which doesn't support postgres." ) - self.inst_db = sql.create_engine(f"sqlite:///{self._sqlite_db_path}") + self.inst_db = create_engine(f"sqlite:///{self._sqlite_db_path}") elif self.loader.meta.database_type == DatabaseType.ASYNCPG: if self.database_engine is None: if os.path.exists(self._sqlite_db_path) or not self.maubot.plugin_postgres_db: @@ -329,7 +329,7 @@ class PluginInstance(DBInstance): async def stop_database(self) -> None: if isinstance(self.inst_db, Database): await self.inst_db.stop() - elif isinstance(self.inst_db, sql.engine.Engine): + elif isinstance(self.inst_db, Engine): self.inst_db.dispose() else: raise RuntimeError(f"Unknown database type {type(self.inst_db).__name__}") diff --git a/maubot/lib/optionalalchemy.py b/maubot/lib/optionalalchemy.py new file mode 100644 index 0000000..ba94271 --- /dev/null +++ b/maubot/lib/optionalalchemy.py @@ -0,0 +1,19 @@ +try: + from sqlalchemy import MetaData, asc, create_engine, desc + from sqlalchemy.engine import Engine + from sqlalchemy.exc import IntegrityError, OperationalError +except ImportError: + + class FakeError(Exception): + pass + + class FakeType: + def __init__(self, *args, **kwargs): + raise Exception("SQLAlchemy is not installed") + + def create_engine(*args, **kwargs): + raise Exception("SQLAlchemy is not installed") + + MetaData = Engine = FakeType + IntegrityError = OperationalError = FakeError + asc = desc = lambda a: a diff --git a/maubot/loader/zip.py b/maubot/loader/zip.py index 70cee5a..8642183 100644 --- a/maubot/loader/zip.py +++ b/maubot/loader/zip.py @@ -31,7 +31,7 @@ from ..config import Config from ..lib.zipimport import ZipImportError, zipimporter from ..plugin_base import Plugin from .abc import IDConflictError, PluginClass, PluginLoader -from .meta import PluginMeta +from .meta import DatabaseType, PluginMeta current_version = Version(__version__) yaml = YAML() @@ -155,9 +155,9 @@ class ZippedPluginLoader(PluginLoader): return file, meta @classmethod - def verify_meta(cls, source) -> tuple[str, Version]: + def verify_meta(cls, source) -> tuple[str, Version, DatabaseType | None]: _, meta = cls._read_meta(source) - return meta.id, meta.version + return meta.id, meta.version, meta.database_type if meta.database else None def _load_meta(self) -> None: file, meta = self._read_meta(self.path) @@ -167,7 +167,7 @@ class ZippedPluginLoader(PluginLoader): if "/" in meta.main_class: self.main_module, self.main_class = meta.main_class.split("/")[:2] else: - self.main_module = meta.modules[0] + self.main_module = meta.modules[-1] self.main_class = meta.main_class self._file = file diff --git a/maubot/management/api/client.py b/maubot/management/api/client.py index 2a8964c..d2ad35d 100644 --- a/maubot/management/api/client.py +++ b/maubot/management/api/client.py @@ -78,8 +78,8 @@ async def _create_client(user_id: UserID | None, data: dict) -> web.Response: ) client.enabled = data.get("enabled", True) client.sync = data.get("sync", True) - client.autojoin = data.get("autojoin", True) - client.online = data.get("online", True) + await client.update_autojoin(data.get("autojoin", True), save=False) + await client.update_online(data.get("online", True), save=False) client.displayname = data.get("displayname", "disable") client.avatar_url = data.get("avatar_url", "disable") await client.update() diff --git a/maubot/management/api/instance_database.py b/maubot/management/api/instance_database.py index 97b2edf..2f8c37a 100644 --- a/maubot/management/api/instance_database.py +++ b/maubot/management/api/instance_database.py @@ -19,12 +19,12 @@ from datetime import datetime from aiohttp import web from asyncpg import PostgresError -from sqlalchemy import asc, desc, engine, exc import aiosqlite from mautrix.util.async_db import Database from ...instance import PluginInstance +from ...lib.optionalalchemy import Engine, IntegrityError, OperationalError, asc, desc from .base import routes from .responses import resp @@ -56,15 +56,17 @@ async def get_table(request: web.Request) -> web.Response: try: order = [tuple(order.split(":")) for order in request.query.getall("order")] order = [ - (asc if sort.lower() == "asc" else desc)(table.columns[column]) - if sort - else table.columns[column] + ( + (asc if sort.lower() == "asc" else desc)(table.columns[column]) + if sort + else table.columns[column] + ) for column, sort in order ] except KeyError: order = [] limit = int(request.query.get("limit", "100")) - if isinstance(instance.inst_db, engine.Engine): + if isinstance(instance.inst_db, Engine): return _execute_query_sqlalchemy(instance, table.select().order_by(*order).limit(limit)) @@ -82,7 +84,7 @@ async def query(request: web.Request) -> web.Response: except KeyError: return resp.query_missing rows_as_dict = data.get("rows_as_dict", False) - if isinstance(instance.inst_db, engine.Engine): + if isinstance(instance.inst_db, Engine): return _execute_query_sqlalchemy(instance, sql_query, rows_as_dict) elif isinstance(instance.inst_db, Database): try: @@ -131,12 +133,12 @@ async def _execute_query_asyncpg( def _execute_query_sqlalchemy( instance: PluginInstance, sql_query: str, rows_as_dict: bool = False ) -> web.Response: - assert isinstance(instance.inst_db, engine.Engine) + assert isinstance(instance.inst_db, Engine) try: res = instance.inst_db.execute(sql_query) - except exc.IntegrityError as e: + except IntegrityError as e: return resp.sql_integrity_error(e, sql_query) - except exc.OperationalError as e: + except OperationalError as e: return resp.sql_operational_error(e, sql_query) data = { "ok": True, diff --git a/maubot/management/api/plugin_upload.py b/maubot/management/api/plugin_upload.py index ea4fd1f..4cd2c47 100644 --- a/maubot/management/api/plugin_upload.py +++ b/maubot/management/api/plugin_upload.py @@ -23,10 +23,17 @@ import traceback from aiohttp import web from packaging.version import Version -from ...loader import MaubotZipImportError, PluginLoader, ZippedPluginLoader +from ...loader import DatabaseType, MaubotZipImportError, PluginLoader, ZippedPluginLoader from .base import get_config, routes from .responses import resp +try: + import sqlalchemy + + has_alchemy = True +except ImportError: + has_alchemy = False + log = logging.getLogger("maubot.server.upload") @@ -36,9 +43,11 @@ async def put_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version = ZippedPluginLoader.verify_meta(file) + pid, version, db_type = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) + if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: + return resp.sqlalchemy_not_installed if pid != plugin_id: return resp.pid_mismatch plugin = PluginLoader.id_cache.get(plugin_id, None) @@ -55,9 +64,11 @@ async def upload_plugin(request: web.Request) -> web.Response: content = await request.read() file = BytesIO(content) try: - pid, version = ZippedPluginLoader.verify_meta(file) + pid, version, db_type = ZippedPluginLoader.verify_meta(file) except MaubotZipImportError as e: return resp.plugin_import_error(str(e), traceback.format_exc()) + if db_type == DatabaseType.SQLALCHEMY and not has_alchemy: + return resp.sqlalchemy_not_installed plugin = PluginLoader.id_cache.get(pid, None) if not plugin: return await upload_new_plugin(content, pid, version) diff --git a/maubot/management/api/responses.py b/maubot/management/api/responses.py index 15f6a96..0f22caa 100644 --- a/maubot/management/api/responses.py +++ b/maubot/management/api/responses.py @@ -15,13 +15,16 @@ # along with this program. If not, see . from __future__ import annotations +from typing import TYPE_CHECKING from http import HTTPStatus from aiohttp import web from asyncpg import PostgresError -from sqlalchemy.exc import IntegrityError, OperationalError import aiosqlite +if TYPE_CHECKING: + from sqlalchemy.exc import IntegrityError, OperationalError + class _Response: @property @@ -324,6 +327,16 @@ class _Response: } ) + @property + def sqlalchemy_not_installed(self) -> web.Response: + return web.json_response( + { + "error": "This plugin requires a legacy database, but SQLAlchemy is not installed", + "errcode": "unsupported_plugin_database", + }, + status=HTTPStatus.NOT_IMPLEMENTED, + ) + @property def table_not_found(self) -> web.Response: return web.json_response( diff --git a/maubot/management/frontend/src/api.js b/maubot/management/frontend/src/api.js index 5c1fd55..d1a51b8 100644 --- a/maubot/management/frontend/src/api.js +++ b/maubot/management/frontend/src/api.js @@ -205,7 +205,7 @@ export const getClients = () => defaultGet("/clients") export const getClient = id => defaultGet(`/clients/${id}`) export async function uploadAvatar(id, data, mime) { - const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/r0/upload`, { + const resp = await fetch(`${BASE_PATH}/proxy/${id}/_matrix/media/v3/upload`, { headers: getHeaders(mime), body: data, method: "POST", @@ -217,8 +217,9 @@ export function getAvatarURL({ id, avatar_url }) { if (!avatar_url?.startsWith("mxc://")) { return null } - avatar_url = avatar_url.substr("mxc://".length) - return `${BASE_PATH}/proxy/${id}/_matrix/media/r0/download/${avatar_url}?access_token=${ + avatar_url = avatar_url.substring("mxc://".length) + // Note: the maubot backend will replace the query param with an authorization header + return `${BASE_PATH}/proxy/${id}/_matrix/client/v1/media/download/${avatar_url}?access_token=${ localStorage.accessToken}` } diff --git a/maubot/management/frontend/src/pages/dashboard/Client.js b/maubot/management/frontend/src/pages/dashboard/Client.js index c60129c..e9d7541 100644 --- a/maubot/management/frontend/src/pages/dashboard/Client.js +++ b/maubot/management/frontend/src/pages/dashboard/Client.js @@ -70,9 +70,9 @@ class Client extends BaseMainView { get initialState() { return { id: "", - displayname: "", + displayname: "disable", homeserver: "", - avatar_url: "", + avatar_url: "disable", access_token: "", device_id: "", fingerprint: null, diff --git a/maubot/matrix.py b/maubot/matrix.py index 0f3e36b..8a811af 100644 --- a/maubot/matrix.py +++ b/maubot/matrix.py @@ -15,7 +15,7 @@ # along with this program. If not, see . from __future__ import annotations -from typing import Awaitable +from typing import Any, Awaitable from html import escape import asyncio @@ -62,7 +62,10 @@ async def parse_formatted( html = message else: return message, escape(message) - return (await MaubotHTMLParser().parse(html)).text, html + text = (await MaubotHTMLParser().parse(html)).text + if len(text) > 100 and len(text) + len(html) > 40000: + text = text[:100] + "[long message cut off]" + return text, html class MaubotMessageEvent(MessageEvent): @@ -85,6 +88,7 @@ class MaubotMessageEvent(MessageEvent): reply: bool | str = False, in_thread: bool | None = None, edits: EventID | MessageEvent | None = None, + extra_content: dict[str, Any] | None = None, ) -> EventID: """ Respond to the message. @@ -104,6 +108,7 @@ class MaubotMessageEvent(MessageEvent): the root if necessary. edits: An event ID or MessageEvent to edit. If set, the reply and in_thread parameters are ignored, as edits can't change the reply or thread status. + extra_content: Extra content to add to the event. Returns: The ID of the response event. @@ -140,6 +145,9 @@ class MaubotMessageEvent(MessageEvent): ) else: content.set_reply(self) + if extra_content: + for k, v in extra_content.items(): + content[k] = v return await self.client.send_message_event(self.room_id, event_type, content) def reply( @@ -149,6 +157,7 @@ class MaubotMessageEvent(MessageEvent): markdown: bool = True, allow_html: bool = False, in_thread: bool | None = None, + extra_content: dict[str, Any] | None = None, ) -> Awaitable[EventID]: """ Reply to the message. The parameters are the same as :meth:`respond`, @@ -166,6 +175,7 @@ class MaubotMessageEvent(MessageEvent): thread. If set to ``False``, the response will never be in a thread. If set to ``True``, the response will always be in a thread, creating one with this event as the root if necessary. + extra_content: Extra content to add to the event. Returns: The ID of the response event. @@ -177,6 +187,7 @@ class MaubotMessageEvent(MessageEvent): reply=True, in_thread=in_thread, allow_html=allow_html, + extra_content=extra_content, ) def mark_read(self) -> Awaitable[None]: @@ -253,14 +264,18 @@ class MaubotMatrixClient(MatrixClient): markdown: str, *, allow_html: bool = False, + render_markdown: bool = True, msgtype: MessageType = MessageType.TEXT, edits: EventID | MessageEvent | None = None, relates_to: RelatesTo | None = None, + extra_content: dict[str, Any] = None, **kwargs, ) -> EventID: content = TextMessageEventContent(msgtype=msgtype, format=Format.HTML) content.body, content.formatted_body = await parse_formatted( - markdown, allow_html=allow_html + markdown, + allow_html=allow_html, + render_markdown=render_markdown, ) if relates_to: if edits: @@ -268,6 +283,9 @@ class MaubotMatrixClient(MatrixClient): content.relates_to = relates_to elif edits: content.set_edit(edits) + if extra_content: + for k, v in extra_content.items(): + content[k] = v return await self.send_message(room_id, content, **kwargs) def dispatch_event(self, event: Event, source: SyncStream) -> list[asyncio.Task]: diff --git a/maubot/plugin_base.py b/maubot/plugin_base.py index d7b6bbd..1be15e0 100644 --- a/maubot/plugin_base.py +++ b/maubot/plugin_base.py @@ -20,14 +20,17 @@ from abc import ABC from asyncio import AbstractEventLoop from aiohttp import ClientSession -from sqlalchemy.engine.base import Engine from yarl import URL from mautrix.util.async_db import Database, UpgradeTable from mautrix.util.config import BaseProxyConfig from mautrix.util.logging import TraceLogger +from .scheduler import BasicScheduler + if TYPE_CHECKING: + from sqlalchemy.engine.base import Engine + from .client import MaubotMatrixClient from .loader import BasePluginLoader from .plugin_server import PluginWebApp @@ -40,6 +43,7 @@ class Plugin(ABC): log: TraceLogger loop: AbstractEventLoop loader: BasePluginLoader + sched: BasicScheduler config: BaseProxyConfig | None database: Engine | Database | None webapp: PluginWebApp | None @@ -53,11 +57,12 @@ class Plugin(ABC): instance_id: str, log: TraceLogger, config: BaseProxyConfig | None, - database: Engine | None, + database: Engine | Database | None, webapp: PluginWebApp | None, webapp_url: str | None, loader: BasePluginLoader, ) -> None: + self.sched = BasicScheduler(log=log.getChild("scheduler")) self.client = client self.loop = loop self.http = http @@ -117,6 +122,7 @@ class Plugin(ABC): self.client.remove_event_handler(event_type, func) if self.webapp is not None: self.webapp.clear() + self.sched.stop() await self.stop() async def stop(self) -> None: diff --git a/maubot/plugin_server.py b/maubot/plugin_server.py index 9dd2df4..e5c246c 100644 --- a/maubot/plugin_server.py +++ b/maubot/plugin_server.py @@ -40,6 +40,8 @@ class PluginWebApp(web.UrlDispatcher): self._resources = [] self._named_resources = {} self._middleware = [] + self._resource_index = {} + self._matched_sub_app_resources = [] async def handle(self, request: web.Request) -> web.StreamResponse: match_info = await self.resolve(request) diff --git a/maubot/scheduler.py b/maubot/scheduler.py new file mode 100644 index 0000000..0cb39ed --- /dev/null +++ b/maubot/scheduler.py @@ -0,0 +1,159 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2024 Tulir Asokan +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from __future__ import annotations + +from typing import Awaitable, Callable +import asyncio +import logging + + +class BasicScheduler: + background_loop: asyncio.Task | None + tasks: set[asyncio.Task] + log: logging.Logger + + def __init__(self, log: logging.Logger) -> None: + self.log = log + self.tasks = set() + + def _find_caller(self) -> str: + try: + file_name, line_number, function_name, _ = self.log.findCaller() + return f"{function_name} at {file_name}:{line_number}" + except ValueError: + return "unknown function" + + def run_periodically( + self, + period: float | int, + func: Callable[[], Awaitable], + run_task_in_background: bool = False, + catch_errors: bool = True, + ) -> asyncio.Task: + """ + Run a function periodically in the background. + + Args: + period: The period in seconds between each call to the function. + func: The function to run. No parameters will be provided, + use :meth:`functools.partial` if you need to pass parameters. + run_task_in_background: If ``True``, the function will be run in a background task. + If ``False`` (the default), the loop will wait for the task to return before + sleeping for the next period. + catch_errors: Whether the scheduler should catch and log any errors. + If ``False``, errors will be raised, and the caller must await the returned task + to find errors. This parameter has no effect if ``run_task_in_background`` + is ``True``. + + Returns: + The asyncio task object representing the background loop. + """ + task = asyncio.create_task( + self._call_periodically( + period, + func, + caller=self._find_caller(), + catch_errors=catch_errors, + run_task_in_background=run_task_in_background, + ) + ) + self._register_task(task) + return task + + def run_later( + self, delay: float | int, coro: Awaitable, catch_errors: bool = True + ) -> asyncio.Task: + """ + Run a coroutine after a delay. + + Examples: + >>> self.sched.run_later(5, self.async_task(meow=True)) + + Args: + delay: The delay in seconds to await the coroutine after. + coro: The coroutine to await. + catch_errors: Whether the scheduler should catch and log any errors. + If ``False``, errors will be raised, and the caller must await the returned task + to find errors. + + Returns: + The asyncio task object representing the scheduled task. + """ + task = asyncio.create_task( + self._call_with_delay( + delay, coro, caller=self._find_caller(), catch_errors=catch_errors + ) + ) + self._register_task(task) + return task + + def _register_task(self, task: asyncio.Task) -> None: + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + async def _call_periodically( + self, + period: float | int, + func: Callable[[], Awaitable], + caller: str, + catch_errors: bool, + run_task_in_background: bool, + ) -> None: + while True: + try: + await asyncio.sleep(period) + if run_task_in_background: + self._register_task( + asyncio.create_task(self._call_periodically_background(func(), caller)) + ) + else: + await func() + except asyncio.CancelledError: + raise + except Exception: + if catch_errors: + self.log.exception(f"Uncaught error in background loop (created in {caller})") + else: + raise + + async def _call_periodically_background(self, coro: Awaitable, caller: str) -> None: + try: + await coro + except asyncio.CancelledError: + raise + except Exception: + self.log.exception(f"Uncaught error in background loop subtask (created in {caller})") + + async def _call_with_delay( + self, delay: float | int, coro: Awaitable, caller: str, catch_errors: bool + ) -> None: + try: + await asyncio.sleep(delay) + await coro + except asyncio.CancelledError: + raise + except Exception: + if catch_errors: + self.log.exception(f"Uncaught error in scheduled task (created in {caller})") + else: + raise + + def stop(self) -> None: + """ + Stop all scheduled tasks and background loops. + """ + for task in self.tasks: + task.cancel(msg="Scheduler stopped") diff --git a/maubot/server.py b/maubot/server.py index 097fe5b..dd1101e 100644 --- a/maubot/server.py +++ b/maubot/server.py @@ -64,14 +64,14 @@ class MaubotServer: if request.path.startswith(path): request = request.clone( rel_url=request.rel_url.with_path( - request.rel_url.path[len(path) :] + request.rel_url.path[len(path) - 1 :] ).with_query(request.query_string) ) return await app.handle(request) return web.Response(status=404) def get_instance_subapp(self, instance_id: str) -> tuple[PluginWebApp, str]: - subpath = self.config["server.plugin_base_path"] + instance_id + subpath = self.config["server.plugin_base_path"] + instance_id + "/" url = self.config["server.public_url"] + subpath try: return self.plugin_routes[subpath], url @@ -82,7 +82,7 @@ class MaubotServer: def remove_instance_webapp(self, instance_id: str) -> None: try: - subpath = self.config["server.plugin_base_path"] + instance_id + subpath = self.config["server.plugin_base_path"] + instance_id + "/" self.plugin_routes.pop(subpath).clear() except KeyError: return diff --git a/maubot/standalone/Dockerfile b/maubot/standalone/Dockerfile index 2db8426..54623f2 100644 --- a/maubot/standalone/Dockerfile +++ b/maubot/standalone/Dockerfile @@ -1,9 +1,8 @@ -FROM docker.io/alpine:3.18 +FROM docker.io/alpine:3.21 RUN apk add --no-cache \ python3 py3-pip py3-setuptools py3-wheel \ py3-aiohttp \ - py3-sqlalchemy \ py3-attrs \ py3-bcrypt \ py3-cffi \ @@ -26,8 +25,8 @@ RUN cd /opt/maubot \ python3-dev \ libffi-dev \ build-base \ - && pip3 install -r requirements.txt -r optional-requirements.txt \ + && pip3 install --break-system-packages -r requirements.txt -r optional-requirements.txt \ && apk del .build-deps COPY . /opt/maubot -RUN cd /opt/maubot && pip3 install . +RUN cd /opt/maubot && pip3 install --break-system-packages . diff --git a/maubot/standalone/__main__.py b/maubot/standalone/__main__.py index 6d3150d..c320af4 100644 --- a/maubot/standalone/__main__.py +++ b/maubot/standalone/__main__.py @@ -115,7 +115,7 @@ with open(args.meta, "r") as meta_file: if "/" in meta.main_class: module, main_class = meta.main_class.split("/", 1) else: - module = meta.modules[0] + module = meta.modules[-1] main_class = meta.main_class if args.meta != "maubot.yaml" and os.path.dirname(args.meta) != "": diff --git a/maubot/testing/__init__.py b/maubot/testing/__init__.py new file mode 100644 index 0000000..1fcdfc0 --- /dev/null +++ b/maubot/testing/__init__.py @@ -0,0 +1,17 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from .bot import TestBot, make_message # noqa: F401 +from .fixtures import * # noqa: F401,F403 diff --git a/maubot/testing/bot.py b/maubot/testing/bot.py new file mode 100644 index 0000000..0519016 --- /dev/null +++ b/maubot/testing/bot.py @@ -0,0 +1,100 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +import asyncio +import time + +from attr import dataclass + +from maubot.matrix import MaubotMatrixClient, MaubotMessageEvent +from mautrix.api import HTTPAPI +from mautrix.types import ( + EventContent, + EventType, + MessageEvent, + MessageType, + RoomID, + TextMessageEventContent, +) + + +@dataclass +class MatrixEvent: + room_id: RoomID + event_type: EventType + content: EventContent + kwargs: dict + + +class TestBot: + """A mocked bot used for testing purposes. + + Send messages to the mock Matrix server with the ``send()`` method. + Look into the ``responded`` list to get what server has replied. + """ + + def __init__(self, mxid="@botname:example.com", mxurl="http://matrix.example.com"): + api = HTTPAPI(base_url=mxurl) + self.client = MaubotMatrixClient(api=api) + self.responded = [] + self.client.mxid = mxid + self.client.send_message_event = self._mock_send_message_event + + async def _mock_send_message_event(self, room_id, event_type, content, txn_id=None, **kwargs): + self.responded.append( + MatrixEvent(room_id=room_id, event_type=event_type, content=content, kwargs=kwargs) + ) + + async def dispatch(self, event_type: EventType, event): + tasks = self.client.dispatch_manual_event(event_type, event, force_synchronous=True) + return await asyncio.gather(*tasks) + + async def send( + self, + content, + html=None, + room_id="testroom", + msg_type=MessageType.TEXT, + sender="@dummy:example.com", + timestamp=None, + ): + event = make_message( + content, + html=html, + room_id=room_id, + msg_type=msg_type, + sender=sender, + timestamp=timestamp, + ) + await self.dispatch(EventType.ROOM_MESSAGE, MaubotMessageEvent(event, self.client)) + + +def make_message( + content, + html=None, + room_id="testroom", + msg_type=MessageType.TEXT, + sender="@dummy:example.com", + timestamp=None, +): + """Make a Matrix message event.""" + return MessageEvent( + type=EventType.ROOM_MESSAGE, + room_id=room_id, + event_id="test", + sender=sender, + timestamp=timestamp or int(time.time() * 1000), + content=TextMessageEventContent(msgtype=msg_type, body=content, formatted_body=html), + ) diff --git a/maubot/testing/fixtures.py b/maubot/testing/fixtures.py new file mode 100644 index 0000000..e975782 --- /dev/null +++ b/maubot/testing/fixtures.py @@ -0,0 +1,135 @@ +# maubot - A plugin-based Matrix bot system. +# Copyright (C) 2023 Aurélien Bompard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +from pathlib import Path +import asyncio +import logging + +from ruamel.yaml import YAML +import aiohttp +import pytest +import pytest_asyncio + +from maubot import Plugin +from maubot.loader import PluginMeta +from maubot.standalone.loader import FileSystemLoader +from mautrix.util.async_db import Database +from mautrix.util.config import BaseProxyConfig, RecursiveDict +from mautrix.util.logging import TraceLogger + +from .bot import TestBot + + +@pytest_asyncio.fixture +async def maubot_test_bot(): + return TestBot() + + +@pytest.fixture +def maubot_upgrade_table(): + return None + + +@pytest.fixture +def maubot_plugin_path(): + return Path(".") + + +@pytest.fixture +def maubot_plugin_meta(maubot_plugin_path): + yaml = YAML() + with open(maubot_plugin_path.joinpath("maubot.yaml")) as fh: + plugin_meta = PluginMeta.deserialize(yaml.load(fh.read())) + return plugin_meta + + +@pytest_asyncio.fixture +async def maubot_plugin_db(tmp_path, maubot_plugin_meta, maubot_upgrade_table): + if not maubot_plugin_meta.get("database", False): + return + db_path = tmp_path.joinpath("maubot-tests.db").as_posix() + db = Database.create( + f"sqlite:{db_path}", + upgrade_table=maubot_upgrade_table, + log=logging.getLogger("db"), + ) + await db.start() + yield db + await db.stop() + + +@pytest.fixture +def maubot_plugin_class(): + return Plugin + + +@pytest.fixture +def maubot_plugin_config_class(): + return BaseProxyConfig + + +@pytest.fixture +def maubot_plugin_config_dict(): + return {} + + +@pytest.fixture +def maubot_plugin_config_overrides(): + return {} + + +@pytest.fixture +def maubot_plugin_config( + maubot_plugin_path, + maubot_plugin_config_class, + maubot_plugin_config_dict, + maubot_plugin_config_overrides, +): + yaml = YAML() + with open(maubot_plugin_path.joinpath("base-config.yaml")) as fh: + base_config = RecursiveDict(yaml.load(fh)) + maubot_plugin_config_dict.update(maubot_plugin_config_overrides) + return maubot_plugin_config_class( + load=lambda: maubot_plugin_config_dict, + load_base=lambda: base_config, + save=lambda c: None, + ) + + +@pytest_asyncio.fixture +async def maubot_plugin( + maubot_test_bot, + maubot_plugin_db, + maubot_plugin_class, + maubot_plugin_path, + maubot_plugin_config, + maubot_plugin_meta, +): + loader = FileSystemLoader(maubot_plugin_path, maubot_plugin_meta) + async with aiohttp.ClientSession() as http: + instance = maubot_plugin_class( + client=maubot_test_bot.client, + loop=asyncio.get_running_loop(), + http=http, + instance_id="tests", + log=TraceLogger("test"), + config=maubot_plugin_config, + database=maubot_plugin_db, + webapp=None, + webapp_url=None, + loader=loader, + ) + await instance.internal_start() + yield instance diff --git a/optional-requirements.txt b/optional-requirements.txt index 6d87db3..f5b378a 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -5,3 +5,10 @@ python-olm>=3,<4 pycryptodome>=3,<4 unpaddedbase64>=1,<3 + +#/testing +pytest +pytest-asyncio + +#/legacydb +SQLAlchemy>1,<1.4 diff --git a/pyproject.toml b/pyproject.toml index 4cee457..6c2ca27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,5 +9,5 @@ skip = ["maubot/management/frontend"] [tool.black] line-length = 99 -target-version = ["py38"] +target-version = ["py310"] force-exclude = "maubot/management/frontend" diff --git a/requirements.txt b/requirements.txt index 4134190..169d053 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -mautrix>=0.20.2,<0.21 +mautrix>=0.20.9rc3,<0.21 aiohttp>=3,<4 yarl>=1,<2 -SQLAlchemy>=1,<1.4 -asyncpg>=0.20,<0.29 -aiosqlite>=0.16,<0.19 +asyncpg>=0.20,<1 +aiosqlite>=0.16,<1 commonmark>=0.9,<1 -ruamel.yaml>=0.15.35,<0.18 +ruamel.yaml>=0.15.35,<0.19 attrs>=18.1.0 bcrypt>=3,<5 packaging>=10 click>=7,<9 colorama>=0.4,<0.5 -questionary>=1,<2 +questionary>=1,<3 jinja2>=2,<4 +setuptools diff --git a/setup.py b/setup.py index 24f9e00..838196f 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ setuptools.setup( install_requires=install_requires, extras_require=extras_require, - python_requires="~=3.9", + python_requires="~=3.10", classifiers=[ "Development Status :: 4 - Beta", @@ -50,13 +50,16 @@ setuptools.setup( "Framework :: AsyncIO", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ], entry_points=""" [console_scripts] mbc=maubot.cli:app + [pytest11] + maubot=maubot.testing """, data_files=[ (".", ["maubot/example-config.yaml"]),