From 696ee3b4d0ddcce147c750df04ace392fd38e017 Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Sun, 13 Jul 2025 17:27:23 -0700 Subject: [PATCH] RSS consumer --- base-config.yaml | 30 ++++----- consumerntfy.py | 111 ------------------------------ consumerrss.py | 171 +++++++++++++++++++++++++++++++++++++++++++++++ maubot.yaml | 9 ++- readme.md | 12 ++-- 5 files changed, 197 insertions(+), 136 deletions(-) delete mode 100644 consumerntfy.py create mode 100644 consumerrss.py diff --git a/base-config.yaml b/base-config.yaml index e8b8b24..ce96758 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -2,6 +2,10 @@ # DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING command_prefix: report +# Feed URL +# This should point to a helpful page about the feeds. Required for RSS to function +feed_url: "https://example.org" + # Which regions the bot should pay attention to. # Pass through empty list to allow all senders for a region. # Add a list of MatrixIDs to whitelist senders for a region. @@ -16,22 +20,14 @@ allowed_regions: buzz: aaa: -# Bot-specific configurations per region -# Code will assume no authentication if either username or password are missing. -# Use __global__ to specify an endpoint for all regions. -# Code will push to both __global__ and region-specific endpoint. -region_configs: - arizona: - # Where and how the plugin will send the data to - server_url: "https://ntfy.sh" - server_topic: "changeme" +# List of regions to serve feeds to. +# YOU MUST SPECIFY EVERY REGION. +# __global__ is a global feed, not serving feeds for all regions. +serve_regions: + - "__global__" + - "Arizona" + - "California" - # If the plugin should use a username and password to send notifications - username: "no" - password: "way" - -# If this bot should send a reaction to the message -# Will send 👍 on successful POST to NTFY, and 👎 on failiure. -# Failiure to post to either the region specific, or __global__ endpoint is -# considered complete failiure. +# If this bot should send a reaction to the message. +# Will send 👍 on successful storage in database, and 👎 on failiure. send_reaction: true \ No newline at end of file diff --git a/consumerntfy.py b/consumerntfy.py deleted file mode 100644 index 5bce54e..0000000 --- a/consumerntfy.py +++ /dev/null @@ -1,111 +0,0 @@ -from maubot import Plugin, MessageEvent -from maubot.handlers import command - -# Needed for configuration -from typing import Type -from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper - -import aiohttp - -# Ensures a running instance gets an updated config from the Maubot interface -class Config(BaseProxyConfig): - def do_update(self, helper: ConfigUpdateHelper) -> None: - helper.copy("command_prefix") - helper.copy("allowed_regions") - helper.copy("region_configs") - helper.copy("send_reaction") - -class ConsumerNTFY(Plugin): - # Get configuration at startup - async def start(self) -> None: - self.config.load_and_update() - - # Get config - @classmethod - def get_config_class(cls) -> Type[BaseProxyConfig]: - return Config - - # Get !command_name setting from config to register it - def get_command_name(self) -> str: - return self.config["command_prefix"] - - # Checks if a sender if allowed to send for a particular region - def validate_sender(self, region: str, sender: str): - # Mautrix isn't documented, like at all, so I'm just gonna catch the - # error because IDK how to see if a map is inside a map. - try: allowed_list = self.config["allowed_regions"][region] - except: return False - - if allowed_list is None: return True # Empty list - if sender in allowed_list: return True - - # Sender not allowed in region config - return False - - # Does the necessary config checks for the given event - # Returns list of recursive configs to process - def validate_report(self, evt: MessageEvent, message: str): - # Split command (minus !command_name) into tokens - tokens = message.split() - region = tokens[0].lower() - - # Each command must have a state/territory designation and a message - if len(tokens) < 2: return [] - # And we must have self.config["region_configs"] - try: trashvariable = self.config["region_configs"] - except: return [] - - configs = [] # To be returned - - allowed_globally = self.validate_sender("__global__", evt.sender) - allowed_region = self.validate_sender(region, evt.sender) - - # If user is allowed globally and/or for a region, - # the plugin should process both the region and __global__ configs - if allowed_globally or allowed_region: - for i in ["__global__", region]: - try: configs.append(self.config["region_configs"][i]) - except: trashvariable = None - - return configs - - # What gets called when !command_name message is sent - @command.new(name=get_command_name, help="Report Something") - @command.argument("message", pass_raw=True) - async def report(self, evt: MessageEvent, message: str) -> None: - # If all have passed - ntfy_posts_passed = None - - # Iterate through each endpoint that the message should be pushed to - # (if any) - for region_config in self.validate_report(evt, message): - # Detect no regions in reaction - if ntfy_posts_passed is None: ntfy_posts_passed = True - - self.log.debug(region_config) - - # Create notification text - split_message = message.split() - text = "[" + split_message[0] + "] " + ' '.join(message.split()[1:]) - - # Build URL - url = region_config["server_url"] + "/" + region_config["server_topic"] - - # Consider authentication - auth = None - if "username" in region_config and "password" in region_config: - auth = aiohttp.BasicAuth(region_config["username"], region_config["password"]) - - # Send notification - async with self.http.post(url, data=text, auth=auth) as response: - if not response.status == 200: - ntfy_posts_passed = False - - # Send reaction based on successful or failedPOSTs - if self.config["send_reaction"]: - if ntfy_posts_passed is True: - await evt.react("👍") - elif ntfy_posts_passed is False: - await evt.react("👎") - -# That's all, folks \ No newline at end of file diff --git a/consumerrss.py b/consumerrss.py new file mode 100644 index 0000000..a8e37e3 --- /dev/null +++ b/consumerrss.py @@ -0,0 +1,171 @@ +from maubot import Plugin, MessageEvent +from maubot.handlers import command +from maubot.handlers import web + +# Needed for configuration +from typing import Type +from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper +from mautrix.util.async_db import UpgradeTable, Scheme, Connection +from aiohttp.web import Request, Response, json_response + +import aiohttp +import datetime + +import xml.etree.ElementTree as et + +upgrade_table = UpgradeTable() +@upgrade_table.register(description="Initial revision") +async def upgrade_v1(conn: Connection) -> None: + # Table contains: + # Unique ID (event ID) (Primary) + # Timestamp (TIMESTAMP type) + # Region (string) + # Message (string) + await conn.execute( + """CREATE TABLE reports ( + id TEXT PRIMARY KEY, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + region TEXT NOT NULL, + message TEXT + )""" + ) + +# Ensures a running instance gets an updated config from the Maubot interface +class Config(BaseProxyConfig): + def do_update(self, helper: ConfigUpdateHelper) -> None: + helper.copy("command_prefix") + helper.copy("allowed_regions") + helper.copy("serve_regions") + helper.copy("send_reaction") + +class ConsumerRSS(Plugin): + # Get configuration at startup + async def start(self) -> None: + self.config.load_and_update() + + # Get config + @classmethod + def get_config_class(cls) -> Type[BaseProxyConfig]: + return Config + + # Maubot system needs this to register the table + @classmethod + def get_db_upgrade_table(cls) -> UpgradeTable | None: + return upgrade_table + + # Get !command_name setting from config to register it + def get_command_name(self) -> str: + return self.config["command_prefix"] + + # Checks if a sender if allowed to send for a particular region + def validate_sender(self, region: str, sender: str): + # Mautrix isn't documented, like at all, so I'm just gonna catch the + # error because IDK how to see if a map is inside a map. + try: allowed_list = self.config["allowed_regions"][region] + except: return False + + if allowed_list is None: return True # Empty list + if sender in allowed_list: return True + + # Sender not allowed in region config + return False + + # Does the necessary config checks for the given event + # Returns a region name if sender is allowed for a region, or None + def validate_report(self, evt: MessageEvent, message: str): + # Split command (minus !command_name) into tokens + tokens = message.split() + region = tokens[0].lower() + + allowed_globally = self.validate_sender("__global__", evt.sender) + allowed_region = self.validate_sender(region, evt.sender) + if allowed_globally or allowed_region: return region + + return None + + # What gets called when !command_name message is sent + @command.new(name=get_command_name, help="Report Something") + @command.argument("message", pass_raw=True) + async def report(self, evt: MessageEvent, message: str) -> None: + try: + # Check if sender is allowed for a region + region = self.validate_report(evt, message) + if region is None: return + + # Feed information + split_message = message.split() + id = evt.event_id + timestamp = dt = datetime.datetime.fromtimestamp(evt.timestamp/1000) + self.log.debug(timestamp) + region = split_message[0] + message = ' '.join(split_message[1:]) + + # Insert into database + self.log.debug(await self.database.execute("INSERT INTO reports (id, timestamp, region, message) VALUES ($1, $2, $3, $4)", id, timestamp, region, message)) + + # Mark as inserted + await evt.react("👍") + except Exception as error: + self.log.fatal( f"An error occurred: {error}") + + @web.get("/feed.rss") + async def getfeed(self, req: Request) -> Response: + try: + # Get region from query + query = req.query + if not "region" in query: + return Response(text="Request must include ?region= query", status=405) + region = query["region"] + + # Check region against served regions + if not region in self.config["serve_regions"]: + return Response(text="Specified region not served on this consumer", status=404) + + if region == "__global__": + feed_items = await self.database.fetch("SELECT * FROM reports ORDER BY timestamp DESC LIMIT 50") + else: + feed_items = await self.database.fetch("SELECT * FROM reports where region=$1 ORDER BY timestamp DESC LIMIT 50", region) + self.log.debug(feed_items) + + # Build feed + feed = et.Element("rss") + feed.set("version", "2.0") + + # Channel metadata + channel = et.SubElement(feed, "channel") + et.SubElement(channel, "title").text = f"ICE reports {region}" + et.SubElement(channel, "description").text = f"ICE reports for {region}" + link = et.SubElement(channel, 'link') + link.text = self.config["feed_url"] + self_link = et.SubElement(channel, 'atom:link') + self_link.set('rel', 'self') + self_link.set('href', self.config["feed_url"]) + self_link.set('type', 'application/rss+xml') + + + # Add items + for item in feed_items: + # Data from feed + id = item["id"] + timestamp = item["timestamp"] + region = item["region"] + message = "[" + region + "] " + item["message"] + # Item + item_element = et.SubElement(channel, "item") + # GUID is different because the isPermaLink has to be false + guid = et.SubElement(item_element, 'guid') + guid.text = id + guid.set('isPermaLink', 'false') + # Other ite,s + et.SubElement(item_element, "title").text = message + et.SubElement(item_element, "description").text = message + et.SubElement(item_element, "pubDate").text = timestamp.strftime("%a, %d %b %Y %H:%M:%S +0000") + + + + # Return feed + return aiohttp.web.Response(body=et.tostring(feed, encoding="unicode"), content_type='application/rss+xml') + + except Exception as error: + self.log.fatal( f"An error occurred: {error}") + return Response(text="Internal server error", status=500) diff --git a/maubot.yaml b/maubot.yaml index d040b22..d1bcd26 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,10 +1,13 @@ maubot: 0.1.0 -id: org.fiftyfiftyonearizona.reports.consumerntfy +id: org.fiftyfiftyonearizona.reports.consumerrss version: 1.2.0 license: MIT modules: - - consumerntfy -main_class: ConsumerNTFY + - consumerrss +main_class: ConsumerRSS config: true +database: true +database_type: asyncpg +webapp: true extra_files: - base-config.yaml \ No newline at end of file diff --git a/readme.md b/readme.md index 5b3d88d..260aa8e 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# FiftyFiftyOneArizona's Matrix-Based Reporting System, NTFY Consumer +# FiftyFiftyOneArizona's Matrix-Based Reporting System, RSS Consumer Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation @@ -6,8 +6,10 @@ This plugin supports the following: * whitelisting regions, or allowing all regions. * allowed senders per region, or all regions. -* reaction receipt upon successful POST to NTFY. -* authenticated NTFY POSTing. -* Different NTFY endpoint, topic, and authentication per region. +* reaction receipt upon successful storage in database. +* Whitelisting which regions to both listen to and serve RSS feeds store. +* Configuring the feed's atom:link -Please see [base-config.yaml](base-config.yaml) for explanation of behavior. \ No newline at end of file +Please see [base-config.yaml](base-config.yaml) for explanation of behavior. + +The feed is fully valid and contains items with non-URL GUIDs. The title and body content are the same. Items are timestamped per the original Matrix event. \ No newline at end of file