From c7b214387dcaa61fa70b37c516538d3f186ee124 Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Thu, 26 Jun 2025 16:40:49 -0700 Subject: [PATCH 1/4] Fork for RSS consumer --- base-config.yaml | 13 ++++--------- consumerntfy.py => consumerrss.py | 4 ++-- maubot.yaml | 8 +++++--- readme.md | 6 ++++-- 4 files changed, 15 insertions(+), 16 deletions(-) rename consumerntfy.py => consumerrss.py (97%) diff --git a/base-config.yaml b/base-config.yaml index 07fedc5..e496539 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -15,15 +15,10 @@ allowed_regions: # Bot-specific configurations per region region_configs: - arizona: - # Where and how the plugin will send the data to - server_url: "https://ntfy.sh" - server_topic: "changeme" - - # If the plugin should use a username and password to send notifications - server_use_authentication: true - server_username: "no" - server_password: "way" + __global__: true + arizona: true + california: true + somethingyoudontwantanrssfor: true # If this bot should send a reaction to the message # if the notification was successful diff --git a/consumerntfy.py b/consumerrss.py similarity index 97% rename from consumerntfy.py rename to consumerrss.py index 822d18d..a075d6e 100644 --- a/consumerntfy.py +++ b/consumerrss.py @@ -15,7 +15,7 @@ class Config(BaseProxyConfig): helper.copy("region_configs") helper.copy("send_reaction") -class ConsumerNTFY(Plugin): +class ConsumerRSS(Plugin): # Get configuration at startup async def start(self) -> None: self.config.load_and_update() @@ -34,7 +34,7 @@ class ConsumerNTFY(Plugin): # Check that config value exists if not self.config["allowed_regions"]: return False # Check that region is allowed - if not self.config["allowed_regions"][region]: return False + if not region in self.config["allowed_regions"]: return False # All senders allowed for this region if len(self.config["allowed_regions"][region]) == 0: return True # Check that sender is allowed for region diff --git a/maubot.yaml b/maubot.yaml index 899e4fe..59ce33b 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,10 +1,12 @@ maubot: 0.1.0 -id: org.fiftyfiftyonearizona.reports.consumerntfy +id: org.fiftyfiftyonearizona.reports.consumerrss version: 1.1.0 license: MIT modules: - - consumerntfy -main_class: ConsumerNTFY + - consumerrss +main_class: ConsumerRSS config: true +database: true +webapp: true extra_files: - base-config.yaml \ No newline at end of file diff --git a/readme.md b/readme.md index 4b621b7..e23b655 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,7 @@ -# FiftyFiftyOneArizona's Matrix-Based Reporting System, NTFY Consumer +# FiftyFiftyOneArizona's Matrix-Based Reporting System, RSS Consumer Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation -This plugin supports whitelisting state/territory designators, allowed senders, reaction receipt, and authenticated NTFY posting. \ No newline at end of file +This plugin supports whitelisting state/territory designators, allowed senders, and reaction receipts. It also has separate RSS feeds for each reigon, and a global feed. + +Feeds will be available at `http://your.maubot.instance/_matrix/maubot/plugin//.rss`. where `reigon` is non-case-sensitive. Please see https://docs.mau.fi/maubot/dev/handlers/web.html to double check what this path should be exactly. \ No newline at end of file From 2193a9ce62a223af784b0848c560d45599e1555e Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Mon, 30 Jun 2025 09:38:59 -0700 Subject: [PATCH 2/4] Heavy improvements. Version 1.2 --- base-config.yaml | 19 +++++++++----- compile.bash | 9 +++++++ consumerntfy.py | 66 +++++++++++++++++++++++++----------------------- maubot.yaml | 2 +- readme.md | 10 +++++++- 5 files changed, 67 insertions(+), 39 deletions(-) create mode 100644 compile.bash diff --git a/base-config.yaml b/base-config.yaml index 07fedc5..e8b8b24 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -2,8 +2,11 @@ # DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING command_prefix: report -# Which regions the bot should pay attention to -# Additionally, optionally whitelist specific accounts for each region +# Which regions the bot should pay attention to. +# Pass through empty list to allow all senders for a region. +# Add a list of MatrixIDs to whitelist senders for a region. +# Use __global__ to refer to all regions. +# Code looks to see if a sender is allowed for either a specific region or __global__. allowed_regions: foo: - "@reports:fiftyfiftyonerarizona.org" @@ -14,6 +17,9 @@ allowed_regions: aaa: # Bot-specific configurations per region +# Code will assume no authentication if either username or password are missing. +# Use __global__ to specify an endpoint for all regions. +# Code will push to both __global__ and region-specific endpoint. region_configs: arizona: # Where and how the plugin will send the data to @@ -21,10 +27,11 @@ region_configs: server_topic: "changeme" # If the plugin should use a username and password to send notifications - server_use_authentication: true - server_username: "no" - server_password: "way" + username: "no" + password: "way" # If this bot should send a reaction to the message -# if the notification was successful +# Will send 👍 on successful POST to NTFY, and 👎 on failiure. +# Failiure to post to either the region specific, or __global__ endpoint is +# considered complete failiure. send_reaction: true \ No newline at end of file diff --git a/compile.bash b/compile.bash new file mode 100644 index 0000000..66c85a3 --- /dev/null +++ b/compile.bash @@ -0,0 +1,9 @@ +#!/bin/bash + +# Get filename +id=$(cat maubot.yaml | grep "id:" | tr ' ' '\n' | tail -n1) +version=$(cat maubot.yaml | grep "version:" | tr ' ' '\n' | tail -n1) +filename="$id-$version.mbp" + +# Compress to zip +zip $filename * \ No newline at end of file diff --git a/consumerntfy.py b/consumerntfy.py index 822d18d..833a057 100644 --- a/consumerntfy.py +++ b/consumerntfy.py @@ -30,39 +30,44 @@ class ConsumerNTFY(Plugin): return self.config["command_prefix"] # Checks if a sender if allowed to send for a particular region - def validateSender(self, region: str, sender: str): - # Check that config value exists - if not self.config["allowed_regions"]: return False - # Check that region is allowed - if not self.config["allowed_regions"][region]: return False - # All senders allowed for this region - if len(self.config["allowed_regions"][region]) == 0: return True - # Check that sender is allowed for region - if not sender in self.config["allowed_regions"][region]: return False - return True + def validate_sender(self, region: str, sender: str): + # Mautrix isn't documented, like at all, so I'm just gonna catch the + # error because IDK how to see if a map is inside a map. + try: allowed_list = self.config["allowed_regions"][region] + except: return False + + if len(allowed_list) == 0: return True + if sender in allowed_list: return True + + # Sender not allowed in region config + return False - # Does the necesary config checks for the given event - # Returns list of regions to process (strings) - # Currently just the specified region and "__global__" - def validateReport(self, evt: MessageEvent, message: str): + # Does the necessary config checks for the given event + # Returns list of recursive configs to process + def validate_report(self, evt: MessageEvent, message: str): # Split command (minus !command_name) into tokens tokens = message.split() region = tokens[0].lower() # Each command must have a state/territory designation and a message - if len(tokens) < 2: return None + if len(tokens) < 2: return [] + # And we must have self.config["region_configs"] + try: trashvariable = self.config["allowed_regions"][region] + except: return [] - self.log.debug(region) + configs = [] # To be returned - # This is a list of regions to process for this specific message - # This is only used to consider __global__ - regions_to_process = [] - if (self.validateSender("__global__", evt.sender)): - regions_to_process.append("__global__") - if (self.validateSender(region, evt.sender)): - regions_to_process.append(region) + allowed_globally = self.validate_sender("__global__", evt.sender) + allowed_region = self.validate_sender(region, evt.sender) - return regions_to_process + # If user is allowed globally and/or for a region, + # the plugin should process both the region and __global__ configs + if allowed_globally or allowed_region: + for i in ["__global__", region]: + try: configs.append(self.config["region_configs"][i]) + except: trashvariable = None + + return configs # What gets called when !command_name message is sent @command.new(name=get_command_name, help="Report Something") @@ -73,12 +78,11 @@ class ConsumerNTFY(Plugin): # Iterate through each endpoint that the message should be pushed to # (if any) - for region in self.validateReport(evt, message): + for region_config in self.validate_report(evt, message): # Detect no regions in reaction if ntfy_posts_passed is None: ntfy_posts_passed = True - # Grab region-specific conrfig - region_config = self.config["region_configs"][region] + self.log.debug(region_config) # Create notification text split_message = message.split() @@ -88,12 +92,12 @@ class ConsumerNTFY(Plugin): url = region_config["server_url"] + "/" + region_config["server_topic"] # Consider authentication - authentication = None - if region_config["server_use_authentication"]: - authentication = aiohttp.BasicAuth(region_config["server_username"], region_config["server_password"]) + auth = None + if "username" in region_config and "password" in region_config: + auth = aiohttp.BasicAuth(region_config["username"], region_config["password"]) # Send notification - async with self.http.post(url, data=text, auth=authentication) as response: + async with self.http.post(url, data=text, auth=auth) as response: if not response.status == 200: ntfy_posts_passed = False diff --git a/maubot.yaml b/maubot.yaml index 899e4fe..d040b22 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,6 +1,6 @@ maubot: 0.1.0 id: org.fiftyfiftyonearizona.reports.consumerntfy -version: 1.1.0 +version: 1.2.0 license: MIT modules: - consumerntfy diff --git a/readme.md b/readme.md index 4b621b7..5b3d88d 100644 --- a/readme.md +++ b/readme.md @@ -2,4 +2,12 @@ Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation -This plugin supports whitelisting state/territory designators, allowed senders, reaction receipt, and authenticated NTFY posting. \ No newline at end of file +This plugin supports the following: + +* whitelisting regions, or allowing all regions. +* allowed senders per region, or all regions. +* reaction receipt upon successful POST to NTFY. +* authenticated NTFY POSTing. +* Different NTFY endpoint, topic, and authentication per region. + +Please see [base-config.yaml](base-config.yaml) for explanation of behavior. \ No newline at end of file From beefb8c73ebb95be4a88cb5c9ac22e4106d7d0f2 Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Mon, 30 Jun 2025 11:08:09 -0700 Subject: [PATCH 3/4] Minor bug fixes. Will re-release 1.2.0 --- consumerntfy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/consumerntfy.py b/consumerntfy.py index 833a057..5bce54e 100644 --- a/consumerntfy.py +++ b/consumerntfy.py @@ -36,7 +36,7 @@ class ConsumerNTFY(Plugin): try: allowed_list = self.config["allowed_regions"][region] except: return False - if len(allowed_list) == 0: return True + if allowed_list is None: return True # Empty list if sender in allowed_list: return True # Sender not allowed in region config @@ -52,7 +52,7 @@ class ConsumerNTFY(Plugin): # Each command must have a state/territory designation and a message if len(tokens) < 2: return [] # And we must have self.config["region_configs"] - try: trashvariable = self.config["allowed_regions"][region] + try: trashvariable = self.config["region_configs"] except: return [] configs = [] # To be returned From 696ee3b4d0ddcce147c750df04ace392fd38e017 Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Sun, 13 Jul 2025 17:27:23 -0700 Subject: [PATCH 4/4] RSS consumer --- base-config.yaml | 30 ++++----- consumerntfy.py | 111 ------------------------------ consumerrss.py | 171 +++++++++++++++++++++++++++++++++++++++++++++++ maubot.yaml | 9 ++- readme.md | 12 ++-- 5 files changed, 197 insertions(+), 136 deletions(-) delete mode 100644 consumerntfy.py create mode 100644 consumerrss.py diff --git a/base-config.yaml b/base-config.yaml index e8b8b24..ce96758 100644 --- a/base-config.yaml +++ b/base-config.yaml @@ -2,6 +2,10 @@ # DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING command_prefix: report +# Feed URL +# This should point to a helpful page about the feeds. Required for RSS to function +feed_url: "https://example.org" + # Which regions the bot should pay attention to. # Pass through empty list to allow all senders for a region. # Add a list of MatrixIDs to whitelist senders for a region. @@ -16,22 +20,14 @@ allowed_regions: buzz: aaa: -# Bot-specific configurations per region -# Code will assume no authentication if either username or password are missing. -# Use __global__ to specify an endpoint for all regions. -# Code will push to both __global__ and region-specific endpoint. -region_configs: - arizona: - # Where and how the plugin will send the data to - server_url: "https://ntfy.sh" - server_topic: "changeme" +# List of regions to serve feeds to. +# YOU MUST SPECIFY EVERY REGION. +# __global__ is a global feed, not serving feeds for all regions. +serve_regions: + - "__global__" + - "Arizona" + - "California" - # If the plugin should use a username and password to send notifications - username: "no" - password: "way" - -# If this bot should send a reaction to the message -# Will send 👍 on successful POST to NTFY, and 👎 on failiure. -# Failiure to post to either the region specific, or __global__ endpoint is -# considered complete failiure. +# If this bot should send a reaction to the message. +# Will send 👍 on successful storage in database, and 👎 on failiure. send_reaction: true \ No newline at end of file diff --git a/consumerntfy.py b/consumerntfy.py deleted file mode 100644 index 5bce54e..0000000 --- a/consumerntfy.py +++ /dev/null @@ -1,111 +0,0 @@ -from maubot import Plugin, MessageEvent -from maubot.handlers import command - -# Needed for configuration -from typing import Type -from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper - -import aiohttp - -# Ensures a running instance gets an updated config from the Maubot interface -class Config(BaseProxyConfig): - def do_update(self, helper: ConfigUpdateHelper) -> None: - helper.copy("command_prefix") - helper.copy("allowed_regions") - helper.copy("region_configs") - helper.copy("send_reaction") - -class ConsumerNTFY(Plugin): - # Get configuration at startup - async def start(self) -> None: - self.config.load_and_update() - - # Get config - @classmethod - def get_config_class(cls) -> Type[BaseProxyConfig]: - return Config - - # Get !command_name setting from config to register it - def get_command_name(self) -> str: - return self.config["command_prefix"] - - # Checks if a sender if allowed to send for a particular region - def validate_sender(self, region: str, sender: str): - # Mautrix isn't documented, like at all, so I'm just gonna catch the - # error because IDK how to see if a map is inside a map. - try: allowed_list = self.config["allowed_regions"][region] - except: return False - - if allowed_list is None: return True # Empty list - if sender in allowed_list: return True - - # Sender not allowed in region config - return False - - # Does the necessary config checks for the given event - # Returns list of recursive configs to process - def validate_report(self, evt: MessageEvent, message: str): - # Split command (minus !command_name) into tokens - tokens = message.split() - region = tokens[0].lower() - - # Each command must have a state/territory designation and a message - if len(tokens) < 2: return [] - # And we must have self.config["region_configs"] - try: trashvariable = self.config["region_configs"] - except: return [] - - configs = [] # To be returned - - allowed_globally = self.validate_sender("__global__", evt.sender) - allowed_region = self.validate_sender(region, evt.sender) - - # If user is allowed globally and/or for a region, - # the plugin should process both the region and __global__ configs - if allowed_globally or allowed_region: - for i in ["__global__", region]: - try: configs.append(self.config["region_configs"][i]) - except: trashvariable = None - - return configs - - # What gets called when !command_name message is sent - @command.new(name=get_command_name, help="Report Something") - @command.argument("message", pass_raw=True) - async def report(self, evt: MessageEvent, message: str) -> None: - # If all have passed - ntfy_posts_passed = None - - # Iterate through each endpoint that the message should be pushed to - # (if any) - for region_config in self.validate_report(evt, message): - # Detect no regions in reaction - if ntfy_posts_passed is None: ntfy_posts_passed = True - - self.log.debug(region_config) - - # Create notification text - split_message = message.split() - text = "[" + split_message[0] + "] " + ' '.join(message.split()[1:]) - - # Build URL - url = region_config["server_url"] + "/" + region_config["server_topic"] - - # Consider authentication - auth = None - if "username" in region_config and "password" in region_config: - auth = aiohttp.BasicAuth(region_config["username"], region_config["password"]) - - # Send notification - async with self.http.post(url, data=text, auth=auth) as response: - if not response.status == 200: - ntfy_posts_passed = False - - # Send reaction based on successful or failedPOSTs - if self.config["send_reaction"]: - if ntfy_posts_passed is True: - await evt.react("👍") - elif ntfy_posts_passed is False: - await evt.react("👎") - -# That's all, folks \ No newline at end of file diff --git a/consumerrss.py b/consumerrss.py new file mode 100644 index 0000000..a8e37e3 --- /dev/null +++ b/consumerrss.py @@ -0,0 +1,171 @@ +from maubot import Plugin, MessageEvent +from maubot.handlers import command +from maubot.handlers import web + +# Needed for configuration +from typing import Type +from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper +from mautrix.util.async_db import UpgradeTable, Scheme, Connection +from aiohttp.web import Request, Response, json_response + +import aiohttp +import datetime + +import xml.etree.ElementTree as et + +upgrade_table = UpgradeTable() +@upgrade_table.register(description="Initial revision") +async def upgrade_v1(conn: Connection) -> None: + # Table contains: + # Unique ID (event ID) (Primary) + # Timestamp (TIMESTAMP type) + # Region (string) + # Message (string) + await conn.execute( + """CREATE TABLE reports ( + id TEXT PRIMARY KEY, + timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, + region TEXT NOT NULL, + message TEXT + )""" + ) + +# Ensures a running instance gets an updated config from the Maubot interface +class Config(BaseProxyConfig): + def do_update(self, helper: ConfigUpdateHelper) -> None: + helper.copy("command_prefix") + helper.copy("allowed_regions") + helper.copy("serve_regions") + helper.copy("send_reaction") + +class ConsumerRSS(Plugin): + # Get configuration at startup + async def start(self) -> None: + self.config.load_and_update() + + # Get config + @classmethod + def get_config_class(cls) -> Type[BaseProxyConfig]: + return Config + + # Maubot system needs this to register the table + @classmethod + def get_db_upgrade_table(cls) -> UpgradeTable | None: + return upgrade_table + + # Get !command_name setting from config to register it + def get_command_name(self) -> str: + return self.config["command_prefix"] + + # Checks if a sender if allowed to send for a particular region + def validate_sender(self, region: str, sender: str): + # Mautrix isn't documented, like at all, so I'm just gonna catch the + # error because IDK how to see if a map is inside a map. + try: allowed_list = self.config["allowed_regions"][region] + except: return False + + if allowed_list is None: return True # Empty list + if sender in allowed_list: return True + + # Sender not allowed in region config + return False + + # Does the necessary config checks for the given event + # Returns a region name if sender is allowed for a region, or None + def validate_report(self, evt: MessageEvent, message: str): + # Split command (minus !command_name) into tokens + tokens = message.split() + region = tokens[0].lower() + + allowed_globally = self.validate_sender("__global__", evt.sender) + allowed_region = self.validate_sender(region, evt.sender) + if allowed_globally or allowed_region: return region + + return None + + # What gets called when !command_name message is sent + @command.new(name=get_command_name, help="Report Something") + @command.argument("message", pass_raw=True) + async def report(self, evt: MessageEvent, message: str) -> None: + try: + # Check if sender is allowed for a region + region = self.validate_report(evt, message) + if region is None: return + + # Feed information + split_message = message.split() + id = evt.event_id + timestamp = dt = datetime.datetime.fromtimestamp(evt.timestamp/1000) + self.log.debug(timestamp) + region = split_message[0] + message = ' '.join(split_message[1:]) + + # Insert into database + self.log.debug(await self.database.execute("INSERT INTO reports (id, timestamp, region, message) VALUES ($1, $2, $3, $4)", id, timestamp, region, message)) + + # Mark as inserted + await evt.react("👍") + except Exception as error: + self.log.fatal( f"An error occurred: {error}") + + @web.get("/feed.rss") + async def getfeed(self, req: Request) -> Response: + try: + # Get region from query + query = req.query + if not "region" in query: + return Response(text="Request must include ?region= query", status=405) + region = query["region"] + + # Check region against served regions + if not region in self.config["serve_regions"]: + return Response(text="Specified region not served on this consumer", status=404) + + if region == "__global__": + feed_items = await self.database.fetch("SELECT * FROM reports ORDER BY timestamp DESC LIMIT 50") + else: + feed_items = await self.database.fetch("SELECT * FROM reports where region=$1 ORDER BY timestamp DESC LIMIT 50", region) + self.log.debug(feed_items) + + # Build feed + feed = et.Element("rss") + feed.set("version", "2.0") + + # Channel metadata + channel = et.SubElement(feed, "channel") + et.SubElement(channel, "title").text = f"ICE reports {region}" + et.SubElement(channel, "description").text = f"ICE reports for {region}" + link = et.SubElement(channel, 'link') + link.text = self.config["feed_url"] + self_link = et.SubElement(channel, 'atom:link') + self_link.set('rel', 'self') + self_link.set('href', self.config["feed_url"]) + self_link.set('type', 'application/rss+xml') + + + # Add items + for item in feed_items: + # Data from feed + id = item["id"] + timestamp = item["timestamp"] + region = item["region"] + message = "[" + region + "] " + item["message"] + # Item + item_element = et.SubElement(channel, "item") + # GUID is different because the isPermaLink has to be false + guid = et.SubElement(item_element, 'guid') + guid.text = id + guid.set('isPermaLink', 'false') + # Other ite,s + et.SubElement(item_element, "title").text = message + et.SubElement(item_element, "description").text = message + et.SubElement(item_element, "pubDate").text = timestamp.strftime("%a, %d %b %Y %H:%M:%S +0000") + + + + # Return feed + return aiohttp.web.Response(body=et.tostring(feed, encoding="unicode"), content_type='application/rss+xml') + + except Exception as error: + self.log.fatal( f"An error occurred: {error}") + return Response(text="Internal server error", status=500) diff --git a/maubot.yaml b/maubot.yaml index d040b22..d1bcd26 100644 --- a/maubot.yaml +++ b/maubot.yaml @@ -1,10 +1,13 @@ maubot: 0.1.0 -id: org.fiftyfiftyonearizona.reports.consumerntfy +id: org.fiftyfiftyonearizona.reports.consumerrss version: 1.2.0 license: MIT modules: - - consumerntfy -main_class: ConsumerNTFY + - consumerrss +main_class: ConsumerRSS config: true +database: true +database_type: asyncpg +webapp: true extra_files: - base-config.yaml \ No newline at end of file diff --git a/readme.md b/readme.md index 5b3d88d..260aa8e 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# FiftyFiftyOneArizona's Matrix-Based Reporting System, NTFY Consumer +# FiftyFiftyOneArizona's Matrix-Based Reporting System, RSS Consumer Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation @@ -6,8 +6,10 @@ This plugin supports the following: * whitelisting regions, or allowing all regions. * allowed senders per region, or all regions. -* reaction receipt upon successful POST to NTFY. -* authenticated NTFY POSTing. -* Different NTFY endpoint, topic, and authentication per region. +* reaction receipt upon successful storage in database. +* Whitelisting which regions to both listen to and serve RSS feeds store. +* Configuring the feed's atom:link -Please see [base-config.yaml](base-config.yaml) for explanation of behavior. \ No newline at end of file +Please see [base-config.yaml](base-config.yaml) for explanation of behavior. + +The feed is fully valid and contains items with non-URL GUIDs. The title and body content are the same. Items are timestamped per the original Matrix event. \ No newline at end of file