RSS consumer

base-config.yaml

@@ -2,8 +2,15 @@
 # DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING
 command_prefix: report
 
-# Which regions the bot should pay attention to
-# Additionally, optionally whitelist specific accounts for each region
+# Feed URL
+# This should point to a helpful page about the feeds. Required for RSS to function
+feed_url: "https://example.org"
+
+# Which regions the bot should pay attention to.
+# Pass through empty list to allow all senders for a region.
+# Add a list of MatrixIDs to whitelist senders for a region.
+# Use __global__ to refer to all regions.
+# Code looks to see if a sender is allowed for either a specific region or __global__.
 allowed_regions:
     foo:
         - "@reports:fiftyfiftyonerarizona.org"
@@ -13,13 +20,14 @@ allowed_regions:
     buzz:
     aaa:
 
-# Bot-specific configurations per region
-region_configs:
-    __global__: true
-    arizona: true
-    california: true
-    somethingyoudontwantanrssfor: true
+# List of regions to serve feeds to.
+# YOU MUST SPECIFY EVERY REGION.
+# __global__ is a global feed, not serving feeds for all regions.
+serve_regions:
+    - "__global__"
+    - "Arizona"
+    - "California"
 
-# If this bot should send a reaction to the message
-# if the notification was successful
+# If this bot should send a reaction to the message.
+# Will send 👍 on successful storage in database, and 👎 on failiure.
 send_reaction: true

compile.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Get filename
+id=$(cat maubot.yaml | grep "id:" | tr ' ' '\n' | tail -n1)
+version=$(cat maubot.yaml | grep "version:" | tr ' ' '\n' | tail -n1)
+filename="$id-$version.mbp"
+
+# Compress to zip
+zip $filename *

consumerrss.py

@@ -1,18 +1,41 @@
 from maubot import Plugin, MessageEvent
 from maubot.handlers import command
+from maubot.handlers import web
 
 # Needed for configuration
 from typing import Type
 from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
+from mautrix.util.async_db import UpgradeTable, Scheme, Connection
+from aiohttp.web import Request, Response, json_response
 
 import aiohttp
+import datetime
+
+import xml.etree.ElementTree as et
+
+upgrade_table = UpgradeTable()
+@upgrade_table.register(description="Initial revision")
+async def upgrade_v1(conn: Connection) -> None:
+	# Table contains:
+	# Unique ID (event ID) (Primary)
+	# Timestamp (TIMESTAMP type)
+	# Region (string)
+	# Message (string)
+	await conn.execute(
+		"""CREATE TABLE reports (
+			id TEXT PRIMARY KEY,
+			timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
+			region TEXT NOT NULL,
+			message TEXT
+		)"""
+	)
 
 # Ensures a running instance gets an updated config from the Maubot interface
 class Config(BaseProxyConfig):
 	def do_update(self, helper: ConfigUpdateHelper) -> None:
 		helper.copy("command_prefix")
 		helper.copy("allowed_regions")
-		helper.copy("region_configs")
+		helper.copy("serve_regions")
 		helper.copy("send_reaction")
 
 class ConsumerRSS(Plugin):
@@ -25,83 +48,124 @@ class ConsumerRSS(Plugin):
 	def get_config_class(cls) -> Type[BaseProxyConfig]:
 		return Config
 	
+	# Maubot system needs this to register the table
+	@classmethod
+	def get_db_upgrade_table(cls) -> UpgradeTable | None:
+		return upgrade_table
+	
 	# Get !command_name setting from config to register it
 	def get_command_name(self) -> str:
 		return self.config["command_prefix"]
 	
 	# Checks if a sender if allowed to send for a particular region
-	def validateSender(self, region: str, sender: str):
-		# Check that config value exists
-		if not self.config["allowed_regions"]: return False
-		# Check that region is allowed
-		if not region in self.config["allowed_regions"]: return False
-		# All senders allowed for this region
-		if len(self.config["allowed_regions"][region]) == 0: return True
-		# Check that sender is allowed for region
-		if not sender in self.config["allowed_regions"][region]: return False
-		return True
+	def validate_sender(self, region: str, sender: str):
+		# Mautrix isn't documented, like at all, so I'm just gonna catch the
+		# error because IDK how to see if a map is inside a map.
+		try: allowed_list = self.config["allowed_regions"][region]
+		except: return False
 		
-	# Does the necesary config checks for the given event
-	# Returns list of regions to process (strings)
-	# Currently just the specified region and "__global__"
-	def validateReport(self, evt: MessageEvent, message: str):
+		if allowed_list is None: return True # Empty list
+		if sender in allowed_list: return True
+		
+		# Sender not allowed in region config
+		return False
+	
+	# Does the necessary config checks for the given event
+	# Returns a region name if sender is allowed for a region, or None
+	def validate_report(self, evt: MessageEvent, message: str):
 		# Split command (minus !command_name) into tokens
 		tokens = message.split()
 		region = tokens[0].lower()
 		
-		# Each command must have a state/territory designation and a message
-		if len(tokens) < 2: return None
+		allowed_globally = self.validate_sender("__global__", evt.sender)
+		allowed_region = self.validate_sender(region, evt.sender)
+		if allowed_globally or allowed_region: return region
 		
-		self.log.debug(region)
-		
-		# This is a list of regions to process for this specific message
-		# This is only used to consider __global__
-		regions_to_process = []
-		if (self.validateSender("__global__", evt.sender)):
-			regions_to_process.append("__global__")
-		if (self.validateSender(region, evt.sender)):
-			regions_to_process.append(region)
-		
-		return regions_to_process
+		return None
 	
 	# What gets called when !command_name message is sent
 	@command.new(name=get_command_name, help="Report Something")
 	@command.argument("message", pass_raw=True)
 	async def report(self, evt: MessageEvent, message: str) -> None:
-		# If all have passed
-		ntfy_posts_passed = None
+		try:
+			# Check if sender is allowed for a region
+			region = self.validate_report(evt, message)
+			if region is None: return
 			
-		# Iterate through each endpoint that the message should be pushed to
-		# (if any)
-		for region in self.validateReport(evt, message):
-			# Detect no regions in reaction
-			if ntfy_posts_passed is None: ntfy_posts_passed = True
-			
-			# Grab region-specific conrfig
-			region_config = self.config["region_configs"][region]
-			
-			# Create notification text
+			# Feed information
 			split_message = message.split()
-			text = "[" + split_message[0] + "] " + ' '.join(message.split()[1:])
+			id = evt.event_id
+			timestamp = dt = datetime.datetime.fromtimestamp(evt.timestamp/1000)
+			self.log.debug(timestamp)
+			region = split_message[0]
+			message = ' '.join(split_message[1:])
 			
-			# Build URL
-			url = region_config["server_url"] + "/" + region_config["server_topic"]
+			# Insert into database
+			self.log.debug(await self.database.execute("INSERT INTO reports (id, timestamp, region, message) VALUES ($1, $2, $3, $4)", id, timestamp, region, message))
 			
-			# Consider authentication
-			authentication = None
-			if region_config["server_use_authentication"]:
-				authentication = aiohttp.BasicAuth(region_config["server_username"], region_config["server_password"])
+			# Mark as inserted
+			await evt.react("👍")
+		except Exception as error:
+			self.log.fatal( f"An error occurred: {error}")
 	
-			# Send notification
-			async with self.http.post(url, data=text, auth=authentication) as response:
-				if not response.status == 200:
-					ntfy_posts_passed = False
+	@web.get("/feed.rss")
+	async def getfeed(self, req: Request) -> Response:
+		try:
+			# Get region from query
+			query = req.query
+			if not "region" in query:
+				return Response(text="Request must include ?region= query", status=405)
+			region = query["region"]
 			
-		# Send reaction based on successful or failedPOSTs
-		if self.config["send_reaction"]:
-			if ntfy_posts_passed is True:
-				await evt.react("👍")
-			elif ntfy_posts_passed is False:
-				await evt.react("👎")
+			# Check region against served regions
+			if not region in self.config["serve_regions"]:
+				return Response(text="Specified region not served on this consumer", status=404)
 			
-# That's all, folks
+			if region == "__global__":
+				feed_items = await self.database.fetch("SELECT * FROM reports ORDER BY timestamp DESC LIMIT 50")
+			else:
+				feed_items = await self.database.fetch("SELECT * FROM reports where region=$1 ORDER BY timestamp DESC LIMIT 50", region)
+			self.log.debug(feed_items)
+			
+			# Build feed
+			feed = et.Element("rss")
+			feed.set("version", "2.0")
+			
+			# Channel metadata
+			channel = et.SubElement(feed, "channel")
+			et.SubElement(channel, "title").text = f"ICE reports {region}"
+			et.SubElement(channel, "description").text = f"ICE reports for {region}"
+			link = et.SubElement(channel, 'link')
+			link.text = self.config["feed_url"]
+			self_link = et.SubElement(channel, 'atom:link')
+			self_link.set('rel', 'self')
+			self_link.set('href', self.config["feed_url"])
+			self_link.set('type', 'application/rss+xml')
+
+			
+			# Add items
+			for item in feed_items:
+				# Data from feed
+				id = item["id"]
+				timestamp = item["timestamp"]
+				region = item["region"]
+				message = "[" + region + "] " + item["message"]
+				# Item
+				item_element = et.SubElement(channel, "item")
+				# GUID is different because the isPermaLink has to be false
+				guid = et.SubElement(item_element, 'guid')
+				guid.text = id
+				guid.set('isPermaLink', 'false')
+				# Other ite,s
+				et.SubElement(item_element, "title").text = message
+				et.SubElement(item_element, "description").text = message
+				et.SubElement(item_element, "pubDate").text = timestamp.strftime("%a, %d %b %Y %H:%M:%S +0000")
+				
+
+				
+			# Return feed
+			return aiohttp.web.Response(body=et.tostring(feed, encoding="unicode"), content_type='application/rss+xml')
+
+		except Exception as error:
+			self.log.fatal( f"An error occurred: {error}")
+			return Response(text="Internal server error", status=500)

maubot.yaml

@@ -1,12 +1,13 @@
 maubot: 0.1.0
 id: org.fiftyfiftyonearizona.reports.consumerrss
-version: 1.1.0
+version: 1.2.0
 license: MIT
 modules:
   - consumerrss
 main_class: ConsumerRSS
 config: true
 database: true
+database_type: asyncpg
 webapp: true
 extra_files:
   - base-config.yaml

readme.md

@@ -2,6 +2,14 @@
 
 Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation
 
-This plugin supports whitelisting state/territory designators, allowed senders, and reaction receipts. It also has separate RSS feeds for each reigon, and a global feed.
+This plugin supports the following:
 
-Feeds will be available at `http://your.maubot.instance/_matrix/maubot/plugin/<instance ID>/<reigon>.rss`. where `reigon` is non-case-sensitive. Please see https://docs.mau.fi/maubot/dev/handlers/web.html to double check what this path should be exactly.
+* whitelisting regions, or allowing all regions.
+* allowed senders per region, or all regions.
+* reaction receipt upon successful storage in database.
+* Whitelisting which regions to both listen to and serve RSS feeds store.
+* Configuring the feed's atom:link
+
+Please see [base-config.yaml](base-config.yaml) for explanation of behavior.
+
+The feed is fully valid and contains items with non-URL GUIDs. The title and body content are the same. Items are timestamped per the original Matrix event.