RSS consumer
This commit is contained in:
parent
beefb8c73e
commit
696ee3b4d0
GPG key ID:
A5FCBAF34E6E8B50
5 changed files with 197 additions and 136 deletions
|
|
@ -2,6 +2,10 @@
|
|||
# DO NOT CHANGE THIS UNLESS YOU KNOW WHAT YOU ARE DOING
|
||||
command_prefix: report
|
||||
|
||||
# Feed URL
|
||||
# This should point to a helpful page about the feeds. Required for RSS to function
|
||||
feed_url: "https://example.org"
|
||||
|
||||
# Which regions the bot should pay attention to.
|
||||
# Pass through empty list to allow all senders for a region.
|
||||
# Add a list of MatrixIDs to whitelist senders for a region.
|
||||
|
|
@ -16,22 +20,14 @@ allowed_regions:
|
|||
buzz:
|
||||
aaa:
|
||||
|
||||
# Bot-specific configurations per region
|
||||
# Code will assume no authentication if either username or password are missing.
|
||||
# Use __global__ to specify an endpoint for all regions.
|
||||
# Code will push to both __global__ and region-specific endpoint.
|
||||
region_configs:
|
||||
arizona:
|
||||
# Where and how the plugin will send the data to
|
||||
server_url: "https://ntfy.sh"
|
||||
server_topic: "changeme"
|
||||
# List of regions to serve feeds to.
|
||||
# YOU MUST SPECIFY EVERY REGION.
|
||||
# __global__ is a global feed, not serving feeds for all regions.
|
||||
serve_regions:
|
||||
- "__global__"
|
||||
- "Arizona"
|
||||
- "California"
|
||||
|
||||
# If the plugin should use a username and password to send notifications
|
||||
username: "no"
|
||||
password: "way"
|
||||
|
||||
# If this bot should send a reaction to the message
|
||||
# Will send 👍 on successful POST to NTFY, and 👎 on failiure.
|
||||
# Failiure to post to either the region specific, or __global__ endpoint is
|
||||
# considered complete failiure.
|
||||
# If this bot should send a reaction to the message.
|
||||
# Will send 👍 on successful storage in database, and 👎 on failiure.
|
||||
send_reaction: true
|
||||
111
consumerntfy.py
111
consumerntfy.py
|
|
@ -1,111 +0,0 @@
|
|||
from maubot import Plugin, MessageEvent
|
||||
from maubot.handlers import command
|
||||
|
||||
# Needed for configuration
|
||||
from typing import Type
|
||||
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
|
||||
|
||||
import aiohttp
|
||||
|
||||
# Ensures a running instance gets an updated config from the Maubot interface
|
||||
class Config(BaseProxyConfig):
|
||||
def do_update(self, helper: ConfigUpdateHelper) -> None:
|
||||
helper.copy("command_prefix")
|
||||
helper.copy("allowed_regions")
|
||||
helper.copy("region_configs")
|
||||
helper.copy("send_reaction")
|
||||
|
||||
class ConsumerNTFY(Plugin):
|
||||
# Get configuration at startup
|
||||
async def start(self) -> None:
|
||||
self.config.load_and_update()
|
||||
|
||||
# Get config
|
||||
@classmethod
|
||||
def get_config_class(cls) -> Type[BaseProxyConfig]:
|
||||
return Config
|
||||
|
||||
# Get !command_name setting from config to register it
|
||||
def get_command_name(self) -> str:
|
||||
return self.config["command_prefix"]
|
||||
|
||||
# Checks if a sender if allowed to send for a particular region
|
||||
def validate_sender(self, region: str, sender: str):
|
||||
# Mautrix isn't documented, like at all, so I'm just gonna catch the
|
||||
# error because IDK how to see if a map is inside a map.
|
||||
try: allowed_list = self.config["allowed_regions"][region]
|
||||
except: return False
|
||||
|
||||
if allowed_list is None: return True # Empty list
|
||||
if sender in allowed_list: return True
|
||||
|
||||
# Sender not allowed in region config
|
||||
return False
|
||||
|
||||
# Does the necessary config checks for the given event
|
||||
# Returns list of recursive configs to process
|
||||
def validate_report(self, evt: MessageEvent, message: str):
|
||||
# Split command (minus !command_name) into tokens
|
||||
tokens = message.split()
|
||||
region = tokens[0].lower()
|
||||
|
||||
# Each command must have a state/territory designation and a message
|
||||
if len(tokens) < 2: return []
|
||||
# And we must have self.config["region_configs"]
|
||||
try: trashvariable = self.config["region_configs"]
|
||||
except: return []
|
||||
|
||||
configs = [] # To be returned
|
||||
|
||||
allowed_globally = self.validate_sender("__global__", evt.sender)
|
||||
allowed_region = self.validate_sender(region, evt.sender)
|
||||
|
||||
# If user is allowed globally and/or for a region,
|
||||
# the plugin should process both the region and __global__ configs
|
||||
if allowed_globally or allowed_region:
|
||||
for i in ["__global__", region]:
|
||||
try: configs.append(self.config["region_configs"][i])
|
||||
except: trashvariable = None
|
||||
|
||||
return configs
|
||||
|
||||
# What gets called when !command_name message is sent
|
||||
@command.new(name=get_command_name, help="Report Something")
|
||||
@command.argument("message", pass_raw=True)
|
||||
async def report(self, evt: MessageEvent, message: str) -> None:
|
||||
# If all have passed
|
||||
ntfy_posts_passed = None
|
||||
|
||||
# Iterate through each endpoint that the message should be pushed to
|
||||
# (if any)
|
||||
for region_config in self.validate_report(evt, message):
|
||||
# Detect no regions in reaction
|
||||
if ntfy_posts_passed is None: ntfy_posts_passed = True
|
||||
|
||||
self.log.debug(region_config)
|
||||
|
||||
# Create notification text
|
||||
split_message = message.split()
|
||||
text = "[" + split_message[0] + "] " + ' '.join(message.split()[1:])
|
||||
|
||||
# Build URL
|
||||
url = region_config["server_url"] + "/" + region_config["server_topic"]
|
||||
|
||||
# Consider authentication
|
||||
auth = None
|
||||
if "username" in region_config and "password" in region_config:
|
||||
auth = aiohttp.BasicAuth(region_config["username"], region_config["password"])
|
||||
|
||||
# Send notification
|
||||
async with self.http.post(url, data=text, auth=auth) as response:
|
||||
if not response.status == 200:
|
||||
ntfy_posts_passed = False
|
||||
|
||||
# Send reaction based on successful or failedPOSTs
|
||||
if self.config["send_reaction"]:
|
||||
if ntfy_posts_passed is True:
|
||||
await evt.react("👍")
|
||||
elif ntfy_posts_passed is False:
|
||||
await evt.react("👎")
|
||||
|
||||
# That's all, folks
|
||||
171
consumerrss.py
Normal file
171
consumerrss.py
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
from maubot import Plugin, MessageEvent
|
||||
from maubot.handlers import command
|
||||
from maubot.handlers import web
|
||||
|
||||
# Needed for configuration
|
||||
from typing import Type
|
||||
from mautrix.util.config import BaseProxyConfig, ConfigUpdateHelper
|
||||
from mautrix.util.async_db import UpgradeTable, Scheme, Connection
|
||||
from aiohttp.web import Request, Response, json_response
|
||||
|
||||
import aiohttp
|
||||
import datetime
|
||||
|
||||
import xml.etree.ElementTree as et
|
||||
|
||||
upgrade_table = UpgradeTable()
|
||||
@upgrade_table.register(description="Initial revision")
|
||||
async def upgrade_v1(conn: Connection) -> None:
|
||||
# Table contains:
|
||||
# Unique ID (event ID) (Primary)
|
||||
# Timestamp (TIMESTAMP type)
|
||||
# Region (string)
|
||||
# Message (string)
|
||||
await conn.execute(
|
||||
"""CREATE TABLE reports (
|
||||
id TEXT PRIMARY KEY,
|
||||
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
region TEXT NOT NULL,
|
||||
message TEXT
|
||||
)"""
|
||||
)
|
||||
|
||||
# Ensures a running instance gets an updated config from the Maubot interface
|
||||
class Config(BaseProxyConfig):
|
||||
def do_update(self, helper: ConfigUpdateHelper) -> None:
|
||||
helper.copy("command_prefix")
|
||||
helper.copy("allowed_regions")
|
||||
helper.copy("serve_regions")
|
||||
helper.copy("send_reaction")
|
||||
|
||||
class ConsumerRSS(Plugin):
|
||||
# Get configuration at startup
|
||||
async def start(self) -> None:
|
||||
self.config.load_and_update()
|
||||
|
||||
# Get config
|
||||
@classmethod
|
||||
def get_config_class(cls) -> Type[BaseProxyConfig]:
|
||||
return Config
|
||||
|
||||
# Maubot system needs this to register the table
|
||||
@classmethod
|
||||
def get_db_upgrade_table(cls) -> UpgradeTable | None:
|
||||
return upgrade_table
|
||||
|
||||
# Get !command_name setting from config to register it
|
||||
def get_command_name(self) -> str:
|
||||
return self.config["command_prefix"]
|
||||
|
||||
# Checks if a sender if allowed to send for a particular region
|
||||
def validate_sender(self, region: str, sender: str):
|
||||
# Mautrix isn't documented, like at all, so I'm just gonna catch the
|
||||
# error because IDK how to see if a map is inside a map.
|
||||
try: allowed_list = self.config["allowed_regions"][region]
|
||||
except: return False
|
||||
|
||||
if allowed_list is None: return True # Empty list
|
||||
if sender in allowed_list: return True
|
||||
|
||||
# Sender not allowed in region config
|
||||
return False
|
||||
|
||||
# Does the necessary config checks for the given event
|
||||
# Returns a region name if sender is allowed for a region, or None
|
||||
def validate_report(self, evt: MessageEvent, message: str):
|
||||
# Split command (minus !command_name) into tokens
|
||||
tokens = message.split()
|
||||
region = tokens[0].lower()
|
||||
|
||||
allowed_globally = self.validate_sender("__global__", evt.sender)
|
||||
allowed_region = self.validate_sender(region, evt.sender)
|
||||
if allowed_globally or allowed_region: return region
|
||||
|
||||
return None
|
||||
|
||||
# What gets called when !command_name message is sent
|
||||
@command.new(name=get_command_name, help="Report Something")
|
||||
@command.argument("message", pass_raw=True)
|
||||
async def report(self, evt: MessageEvent, message: str) -> None:
|
||||
try:
|
||||
# Check if sender is allowed for a region
|
||||
region = self.validate_report(evt, message)
|
||||
if region is None: return
|
||||
|
||||
# Feed information
|
||||
split_message = message.split()
|
||||
id = evt.event_id
|
||||
timestamp = dt = datetime.datetime.fromtimestamp(evt.timestamp/1000)
|
||||
self.log.debug(timestamp)
|
||||
region = split_message[0]
|
||||
message = ' '.join(split_message[1:])
|
||||
|
||||
# Insert into database
|
||||
self.log.debug(await self.database.execute("INSERT INTO reports (id, timestamp, region, message) VALUES ($1, $2, $3, $4)", id, timestamp, region, message))
|
||||
|
||||
# Mark as inserted
|
||||
await evt.react("👍")
|
||||
except Exception as error:
|
||||
self.log.fatal( f"An error occurred: {error}")
|
||||
|
||||
@web.get("/feed.rss")
|
||||
async def getfeed(self, req: Request) -> Response:
|
||||
try:
|
||||
# Get region from query
|
||||
query = req.query
|
||||
if not "region" in query:
|
||||
return Response(text="Request must include ?region= query", status=405)
|
||||
region = query["region"]
|
||||
|
||||
# Check region against served regions
|
||||
if not region in self.config["serve_regions"]:
|
||||
return Response(text="Specified region not served on this consumer", status=404)
|
||||
|
||||
if region == "__global__":
|
||||
feed_items = await self.database.fetch("SELECT * FROM reports ORDER BY timestamp DESC LIMIT 50")
|
||||
else:
|
||||
feed_items = await self.database.fetch("SELECT * FROM reports where region=$1 ORDER BY timestamp DESC LIMIT 50", region)
|
||||
self.log.debug(feed_items)
|
||||
|
||||
# Build feed
|
||||
feed = et.Element("rss")
|
||||
feed.set("version", "2.0")
|
||||
|
||||
# Channel metadata
|
||||
channel = et.SubElement(feed, "channel")
|
||||
et.SubElement(channel, "title").text = f"ICE reports {region}"
|
||||
et.SubElement(channel, "description").text = f"ICE reports for {region}"
|
||||
link = et.SubElement(channel, 'link')
|
||||
link.text = self.config["feed_url"]
|
||||
self_link = et.SubElement(channel, 'atom:link')
|
||||
self_link.set('rel', 'self')
|
||||
self_link.set('href', self.config["feed_url"])
|
||||
self_link.set('type', 'application/rss+xml')
|
||||
|
||||
|
||||
# Add items
|
||||
for item in feed_items:
|
||||
# Data from feed
|
||||
id = item["id"]
|
||||
timestamp = item["timestamp"]
|
||||
region = item["region"]
|
||||
message = "[" + region + "] " + item["message"]
|
||||
# Item
|
||||
item_element = et.SubElement(channel, "item")
|
||||
# GUID is different because the isPermaLink has to be false
|
||||
guid = et.SubElement(item_element, 'guid')
|
||||
guid.text = id
|
||||
guid.set('isPermaLink', 'false')
|
||||
# Other ite,s
|
||||
et.SubElement(item_element, "title").text = message
|
||||
et.SubElement(item_element, "description").text = message
|
||||
et.SubElement(item_element, "pubDate").text = timestamp.strftime("%a, %d %b %Y %H:%M:%S +0000")
|
||||
|
||||
|
||||
|
||||
# Return feed
|
||||
return aiohttp.web.Response(body=et.tostring(feed, encoding="unicode"), content_type='application/rss+xml')
|
||||
|
||||
except Exception as error:
|
||||
self.log.fatal( f"An error occurred: {error}")
|
||||
return Response(text="Internal server error", status=500)
|
||||
|
|
@ -1,10 +1,13 @@
|
|||
maubot: 0.1.0
|
||||
id: org.fiftyfiftyonearizona.reports.consumerntfy
|
||||
id: org.fiftyfiftyonearizona.reports.consumerrss
|
||||
version: 1.2.0
|
||||
license: MIT
|
||||
modules:
|
||||
- consumerntfy
|
||||
main_class: ConsumerNTFY
|
||||
- consumerrss
|
||||
main_class: ConsumerRSS
|
||||
config: true
|
||||
database: true
|
||||
database_type: asyncpg
|
||||
webapp: true
|
||||
extra_files:
|
||||
- base-config.yaml
|
||||
10
readme.md
10
readme.md
|
|
@ -1,4 +1,4 @@
|
|||
# FiftyFiftyOneArizona's Matrix-Based Reporting System, NTFY Consumer
|
||||
# FiftyFiftyOneArizona's Matrix-Based Reporting System, RSS Consumer
|
||||
|
||||
Please see https://git.fiftyfiftyonearizona.org/webmaster/matrix-report-documentation
|
||||
|
||||
|
|
@ -6,8 +6,10 @@ This plugin supports the following:
|
|||
|
||||
* whitelisting regions, or allowing all regions.
|
||||
* allowed senders per region, or all regions.
|
||||
* reaction receipt upon successful POST to NTFY.
|
||||
* authenticated NTFY POSTing.
|
||||
* Different NTFY endpoint, topic, and authentication per region.
|
||||
* reaction receipt upon successful storage in database.
|
||||
* Whitelisting which regions to both listen to and serve RSS feeds store.
|
||||
* Configuring the feed's atom:link
|
||||
|
||||
Please see [base-config.yaml](base-config.yaml) for explanation of behavior.
|
||||
|
||||
The feed is fully valid and contains items with non-URL GUIDs. The title and body content are the same. Items are timestamped per the original Matrix event.
|
||||
Loading…
Add table
Add a link
Reference in a new issue