From 5b213eb72f1549fe2b2aed6f677ac93385149582 Mon Sep 17 00:00:00 2001 From: 50501AZ Webmaster Date: Fri, 1 Aug 2025 15:16:25 -0700 Subject: [PATCH] Finally finished --- auth/readme.md | 22 ++++++++++++-------- dummy.py | 5 ++++- main.go | 4 ++-- readme.md | 8 ++++---- subprocess/incoming.go | 43 ++++++++++++++++++++-------------------- subprocess/readme.md | 4 +++- subprocess/request.go | 30 ++++++++++++++++++++++++---- subprocess/subprocess.go | 14 +++++++++++++ web/listen.go | 28 ++++++++++++++------------ web/readme.md | 14 +++++++------ 10 files changed, 112 insertions(+), 60 deletions(-) diff --git a/auth/readme.md b/auth/readme.md index c387cf5..16c96e9 100644 --- a/auth/readme.md +++ b/auth/readme.md @@ -1,6 +1,6 @@ # Auth - Signal-CLI HTTP -This module handles the reading and parsing of the auth JSON file. It also acts as a verifier in relation to that information. The file is a JSON object. It acts as a whitelist for which bearer token can do what action. It is passed to the HTTP endpoint via the `Authorization: Bearer ` header. +This module handles the reading and parsing of the auth JSON file. It also acts as a verifier in relation to that information. The file is a JSON object. It acts as a whitelist for which bearer token can do what action. It is passed to the HTTP endpoint via the `Authorization: ` header. Nore that this is not `Authorization: Bearer ` Here's a sample auth JSON: @@ -16,20 +16,26 @@ Here's a sample auth JSON: } ``` -When an HTTP request comes in, this software will do the following: +When an HTTP request comes in, this software will do the following (sending error responses when appropriate): 1. Check that there's an `Authorization` header 2. Get the authorization header's value (bearer token) 3. Read the JSON array corresponding to the bearer token. -4. See if any JSON object in that array (called a filter) does not have any data the request JSON doesn't, except for arrays which must match excactly. -5. If the statement in step 4 is true, forward the request into the signal-cli process and return the response. +4. See if any JSON object in that array "matches" the request JSON +5. Forward the request to the subprocess, and return the result. -So for example, the reqest `{"method":"send","params":{"recipient":["+16028675309"],"message":"message"},"id":"SomeID"},` would be allowed by the filter `{"method":"send","params":{"recipient":["+16028675309"]}}` because the filter does not have any data the request does not. But `{"method":"send","params":{"recipient":["+5555555555"],"message":"message"},"id":"SomeID"},` would not because the phone number differs. +The rules for matching a request JSON to a filter JSON is a recursive process. At each step it goes through the following checks: +1. The types (map JSON, array JSON, value literal, etc) must be the same +2. For the map JSON type, each key inside the filter json must be present inside the request JSON. Each key-value pair in the filter JSON must also match (recursively). +3. For the array JSON type, each key inside the filter json must be present inside the request JSON **and vice-versa**. Each key-value pair must also match (recursively). +4. For anything else, it matches the object directly. This is invoked when checking equality of a value literal. -Note: items in arrays must "match" exactly, but items in items in arrays follow normal rules. So the request `{"method":"send","params":{"recipient":["+16028675309","someBadNumber"]}}` would NOT match the filter `{"method":"send","params":{"recipient":["+16028675309",]}}` - -These filters can be as granular as you want. +Here's some examples for each case: +1. the request `{"method":"send","params":{"recipient":["+16028675309"],"message":"message"},"id":"SomeID"},` would not match the filter `["+5555555555"]` because one is a JSON map and the other a JSON array. +2. the request `{"method":"something","params":{"recipient":["+16028675309"],"message":"message"},"id":"SomeID"},` would not match the filter `{"method":"send","params":{"recipient":["+16028675309"],"message":"message"}}` because the "method" differs. This would also fail to match if the `method` key was missing in the request JSON. +3. `{"method":"send","params":{"recipient":["+16028675309","someBadNumber"]}}` would not match the filter `{"method":"send","params":{"recipient":["+16028675309",]}}` because of the `someBadNumber` number in the request. This rule exists so that a malicious request cant send a message to both a room/concact that it's whitelisted for, and one that it isn't. +4. `"+16028675309"` would not match the filter `"+15555555555"` because their values differ. Here's what each filter JSON object in the above sample JSON does: `{"method":"send","params":{"recipient":["+16028675309"]}}` allows sending to `+16028675309` (any message, timestamp, etc.) diff --git a/dummy.py b/dummy.py index 6dabd61..76d6a2e 100755 --- a/dummy.py +++ b/dummy.py @@ -6,8 +6,11 @@ import sys import json -print(json.dumps({"method":"receive","params":{"envelope":{"source":"67a13c3e-8d29-2539-ce8e-41129c349d6d"},"data":"stuff"}})) +# Stress-testing incoming messages +for i in range(2000): + print(json.dumps({"method":"receive","params":{"envelope":{"source":"67a13c3e-8d29-2539-ce8e-41129c349d6d"},"data":i}})) +# Reply to incoming messages for line in sys.stdin: try: data = json.loads(line.strip()) diff --git a/main.go b/main.go index c7d46d8..426d7d8 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main /* This is the main function of this program. It handles setting everything up */ import ( - "fmt" "signal-cli-http/args" "signal-cli-http/auth" "signal-cli-http/subprocess" @@ -37,6 +36,7 @@ func main() { err = subprocess.SetupCMD(binary); if err != nil {log.Default().Print("Error running subprocess at ", binary, ": ", err); return;}; log.Default().Print("Started subprocess at ", binary); + subprocess.StartCacheClear(); // HTTP Listen port, portSet := args.GetHTTPPort(); @@ -50,7 +50,7 @@ func main() { log.Default().Print("Startup tasks complete!"); time.Sleep(time.Millisecond * 500); - fmt.Println(subprocess.GetIMC()) + // Needed to not kill program wg.Wait(); } \ No newline at end of file diff --git a/readme.md b/readme.md index cb0c61b..432dd95 100644 --- a/readme.md +++ b/readme.md @@ -8,7 +8,7 @@ Please see the JSONRPC documentation for `signal-cli`: [https://github.com/AsamK Please also read the following README files for the individual modules to understand how to configure and interact with this program: -* [args](args/readme.md) handles command line arguments. -* [auth](auth/readme.md) handles the authentication JSON and checking requests. -* [subprocess](subprocess/readme.md) manages the underlying `signal-cli` JSONRPC process, along with caching incoming messages. -* [web](web/readme.md) - handles the HTTP requests to this program, including the necessary edge cases. \ No newline at end of file +* [args](args/readme.md) handles command line arguments. Go here to learn how to run the program. +* [auth](auth/readme.md) handles the authentication JSON and checking requests. Go here to learn how to secure the program, and whitelist authentication keys. +* [subprocess](subprocess/readme.md) manages the underlying `signal-cli` JSONRPC process, along with caching incoming messages. Go here to understand how this program relays and returns the JSON objects. +* [web](web/readme.md) - handles the HTTP requests to this program, including the necessary edge cases. Go here to understand how to send and understand the responses to the program's HTTP endpoint. \ No newline at end of file diff --git a/subprocess/incoming.go b/subprocess/incoming.go index d1e122b..bedc05a 100644 --- a/subprocess/incoming.go +++ b/subprocess/incoming.go @@ -1,9 +1,8 @@ package subprocess -/* This file manages incoming messages*/ +/* This file manages incoming messages, marked with "method":"receive" in the JSON */ import ( - "fmt" "signal-cli-http/auth" "sort" "sync" @@ -22,25 +21,29 @@ type IncomingMessage struct { func newIncomingMessage() *IncomingMessage {return &IncomingMessage{}} -/* Stores incoming messages in a room up to at least 15 minutes old 10,000 - This is intentionally an array of pointers so cache-clearing is faster */ +/* Stores unlimited incoming messages up to 5 minutes old. + This is intentionally an array of pointers so cache-clearing and appending + don't require a copy of unmarshaledJSONMap. */ var incomingMessageCache []*IncomingMessage; +/* Locks incomingMessageCache */ var incomingMessageCacheLock sync.RWMutex; +/* Here exclusively for testing purposes */ func GetIMC() []*IncomingMessage {return incomingMessageCache} /* Handler for incoming JSON objects which have "method":"receive" */ func handleIncoming(body string, unmarshaledJSONMap map[string]any) (ok bool) { + // Check that the message's method is "receive" if val, ok := unmarshaledJSONMap["method"]; !ok || val != "receive" {return false} - fmt.Println(body) + // Create new message structure var newMessage *IncomingMessage = newIncomingMessage(); - // Using time.Now to ensure that pre/post-dated messages don't have issue + // Using time.Now() to ensure that pre/post-dated messages don't have issue newMessage.receivedTime = time.Now(); newMessage.body = body; newMessage.unmarshaledJSONMap = unmarshaledJSONMap; - // Obtain read-write lock + // Add message into cache incomingMessageCacheLock.Lock(); incomingMessageCache = append(incomingMessageCache, newMessage); incomingMessageCacheLock.Unlock(); @@ -48,26 +51,23 @@ func handleIncoming(body string, unmarshaledJSONMap map[string]any) (ok bool) { } /* Handles clearing space in incomingMessageCache */ -func main() {go cacheClear()} - +func StartCacheClear() {go cacheClear()} /* Runs in an infinite loop to try and clear the cache when needed */ func cacheClear() { for { // More than reasonable delay - time.Sleep(time.Millisecond); - // Only clear when it's 1000 items over + time.Sleep(time.Millisecond * 25); + // Only attempt to clear when it's 1000 items or more if len(incomingMessageCache) < 1000 {continue} incomingMessageCacheLock.Lock(); - // Don't clear anything after this time - fifteenMinutesAgo := time.Now().Add(-15 * time.Minute); + fiveMinutesAgo := time.Now().Add(time.Minute*(-5)); - // Find index in incomingMessageCache that is closest above 15 minutes ago + // Find first index in incomingMessageCache that is at most 15 minutes old i := sort.Search(len(incomingMessageCache), func(i int) bool { - return incomingMessageCache[i].receivedTime.After(fifteenMinutesAgo) + return incomingMessageCache[i].receivedTime.After(fiveMinutesAgo) }) - incomingMessageCache = incomingMessageCache[i:] incomingMessageCacheLock.Unlock(); @@ -75,28 +75,29 @@ func cacheClear() { } /* Returns a list of encoded JSON strings from incomingMessageCache that match - the filter from */ + the filter from + @return always valid JSON array object. Can be empty. */ func GetIncoming(filter map[string]any) string { - var list []string; - // Create copy of incomingMessageCache for efficency + // Create copy of incomingMessageCache as the following loop can be slow incomingMessageCacheLock.RLock(); incomingMessageCacheCopy := incomingMessageCache; incomingMessageCacheLock.RUnlock(); // Create list of messages that match the filter + var list []string; for _, message := range incomingMessageCacheCopy { if !auth.Match(message.unmarshaledJSONMap, filter) {continue} - fmt.Println(message.body) list = append(list, message.body) } + // Constructs the JSON string without considering the JSON object var encoded string = "[" for index, object := range list { encoded += object if index == len(list) - 1 {continue} encoded += "," } - encoded += "]" + encoded += "]\n" return encoded; } \ No newline at end of file diff --git a/subprocess/readme.md b/subprocess/readme.md index 8728230..20d8ad6 100644 --- a/subprocess/readme.md +++ b/subprocess/readme.md @@ -4,4 +4,6 @@ This module spawns and handles IO for the signal-cli process. Do not pass an object with the "id" key into this module's methods. It will reject the request for that reason. -This system works with multiple requests at the same time safely. \ No newline at end of file +This system works with multiple requests at the same time safely. + +This system also caches incoming messages up to at least 5 minutes old for later querying. This process takes in a filter JSON and goes through this list and finds any incoming message JSON objects that match to the filter JSON as outlined in the [auth module](../auth/readme.md). \ No newline at end of file diff --git a/subprocess/request.go b/subprocess/request.go index 0e8cc33..4fd935a 100644 --- a/subprocess/request.go +++ b/subprocess/request.go @@ -12,7 +12,8 @@ import ( ) /* Stores a map between job ID and a string pointer. Nil for job not done yet */ -var waitingJobs map[string]*string = make (map[string]*string); +var waitingJobs map[string]*string = make(map[string]*string); +/* Mutex for waitingJobs list */ var waitingJobsMutex sync.RWMutex; /* Method which module http calls to forward the request to the subprocess*/ @@ -23,6 +24,7 @@ func Request(body map[string]any) (responseJSON string, err error) { } // Generate ID and store it + // 32 chars of base 64 is secure enough to assume no accidental collision var id string = genID(); if len(id) == 0 { err = errors.New("Error generating ID!"); @@ -42,35 +44,55 @@ func Request(body map[string]any) (responseJSON string, err error) { waitingJobs[id] = nil; waitingJobsMutex.Unlock(); + // Timeout waiting for response after 30 seconds + timeoutTime := time.Now().Add(time.Second * 30); + // Wait for request to return for { + // Check every milisecond time.Sleep(time.Millisecond) + + // check if job is fufiled waitingJobsMutex.RLock(); if waitingJobs[id] != nil { waitingJobsMutex.RUnlock(); break; } waitingJobsMutex.RUnlock(); + + // If request takes more than 30 seconds + /* This is technically a race condition, as the job could be fufiled + between here and the job fufiled check, but it's the difference of + a milisecond or two out of 30 seconds */ + if time.Now().After(timeoutTime) { + // Delete job from queue + waitingJobsMutex.Lock(); + delete(waitingJobs, id) + waitingJobsMutex.Unlock(); + return "", errors.New("Request timed out!"); + } } // Lock job when dequeueing waitingJobsMutex.Lock(); responseJSON = *waitingJobs[id]; - delete(waitingJobs, id) + delete(waitingJobs, id) // Remove job from "queue" waitingJobsMutex.Unlock(); - err = nil; + err = nil; // No problems return; } /* Handles putting a response into waitingJobs. Returns false on error */ func handleResponse(body string, unmarshaledJSONMap map[string]any) { + waitingJobsMutex.RLock(); // Read lock val, ok := unmarshaledJSONMap["id"]; + waitingJobsMutex.RUnlock(); if !ok {return} id, ok := val.(string) if !ok {return} - // Read-Write lock the mutex + // Read-Write lock the mutex when writing job result waitingJobsMutex.Lock(); defer waitingJobsMutex.Unlock(); diff --git a/subprocess/subprocess.go b/subprocess/subprocess.go index c9f7859..d8167ae 100644 --- a/subprocess/subprocess.go +++ b/subprocess/subprocess.go @@ -8,7 +8,9 @@ import ( "errors" "os" "os/exec" + "os/signal" "sync" + "syscall" "github.com/creack/pty" ) @@ -36,10 +38,22 @@ func SetupCMD(binaryLocation string) error { reader = bufio.NewScanner(f); go readCMD(); + // Make sure cmd is killed when this program is + go catchKillSignal(); + // No problem return nil; } +/* Catch signals from this program to kill child */ +func catchKillSignal() { + signalCatcher := make(chan os.Signal, 1) + signal.Notify(signalCatcher, syscall.SIGINT, syscall.SIGTERM, syscall.SIGILL) + <- signalCatcher; + cmd.Process.Kill(); + os.Exit(0); // Without this the process never exits +} + /* Continuously reads the next line up to 40960 bytes and forwards it to response */ func readCMD() { var maxCapacity int = 40960; diff --git a/web/listen.go b/web/listen.go index 795c164..fef3f8d 100644 --- a/web/listen.go +++ b/web/listen.go @@ -14,16 +14,19 @@ import ( "time" ) +/* Starts the listener */ func StartWebserver(port int) error { http.HandleFunc("/", getRoot); return http.ListenAndServe(":"+fmt.Sprint(port), nil); } +/* Writes data to the log */ func writeLog(method string, status int, start time.Time) { duration := time.Now().Sub(start); log.Default().Printf("%s %d %s", method, status, duration.String()) } +/* Handler for all requests */ func getRoot(w http.ResponseWriter, r *http.Request) { startTime := time.Now() @@ -31,8 +34,8 @@ func getRoot(w http.ResponseWriter, r *http.Request) { authArr, ok := r.Header["Authentication"] if (!ok) || (len(authArr) == 0) { w.WriteHeader(400); - w.Write([]byte("Authentication header missing\n")) - writeLog(r.Method, 400, startTime) + w.Write([]byte("Authentication header missing\n")); + writeLog(r.Method, 400, startTime); return; } bearer := authArr[0]; @@ -42,7 +45,7 @@ func getRoot(w http.ResponseWriter, r *http.Request) { if err != nil { w.WriteHeader(500); w.Write([]byte("Error reading body\n")); - writeLog(r.Method, 500, startTime) + writeLog(r.Method, 500, startTime); return; } @@ -50,7 +53,7 @@ func getRoot(w http.ResponseWriter, r *http.Request) { if !auth.Authenticate(bearer, body) { w.WriteHeader(403); w.Write([]byte("Bearer key not whitelisted for this request type\n")); - writeLog(r.Method, 403, startTime) + writeLog(r.Method, 403, startTime); return; } @@ -59,7 +62,7 @@ func getRoot(w http.ResponseWriter, r *http.Request) { if err := json.Unmarshal(body, &bodyUnmarshaled); err != nil { w.WriteHeader(400); w.Write([]byte("Body content is not a valid JSON")); - writeLog(r.Method, 400, startTime) + writeLog(r.Method, 400, startTime); return; } @@ -68,7 +71,7 @@ func getRoot(w http.ResponseWriter, r *http.Request) { if !ok { w.WriteHeader(400); w.Write([]byte("Body content is not of the write format")); - writeLog(r.Method, 400, startTime) + writeLog(r.Method, 400, startTime); return; } @@ -79,22 +82,21 @@ func getRoot(w http.ResponseWriter, r *http.Request) { incoming := subprocess.GetIncoming(b) w.WriteHeader(200); w.Write([]byte(incoming)); - writeLog(r.Method, 200, startTime) - return + writeLog(r.Method, 200, startTime); + return; } // Run request - bodyContent, err := subprocess.Request(b) + bodyContent, err := subprocess.Request(b); if err != nil { w.WriteHeader(500); w.Write([]byte("Internal server error: " + err.Error() + "\n")); - return + writeLog(r.Method, 500, startTime); + return; } // Request returned something w.WriteHeader(200); w.Write([]byte(bodyContent)); - - // Log the request - log.Default().Print("HTTP Request: ", bearer, " " , 200, " ", string(body)) + log.Default().Print("HTTP Request: ", bearer, " " , 200, " ", string(body)); } \ No newline at end of file diff --git a/web/readme.md b/web/readme.md index 590a962..63c1e87 100644 --- a/web/readme.md +++ b/web/readme.md @@ -1,17 +1,19 @@ # Web - Signal-CLI HTTP -This module handles the HTTP requests. The path used for requests genuinely does not matter. However, every request must have a JSON object body, and an `Authorization: bearer ` header. The response will also always be a JSON. +This module handles the HTTP requests. The path used for requests genuinely does not matter. However, every request must have a JSON object body, and an `Authorization: bearer ` header. Possible response codes: -* 400 Bad Request: Bad JSON, missing Authorization header, etc. +* 400 Bad Request: Bad JSON, missing Authorization header, **sent a request with a defined id**, etc. * 401 Unauthorized: bearer token not allowed for presented request. * 500 Internal Server Error: Self explanitory. * 501 Not Implemented: Will be returned for message receiving. * 200 OK: Request was forwarded to JSONRPC subprocess and that process returned something, including if the "error" key is present in the returned JSON. -Any non-200 response code will come with a JSON with the following format: +There is no body content with non-200 response codes. With 200 the response is a valid JSON map or array. -```json -{"error":"Error message string"} -``` \ No newline at end of file +This program simply relays requests to the signal-cli program. **It will not prevent you from breaking anything, outside of not whitelisting certain requests. This program does not understand what requests mean.** Each request comes formatted as a JSON object outlined in the [JSON-RPC documentation](https://github.com/AsamK/signal-cli/blob/master/man/signal-cli-jsonrpc.5.adoc](https://github.com/AsamK/signal-cli/blob/master/man/signal-cli-jsonrpc.5.adoc). + +The program will ensure that the request object is a JSON map, and that the `request` key is present. For any request type that is not `receive`, the program will generate an ID for your request (do not put on in the request, it will return an error) and return the program's response. + +For `receive` it will return a JSON list of JSON maps in the incoming message cache that match to your request. There is no limit to how many messages it returns so be careful. \ No newline at end of file