Add preliminary code for database (this really needs to be cleaned)

Makefile

@@ -1,5 +1,5 @@
 CFLAGS = -Wall -Wextra -std=gnu11 -O2
-LDFLAGS = -lpthread -lcurl -lmrss
+LDFLAGS = -lpthread -lcurl -lmrss -lpq
 
 SRC = $(wildcard *.c)
 OBJ = $(SRC:.c=.o)

README.md

@@ -5,12 +5,13 @@ A simple, lightweight Discord RSS bot
 - libcurl
 - [concord](https://github.com/Cogmasters/concord/)
 - libmrss
+- postgresql
+- libpq
 
 ## TODO
 - [ ] Add build instructions
 - [x] Get all new feeds, not just the first one
 - [ ] Set permissions for add and remove command
-- [ ] Import feeds from disk on startup
 - [ ] Remove all feeds if bot is removed from a guild
 - [ ] Implement list command
 - [ ] Implement remove command

config.c

@@ -6,6 +6,6 @@ struct zblock_config zblock_config;
 
 int zblock_config_load(struct discord *client) {
 	// TODO: actually load config
-	zblock_config.database_path = "feeds";
+	zblock_config.conninfo = "feeds";
 	return 0;
 }

config.h

@@ -5,7 +5,7 @@
 
 // the current zblock config
 extern struct zblock_config {
-	char *database_path;
+	char *conninfo;
 } zblock_config;
 
 int zblock_config_load(struct discord *client);

config.json

@@ -19,6 +19,6 @@
     }
   },
   "zblock": {
-    "database_path": "feeds"
+    "conninfo": "YOUR-DB-CONNINFO"
   }
 }

feed_info.c

@@ -1,3 +1,6 @@
+#define _GNU_SOURCE
+#define _XOPEN_SOURCE
+
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -7,50 +10,42 @@
 #include <concord/discord.h>
 #include <concord/log.h>
 
+#include <postgresql/libpq-fe.h>
+
 #include "config.h"
 #include "feed_info.h"
 
-void feed_info_free(feed_info *feed) {
+// returns a string about the result of a feed_info function
-	free(feed->title);
+const char *zblock_feed_info_strerror(zblock_feed_info_err error) {
-	free(feed->url);
+	static_assert(ZBLOCK_FEED_INFO_ERRORCOUNT == 3, "Not all feed info errors implemented");
-	free(feed->last_pubDate);
-	free(feed);
-}
-
-const char *feed_info_strerror(feed_info_err error) {
-	static_assert(FEED_INFO_ERRORCOUNT == 3, "Not all feed info errors implemented");
 	switch (error) {
-		case FEED_INFO_OK: {
+		case ZBLOCK_FEED_INFO_OK: {
 			return "OK";
 		}
-		case FEED_INFO_FILEERROR: {
+		case ZBLOCK_FEED_INFO_NULL: {
-			return strerror(errno);
-		}
-		case FEED_INFO_NULL: {
 			return "No feed info was provided";
 		}
+		case ZBLOCK_FEED_INFO_POSTGRES: {
+			return "An error was encountered with the feed database";
+		}
 		default: {
 			return "Unspecified error";
 		}
 	}
 }
 
-/*
+// format string for for the time format of pubDate
- * Reads feed info for given information from file and puts it in the provided struct
+#define PUBDATE_FMT "%a, %d %b %Y %T %z"
- */
+
-feed_info_err feed_info_save_file(feed_info *feed) {
+time_t pubDate_to_time_t(char *s) {
-	if (!feed) return FEED_INFO_NULL;
+	struct tm tm;
+	char *res = strptime(s, PUBDATE_FMT, &tm);
+	if (!res || !*res) return 0; // invalid time
 	
-	char file_path[PATH_MAX];
+	return mktime(&tm);
-	// maybe check if we ran out of characters for path?
-	snprintf(file_path, sizeof(file_path), "%s/%lu/%lu/%x", zblock_config.database_path, feed->guild_id, feed->channel_id, feed->feed_id);
-	
-	FILE *fp = fopen(file_path, "w");
-	if (!fp) return FEED_INFO_FILEERROR;
-	fprintf(fp, "title=%s\nurl=%s\nlast_pubDate=%s\n", feed->title, feed->url, feed->last_pubDate);
-	fclose(fp);
 }
 
-feed_info_err feed_info_load_file(u64snowflake guild_id, u64snowflake channel_id, unsigned feed_id, feed_info *feed) {
+// Insert new feed into the database
+zblock_feed_info_err zblock_feed_info_insert(PGconn *conn, zblock_feed_info *feed) {
 	assert(0 && "not implemented yet");
 }

feed_info.h

@@ -3,42 +3,38 @@
 
 #include <concord/discord.h>
 
-// TODO: last_pubDate doesn't actually work properly yet
+#include <postgresql/libpq-fe.h>
-typedef struct feed_info {
+
-	char *title;
+typedef struct {
 	char *url;
 	char *last_pubDate;
-	u64snowflake guild_id;
 	u64snowflake channel_id;
-	unsigned timer_id;
+} zblock_feed_info_minimal;
-	unsigned feed_id;
+
-} feed_info;
+typedef struct {
+	// same definition as feed_info_minimal
+	char *url;
+	char *last_pubDate;
+	u64snowflake channel_id;
+	// extra things
+	char *title;
+	u64snowflake guild_id;
+} zblock_feed_info;
 
 typedef enum {
-	FEED_INFO_OK,
+	ZBLOCK_FEED_INFO_OK,
-	FEED_INFO_NULL,
+	ZBLOCK_FEED_INFO_NULL,
-	FEED_INFO_FILEERROR,
+	ZBLOCK_FEED_INFO_POSTGRES,
-	FEED_INFO_ERRORCOUNT
+	ZBLOCK_FEED_INFO_ERRORCOUNT
-} feed_info_err;
+} zblock_feed_info_err;
 
-/*
+// maybe change the function signature so you can actually do error handling with the result?
- * Free the feed info struct
+time_t pubDate_to_time_t(char *s);
- */
-void feed_info_free(struct feed_info *feed);
 
-/*
+// returns a string about the result of a feed_info function
- * Get a string explaining a feed info error
+const char *zblock_feed_info_strerror(zblock_feed_info_err error);
- */
-const char *feed_info_strerror(feed_info_err error);
 
-/*
+// Insert new feed into the database
- * Saves feed info to given file
+zblock_feed_info_err zblock_feed_info_insert(PGconn *conn, zblock_feed_info *feed);
- */
-feed_info_err feed_info_save_file(feed_info *feed);
-
-/*
- * Reads feed info for given information from file and puts it in the provided struct
- */
-feed_info_err feed_info_load_file(u64snowflake guild_id, u64snowflake channel_id, unsigned feed_id, feed_info *feed);
 
 #endif

main.c

@@ -1,6 +1,3 @@
-#define _GNU_SOURCE
-#define _XOPEN_SOURCE
-
 #include <stdio.h>
 #include <stdbool.h>
 #include <stdlib.h>
@@ -10,11 +7,15 @@
 #include <unistd.h>
 #include <time.h>
 
+#include <curl/curl.h>
+
 #include <concord/discord.h>
 #include <concord/log.h>
 
 #include <mrss.h>
 
+#include <postgresql/libpq-fe.h>
+
 #include "config.h"
 #include "feed_info.h"
 
@@ -42,21 +43,169 @@ struct bot_command {
 	discord_create_interaction_response(client, event->id, event->token, &res, NULL); \
 } while (0)
 
-// format string for for the time format of pubDate
-#define PUBDATE_FMT "%a, %d %b %Y %T %z"
-
-// maybe change the function signature so you can actually do error handling with the result?
-static time_t pubDate_to_time_t(char *s) {
-	struct tm tm;
-	char *res = strptime(s, PUBDATE_FMT, &tm);
-	if (!res || !*res) return 0; // invalid time
-	
-	return mktime(&tm);
-}
-
 // default interval for the feed retrieval timer
 #define TIMER_INTERVAL 600
 
+typedef struct {
+	zblock_feed_info_minimal info;
+	FILE *fp;
+	char *buf;
+	size_t bufsize;
+} zblock_feed_buffer;
+
+// the database connection
+static PGconn *database_conn;
+
+// this does not account for large-scale usage yet.
+static void timer_retrieve_feeds(struct discord *client, struct discord_timer *timer) {
+	// not doing anything with the timer yet
+	(void) timer;
+
+	// all this SQL stuff should *really* be extracted somewhere else
+	// maybe make a function where you can do a lookup with a quantity and offset
+	PGresult *database_res = PQexec(database_conn, "SELECT url, last_pubDate, channel_id from feeds");
+	if (PQresultStatus(database_res) != PGRES_COMMAND_OK) {
+		log_error("Unable to retrieve feed list: %s", PQerrorMessage(database_conn));
+		return;
+	}
+	
+	// get all the required feed info to send messages
+	int nfeeds = PQntuples(database_res);
+	zblock_feed_buffer *feed_list = malloc(nfeeds * sizeof(*feed_list));
+	if (!feed_list) {
+		// well there goes that idea
+		PQclear(database_res);
+		return;
+	}
+	
+	for (int i = 0; i < nfeeds; ++i) {
+		feed_list[i].info.url = PQgetvalue(database_res, i, 0);
+		feed_list[i].info.last_pubDate = PQgetvalue(database_res, i, 1);
+		feed_list[i].info.channel_id = *(u64snowflake *) PQgetvalue(database_res, i, 2);
+	}
+	
+	// get all those feeds
+	CURLM *multi = curl_multi_init();
+	if (!multi) {
+		// oh no
+		goto all_done;
+	}
+	
+	for (int i = 0; i < nfeeds; ++i) {
+		feed_list[i].fp = open_memstream(&feed_list[i].buf, &feed_list[i].bufsize);
+		if (!feed_list[i].fp) continue; // fail gracefully
+	
+		CURL *feed_handle = curl_easy_init();
+		if (!feed_handle) {
+			fclose(feed_list[i].fp);
+			free(feed_list[i].buf);
+			continue;
+		}
+		
+		curl_easy_setopt(feed_handle, CURLOPT_URL, feed_list[i].info.url);
+		curl_easy_setopt(feed_handle, CURLOPT_WRITEDATA, feed_list[i].fp);
+		curl_easy_setopt(feed_handle, CURLOPT_PRIVATE, &feed_list[i]);
+		CURLMcode mc = curl_multi_add_handle(multi, feed_handle);
+		if (mc) {
+			curl_easy_cleanup(feed_handle);
+			fclose(feed_list[i].fp);
+			free(feed_list[i].buf);
+			continue;
+		}
+	}
+	
+	// it's time
+	int running_handles_prev = 0;
+	int running_handles;
+	do {
+		CURLMcode mc = curl_multi_perform(multi, &running_handles);
+		if (running_handles < running_handles_prev) {
+			running_handles_prev = running_handles;
+			
+			CURLMsg *msg;
+			int msgs_in_queue;
+			do {
+				msg = curl_multi_info_read(multi, &msgs_in_queue);
+				if (msg && msg->msg == CURLMSG_DONE) {
+					CURL *handle = msg->easy_handle;
+					// get our buffer out
+					zblock_feed_buffer *feed_buffer;
+					curl_easy_getinfo(handle, CURLINFO_PRIVATE, &feed_buffer);
+					if (!msg->data.result) {
+						// hell yeah parse that RSS feed
+						mrss_t *mrss_feed;
+						mrss_error_t mrss_err = mrss_parse_buffer(feed_buffer->buf, feed_buffer->bufsize, &mrss_feed);
+						if (!mrss_err) {
+							// get publication date of entries send any new ones
+							time_t last_pubDate_time = pubDate_to_time_t(feed_buffer->info.last_pubDate);
+							mrss_item_t *item = mrss_feed->item;
+							bool update_pubDate = false;
+							while (item && pubDate_to_time_t(item->pubDate) > last_pubDate_time) {
+								update_pubDate = true;
+								
+								// Send new entry in the feed
+								char msg[DISCORD_MAX_MESSAGE_LEN];
+								snprintf(msg, sizeof(msg), "## %s\n### %s\n%s", mrss_feed->title, mrss_feed->item->title, mrss_feed->item->link);
+								struct discord_create_message res = { .content = msg };
+								discord_create_message(client, feed_buffer->info.channel_id, &res, NULL);
+								item = item->next;
+							}
+							
+							if (update_pubDate) {
+								char *current_pubDate = mrss_feed->item->pubDate;
+								char channel_id_str[21]; // hold a 64-bit int in decimal form
+								snprintf(channel_id_str, sizeof(channel_id_str), "%ld", feed_buffer->info.channel_id);
+								
+								const char *const update_params[] = {current_pubDate, feed_buffer->info.url, channel_id_str};
+								// save the updated pubDate to disk once that's implemented
+								PGresult *update_res = PQexecParams(database_conn,
+									"UPDATE feeds SET last_pubDate = $1 WHERE url = $2 AND channel_id = $3",
+									3, NULL, update_params, NULL, NULL, 0
+								);
+								ExecStatusType update_status = PQresultStatus(update_res);
+								if (update_status != PGRES_COMMAND_OK) {
+									log_error("Failed to update pubDate: %s", PQresStatus(update_status)); // cry
+								}
+								PQclear(update_res);
+							}
+							
+							// done with our feed!
+							mrss_free(mrss_feed);
+						} else {
+							log_error("Error parsing feed: %s\n", mrss_strerror(mrss_err));
+						}
+					} else {
+						log_error("Error downloading RSS feed: %s\n", msg->data.result);
+					}
+					
+					// free our buffers
+					curl_easy_cleanup(handle);
+					fclose(feed_buffer->fp);
+					free(feed_buffer->buf);
+				}
+			} while (msg);
+		}
+		
+		if (!mc && running_handles > 0) {
+			mc = curl_multi_poll(multi, NULL, 0, 300, NULL);
+		}
+		if (mc) {
+			// figure out how to free all resources instead of crashing
+			log_fatal("curl_multi_poll(): %s", curl_multi_strerror(mc));
+			exit(1);
+		}
+	} while (running_handles > 0);
+	
+	curl_multi_cleanup(multi);
+	
+	// processing is done
+	all_done:
+	free(feed_list);
+	PQclear(database_res);
+	
+}
+
+#if 0
 // this just barely works at the moment
 static void timer_retrieve_feeds(struct discord *client, struct discord_timer *timer) {
 	struct feed_info *feed = timer->data;
@@ -87,45 +236,37 @@ static void timer_retrieve_feeds(struct discord *client, struct discord_timer *t
 	
 	mrss_free(mrss_feed);
 }
+#endif
 
 static void bot_command_add(struct discord *client, const struct discord_interaction *event) {
 	char msg[DISCORD_MAX_MESSAGE_LEN];
-	struct feed_info *feed = calloc(1, sizeof(struct feed_info));
+	zblock_feed_info feed;
-	if (!feed) {
-		snprintf(msg, sizeof(msg), "Error adding feed: %s", strerror(errno));
-		goto send_msg;
-	}
 	
-	feed->url = strdup(event->data->options->array[0].value);
+	feed.url = event->data->options->array[0].value;
-	if (!feed->url) {
+	if (!feed.url) {
 		snprintf(msg, sizeof(msg), "Error adding feed: %s", strerror(errno));
-		feed_info_free(feed);
 		goto send_msg;
 	}
 	
 	mrss_t *mrss_feed = NULL;
-	if(mrss_parse_url(feed->url, &mrss_feed)) {
+	if(mrss_parse_url(feed.url, &mrss_feed)) {
 		// error here figure this out
-		feed_info_free(feed);
 		goto send_msg;
 	}
 	
-	feed->title = mrss_feed->title;
+	feed.title = mrss_feed->title;
-	feed->last_pubDate = mrss_feed->item->pubDate;
+	feed.last_pubDate = mrss_feed->item->pubDate;
-	feed->feed_id = rand();
+	feed.guild_id = event->guild_id;
-	feed->guild_id = event->guild_id;
+	feed.channel_id = event->channel_id;
-	feed->channel_id = event->channel_id;
 	
 	
-	feed_info_err feed_error = feed_info_save_file(feed);
+	zblock_feed_info_err feed_error = zblock_feed_info_insert(database_conn, &feed);
 	if (feed_error) {
 		// write error message
-		snprintf(msg, sizeof(msg), "Error adding feed: %s", feed_info_strerror(feed_error));
+		snprintf(msg, sizeof(msg), "Error adding feed: %s", zblock_feed_info_strerror(feed_error));
 	} else {
-		// spawn the timer for this feed
-		feed->timer_id = discord_timer_interval(client, timer_retrieve_feeds, NULL, feed, 0, TIMER_INTERVAL, -1);	
 		// write the confirmation message
-		snprintf(msg, sizeof(msg), "The following feed has been successfully added to this channel:\n`%s`", feed->url);
+		snprintf(msg, sizeof(msg), "The following feed has been successfully added to this channel:\n`%s`", feed.url);
 	}
 		
 	mrss_free(mrss_feed);
@@ -211,7 +352,7 @@ static void on_ready(struct discord *client, const struct discord_ready *event)
 	}
 	
 	// create feed retrieval timers
-	
+	discord_timer_interval(client, timer_retrieve_feeds, NULL, NULL, 0, TIMER_INTERVAL, -1);
 
 	log_info("Ready!");
 }
@@ -242,9 +383,19 @@ int main(void) {
 	srand(time(NULL));
 	struct discord *client = discord_config_init("config.json");
 	zblock_config_load(client);
+	
+	// connect to database
+	database_conn = PQconnectdb(zblock_config.conninfo);
+	if (!database_conn) {
+		log_fatal("Failed to connect to database.");
+		goto cleanup;
+	}
+	
 	discord_set_on_ready(client, &on_ready);
 	discord_set_on_interaction_create(client, &on_interaction);
 	discord_run(client);
+	
+	cleanup:
 	discord_cleanup(client);
 	ccord_global_cleanup();
 }