From ae2ec6f66202a1490d5da6477df865df2031680d Mon Sep 17 00:00:00 2001 From: Will Brown Date: Fri, 22 Nov 2024 19:20:44 -0500 Subject: [PATCH] Add preliminary code for database (this really needs to be cleaned) --- Makefile | 2 +- README.md | 3 +- config.c | 2 +- config.h | 2 +- config.json | 2 +- feed_info.c | 51 ++++++------ feed_info.h | 54 ++++++------- main.c | 223 +++++++++++++++++++++++++++++++++++++++++++--------- 8 files changed, 241 insertions(+), 98 deletions(-) diff --git a/Makefile b/Makefile index 8ea5b99..7074a0e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ CFLAGS = -Wall -Wextra -std=gnu11 -O2 -LDFLAGS = -lpthread -lcurl -lmrss +LDFLAGS = -lpthread -lcurl -lmrss -lpq SRC = $(wildcard *.c) OBJ = $(SRC:.c=.o) diff --git a/README.md b/README.md index 0aff2f1..58961c1 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,13 @@ A simple, lightweight Discord RSS bot - libcurl - [concord](https://github.com/Cogmasters/concord/) - libmrss +- postgresql +- libpq ## TODO - [ ] Add build instructions - [x] Get all new feeds, not just the first one - [ ] Set permissions for add and remove command -- [ ] Import feeds from disk on startup - [ ] Remove all feeds if bot is removed from a guild - [ ] Implement list command - [ ] Implement remove command diff --git a/config.c b/config.c index 2dfdc97..f7e2d3f 100644 --- a/config.c +++ b/config.c @@ -6,6 +6,6 @@ struct zblock_config zblock_config; int zblock_config_load(struct discord *client) { // TODO: actually load config - zblock_config.database_path = "feeds"; + zblock_config.conninfo = "feeds"; return 0; } diff --git a/config.h b/config.h index 44caa1d..15f8542 100644 --- a/config.h +++ b/config.h @@ -5,7 +5,7 @@ // the current zblock config extern struct zblock_config { - char *database_path; + char *conninfo; } zblock_config; int zblock_config_load(struct discord *client); diff --git a/config.json b/config.json index 5ea7951..e282bf8 100644 --- a/config.json +++ b/config.json @@ -19,6 +19,6 @@ } }, "zblock": { - "database_path": "feeds" + "conninfo": "YOUR-DB-CONNINFO" } } diff --git a/feed_info.c b/feed_info.c index 36159a7..a4c68b8 100644 --- a/feed_info.c +++ b/feed_info.c @@ -1,3 +1,6 @@ +#define _GNU_SOURCE +#define _XOPEN_SOURCE + #include #include #include @@ -7,50 +10,42 @@ #include #include +#include + #include "config.h" #include "feed_info.h" -void feed_info_free(feed_info *feed) { - free(feed->title); - free(feed->url); - free(feed->last_pubDate); - free(feed); -} - -const char *feed_info_strerror(feed_info_err error) { - static_assert(FEED_INFO_ERRORCOUNT == 3, "Not all feed info errors implemented"); +// returns a string about the result of a feed_info function +const char *zblock_feed_info_strerror(zblock_feed_info_err error) { + static_assert(ZBLOCK_FEED_INFO_ERRORCOUNT == 3, "Not all feed info errors implemented"); switch (error) { - case FEED_INFO_OK: { + case ZBLOCK_FEED_INFO_OK: { return "OK"; } - case FEED_INFO_FILEERROR: { - return strerror(errno); - } - case FEED_INFO_NULL: { + case ZBLOCK_FEED_INFO_NULL: { return "No feed info was provided"; } + case ZBLOCK_FEED_INFO_POSTGRES: { + return "An error was encountered with the feed database"; + } default: { return "Unspecified error"; } } } -/* - * Reads feed info for given information from file and puts it in the provided struct - */ -feed_info_err feed_info_save_file(feed_info *feed) { - if (!feed) return FEED_INFO_NULL; +// format string for for the time format of pubDate +#define PUBDATE_FMT "%a, %d %b %Y %T %z" + +time_t pubDate_to_time_t(char *s) { + struct tm tm; + char *res = strptime(s, PUBDATE_FMT, &tm); + if (!res || !*res) return 0; // invalid time - char file_path[PATH_MAX]; - // maybe check if we ran out of characters for path? - snprintf(file_path, sizeof(file_path), "%s/%lu/%lu/%x", zblock_config.database_path, feed->guild_id, feed->channel_id, feed->feed_id); - - FILE *fp = fopen(file_path, "w"); - if (!fp) return FEED_INFO_FILEERROR; - fprintf(fp, "title=%s\nurl=%s\nlast_pubDate=%s\n", feed->title, feed->url, feed->last_pubDate); - fclose(fp); + return mktime(&tm); } -feed_info_err feed_info_load_file(u64snowflake guild_id, u64snowflake channel_id, unsigned feed_id, feed_info *feed) { +// Insert new feed into the database +zblock_feed_info_err zblock_feed_info_insert(PGconn *conn, zblock_feed_info *feed) { assert(0 && "not implemented yet"); } diff --git a/feed_info.h b/feed_info.h index 4ebfb0a..ca260d5 100644 --- a/feed_info.h +++ b/feed_info.h @@ -3,42 +3,38 @@ #include -// TODO: last_pubDate doesn't actually work properly yet -typedef struct feed_info { - char *title; +#include + +typedef struct { char *url; char *last_pubDate; - u64snowflake guild_id; u64snowflake channel_id; - unsigned timer_id; - unsigned feed_id; -} feed_info; +} zblock_feed_info_minimal; + +typedef struct { + // same definition as feed_info_minimal + char *url; + char *last_pubDate; + u64snowflake channel_id; + // extra things + char *title; + u64snowflake guild_id; +} zblock_feed_info; typedef enum { - FEED_INFO_OK, - FEED_INFO_NULL, - FEED_INFO_FILEERROR, - FEED_INFO_ERRORCOUNT -} feed_info_err; + ZBLOCK_FEED_INFO_OK, + ZBLOCK_FEED_INFO_NULL, + ZBLOCK_FEED_INFO_POSTGRES, + ZBLOCK_FEED_INFO_ERRORCOUNT +} zblock_feed_info_err; -/* - * Free the feed info struct - */ -void feed_info_free(struct feed_info *feed); +// maybe change the function signature so you can actually do error handling with the result? +time_t pubDate_to_time_t(char *s); -/* - * Get a string explaining a feed info error - */ -const char *feed_info_strerror(feed_info_err error); +// returns a string about the result of a feed_info function +const char *zblock_feed_info_strerror(zblock_feed_info_err error); -/* - * Saves feed info to given file - */ -feed_info_err feed_info_save_file(feed_info *feed); - -/* - * Reads feed info for given information from file and puts it in the provided struct - */ -feed_info_err feed_info_load_file(u64snowflake guild_id, u64snowflake channel_id, unsigned feed_id, feed_info *feed); +// Insert new feed into the database +zblock_feed_info_err zblock_feed_info_insert(PGconn *conn, zblock_feed_info *feed); #endif diff --git a/main.c b/main.c index 1f3d8dd..cb987a5 100644 --- a/main.c +++ b/main.c @@ -1,6 +1,3 @@ -#define _GNU_SOURCE -#define _XOPEN_SOURCE - #include #include #include @@ -10,11 +7,15 @@ #include #include +#include + #include #include #include +#include + #include "config.h" #include "feed_info.h" @@ -42,21 +43,169 @@ struct bot_command { discord_create_interaction_response(client, event->id, event->token, &res, NULL); \ } while (0) -// format string for for the time format of pubDate -#define PUBDATE_FMT "%a, %d %b %Y %T %z" - -// maybe change the function signature so you can actually do error handling with the result? -static time_t pubDate_to_time_t(char *s) { - struct tm tm; - char *res = strptime(s, PUBDATE_FMT, &tm); - if (!res || !*res) return 0; // invalid time - - return mktime(&tm); -} - // default interval for the feed retrieval timer #define TIMER_INTERVAL 600 +typedef struct { + zblock_feed_info_minimal info; + FILE *fp; + char *buf; + size_t bufsize; +} zblock_feed_buffer; + +// the database connection +static PGconn *database_conn; + +// this does not account for large-scale usage yet. +static void timer_retrieve_feeds(struct discord *client, struct discord_timer *timer) { + // not doing anything with the timer yet + (void) timer; + + // all this SQL stuff should *really* be extracted somewhere else + // maybe make a function where you can do a lookup with a quantity and offset + PGresult *database_res = PQexec(database_conn, "SELECT url, last_pubDate, channel_id from feeds"); + if (PQresultStatus(database_res) != PGRES_COMMAND_OK) { + log_error("Unable to retrieve feed list: %s", PQerrorMessage(database_conn)); + return; + } + + // get all the required feed info to send messages + int nfeeds = PQntuples(database_res); + zblock_feed_buffer *feed_list = malloc(nfeeds * sizeof(*feed_list)); + if (!feed_list) { + // well there goes that idea + PQclear(database_res); + return; + } + + for (int i = 0; i < nfeeds; ++i) { + feed_list[i].info.url = PQgetvalue(database_res, i, 0); + feed_list[i].info.last_pubDate = PQgetvalue(database_res, i, 1); + feed_list[i].info.channel_id = *(u64snowflake *) PQgetvalue(database_res, i, 2); + } + + // get all those feeds + CURLM *multi = curl_multi_init(); + if (!multi) { + // oh no + goto all_done; + } + + for (int i = 0; i < nfeeds; ++i) { + feed_list[i].fp = open_memstream(&feed_list[i].buf, &feed_list[i].bufsize); + if (!feed_list[i].fp) continue; // fail gracefully + + CURL *feed_handle = curl_easy_init(); + if (!feed_handle) { + fclose(feed_list[i].fp); + free(feed_list[i].buf); + continue; + } + + curl_easy_setopt(feed_handle, CURLOPT_URL, feed_list[i].info.url); + curl_easy_setopt(feed_handle, CURLOPT_WRITEDATA, feed_list[i].fp); + curl_easy_setopt(feed_handle, CURLOPT_PRIVATE, &feed_list[i]); + CURLMcode mc = curl_multi_add_handle(multi, feed_handle); + if (mc) { + curl_easy_cleanup(feed_handle); + fclose(feed_list[i].fp); + free(feed_list[i].buf); + continue; + } + } + + // it's time + int running_handles_prev = 0; + int running_handles; + do { + CURLMcode mc = curl_multi_perform(multi, &running_handles); + if (running_handles < running_handles_prev) { + running_handles_prev = running_handles; + + CURLMsg *msg; + int msgs_in_queue; + do { + msg = curl_multi_info_read(multi, &msgs_in_queue); + if (msg && msg->msg == CURLMSG_DONE) { + CURL *handle = msg->easy_handle; + // get our buffer out + zblock_feed_buffer *feed_buffer; + curl_easy_getinfo(handle, CURLINFO_PRIVATE, &feed_buffer); + if (!msg->data.result) { + // hell yeah parse that RSS feed + mrss_t *mrss_feed; + mrss_error_t mrss_err = mrss_parse_buffer(feed_buffer->buf, feed_buffer->bufsize, &mrss_feed); + if (!mrss_err) { + // get publication date of entries send any new ones + time_t last_pubDate_time = pubDate_to_time_t(feed_buffer->info.last_pubDate); + mrss_item_t *item = mrss_feed->item; + bool update_pubDate = false; + while (item && pubDate_to_time_t(item->pubDate) > last_pubDate_time) { + update_pubDate = true; + + // Send new entry in the feed + char msg[DISCORD_MAX_MESSAGE_LEN]; + snprintf(msg, sizeof(msg), "## %s\n### %s\n%s", mrss_feed->title, mrss_feed->item->title, mrss_feed->item->link); + struct discord_create_message res = { .content = msg }; + discord_create_message(client, feed_buffer->info.channel_id, &res, NULL); + item = item->next; + } + + if (update_pubDate) { + char *current_pubDate = mrss_feed->item->pubDate; + char channel_id_str[21]; // hold a 64-bit int in decimal form + snprintf(channel_id_str, sizeof(channel_id_str), "%ld", feed_buffer->info.channel_id); + + const char *const update_params[] = {current_pubDate, feed_buffer->info.url, channel_id_str}; + // save the updated pubDate to disk once that's implemented + PGresult *update_res = PQexecParams(database_conn, + "UPDATE feeds SET last_pubDate = $1 WHERE url = $2 AND channel_id = $3", + 3, NULL, update_params, NULL, NULL, 0 + ); + ExecStatusType update_status = PQresultStatus(update_res); + if (update_status != PGRES_COMMAND_OK) { + log_error("Failed to update pubDate: %s", PQresStatus(update_status)); // cry + } + PQclear(update_res); + } + + // done with our feed! + mrss_free(mrss_feed); + } else { + log_error("Error parsing feed: %s\n", mrss_strerror(mrss_err)); + } + } else { + log_error("Error downloading RSS feed: %s\n", msg->data.result); + } + + // free our buffers + curl_easy_cleanup(handle); + fclose(feed_buffer->fp); + free(feed_buffer->buf); + } + } while (msg); + } + + if (!mc && running_handles > 0) { + mc = curl_multi_poll(multi, NULL, 0, 300, NULL); + } + if (mc) { + // figure out how to free all resources instead of crashing + log_fatal("curl_multi_poll(): %s", curl_multi_strerror(mc)); + exit(1); + } + } while (running_handles > 0); + + curl_multi_cleanup(multi); + + // processing is done + all_done: + free(feed_list); + PQclear(database_res); + +} + +#if 0 // this just barely works at the moment static void timer_retrieve_feeds(struct discord *client, struct discord_timer *timer) { struct feed_info *feed = timer->data; @@ -87,45 +236,37 @@ static void timer_retrieve_feeds(struct discord *client, struct discord_timer *t mrss_free(mrss_feed); } +#endif static void bot_command_add(struct discord *client, const struct discord_interaction *event) { char msg[DISCORD_MAX_MESSAGE_LEN]; - struct feed_info *feed = calloc(1, sizeof(struct feed_info)); - if (!feed) { - snprintf(msg, sizeof(msg), "Error adding feed: %s", strerror(errno)); - goto send_msg; - } + zblock_feed_info feed; - feed->url = strdup(event->data->options->array[0].value); - if (!feed->url) { + feed.url = event->data->options->array[0].value; + if (!feed.url) { snprintf(msg, sizeof(msg), "Error adding feed: %s", strerror(errno)); - feed_info_free(feed); goto send_msg; } mrss_t *mrss_feed = NULL; - if(mrss_parse_url(feed->url, &mrss_feed)) { + if(mrss_parse_url(feed.url, &mrss_feed)) { // error here figure this out - feed_info_free(feed); goto send_msg; } - feed->title = mrss_feed->title; - feed->last_pubDate = mrss_feed->item->pubDate; - feed->feed_id = rand(); - feed->guild_id = event->guild_id; - feed->channel_id = event->channel_id; + feed.title = mrss_feed->title; + feed.last_pubDate = mrss_feed->item->pubDate; + feed.guild_id = event->guild_id; + feed.channel_id = event->channel_id; - feed_info_err feed_error = feed_info_save_file(feed); + zblock_feed_info_err feed_error = zblock_feed_info_insert(database_conn, &feed); if (feed_error) { // write error message - snprintf(msg, sizeof(msg), "Error adding feed: %s", feed_info_strerror(feed_error)); + snprintf(msg, sizeof(msg), "Error adding feed: %s", zblock_feed_info_strerror(feed_error)); } else { - // spawn the timer for this feed - feed->timer_id = discord_timer_interval(client, timer_retrieve_feeds, NULL, feed, 0, TIMER_INTERVAL, -1); // write the confirmation message - snprintf(msg, sizeof(msg), "The following feed has been successfully added to this channel:\n`%s`", feed->url); + snprintf(msg, sizeof(msg), "The following feed has been successfully added to this channel:\n`%s`", feed.url); } mrss_free(mrss_feed); @@ -211,7 +352,7 @@ static void on_ready(struct discord *client, const struct discord_ready *event) } // create feed retrieval timers - + discord_timer_interval(client, timer_retrieve_feeds, NULL, NULL, 0, TIMER_INTERVAL, -1); log_info("Ready!"); } @@ -242,9 +383,19 @@ int main(void) { srand(time(NULL)); struct discord *client = discord_config_init("config.json"); zblock_config_load(client); + + // connect to database + database_conn = PQconnectdb(zblock_config.conninfo); + if (!database_conn) { + log_fatal("Failed to connect to database."); + goto cleanup; + } + discord_set_on_ready(client, &on_ready); discord_set_on_interaction_create(client, &on_interaction); discord_run(client); + + cleanup: discord_cleanup(client); ccord_global_cleanup(); }