fix: advance DM pagination cursor by full chunk, not single message

scripts/backfill_reviews.rs

@@ -33,7 +33,6 @@ async fn main() {
 
     let config = load_config();
     let discord = DiscordAPI::default().await;
-    let channel_ids = [config.teatro_channel_id, config.artes_channel_id];
     let total = records.len();
 
     async {
@@ -55,11 +54,18 @@ async fn main() {
             let vote_emoji = &config.voting_emojis[(record.rating - 1) as usize];
 
             match discord
-                .send_backfill_review(user_id, &record.url, vote_emoji, comment, &channel_ids)
+                .send_backfill_review(
+                    user_id,
+                    &record.url,
+                    vote_emoji,
+                    comment,
+                    &config.venue_ticket_shop_url,
+                    &config.ticket_shop_icon_url,
+                )
                 .await
             {
                 Ok(true) => info!("Sent review for {}", record.url),
-                Ok(false) => warn!("Skipped (already sent or event not found): {}", record.url),
+                Ok(false) => warn!("Skipped (already sent): {}", record.url),
                 Err(_) => error!("Failed to send review for {}", record.url),
             }
         }

src/agenda_cultural/api.rs

@@ -1,5 +1,5 @@
 use super::{dto::EventResponse, model::Event};
-use crate::agenda_cultural::model::Category;
+use crate::agenda_cultural::model::{Category, EventDetails, Schedule};
 use chrono::{Datelike, NaiveDate, TimeDelta, Utc};
 use futures::TryFutureExt;
 use lazy_static::lazy_static;
@@ -32,6 +32,10 @@ lazy_static! {
         .build();
     static ref EVENT_DESCRIPTION_SELECTOR: Selector =
         Selector::parse(".entry-container > :not(.event__extra-info):not(.section-title):not(.section-title--venue):not(.venue):not(.post__share)").unwrap();
+    static ref OG_TITLE_SELECTOR: Selector = Selector::parse(r#"meta[property="og:title"]"#).unwrap();
+    static ref OG_IMAGE_SELECTOR: Selector = Selector::parse(r#"meta[property="og:image"]"#).unwrap();
+    static ref VENUE_NAME_SELECTOR: Selector = Selector::parse(".venue__name").unwrap();
+    static ref EVENT_DATES_SELECTOR: Selector = Selector::parse(".signpost__date").unwrap();
 }
 
 pub struct AgendaCulturalAPI;
@@ -215,9 +219,79 @@ impl AgendaCulturalAPI {
     fn clean_description(description: &str) -> String {
         strip_tags(description)
             .replace(" ", " ")
-            .trim_end_matches("\n")
+            .trim()
             .to_owned()
     }
+
+    /// Scrapes title, venue, dates and image directly off the event page, for events
+    /// no longer present in the upcoming-events API (e.g. when backfilling old reviews).
+    pub async fn scrape_event(link: &str) -> Option<Event> {
+        let full_page: Result<Response, _> = REST_CLIENT.get(link).send().await;
+
+        let body = match full_page {
+            Ok(full_page) => full_page
+                .text()
+                .await
+                .inspect_err(|err| warn!("Failed to get event page text: {}", err))
+                .ok()?,
+            Err(err) => {
+                warn!("Failed to get event page: {:?}", err);
+                return None;
+            }
+        };
+
+        let description = Self::extract_full_description(&body).unwrap_or_else(|| {
+            warn!("Unable to extract description for '{}'", link);
+            String::new()
+        });
+
+        let document = Html::parse_document(&body);
+
+        let title = Self::extract_meta_content(&document, &OG_TITLE_SELECTOR).unwrap_or_else(|| {
+            warn!("Unable to extract title for '{}'", link);
+            String::new()
+        });
+        let image_url = Self::extract_meta_content(&document, &OG_IMAGE_SELECTOR)
+            .unwrap_or_else(|| {
+                warn!("Unable to extract image for '{}'", link);
+                String::new()
+            });
+        let venue = Self::extract_text(&document, &VENUE_NAME_SELECTOR).unwrap_or_else(|| {
+            warn!("Unable to extract venue for '{}'", link);
+            String::new()
+        });
+        let dates = Self::extract_text(&document, &EVENT_DATES_SELECTOR).unwrap_or_else(|| {
+            warn!("Unable to extract dates for '{}'", link);
+            String::new()
+        });
+
+        Some(Event::new(
+            title,
+            EventDetails::new(String::new(), description, image_url),
+            link.to_string(),
+            Schedule::new(dates, String::new()),
+            venue,
+            Vec::new(),
+        ))
+    }
+
+    fn extract_meta_content(document: &Html, selector: &Selector) -> Option<String> {
+        document
+            .select(selector)
+            .next()
+            .and_then(|element| element.value().attr("content"))
+            .map(str::to_string)
+    }
+
+    fn extract_text(document: &Html, selector: &Selector) -> Option<String> {
+        document.select(selector).next().map(|element| {
+            element
+                .text()
+                .collect::<String>()
+                .trim()
+                .to_string()
+        })
+    }
 }
 
 #[cfg(test)]

src/discord/api.rs

@@ -1,3 +1,4 @@
+use crate::agenda_cultural::api::AgendaCulturalAPI;
 use crate::agenda_cultural::model::Event;
 use crate::config::model::EmojiConfig;
 use crate::metrics::{record_dm_review_rewrite, record_dm_review_sent, MetricResult};
@@ -17,6 +18,7 @@ use serenity::cache::Settings;
 use serenity::model::id::ChannelId;
 use serenity::prelude::SerenityError;
 use serenity::Client;
+use std::collections::HashMap;
 use std::env;
 use std::fmt::Debug;
 use tracing::field::debug;
@@ -104,6 +106,25 @@ impl DiscordAPI {
     ) -> Result<Message, DiscordError> {
         info!(channel_id = %channel_id, event = %event.title, "Sending event");
 
+        let title = event.title.clone();
+        let embed = Self::build_event_embed(event, ticket_shop_url, ticket_shop_icon_url);
+
+        let message_builder = CreateMessage::new().add_embed(embed.clone());
+
+        channel_id
+            .send_message(&self.client.http, message_builder)
+            .await
+            .map_err(|err| {
+                error!("Failed sending event '{}' due to '{}'", title, err);
+                DiscordError::Api
+            })
+    }
+
+    fn build_event_embed(
+        event: Event,
+        ticket_shop_url: Option<String>,
+        ticket_shop_icon_url: &str,
+    ) -> CreateEmbed {
         let mut description = event.details.description;
 
         if event.is_for_children {
@@ -117,26 +138,15 @@ impl DiscordAPI {
         }
 
         let embed_description = Self::truncate_embed_description(description);
-        let embed_title = event.title.clone();
 
-        let embed = CreateEmbed::new()
-            .title(embed_title)
+        CreateEmbed::new()
+            .title(event.title)
             .url(event.link)
             .description(embed_description)
             .author(author)
             .color(Colour::new(0x005eeb))
             .field("Datas", event.occurring_at.dates, true)
-            .image(event.details.image_url);
-
-        let message_builder = CreateMessage::new().add_embed(embed.clone());
-
-        channel_id
-            .send_message(&self.client.http, message_builder)
-            .await
-            .map_err(|err| {
-                error!("Failed sending event '{}' due to '{}'", event.title, err);
-                DiscordError::Api
-            })
+            .image(event.details.image_url)
     }
 
     pub async fn add_custom_reaction(&self, message: &Message, emoji: &EmojiConfig) {
@@ -575,32 +585,22 @@ impl DiscordAPI {
         }
     }
 
-    /// Sends a review DM identical in format to `send_user_review_in_dm`, but with an
-    /// explicit comment string instead of reading the user's last DM message.
-    /// Used to backfill historical reviews that predate (or bypassed) the normal
-    /// reaction -> DM flow.
     pub async fn send_backfill_review(
         &self,
         user_id: UserId,
         event_url: &str,
         vote_emoji: &EmojiConfig,
         comment: Option<&str>,
-        channel_ids: &[ChannelId],
+        venue_ticket_shop_url: &HashMap<String, String>,
+        ticket_shop_icon_url: &str,
     ) -> Result<bool, ()> {
-        let mut event_embed = None;
-
-        for channel_id in channel_ids {
-            if let Some(found) = self.find_event_embed(*channel_id, event_url).await {
-                event_embed = Some(found);
-                break;
-            }
-        }
-
-        let Some(event_embed) = event_embed else {
-            warn!("Could not find event message for url '{}'", event_url);
+        let Some(event) = AgendaCulturalAPI::scrape_event(event_url).await else {
+            warn!("Could not scrape event details for '{}'", event_url);
             return Ok(false);
         };
 
+        let ticket_shop_url = venue_ticket_shop_url.get(&event.venue).cloned();
+
         let dm = match user_id.create_dm_channel(&self.client.http).await {
             Ok(dm) => dm,
             Err(e) => {
@@ -618,7 +618,13 @@ impl DiscordAPI {
             Ok(false) => {}
         }
 
-        let embed = Self::create_user_review_embed(vote_emoji, event_embed, comment);
+        let mut embed =
+            Self::build_event_embed(event, ticket_shop_url, ticket_shop_icon_url)
+                .field("Voto", vote_emoji.to_string(), true);
+
+        if let Some(comment) = comment {
+            embed = embed.field("Comentários", comment, true);
+        }
 
         match dm
             .send_message(&self.client.http, CreateMessage::new().embed(embed))
@@ -651,9 +657,10 @@ impl DiscordAPI {
             Err(_) => return 0,
         };
 
-        // Snowflake IDs are chronologically ordered; sort ascending so the latest
-        // reply to any given review embed is always processed last.
-        messages.sort_by_key(|message| message.id);
+        // fetch_all_dm_messages returns newest-to-oldest with no gaps or overlaps;
+        // reverse to oldest-first so the latest reply to any given review embed is
+        // always processed last.
+        messages.reverse();
 
         let mut rewritten_count = 0;
 
@@ -711,9 +718,9 @@ impl DiscordAPI {
                 e
             })?;
 
-            match page.first() {
+            match page.last() {
                 None => break,
-                Some(message) => last_message_id = Some(message.id),
+                Some(oldest_in_page) => last_message_id = Some(oldest_in_page.id),
             }
 
             all_messages.extend(page);
@@ -782,23 +789,6 @@ impl DiscordAPI {
         }
     }
 
-    async fn find_event_embed(&self, channel_id: ChannelId, event_url: &str) -> Option<Embed> {
-        let guild = self.get_guild(channel_id).await;
-        let threads = self.get_channel_threads(&guild, channel_id).await;
-
-        for thread in threads {
-            let messages = self.get_all_messages(thread.id).await;
-
-            if let Some(message) = messages.into_iter().find(|m| {
-                m.embeds.first().and_then(|e| e.url.clone()).as_deref() == Some(event_url)
-            }) {
-                return message.embeds.into_iter().next();
-            }
-        }
-
-        None
-    }
-
     async fn get_user_last_comment(&self, dm: &PrivateChannel) -> Option<Message> {
         match dm.last_message_id {
             Some(last_message_id) => {
@@ -859,11 +849,11 @@ impl DiscordAPI {
                 return Ok(true);
             }
 
-            match messages_iter.first() {
+            match messages_iter.last() {
                 None => {
                     searched_all_dms = true;
                 }
-                Some(last_message) => last_message_id = Some(last_message.id),
+                Some(oldest_in_page) => last_message_id = Some(oldest_in_page.id),
             }
         }
 

tests/agenda_cultural_api_tests.rs

@@ -25,4 +25,18 @@ mod agenda_cultural {
 
         assert_eq!(res.len(), 2);
     }
+
+    #[test_log::test(tokio::test)]
+    async fn should_scrape_single_event_page() {
+        let event = AgendaCulturalAPI::scrape_event("https://www.agendalx.pt/events/event/maes/")
+            .await
+            .expect("Failed to scrape event");
+
+        assert_eq!(event.title, "Mães");
+        assert_eq!(event.link, "https://www.agendalx.pt/events/event/maes/");
+        assert_eq!(event.venue, "Teatro Villaret");
+        assert_eq!(event.occurring_at.dates, "14 março a 30 junho 2024");
+        assert_eq!(event.details.description, "Três mães e uma grávida juntas num musical hilariante e ternurento onde ficamos a conhecer a poderosa amizade de quatro mulheres…");
+        assert_eq!(event.details.image_url, "https://www.agendalx.pt/content/uploads/2024/02/Maes.jpg");
+    }
 }