diff --git a/atcoder-problems-backend/src/crawler_utils.rs b/atcoder-problems-backend/src/crawler_utils.rs index e137d91e..360471f7 100644 --- a/atcoder-problems-backend/src/crawler_utils.rs +++ b/atcoder-problems-backend/src/crawler_utils.rs @@ -8,6 +8,15 @@ use sea_orm::{ }; const ATCODER_WEEKDAY_CONTEST_CATEGORY: u32 = 20; +const ATCODER_DAILY_TRAINING_CATEGORY: u32 = 60; + +/// Contest categories that AtCoder excludes from the unfiltered archive and +/// must therefore be crawled from their category-filtered archive pages. +/// Each entry is `(category id, human-readable label for logging)`. +const FILTERED_ARCHIVE_CATEGORIES: &[(u32, &str)] = &[ + (ATCODER_WEEKDAY_CONTEST_CATEGORY, "AtCoder Weekday Contest"), + (ATCODER_DAILY_TRAINING_CATEGORY, "AtCoder Daily Training"), +]; pub async fn fetch_submissions( crawler: &CrawlerClient, @@ -283,7 +292,8 @@ async fn upsert_problems( /// This function: /// 1. Fetches permanent contests (practice, APG4b, etc.) /// 2. Fetches contests from archive pages (paginated) -/// 3. Fetches AtCoder Weekday Contests from their category archive +/// 3. Fetches category-filtered contests (Weekday Contests, Daily Training) +/// from their category archives /// 4. Upserts all contests into the database /// /// Returns the total number of contests inserted/updated. @@ -319,31 +329,26 @@ pub async fn crawl_contests( tokio::time::sleep(std::time::Duration::from_millis(500)).await; } - // AtCoder excludes Weekday Contests from the unfiltered archive. - let mut page = 1; - loop { - tracing::info!( - "Fetching AtCoder Weekday Contests from archive page {}...", - page - ); - let contests = - fetch_contests_in_category_with_retry(fetcher, page, ATCODER_WEEKDAY_CONTEST_CATEGORY) - .await; - - if contests.is_empty() { - tracing::info!("No more AtCoder Weekday Contests found on page {}", page); - break; - } + // AtCoder excludes some contest categories (e.g. Weekday Contests and + // Daily Training) from the unfiltered archive, so crawl them from their + // category-filtered archive pages. + for &(category, label) in FILTERED_ARCHIVE_CATEGORIES { + let mut page = 1; + loop { + tracing::info!("Fetching {} from archive page {}...", label, page); + let contests = fetch_contests_in_category_with_retry(fetcher, page, category).await; + + if contests.is_empty() { + tracing::info!("No more {} found on page {}", label, page); + break; + } - tracing::info!( - "Fetched {} AtCoder Weekday Contests from page {}", - contests.len(), - page - ); - all_contests.extend(contests); - page += 1; + tracing::info!("Fetched {} {} from page {}", contests.len(), label, page); + all_contests.extend(contests); + page += 1; - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } } tracing::info!("Total contests fetched: {}", all_contests.len()); diff --git a/atcoder-problems-backend/tests/test_crawler_utils.rs b/atcoder-problems-backend/tests/test_crawler_utils.rs index 913b2a89..045f593b 100644 --- a/atcoder-problems-backend/tests/test_crawler_utils.rs +++ b/atcoder-problems-backend/tests/test_crawler_utils.rs @@ -322,7 +322,7 @@ async fn test_crawl_problems_generates_correct_title() { } #[tokio::test] -async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() { +async fn test_crawl_contests_fetches_filtered_archive_categories() { let db = setup_db().await.unwrap(); let mut mock_fetcher = MockContestFetcher::new(); @@ -366,16 +366,39 @@ async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() { .with(mockall::predicate::eq(2), mockall::predicate::eq(20)) .times(1) .returning(|_, _| Ok(vec![])); + mock_fetcher + .expect_fetch_contests_in_category() + .with(mockall::predicate::eq(1), mockall::predicate::eq(60)) + .times(1) + .returning(|_, _| { + Ok(vec![Contest { + id: "adt_all_20260612_2".to_string(), + start_epoch_second: 1_781_260_800, + duration_second: 6_000, + title: "AtCoder Daily Training 2026/06/12 All".to_string(), + rate_change: "-".to_string(), + }]) + }); + mock_fetcher + .expect_fetch_contests_in_category() + .with(mockall::predicate::eq(2), mockall::predicate::eq(60)) + .times(1) + .returning(|_, _| Ok(vec![])); let inserted = atcoder_problems_backend::crawler_utils::crawl_contests(&mock_fetcher, &db) .await .unwrap(); - assert_eq!(inserted, 2); + assert_eq!(inserted, 3); let contests = sql_entities::contests::Entity::find() .all(&db) .await .unwrap(); assert!(contests.iter().any(|contest| contest.id == "abc461")); assert!(contests.iter().any(|contest| contest.id == "awc0090")); + assert!( + contests + .iter() + .any(|contest| contest.id == "adt_all_20260612_2") + ); } diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts index 8e803843..dcc86212 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts @@ -52,6 +52,17 @@ describe("test function classifyContest", () => { expect(classifyContest(abcContest)).toBe("ABC" as ContestCategory); }); + it("when ADT", () => { + const adtContest: Contest = { + duration_second: 6000, + id: "adt_all_20260612_2", + rate_change: "-", + start_epoch_second: 1781260800, + title: "AtCoder Daily Training 2026/06/12 All", + }; + expect(classifyContest(adtContest)).toBe("ADT" as ContestCategory); + }); + it("when ABC-like", () => { const abcLikeContest: Contest = { duration_second: 6000, diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.ts index 79f0760e..ffe342c4 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.ts @@ -6,6 +6,7 @@ export const ContestCategories = [ "ARC", "AGC", "AWC", + "ADT", "ABC-Like", "ARC-Like", "AGC-Like", @@ -60,6 +61,9 @@ export const classifyContest = ( if (/^awc\d{4}$/.exec(contest.id)) { return "AWC"; } + if (/^adt_/.exec(contest.id)) { + return "ADT"; + } if ( /^ahc\d{3}$/.exec(contest.id) || ["toyota2023summer-final"].includes(contest.id)