From 2f66a8f1f36da7b48e29a7b922d02ef071aee659 Mon Sep 17 00:00:00 2001 From: kenkoooo Date: Sun, 14 Jun 2026 09:52:35 +0900 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20AtCoder=20Daily=20Training=20?= =?UTF-8?q?=E3=82=92=E3=82=AF=E3=83=AD=E3=83=BC=E3=83=AB=E3=83=BB=E5=88=86?= =?UTF-8?q?=E9=A1=9E=E3=81=97=E3=81=A6=E8=A1=A8=E7=A4=BA=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AtCoder Daily Training (アーカイブ category=60, コンテストID `adt_*`) は Weekday Contest と同様にフィルタなしのアーカイブ一覧から除外されているため、 クローラが取得できず一覧に表示されていなかった。 - backend: カテゴリ別アーカイブ巡回を Weekday Contest 専用ループから カテゴリ配列ベースに変更し、Daily Training (category=60) も巡回する - frontend: `adt_` を "Daily Training" カテゴリに分類し、独立タブで表示する Co-Authored-By: Claude Opus 4.8 (1M context) --- atcoder-problems-backend/src/crawler_utils.rs | 50 ++++++++++--------- .../tests/test_crawler_utils.rs | 27 +++++++++- .../src/utils/ContestClassifier.test.ts | 13 +++++ .../src/utils/ContestClassifier.ts | 4 ++ 4 files changed, 69 insertions(+), 25 deletions(-) diff --git a/atcoder-problems-backend/src/crawler_utils.rs b/atcoder-problems-backend/src/crawler_utils.rs index e137d91e6..3471b9b4e 100644 --- a/atcoder-problems-backend/src/crawler_utils.rs +++ b/atcoder-problems-backend/src/crawler_utils.rs @@ -8,6 +8,15 @@ use sea_orm::{ }; const ATCODER_WEEKDAY_CONTEST_CATEGORY: u32 = 20; +const ATCODER_DAILY_TRAINING_CATEGORY: u32 = 60; + +/// Contest categories that AtCoder excludes from the unfiltered archive and +/// must therefore be crawled from their category-filtered archive pages. +/// Each entry is `(category id, human-readable label for logging)`. +const FILTERED_ARCHIVE_CATEGORIES: &[(u32, &str)] = &[ + (ATCODER_WEEKDAY_CONTEST_CATEGORY, "AtCoder Weekday Contest"), + (ATCODER_DAILY_TRAINING_CATEGORY, "AtCoder Daily Training"), +]; pub async fn fetch_submissions( crawler: &CrawlerClient, @@ -319,31 +328,26 @@ pub async fn crawl_contests( tokio::time::sleep(std::time::Duration::from_millis(500)).await; } - // AtCoder excludes Weekday Contests from the unfiltered archive. - let mut page = 1; - loop { - tracing::info!( - "Fetching AtCoder Weekday Contests from archive page {}...", - page - ); - let contests = - fetch_contests_in_category_with_retry(fetcher, page, ATCODER_WEEKDAY_CONTEST_CATEGORY) - .await; - - if contests.is_empty() { - tracing::info!("No more AtCoder Weekday Contests found on page {}", page); - break; - } + // AtCoder excludes some contest categories (e.g. Weekday Contests and + // Daily Training) from the unfiltered archive, so crawl them from their + // category-filtered archive pages. + for &(category, label) in FILTERED_ARCHIVE_CATEGORIES { + let mut page = 1; + loop { + tracing::info!("Fetching {} from archive page {}...", label, page); + let contests = fetch_contests_in_category_with_retry(fetcher, page, category).await; + + if contests.is_empty() { + tracing::info!("No more {} found on page {}", label, page); + break; + } - tracing::info!( - "Fetched {} AtCoder Weekday Contests from page {}", - contests.len(), - page - ); - all_contests.extend(contests); - page += 1; + tracing::info!("Fetched {} {} from page {}", contests.len(), label, page); + all_contests.extend(contests); + page += 1; - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + } } tracing::info!("Total contests fetched: {}", all_contests.len()); diff --git a/atcoder-problems-backend/tests/test_crawler_utils.rs b/atcoder-problems-backend/tests/test_crawler_utils.rs index 913b2a89c..045f593be 100644 --- a/atcoder-problems-backend/tests/test_crawler_utils.rs +++ b/atcoder-problems-backend/tests/test_crawler_utils.rs @@ -322,7 +322,7 @@ async fn test_crawl_problems_generates_correct_title() { } #[tokio::test] -async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() { +async fn test_crawl_contests_fetches_filtered_archive_categories() { let db = setup_db().await.unwrap(); let mut mock_fetcher = MockContestFetcher::new(); @@ -366,16 +366,39 @@ async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() { .with(mockall::predicate::eq(2), mockall::predicate::eq(20)) .times(1) .returning(|_, _| Ok(vec![])); + mock_fetcher + .expect_fetch_contests_in_category() + .with(mockall::predicate::eq(1), mockall::predicate::eq(60)) + .times(1) + .returning(|_, _| { + Ok(vec![Contest { + id: "adt_all_20260612_2".to_string(), + start_epoch_second: 1_781_260_800, + duration_second: 6_000, + title: "AtCoder Daily Training 2026/06/12 All".to_string(), + rate_change: "-".to_string(), + }]) + }); + mock_fetcher + .expect_fetch_contests_in_category() + .with(mockall::predicate::eq(2), mockall::predicate::eq(60)) + .times(1) + .returning(|_, _| Ok(vec![])); let inserted = atcoder_problems_backend::crawler_utils::crawl_contests(&mock_fetcher, &db) .await .unwrap(); - assert_eq!(inserted, 2); + assert_eq!(inserted, 3); let contests = sql_entities::contests::Entity::find() .all(&db) .await .unwrap(); assert!(contests.iter().any(|contest| contest.id == "abc461")); assert!(contests.iter().any(|contest| contest.id == "awc0090")); + assert!( + contests + .iter() + .any(|contest| contest.id == "adt_all_20260612_2") + ); } diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts index 8e8038439..6d76f7e7b 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts @@ -52,6 +52,19 @@ describe("test function classifyContest", () => { expect(classifyContest(abcContest)).toBe("ABC" as ContestCategory); }); + it("when Daily Training", () => { + const dailyTrainingContest: Contest = { + duration_second: 6000, + id: "adt_all_20260612_2", + rate_change: "-", + start_epoch_second: 1781260800, + title: "AtCoder Daily Training 2026/06/12 All", + }; + expect(classifyContest(dailyTrainingContest)).toBe( + "Daily Training" as ContestCategory + ); + }); + it("when ABC-like", () => { const abcLikeContest: Contest = { duration_second: 6000, diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.ts index 79f0760e9..d4fc19f79 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.ts @@ -6,6 +6,7 @@ export const ContestCategories = [ "ARC", "AGC", "AWC", + "Daily Training", "ABC-Like", "ARC-Like", "AGC-Like", @@ -60,6 +61,9 @@ export const classifyContest = ( if (/^awc\d{4}$/.exec(contest.id)) { return "AWC"; } + if (/^adt_/.exec(contest.id)) { + return "Daily Training"; + } if ( /^ahc\d{3}$/.exec(contest.id) || ["toyota2023summer-final"].includes(contest.id) From 63f43355dbe9242339fe93416529d3a1007e866e Mon Sep 17 00:00:00 2001 From: kenkoooo Date: Sun, 14 Jun 2026 09:56:35 +0900 Subject: [PATCH 2/3] =?UTF-8?q?docs:=20crawl=5Fcontests=20=E3=81=AE?= =?UTF-8?q?=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92?= =?UTF-8?q?=E3=82=AB=E3=83=86=E3=82=B4=E3=83=AA=E5=B7=A1=E5=9B=9E=E3=81=AE?= =?UTF-8?q?=E4=B8=80=E8=88=AC=E5=8C=96=E3=81=AB=E5=90=88=E3=82=8F=E3=81=9B?= =?UTF-8?q?=E3=81=A6=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit レビュー指摘 (Nit): ステップ3のコメントが Weekday Contest 専用のままだったため、 Daily Training を含むカテゴリ別アーカイブ巡回に合わせて修正。 Co-Authored-By: Claude Opus 4.8 (1M context) --- atcoder-problems-backend/src/crawler_utils.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/atcoder-problems-backend/src/crawler_utils.rs b/atcoder-problems-backend/src/crawler_utils.rs index 3471b9b4e..360471f7d 100644 --- a/atcoder-problems-backend/src/crawler_utils.rs +++ b/atcoder-problems-backend/src/crawler_utils.rs @@ -292,7 +292,8 @@ async fn upsert_problems( /// This function: /// 1. Fetches permanent contests (practice, APG4b, etc.) /// 2. Fetches contests from archive pages (paginated) -/// 3. Fetches AtCoder Weekday Contests from their category archive +/// 3. Fetches category-filtered contests (Weekday Contests, Daily Training) +/// from their category archives /// 4. Upserts all contests into the database /// /// Returns the total number of contests inserted/updated. From 10b58646adf09341e984d5811699dbba1b3e6d6f Mon Sep 17 00:00:00 2001 From: kenkoooo Date: Sun, 14 Jun 2026 10:23:31 +0900 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20Daily=20Training=20=E3=82=AB?= =?UTF-8?q?=E3=83=86=E3=82=B4=E3=83=AA=E3=81=AE=E8=A1=A8=E7=A4=BA=E5=90=8D?= =?UTF-8?q?=E3=82=92=20ADT=20=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR レビューコメント対応: 表示名は "Daily Training" よりも "ADT" の方が 適切との指摘を受け、フロントエンドのカテゴリ名を ADT に変更。 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/utils/ContestClassifier.test.ts | 8 +++----- atcoder-problems-frontend/src/utils/ContestClassifier.ts | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts index 6d76f7e7b..dcc862124 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.test.ts @@ -52,17 +52,15 @@ describe("test function classifyContest", () => { expect(classifyContest(abcContest)).toBe("ABC" as ContestCategory); }); - it("when Daily Training", () => { - const dailyTrainingContest: Contest = { + it("when ADT", () => { + const adtContest: Contest = { duration_second: 6000, id: "adt_all_20260612_2", rate_change: "-", start_epoch_second: 1781260800, title: "AtCoder Daily Training 2026/06/12 All", }; - expect(classifyContest(dailyTrainingContest)).toBe( - "Daily Training" as ContestCategory - ); + expect(classifyContest(adtContest)).toBe("ADT" as ContestCategory); }); it("when ABC-like", () => { diff --git a/atcoder-problems-frontend/src/utils/ContestClassifier.ts b/atcoder-problems-frontend/src/utils/ContestClassifier.ts index d4fc19f79..ffe342c43 100644 --- a/atcoder-problems-frontend/src/utils/ContestClassifier.ts +++ b/atcoder-problems-frontend/src/utils/ContestClassifier.ts @@ -6,7 +6,7 @@ export const ContestCategories = [ "ARC", "AGC", "AWC", - "Daily Training", + "ADT", "ABC-Like", "ARC-Like", "AGC-Like", @@ -62,7 +62,7 @@ export const classifyContest = ( return "AWC"; } if (/^adt_/.exec(contest.id)) { - return "Daily Training"; + return "ADT"; } if ( /^ahc\d{3}$/.exec(contest.id) ||