Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 29 additions & 24 deletions atcoder-problems-backend/src/crawler_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,15 @@ use sea_orm::{
};

const ATCODER_WEEKDAY_CONTEST_CATEGORY: u32 = 20;
const ATCODER_DAILY_TRAINING_CATEGORY: u32 = 60;

/// Contest categories that AtCoder excludes from the unfiltered archive and
/// must therefore be crawled from their category-filtered archive pages.
/// Each entry is `(category id, human-readable label for logging)`.
const FILTERED_ARCHIVE_CATEGORIES: &[(u32, &str)] = &[
(ATCODER_WEEKDAY_CONTEST_CATEGORY, "AtCoder Weekday Contest"),
(ATCODER_DAILY_TRAINING_CATEGORY, "AtCoder Daily Training"),
];

pub async fn fetch_submissions(
crawler: &CrawlerClient,
Expand Down Expand Up @@ -283,7 +292,8 @@ async fn upsert_problems(
/// This function:
/// 1. Fetches permanent contests (practice, APG4b, etc.)
/// 2. Fetches contests from archive pages (paginated)
/// 3. Fetches AtCoder Weekday Contests from their category archive
/// 3. Fetches category-filtered contests (Weekday Contests, Daily Training)
/// from their category archives
/// 4. Upserts all contests into the database
///
/// Returns the total number of contests inserted/updated.
Expand Down Expand Up @@ -319,31 +329,26 @@ pub async fn crawl_contests(
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}

// AtCoder excludes Weekday Contests from the unfiltered archive.
let mut page = 1;
loop {
tracing::info!(
"Fetching AtCoder Weekday Contests from archive page {}...",
page
);
let contests =
fetch_contests_in_category_with_retry(fetcher, page, ATCODER_WEEKDAY_CONTEST_CATEGORY)
.await;

if contests.is_empty() {
tracing::info!("No more AtCoder Weekday Contests found on page {}", page);
break;
}
// AtCoder excludes some contest categories (e.g. Weekday Contests and
// Daily Training) from the unfiltered archive, so crawl them from their
// category-filtered archive pages.
for &(category, label) in FILTERED_ARCHIVE_CATEGORIES {
let mut page = 1;
loop {
tracing::info!("Fetching {} from archive page {}...", label, page);
let contests = fetch_contests_in_category_with_retry(fetcher, page, category).await;

if contests.is_empty() {
tracing::info!("No more {} found on page {}", label, page);
break;
}

tracing::info!(
"Fetched {} AtCoder Weekday Contests from page {}",
contests.len(),
page
);
all_contests.extend(contests);
page += 1;
tracing::info!("Fetched {} {} from page {}", contests.len(), label, page);
all_contests.extend(contests);
page += 1;

tokio::time::sleep(std::time::Duration::from_millis(500)).await;
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
}
}

tracing::info!("Total contests fetched: {}", all_contests.len());
Expand Down
27 changes: 25 additions & 2 deletions atcoder-problems-backend/tests/test_crawler_utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -322,7 +322,7 @@ async fn test_crawl_problems_generates_correct_title() {
}

#[tokio::test]
async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() {
async fn test_crawl_contests_fetches_filtered_archive_categories() {
let db = setup_db().await.unwrap();
let mut mock_fetcher = MockContestFetcher::new();

Expand Down Expand Up @@ -366,16 +366,39 @@ async fn test_crawl_contests_fetches_atcoder_weekday_contest_category() {
.with(mockall::predicate::eq(2), mockall::predicate::eq(20))
.times(1)
.returning(|_, _| Ok(vec![]));
mock_fetcher
.expect_fetch_contests_in_category()
.with(mockall::predicate::eq(1), mockall::predicate::eq(60))
.times(1)
.returning(|_, _| {
Ok(vec![Contest {
id: "adt_all_20260612_2".to_string(),
start_epoch_second: 1_781_260_800,
duration_second: 6_000,
title: "AtCoder Daily Training 2026/06/12 All".to_string(),
rate_change: "-".to_string(),
}])
});
mock_fetcher
.expect_fetch_contests_in_category()
.with(mockall::predicate::eq(2), mockall::predicate::eq(60))
.times(1)
.returning(|_, _| Ok(vec![]));

let inserted = atcoder_problems_backend::crawler_utils::crawl_contests(&mock_fetcher, &db)
.await
.unwrap();

assert_eq!(inserted, 2);
assert_eq!(inserted, 3);
let contests = sql_entities::contests::Entity::find()
.all(&db)
.await
.unwrap();
assert!(contests.iter().any(|contest| contest.id == "abc461"));
assert!(contests.iter().any(|contest| contest.id == "awc0090"));
assert!(
contests
.iter()
.any(|contest| contest.id == "adt_all_20260612_2")
);
}
11 changes: 11 additions & 0 deletions atcoder-problems-frontend/src/utils/ContestClassifier.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,17 @@ describe("test function classifyContest", () => {
expect(classifyContest(abcContest)).toBe("ABC" as ContestCategory);
});

it("when ADT", () => {
const adtContest: Contest = {
duration_second: 6000,
id: "adt_all_20260612_2",
rate_change: "-",
start_epoch_second: 1781260800,
title: "AtCoder Daily Training 2026/06/12 All",
};
expect(classifyContest(adtContest)).toBe("ADT" as ContestCategory);
});

it("when ABC-like", () => {
const abcLikeContest: Contest = {
duration_second: 6000,
Expand Down
4 changes: 4 additions & 0 deletions atcoder-problems-frontend/src/utils/ContestClassifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ export const ContestCategories = [
"ARC",
"AGC",
"AWC",
"ADT",
"ABC-Like",
"ARC-Like",
"AGC-Like",
Expand Down Expand Up @@ -60,6 +61,9 @@ export const classifyContest = (
if (/^awc\d{4}$/.exec(contest.id)) {
return "AWC";
}
if (/^adt_/.exec(contest.id)) {
return "ADT";
}
if (
/^ahc\d{3}$/.exec(contest.id) ||
["toyota2023summer-final"].includes(contest.id)
Expand Down
Loading