qualbench/Makefile at main · semcod/qualbench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
.PHONY: all setup benchmark evaluate score leaderboard test lint serve clean

DATASET = dataset/qualbench-v0.json
RESULTS_DIR = results
TIMEOUT = 900

all: test lint benchmark evaluate leaderboard

# --- Development ---

install:
	pip install -e ".[test]"

test:
	pytest tests/ -v --tb=short

lint:
	ruff check qualbench/ tests/

# --- Quick start ---

quickstart:
	qualbench quickstart

run:
	qualbench run --tool prollama

run-json:
	qualbench run --tool prollama --json

# --- Full benchmark ---

setup-repos:
	qualbench setup --dataset $(DATASET)

benchmark: run-copilot run-openhands run-prollama

run-copilot:
	python runners/copilot_runner.py --dataset $(DATASET) --output $(RESULTS_DIR)/copilot/ --timeout $(TIMEOUT)

run-openhands:
	python runners/openhands_runner.py --dataset $(DATASET) --output $(RESULTS_DIR)/openhands/ --timeout $(TIMEOUT)

run-prollama:
	python runners/prollama_runner.py --dataset $(DATASET) --output $(RESULTS_DIR)/prollama/ --timeout $(TIMEOUT)

run-custom:
	@test -n "$(TOOL)" || (echo "Usage: make run-custom TOOL=my_tool" && exit 1)
	python runners/$(TOOL).py --dataset $(DATASET) --output $(RESULTS_DIR)/$(TOOL)/ --timeout $(TIMEOUT)

# --- Evaluate ---

evaluate:
	python scripts/evaluate.py --results-dir $(RESULTS_DIR) --dataset $(DATASET)

leaderboard:
	python scripts/score.py --evaluation $(RESULTS_DIR)/evaluation.json --leaderboard LEADERBOARD.md
	@cat LEADERBOARD.md

# --- Docker ---

docker-build:
	docker build -t semcod/qualbench-action:latest action/

docker-run:
	docker run --rm -v $(PWD):/workspace -w /workspace semcod/qualbench-action:latest

# --- Web Server ---

serve:
	python server.py

serve-api:
	uvicorn qualbench.api:app --reload --port 8000

# --- Leaderboard API ---

submit:
	qualbench submit --tool prollama

view-leaderboard:
	qualbench leaderboard

# --- Cleanup ---

clean:
	rm -rf repos/ $(RESULTS_DIR)/ reviews/ dist/ build/ *.egg-info