diff --git a/studies/study_017/README.md b/studies/study_017/README.md new file mode 100644 index 0000000..465c62a --- /dev/null +++ b/studies/study_017/README.md @@ -0,0 +1,7 @@ +# Analyzing Frictions in Generalized Second-Price Auction Markets + +This HumanStudy-Hub package was reconstructed from a published paper. + +- Review `study.json` for the study overview and readiness status. +- Review `audit/missing_information.json` before running the study. +- Run `python task/adapter.py --smoke-test` to check the package entry point. diff --git a/studies/study_017/audit/missing_information.json b/studies/study_017/audit/missing_information.json new file mode 100644 index 0000000..862902f --- /dev/null +++ b/studies/study_017/audit/missing_information.json @@ -0,0 +1,82 @@ +{ + "_purpose": "Authoritative researcher checklist. Every entry names a study, the affected field, why it is missing/uncertain, its likely effect on faithfulness or findings, and a concrete suggested action. Nothing here has been silently filled in with invented content anywhere else in the package.", + "items": [ + { + "study": "study_1_main_experiment", + "field": "materials/materials.json:study_1_main_experiment.instructions_handout (verbatim text)", + "reason": "The instructions handout is printed only in Online Appendix L. Every retrieval attempt against the one authorized external URL (the ISR supplemental PDF) returned an HTTP 403 Cloudflare bot-check page, not the document.", + "impact": "Agent participants are onboarded using the paper's own verbatim rule statements (payoff rule, Remarks 1-4) rather than the exact handout wording and framing examples. Minor wording/emphasis differences from the original handout could affect how quickly participants understand the mechanism, though the substantive rules implemented are unchanged.", + "suggested_action": "If the researcher can access Online Appendix L (e.g., via institutional access to the publisher site, which may not trigger the same bot challenge), supply the verbatim handout text and it can be inserted into materials/materials.json without any change to task/adapter.py's mechanics." + }, + { + "study": "study_1_main_experiment", + "field": "materials/materials.json:study_1_main_experiment.practice_tasks (exact content)", + "reason": "The six practice tasks and five randomized examples are described only structurally in the main text; their exact numeric content is in Online Appendix L (inaccessible, see above).", + "impact": "Low: practice tasks are a comprehension check before the paid matches begin and are not part of the measured experiment. Their omission does not affect the implemented auction mechanics, but a fully faithful onboarding flow is not yet implemented in task/adapter.py.", + "suggested_action": "Supply the six practice-task specifications and five example templates from Online Appendix L if a faithful onboarding/comprehension-check phase is required before data collection." + }, + { + "study": "study_1_main_experiment; study_2_mechanism_experiment", + "field": "task/task.json:studies[].procedure.payment_to_participant / points-to-currency exchange rate", + "reason": "The main text states participants were 'paid in cash' with earnings equal to the sum (Study 1) or last-random-period payoff (both studies) but never states the numeric points-to-dollars conversion rate or any show-up fee.", + "impact": "Payoffs computed by task/adapter.py and evaluation/evaluation.py are in the paper's abstract point units (the same units as v_j in [1,10]), not real currency. This does not affect the relative comparisons the paper's Results 6-10 rely on, but it blocks any research question about real monetary stakes or absolute cash amounts.", + "suggested_action": "If real-currency payment is required, supply the exchange rate/show-up fee from Online Appendix L or M; otherwise treat all reported payoffs as point units, consistent with how Tables 5-7 in the paper report them." + }, + { + "study": "study_1_main_experiment", + "field": "task/task.json:studies[0].procedure.second_good_value (rounding convention)", + "reason": "Section 4.2 states the value of good 2 = alpha * value of good 1, but never states whether the product is rounded (e.g., 7 * 0.2 = 1.4) or displayed/used with decimals.", + "impact": "Low-to-moderate: rounding could shift bid-to-value ratios and payoffs at the margin, especially for alpha=0.2. task/adapter.py currently uses the unrounded float.", + "suggested_action": "Confirm the rounding convention from Online Appendix L screenshots (if recoverable) or from the authors; update compute_match_payoffs / run_study1_match if rounding is required." + }, + { + "study": "study_1_main_experiment", + "field": "task/task.json:studies[0].procedure.value_generation and .bid_domain (distribution/bounds)", + "reason": "Section 4 does not restate the U{1,10} valuation distribution or the [0,10] bid ceiling used by the computational model in Section 3.2.1; it only says the human experimental design implements 'the auction presented in Section 3.2.1'.", + "impact": "Low: this is a textually well-supported inference (evidence_label 'derived', not 'verbatim'), but it is technically an inference rather than a direct quote, so it is flagged for researcher confirmation before treating simulated data as a strict replication.", + "suggested_action": "Confirm against Online Appendix L (if recoverable) that the human experiment used the identical U{1,10} value range and [0,10] bid domain as the computational model." + }, + { + "study": "study_1_main_experiment", + "field": "task/task.json:studies[0].conditions.within_subjects (Remark 2 cross-session common random numbers)", + "reason": "Remark 2's comparability control operates ACROSS multiple sessions run under different treatments sharing the same drawn valuations in matches 5-10. A single package run has no other session to share seeds with.", + "impact": "Moderate for any downstream comparison across two separately-run task packages: without deliberately sharing a seed/value schedule between a condition_c=0.0 run and a condition_c=0.1 run, the two runs' matches 5-10 will not have matched valuations, which the paper used specifically to strengthen its Result 6/8/10 comparisons.", + "suggested_action": "When running paired condition_c=0.0 and condition_c=0.1 sessions for comparison, pass the same --seed and, if exact value-matching is required, pre-generate and share a fixed values-per-match schedule across both adapter calls (task/adapter.py's run_study1_match accepts explicit values via minor code modification if this is needed)." + }, + { + "study": "study_1_main_experiment; study_2_mechanism_experiment", + "field": "task/task.json known_departures_from_original_design: max_periods safety cap", + "reason": "The paper's random-termination process (Remark 1, and its discrete-time analogue in Study 2) is unbounded in principle (geometric distribution). Each period in a package run is a real agent decision call, so an unbounded process is not computationally tractable.", + "impact": "Low-to-moderate: the cap (default 300 periods for Study 1, ~2.5x the expected 120; default 60 for Study 2, ~6x the expected 10) is set far past the expectation, so it should rarely bind, but it removes the small-probability long tail of very long matches, which slightly reduces the expected number of measured bid adjustments relative to a truly unbounded process.", + "suggested_action": "Increase max_periods in Study1Config/Study2Config if a research question specifically concerns tail behavior of very long matches; otherwise the default is a disclosed, documented approximation." + }, + { + "study": "study_2_mechanism_experiment", + "field": "materials/materials.json:study_2_mechanism_experiment.instructions_and_screens; Online Table N.1 (four duration sequences)", + "reason": "Online Appendix N.1 (instructions/decision screens), N.2 (info-plot screenshots), and Online Table N.1 (the four exact supergame-length sequences) are in the same inaccessible supplemental PDF.", + "impact": "Moderate: task/adapter.py substitutes a geometric-hazard generator (p=0.1 per period) for the four unspecified duration sequences, which is mathematically consistent with the paper's stated 'probability of termination was 0.1' but does not reproduce the exact four realized sequences the original subjects experienced, so period-by-period comparisons to the original data will not line up exactly.", + "suggested_action": "Supply Online Table N.1's four sequences (as explicit period counts) and Online Appendix N.1/N.2 screenshots if exact replication of Study 2's timing and interface is required; update run_study2_session to draw from the supplied fixed sequences instead of the hazard-process substitute." + }, + { + "study": "study_1_main_experiment; study_2_mechanism_experiment", + "field": "task/task.json:studies[].num_agents (population size)", + "reason": "The paper ran 138 (Study 1) and 183 (Study 2) human subjects across 8+ sessions with random rematching into triads each match; the package defaults to a single triad (3 agents) as the minimal faithful unit.", + "impact": "Moderate for statistical power and for the random-rematching dynamic itself: with only 3 agents, 'rematching' each match is a no-op (the same three agents are always the only possible triad). The rematching code path is implemented and exercised only if num_agents is increased to a larger multiple of 3.", + "suggested_action": "Set num_agents to a larger multiple of 3 (e.g., 15-18 to mirror one session) if the random-rematching dynamic across many participants, rather than a single fixed triad, is part of the research question." + }, + { + "study": "study_1_main_experiment", + "field": "Remark 4 (silent protocol)", + "reason": "The silent protocol addressed a physical-room information leak (audible mouse clicks) specific to co-located human subjects in the same physical lab.", + "impact": "None for a text-based agent replay: agents are not co-located and cannot overhear each other's actions regardless. The information-visibility guarantee the protocol protected (opponents cannot infer bid changes except through the screen) is preserved directly by task/task.json's visibility rules.", + "suggested_action": "No action needed; recorded here only so the omission is explicit rather than silent." + }, + { + "study": "study_1_main_experiment", + "field": "Footnote 19 (disclosed software bug affecting seed increments in sessions 1-4, matches 5-10)", + "reason": "The authors themselves disclose a software bug in the original study's random-seed generation, not a gap in this extraction.", + "impact": "None for this package (the bug was in the original study's software, not reproduced here); recorded for researcher awareness when comparing this package's simulated data against the original published data from sessions 1-4.", + "suggested_action": "No action needed for package construction; keep in mind only when benchmarking against the original paper's raw data tables." + } + ] +} diff --git a/studies/study_017/evaluation/evaluation.py b/studies/study_017/evaluation/evaluation.py new file mode 100644 index 0000000..dc380d6 --- /dev/null +++ b/studies/study_017/evaluation/evaluation.py @@ -0,0 +1,238 @@ +""" +Evaluation for Kannan, Pamuru & Rosokha (2023) task runs. + +Consumes one or more session-record JSON files produced by +task/adapter.py (run_study1_session / run_study2_session output) and checks +the paper's reported findings (Results 6-10, Section 4.6). Each Result +requires specific conditions to be present across the supplied session files; +when a required condition/comparison is absent, that check returns a +structured `not_ready` result naming exactly what is missing -- it never +fabricates a comparison from data that was not run. + +No network access is used or required. +""" + +import argparse +import json +import random +from statistics import mean + + +def _mean(xs): + xs = list(xs) + return sum(xs) / len(xs) if xs else None + + +def _rank_label(rank): + return {1: "highest_valued", 2: "medium_valued", 3: "lowest_valued"}.get(rank) + + +def load_sessions(paths): + sessions = [] + for p in paths: + with open(p) as f: + sessions.append(json.load(f)) + return sessions + + +def _matches_in_window(session, window): + return [m for m in session["matches"] if window is None or m["match_index"] in window] + + +def extract_rows(session, window=None): + """One row per (match, participant): rank label, num_adjustments, + bid_to_value_ratio, allocative_efficiency (match-level, repeated per row), + alpha, cost_c.""" + rows = [] + for m in _matches_in_window(session, window): + for pid, payoff in m["payoffs"].items(): + rows.append({ + "match_index": m["match_index"], + "rank": payoff["rank"], + "rank_label": _rank_label(payoff["rank"]), + "num_adjustments": payoff["num_adjustments"], + "bid_to_value_ratio": payoff["bid_to_value_ratio"], + "allocative_efficiency": m["allocative_efficiency"], + "alpha": m["alpha"], + "cost_c": m["cost_c"], + }) + return rows + + +def permutation_test(sample_a, sample_b, n_permutations=10000, seed=0): + """Two-tailed permutation test on the difference in means (Good 2013), + matching the statistical test cited throughout Section 4.6 of the paper.""" + a = [x for x in sample_a if x is not None] + b = [x for x in sample_b if x is not None] + if len(a) == 0 or len(b) == 0: + return None + rng = random.Random(seed) + observed = mean(a) - mean(b) if a and b else None + if observed is None: + return None + pooled = a + b + n_a = len(a) + count_extreme = 0 + for _ in range(n_permutations): + rng.shuffle(pooled) + perm_a = pooled[:n_a] + perm_b = pooled[n_a:] + diff = mean(perm_a) - mean(perm_b) + if abs(diff) >= abs(observed): + count_extreme += 1 + p_value = count_extreme / n_permutations + return {"observed_difference": observed, "p_value": p_value, "n_permutations": n_permutations} + + +def bootstrap_se(sample, n_boot=2000, seed=0): + xs = [x for x in sample if x is not None] + if not xs: + return None + rng = random.Random(seed) + boot_means = [] + n = len(xs) + for _ in range(n_boot): + resample = [xs[rng.randrange(n)] for _ in range(n)] + boot_means.append(mean(resample)) + m = mean(boot_means) + var = sum((x - m) ** 2 for x in boot_means) / (len(boot_means) - 1) if len(boot_means) > 1 else 0.0 + return var ** 0.5 + + +def _find_condition_sessions(sessions, key, value_a, value_b): + a = [s for s in sessions if s.get(key) == value_a] + b = [s for s in sessions if s.get(key) == value_b] + return a, b + + +def check_result_6_lower_cost_more_exploration(sessions, window=None): + """Result 6: 'Lower costs lead to more exploration.' Requires at least one + session with cost_c == 0.0 and one with cost_c == 0.1 (same study_id).""" + c0, c1 = _find_condition_sessions(sessions, "condition_c", 0.0, 0.1) + if not c0 or not c1: + return {"result": "Result 6", "status": "not_ready", + "missing_requirement": "Need at least one session run with condition_c=0.0 " + "and at least one with condition_c=0.1 to compare."} + rows0 = [r for s in c0 for r in extract_rows(s, window)] + rows1 = [r for s in c1 for r in extract_rows(s, window)] + test = permutation_test([r["num_adjustments"] for r in rows0], + [r["num_adjustments"] for r in rows1]) + supported = test is not None and mean([r["num_adjustments"] for r in rows0]) > \ + mean([r["num_adjustments"] for r in rows1]) + return {"result": "Result 6", "status": "evaluated", + "mean_adjustments_c0.0": _mean(r["num_adjustments"] for r in rows0), + "mean_adjustments_c0.1": _mean(r["num_adjustments"] for r in rows1), + "permutation_test": test, "directionally_consistent_with_paper": supported} + + +def check_result_7_lowest_valued_overbid_most(sessions, window=None): + """Result 7: 'The bid-to-value ratio is higher for lower valued agents.' + Evaluable within any single session that has at least one match.""" + rows = [r for s in sessions for r in extract_rows(s, window)] + if not rows: + return {"result": "Result 7", "status": "not_ready", + "missing_requirement": "No match data supplied."} + by_rank = {label: [r["bid_to_value_ratio"] for r in rows if r["rank_label"] == label] + for label in ("highest_valued", "medium_valued", "lowest_valued")} + means = {k: _mean(v) for k, v in by_rank.items()} + test = permutation_test(by_rank["lowest_valued"], by_rank["highest_valued"]) + supported = (means["lowest_valued"] is not None and means["highest_valued"] is not None + and means["lowest_valued"] > means["highest_valued"]) + return {"result": "Result 7", "status": "evaluated", "mean_bid_to_value_by_rank": means, + "permutation_test_lowest_vs_highest": test, "directionally_consistent_with_paper": supported} + + +def check_result_8_lowest_valued_friction_effect(sessions, window=None): + """Result 8: 'For the lowest valued agents, the bid-to-value ratio + increases as friction decreases.' Requires both cost_c conditions.""" + c0, c1 = _find_condition_sessions(sessions, "condition_c", 0.0, 0.1) + if not c0 or not c1: + return {"result": "Result 8", "status": "not_ready", + "missing_requirement": "Need sessions at both condition_c=0.0 and condition_c=0.1."} + r0 = [r for s in c0 for r in extract_rows(s, window) if r["rank_label"] == "lowest_valued"] + r1 = [r for s in c1 for r in extract_rows(s, window) if r["rank_label"] == "lowest_valued"] + test = permutation_test([r["bid_to_value_ratio"] for r in r0], + [r["bid_to_value_ratio"] for r in r1]) + m0, m1 = _mean(r["bid_to_value_ratio"] for r in r0), _mean(r["bid_to_value_ratio"] for r in r1) + supported = m0 is not None and m1 is not None and m0 > m1 + return {"result": "Result 8", "status": "evaluated", + "mean_bid_to_value_lowest_valued_c0.0": m0, "mean_bid_to_value_lowest_valued_c0.1": m1, + "permutation_test": test, "directionally_consistent_with_paper": supported} + + +def check_result_9_alpha_effect(sessions, window=None): + """Result 9: 'The bid-to-value ratio decreases as alpha increases for + medium and highest valued agents.' Requires within-subjects alpha + variation (Study 1 only; Study 2 fixes alpha=0.5).""" + rows = [r for s in sessions for r in extract_rows(s, window)] + alphas_present = sorted(set(r["alpha"] for r in rows)) + if len(alphas_present) < 2: + return {"result": "Result 9", "status": "not_ready", + "missing_requirement": "Need matches spanning at least two distinct alpha values " + "(Study 1 varies alpha within-subjects; Study 2 fixes alpha=0.5 " + "and cannot support this check)."} + low_alpha, high_alpha = alphas_present[0], alphas_present[-1] + out = {} + for label in ("medium_valued", "highest_valued"): + low = [r["bid_to_value_ratio"] for r in rows if r["rank_label"] == label and r["alpha"] == low_alpha] + high = [r["bid_to_value_ratio"] for r in rows if r["rank_label"] == label and r["alpha"] == high_alpha] + test = permutation_test(low, high) + m_low, m_high = _mean(low), _mean(high) + out[label] = {"mean_at_low_alpha": m_low, "mean_at_high_alpha": m_high, + "permutation_test": test, + "directionally_consistent_with_paper": (m_low is not None and m_high is not None + and m_low > m_high)} + return {"result": "Result 9", "status": "evaluated", "low_alpha": low_alpha, "high_alpha": high_alpha, + "by_rank": out} + + +def check_result_10_efficiency_increases_with_friction(sessions, window=None): + """Result 10: 'Allocative efficiency of the market increases as friction + costs increase.' Requires both cost_c conditions.""" + c0, c1 = _find_condition_sessions(sessions, "condition_c", 0.0, 0.1) + if not c0 or not c1: + return {"result": "Result 10", "status": "not_ready", + "missing_requirement": "Need sessions at both condition_c=0.0 and condition_c=0.1."} + eff0 = [m["allocative_efficiency"] for s in c0 for m in _matches_in_window(s, window)] + eff1 = [m["allocative_efficiency"] for s in c1 for m in _matches_in_window(s, window)] + test = permutation_test(eff0, eff1) + m0, m1 = _mean(eff0), _mean(eff1) + supported = m0 is not None and m1 is not None and m1 > m0 + return {"result": "Result 10", "status": "evaluated", + "mean_efficiency_c0.0": m0, "mean_efficiency_c0.1": m1, + "permutation_test": test, "directionally_consistent_with_paper": supported} + + +ALL_CHECKS = [ + check_result_6_lower_cost_more_exploration, + check_result_7_lowest_valued_overbid_most, + check_result_8_lowest_valued_friction_effect, + check_result_9_alpha_effect, + check_result_10_efficiency_increases_with_friction, +] + + +def evaluate(sessions, matches_window=range(5, 11)): + """matches_window mirrors the paper's Section 4.6 analysis window + (matches 5-10, evidence E21). Pass None to use all matches.""" + return {"analysis_window_match_indices": list(matches_window) if matches_window else "all", + "num_sessions_supplied": len(sessions), + "checks": [check(sessions, matches_window) for check in ALL_CHECKS]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--session", action="append", required=True, dest="sessions", + help="Path to a session-record JSON file from task/adapter.py. " + "May be repeated to supply multiple conditions/runs.") + parser.add_argument("--all-matches", action="store_true", + help="Use all matches instead of the paper's matches-5-10 analysis window.") + args = parser.parse_args() + + sessions = load_sessions(args.sessions) + window = None if args.all_matches else range(5, 11) + print(json.dumps(evaluate(sessions, window), indent=2, default=str)) + + +if __name__ == "__main__": + main() diff --git a/studies/study_017/materials/materials.json b/studies/study_017/materials/materials.json new file mode 100644 index 0000000..230464c --- /dev/null +++ b/studies/study_017/materials/materials.json @@ -0,0 +1,110 @@ +{ + "_purpose": "Participant-visible material, grouped by study and condition. Every item carries an evidence_label and a source pointer into source/evidence.json. Items labeled 'missing' are NOT filled with invented text; the field is left explicit so a researcher can supply the real wording from the paper's Online Appendix.", + + "study_1_main_experiment": { + "paper_section": "Section 4 (pp. 1443-1448)", + "participant_facing_terminology": { + "evidence_label": "verbatim", + "evidence_ref": "E01,E04", + "note": "The paper deliberately uses lab-friendly terms with participants instead of the theoretical vocabulary. Use these terms in all agent-facing prompts.", + "mapping": { + "advertiser (theory)": "participant", + "ad slot (theory)": "good", + "auction (theory)": "match", + "bid-adjustment cost C (theory)": "cost of changing your bid" + } + }, + "treatments_table_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E03", + "table_title": "Table 4. Treatments Summary", + "rows": [ + { "treatments": "Between-Subjects", "parameter_varied": "C in {0.0, 0.1}", "description": "Costless or costly bid adjustment within matches" }, + { "treatments": "Within-Subjects", "parameter_varied": "alpha in {0.2, 0.5, 0.8}", "description": "Value of the second good as a fraction of the first good" } + ] + }, + "payoff_rule_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E07", + "text": "At the beginning of a match, each participant j submitted a bid bj at no cost. Participants could then revise their bids in continuous time during the match. Participants could lock the bids to see the associated outcome and payoff with the current combination of bids. Specifically, if a participant's bid, bj, was the highest, then the participant would get the first good and pay the amount equal to the second-highest bid minus the friction cost incurred during the match, (v(1) - b(2)) - C(1). If the participant's bid was the second highest, then the participant would get the second good and pay the amount equal to the third-highest bid times alpha minus the friction cost incurred during the match, that is, alpha(v(2) - b(3)) - C(2). If the participant's bid was the lowest, the participant would not receive any good and would pay only the friction cost incurred. We announced that alpha would be the same for all participants in the group during the instructions and had a reminder on the screen to ensure its common knowledge." + }, + "design_remarks_verbatim": { + "evidence_label": "verbatim", + "items": [ + { "remark": "Remark 1", "text": "We used a random-termination protocol.", "evidence_ref": "E05" }, + { "remark": "Remark 2", "text": "We used common random numbers in matches 5-10 such that, for each match, m in {5,...,10}, all groups had the same three valuations {v1m, v2m, v3m}; m in {5,...,10}, there was at least one group for each value of alpha.", "evidence_ref": "E06" }, + { "remark": "Remark 3", "text": "We used a last (randomly occurring) period compensation mechanism.", "evidence_ref": "E08" }, + { "remark": "Remark 4", "text": "We used a silent protocol.", "evidence_ref": "E09" } + ] + }, + "interface_visibility_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E10", + "text": "The participant's screen summarized information provided for that match (#1) as well as the current action and the outcome associated with this action (#2). The outcomes are updated live and depend on the actions of all three participants in the group. Scale that is used to place and revise bids (#3). Scales for the other two participants (#4) remain blank until the match is over, at which point the actions of the other participants are revealed. Reminders about the rules of the experiment (#5)." + }, + "instructions_handout": { + "evidence_label": "missing", + "evidence_ref": "E12", + "reason": "Verbatim instructions live in Online Appendix L, which could not be retrieved (Cloudflare 403 on the authorized supplement URL).", + "researcher_action_needed": "Supply the verbatim instructions handout text (Online Appendix L) if exact wording is required for participant onboarding. Until supplied, task/adapter.py uses only the paper's own verbatim description of the rules (payoff_rule_verbatim, design_remarks_verbatim above) as the operative instructions." + }, + "practice_tasks": { + "evidence_label": "reported", + "evidence_ref": "E11", + "text": "Six practice tasks dealing with placing and modifying the bid (proceed to next only after correctly completing the previous), followed by five randomly generated examples practicing bid placement/revision against hypothetical opponent actions.", + "exact_content": { + "evidence_label": "missing", + "reason": "The specific numeric values and correct answers used in the six practice tasks and five examples are only in Online Appendix L." + } + }, + "point_to_currency_conversion": { + "evidence_label": "missing", + "evidence_ref": "E22", + "reason": "No explicit points-to-dollars exchange rate or show-up fee is stated in the accessible main text." + } + }, + + "study_2_mechanism_experiment": { + "paper_section": "Section 5 (pp. 1448-1450)", + "design_summary_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E14,E16", + "text": "We implemented a 2x2 factorial design in which we varied the cost of bid adjustment (as is the main treatment variation in the original set of experiment) and the cost of the information. For all four treatments, we fixed the same alpha = 0.5. The differences from the original experiments include (i) discrete-time setup, (ii) duration of the interaction, (iii) multiple duration realizations, and (iv) independent random draws for each subject in each match. Specifically, in the discrete-time setting, subjects faced an uncertain number of decision periods. Within each period, they had unlimited time to make a desired bid adjustment (if any). Once all participants in their group submitted their decisions, each saw the outcome for that period." + }, + "information_feature_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E15", + "text": "The information consisted of an interactive plot that displayed possible payoffs for any bid conditional on the other two group members holding the bids as they did in the previous period. The treatment variation included the cost of accessing this plot. In the costly information treatment, subjects incurred a cost of $0.5 every period in which they accessed this information, whereas in the free information treatment, subjects did not incur any cost." + }, + "termination_redesign_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E14", + "text": "To make the discrete-time setup feasible within the time constraints offered by the laboratory experiment, we reduced the number of expected interactions to 10 (i.e., probability of termination was 0.1 instead of 0.01). In addition, to ensure that one specific sequence of match durations did not bias the results, we now consider four sequences of supergame length (see Online Table N.1)." + }, + "instructions_and_screens": { + "evidence_label": "missing", + "evidence_ref": "E17", + "reason": "Online Appendix N.1 (instructions/decision screens) and N.2 (info-plot screenshots) and Online Table N.1 (the four duration sequences) could not be retrieved." + }, + "software_platform": { + "evidence_label": "verbatim", + "evidence_ref": "E13", + "text": "The new experiment was programmed in oTree software (Chen et al. 2016a)." + } + }, + + "shared_theoretical_material": { + "paper_section": "Section 3.1 (pp. 1440-1441)", + "gsp_payoff_formula_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E18", + "text": "The payoff for the kth highest bidding advertiser (allotted to the kth slot) is given by alpha_k(v(k) - b(k+1)). Frictions are modeled as an additional cost Cj incurred by advertiser j from revising bids: pi_j = alpha_k(vj - b(k+1)) - Cj." + }, + "metrics_verbatim": { + "evidence_label": "verbatim", + "evidence_ref": "E19", + "allocative_efficiency": "Psi = sum_{k=1}^{K} alpha_k*v(k) divided by sum_{j=1}^{K} alpha_j*v^j", + "bid_to_value_ratio": "Omega_j = b_j / v_j" + } + } +} diff --git a/studies/study_017/source/evidence.json b/studies/study_017/source/evidence.json new file mode 100644 index 0000000..6bb0f38 --- /dev/null +++ b/studies/study_017/source/evidence.json @@ -0,0 +1,240 @@ +{ + "_purpose": "Complete evidence and provenance record. Every substantive claim or material used elsewhere in this package is listed here with its source location in input/paper.pdf, an evidence label, and (for derived items) the derivation logic. Page numbers refer to the printed journal pagination (1437-1454), which pdftotext preserves in the extracted text.", + "evidence_labels_used": { + "verbatim": "directly quoted or transcribed from the paper", + "reported": "faithfully paraphrased from the paper", + "derived": "transformed from reported information, with the derivation stated", + "missing": "unavailable in the accessible source (the Online Appendix could not be retrieved); requires researcher input" + }, + "studies_identified": [ + { + "study_id": "computational_main", + "label": "not_human_subject", + "paper_section": "3. Computational Analysis", + "reported": "Q-learning agent-based simulation of J=3 bidders competing for K=2 ad slots over M=2000 matches, T=100 periods each. This is a machine-simulation study, not a human-subject study, so it is out of scope for the agent-participant package but is recorded here for completeness and because Study 1's design mirrors its parameters (J=3, K=2, U{1,10} valuations).", + "evidence_label": "reported", + "source_location": "pp. 1440-1444" + }, + { + "study_id": "study_1_main_experiment", + "label": "human_subject", + "paper_section": "4. Experimental Analysis", + "reported": "Laboratory human-subject experiment with 138 Purdue participants recruited via ORSEE, 8 sessions of 15-18 participants each. Between-subjects treatment C in {0.0, 0.1} (bid-adjustment friction cost); within-subjects treatment alpha in {0.2, 0.5, 0.8} (slot-2/slot-1 value ratio). 10 matches per session, random regrouping into triads (J=3) each match, continuous-time bidding within each match with random termination.", + "evidence_label": "reported", + "source_location": "pp. 1443-1448", + "used_in": ["task/task.json:studies[0]", "materials/materials.json:study_1_main_experiment"] + }, + { + "study_id": "study_2_mechanism_experiment", + "label": "human_subject", + "paper_section": "5. Experiment to Study the Underlying Mechanism", + "reported": "Follow-up laboratory human-subject experiment with 183 new Purdue participants (no overlap with Study 1), programmed in oTree. 2x2 between-subjects factorial design: bid-adjustment cost C in {0.0, 0.1} crossed with a new treatment, cost of accessing counterfactual-payoff information (free vs. $0.5/period 'InfoCosts'). Discrete-time periods (unlike Study 1's continuous time), four pre-set supergame-length sequences, independently redrawn private values each match (no shared seeds across groups).", + "evidence_label": "reported", + "source_location": "pp. 1448-1450", + "used_in": ["task/task.json:studies[1]", "materials/materials.json:study_2_mechanism_experiment"] + }, + { + "study_id": "computational_robustness_section6", + "label": "not_human_subject", + "paper_section": "6. Additional Insights", + "reported": "Two additional computational (Q-learning) studies calibrated to the human-subject data: one recalibrating the learning model, one varying the number of advertisers J in {3,4,5,6}. Neither involves human or agent participants, so neither is represented in task/task.json.", + "evidence_label": "reported", + "source_location": "pp. 1450-1451" + } + ], + "evidence_items": [ + { + "id": "E01", + "claim": "Sample size and recruitment for Study 1", + "quote": "For the experiment, we recruited 138 participants using ORSEE software (Greiner 2015) on the campus of Purdue University. We administered eight sessions of the experiment with 15-18 participants in each session.", + "evidence_label": "verbatim", + "source_location": "Section 4.5, p. 1446", + "used_in": "study.json:studies[0].sample; materials/materials.json" + }, + { + "id": "E02", + "claim": "Physical isolation and anonymity of participants", + "quote": "Upon entering the laboratory, participants were assigned to a computer terminal. All terminals were separated by physical barriers such that participants could not see choices made by other participants in the room. Participants remained anonymous throughout the experiment.", + "evidence_label": "verbatim", + "source_location": "Section 4.5, p. 1446", + "used_in": "task/task.json visibility rules" + }, + { + "id": "E03", + "claim": "Treatments summary (Table 4)", + "quote": "Table 4. Treatments Summary | Treatments: Between-Subjects, Parameter varied: C in {0.0, 0.1}, Description: Costless or costly bid adjustment within matches | Treatments: Within-Subjects, Parameter varied: alpha in {0.2, 0.5, 0.8}, Description: Value of the second good as a fraction of the first good", + "evidence_label": "verbatim", + "source_location": "Table 4, p. 1445", + "used_in": "task/task.json condition assignment" + }, + { + "id": "E04", + "claim": "Matches per session and earnings rule", + "quote": "Each session consisted of M = 10 matches. At the beginning of each match, participants were randomly split into groups of three (J = 3) and remained so until the end of the match. The regrouping for the next match was random to avoid any systematic learning about participant behaviors. Earnings for the experiment were the sum of payoffs across all 10 matches.", + "evidence_label": "verbatim", + "source_location": "Section 4.2, p. 1443", + "used_in": "task/task.json:studies[0].procedure" + }, + { + "id": "E05", + "claim": "Random-termination protocol (Remark 1)", + "quote": "To avoid the end-of-match effects associated with the fixed duration of a match, we opted for random termination. Specifically, each match lasted at least 20 seconds, after which the chance of the match terminating each second was 1%. Thus, the expected duration of each match was two minutes. To ensure a valid comparison across sessions, we used the same sequence of seconds across matches in every session. Remark 1. We used a random-termination protocol.", + "evidence_label": "verbatim", + "source_location": "Section 4.2, p. 1443", + "used_in": "task/task.json:studies[0].procedure.match_termination" + }, + { + "id": "E06", + "claim": "Private-value / slot-value generation and common-seed design (Remark 2)", + "quote": "For each match, the participants were provided with randomly drawn private values for good 1. We then obtained the value of good 2 by multiplying the value of good 1 and alpha. Parameters for the initial four matches were drawn at random without any restriction. However, in matches 5-10, we aimed to provide a clean comparison among the treatments. Therefore, we used common seeds to generate the same random values across the two treatment dimensions. Remark 2. We used common random numbers in matches 5-10 such that, for each match, m in {5,...,10}, all groups had the same three valuations {v1m, v2m, v3m}; m in {5,...,10}, there was at least one group for each value of alpha.", + "evidence_label": "verbatim", + "source_location": "Section 4.2, p. 1443-1444", + "used_in": "task/task.json:studies[0].procedure.value_generation", + "note_on_derivation": "The cross-session common-seed comparison (Remark 2) requires multiple independently run sessions/groups and is a between-session experimental-design control, not a within-run mechanic. A single package run (one triad or one small pool of agents) implements the private-value draw and the alpha-sharing rule but cannot itself realize the cross-session seed-matching; this is recorded as a departure in audit/missing_information.json." + }, + { + "id": "E07", + "claim": "GSP payoff/auction rule as implemented for participants", + "quote": "At the beginning of a match, each participant j submitted a bid bj at no cost. Participants could then revise their bids in continuous time during the match. Participants could lock the bids to see the associated outcome and payoff with the current combination of bids. Specifically, if a participant's bid, bj, was the highest (i.e., j -> (1)), then the participant would get the first good and pay the amount equal to the second-highest bid minus the friction cost incurred during the match, (v(1) - b(2)) - C(1). If the participant's bid was the second highest, then the participant would get the second good and pay the amount equal to the third-highest bid times alpha minus the friction cost incurred during the match, that is, alpha(v(2) - b(3)) - C(2). If the participant's bid was the lowest, the participant would not receive any good and would pay only the friction cost incurred. We announced that alpha would be the same for all participants in the group during the instructions and had a reminder on the screen to ensure its common knowledge.", + "evidence_label": "verbatim", + "source_location": "Section 4.3, p. 1444-1445", + "used_in": "task/adapter.py payoff function; task/task.json payoff_rule" + }, + { + "id": "E08", + "claim": "Last (randomly occurring) period compensation mechanism (Remark 3)", + "quote": "the actual compensation in each match is based on the last auction in a sequence. ... Remark 3. We used a last (randomly occurring) period compensation mechanism.", + "evidence_label": "verbatim", + "source_location": "Section 4.3, p. 1445", + "used_in": "task/task.json:studies[0].procedure.payment_rule" + }, + { + "id": "E09", + "claim": "Silent protocol for bid revision (Remark 4)", + "quote": "To resolve this issue, we implemented a silent protocol. Specifically, instead of clicking to select a new bid, subjects placed and adjusted their bids by moving the mouse back and forth across the scale border. ... Remark 4. We used a silent protocol.", + "evidence_label": "verbatim", + "source_location": "Section 4.4, p. 1445-1446", + "used_in": "audit/missing_information.json (not applicable to text-based agents; recorded as inapplicable rather than silently dropped)" + }, + { + "id": "E10", + "claim": "Experimental interface information structure", + "quote": "The participant's screen summarized information provided for that match (#1 in Figure 2) as well as the current action and the outcome associated with this action (#2 in Figure 2). ... (4) Scales for the other two participants. These scales remain blank until the match is over, at which point the actions of the other participants are revealed.", + "evidence_label": "verbatim", + "source_location": "Section 4.4, p. 1445, Figure 2 notes", + "used_in": "task/task.json:studies[0].visibility" + }, + { + "id": "E11", + "claim": "Instructions and practice-task administration", + "quote": "Subjects received a handout containing the instructions (Online Appendix L). An experimenter read them out loud to ensure common knowledge of the environment. Second, subjects had to complete six practice tasks that dealt with placing and modifying the bid. Subjects could proceed to the next task only after correctly completing the previous task. Third, the subjects had to go through five examples, which, to eliminate any bias, were generated at random. ... Finally, subjects were provided with a calculator, pen, and paper for the duration of the instructions and the experiment.", + "evidence_label": "verbatim", + "source_location": "Section 4.5, p. 1446", + "used_in": "materials/materials.json study_1_main_experiment.instructions_handout (marked missing for verbatim text) and .practice_tasks" + }, + { + "id": "E12", + "claim": "Verbatim instructions text (Online Appendix L) is unavailable", + "evidence_label": "missing", + "source_location": "Referenced at Section 4.5, p. 1446; content resides in Online Appendix L", + "reason": "Online Appendix L is only in the supplemental PDF, which returned HTTP 403 (Cloudflare bot challenge) on every retrieval attempt.", + "used_in": "materials/materials.json; audit/missing_information.json" + }, + { + "id": "E13", + "claim": "Sample size, recruitment, and platform for Study 2", + "quote": "For the new experiments, we recruited 183 participants on the campus of Purdue University who had not participated in the original set of experiments. The new experiment was programmed in oTree software (Chen et al. 2016a). Instructions and decision screens are presented in Online Appendix N.1.", + "evidence_label": "verbatim", + "source_location": "Section 5, p. 1449", + "used_in": "study.json:studies[1].sample" + }, + { + "id": "E14", + "claim": "Study 2 discrete-time redesign and rationale", + "quote": "we run a new set of experiments in which we consider a discrete-time setting ... to make the discrete-time setup feasible within the time constraints offered by the laboratory experiment, we reduced the number of expected interactions to 10 (i.e., probability of termination was 0.1 instead of 0.01). In addition, to ensure that one specific sequence of match durations did not bias the results, we now consider four sequences of supergame length (see Online Table N.1). Finally ... we did not fix or match the private values in the new experiment. Instead, private values were drawn at random at the beginning of each match for each participant.", + "evidence_label": "verbatim", + "source_location": "Section 5, p. 1448-1449", + "used_in": "task/task.json:studies[1].procedure" + }, + { + "id": "E15", + "claim": "Study 2 information-cost treatment (interactive counterfactual-payoff plot)", + "quote": "the information consisted of an interactive plot (see screenshots in Online Appendix N.2) that displayed possible payoffs for any bid conditional on the other two group members holding the bids as they did in the previous period. The treatment variation included the cost of accessing this plot. Specifically, in the costly information treatment, subjects incurred a cost of $0.5 every period in which they accessed this information, whereas in the free information treatment, subjects did not incur any cost.", + "evidence_label": "verbatim", + "source_location": "Section 5, p. 1449", + "used_in": "task/task.json:studies[1].information_feature; materials/materials.json" + }, + { + "id": "E16", + "claim": "Study 2 factorial design and discrete-time mechanics", + "quote": "we implemented a 2x2 factorial design in which we varied the cost of bid adjustment (as is the main treatment variation in the original set of experiment) and the cost of the information. For all four treatments, we fixed the same alpha = 0.5. The differences from the original experiments include (i) discrete-time setup, (ii) duration of the interaction, (iii) multiple duration realizations, and (iv) independent random draws for each subject in each match. Specifically, in the discrete-time setting, subjects faced an uncertain number of decision periods. Within each period, they had unlimited time to make a desired bid adjustment (if any). Once all participants in their group submitted their decisions, each saw the outcome for that period.", + "evidence_label": "verbatim", + "source_location": "Section 5, p. 1449", + "used_in": "task/task.json:studies[1].procedure" + }, + { + "id": "E17", + "claim": "Study 2 verbatim instructions/screens and duration-sequence table are unavailable", + "evidence_label": "missing", + "source_location": "Referenced at Section 5, p. 1448-1449; content in Online Appendix N.1 (instructions/screens), Online Appendix N.2 (info-plot screenshots), Online Table N.1 (four duration sequences)", + "reason": "Same Online Appendix retrieval failure as E12.", + "used_in": "materials/materials.json; audit/missing_information.json" + }, + { + "id": "E18", + "claim": "Core GSP model and payoff formula (theoretical basis shared by both studies)", + "quote": "the payoff for the kth highest bidding advertiser (allotted to the kth slot) is given by alpha_k(v(k) - b(k+1)). ... we model frictions as an additional cost Cj incurred by advertiser j from revising the advertiser's bids. That is, the payoff for advertiser j who places the kth highest bid is pi_j = alpha_k(vj - b(k+1)) - Cj", + "evidence_label": "verbatim", + "source_location": "Section 3.1, p. 1440-1441, equations (1)-(3)", + "used_in": "task/adapter.py payoff function" + }, + { + "id": "E19", + "claim": "Outcome metrics: allocative efficiency and bid-to-value ratio", + "quote": "The first metric of interest is the allocative efficiency of the auction. ... Psi = sum_{k=1}^{K} alpha_k v(k) / sum_{j=1}^{K} alpha_j v^j ... The second metric of interest is the bid-to-value ratio for each of the advertisers ... Omega_j = b_j / v_j", + "evidence_label": "verbatim", + "source_location": "Section 3.1, p. 1441, equations (1)-(2)", + "used_in": "evaluation/evaluation.py" + }, + { + "id": "E20", + "claim": "Study 1 headline results (Results 5-10) used as evaluation targets", + "quote": "Result 6. Lower costs lead to more exploration. Result 7. The bid-to-value ratio is higher for lower valued agents. Result 8. For the lowest valued agents, the bid-to-value ratio increases as friction decreases. Result 9. The bid-to-value ratio decreases as alpha increases for medium and highest valued agents. Result 10. Allocative efficiency of the market increases as friction costs increase.", + "evidence_label": "verbatim", + "source_location": "Section 4.6, pp. 1447-1448", + "used_in": "study.json:studies[0].outcomes; evaluation/evaluation.py" + }, + { + "id": "E21", + "claim": "Analysis window: matches 5-10, statistical test used", + "quote": "given our design, we focus on matches 5-10 ... p-values are determined using two-tailed permutation tests (Good 2013).", + "evidence_label": "verbatim", + "source_location": "Section 4.6.1 note, Table 5 notes, p. 1446-1447", + "used_in": "evaluation/evaluation.py analysis_window; not_ready permutation-test note" + }, + { + "id": "E22", + "claim": "Point-to-currency exchange rate for participant payment is not stated in the main text", + "evidence_label": "missing", + "source_location": "Not found in Sections 4.2, 4.5 of input/paper.pdf; possibly in Online Appendix L or M", + "reason": "Main text states participants were 'paid in cash' and earnings are the sum of match payoffs, but no explicit points-to-dollars conversion rate or show-up fee is given in the accessible text.", + "used_in": "audit/missing_information.json" + }, + { + "id": "E23", + "claim": "Rounding convention for good-2 (slot-2) value = alpha * value of good 1", + "evidence_label": "missing", + "source_location": "Section 4.2, p. 1443", + "reason": "The paper states good-2 value is obtained by multiplying good-1 value by alpha (e.g., a value of 7 with alpha=0.2 gives 1.4), but does not state whether displayed/used values are rounded or shown with decimals.", + "used_in": "audit/missing_information.json; task/adapter.py uses unrounded float and flags the assumption" + }, + { + "id": "E24", + "claim": "Session software-bug affecting matches 5-10 seed increment (disclosed by authors)", + "quote": "After running the first four sessions (two for C=0.0 and two for C=1.0), we discovered an error in the way the software generated random seeds ... The bug was the same across the treatments, so ... no systematic effect should exist between treatments.", + "evidence_label": "verbatim", + "source_location": "Footnote 19, p. 1452", + "used_in": "study.json known_limitations" + } + ] +} diff --git a/studies/study_017/source/paper_metadata.json b/studies/study_017/source/paper_metadata.json new file mode 100644 index 0000000..d23854c --- /dev/null +++ b/studies/study_017/source/paper_metadata.json @@ -0,0 +1,35 @@ +{ + "_purpose": "Bibliographic metadata for the source paper and a record of any externally authorized sources actually consulted. All fields except external_sources describe the uploaded PDF only.", + "title": "Analyzing Frictions in Generalized Second-Price Auction Markets", + "authors": [ + { "name": "Karthik Kannan", "affiliation": "Eller College of Management, University of Arizona" }, + { "name": "Vandith Pamuru", "affiliation": "Indian School of Business, Hyderabad" }, + { "name": "Yaroslav Rosokha", "affiliation": "Krannert School of Management, Purdue University" } + ], + "journal": "Information Systems Research", + "volume": "34", + "issue": "4", + "pages": "1437-1454", + "year": 2023, + "published_online": "2022-12-14", + "doi": "10.1287/isre.2022.1187", + "publisher": "INFORMS", + "issn": { "print": "1047-7047", "online": "1526-5536" }, + "source_file": "input/paper.pdf", + "source_file_original_name": "EBSCO-FullText-08_06_2026.pdf", + "evidence_label": "verbatim", + "extraction_tool": "pdftotext -layout", + "external_sources": [ + { + "url": "https://pubsonline.informs.org/doi/suppl/10.1287/isre.2022.1187/suppl_file/isre.2022.1187.sm1.pdf", + "authorized_by": "job.json osfUrl field, explicitly supplied by contributor Yuanjun", + "purpose": "Retrieve the paper's Online Appendix, which contains the verbatim participant instructions (Appendix L for the main experiment, Appendix N.1-N.2 for the second experiment) and additional robustness tables.", + "access_attempted": true, + "access_result": "blocked", + "access_detail": "Requests (via the WebFetch tool and via curl with standard browser user-agent headers, on 2026-08-10) received HTTP 403 responses. The response body was a Cloudflare 'Just a moment...' interactive bot-check challenge page, not the PDF. No content from this URL was retrieved or used.", + "links_followed_from_this_source": [], + "content_incorporated": false + } + ], + "external_research_policy_note": "Per job policy, only the single URL above was authorized. No web search, general web fetch, DOI lookup, or author-page lookup was performed. The supplemental PDF could not be retrieved (Cloudflare bot challenge), so this package is built entirely from input/paper.pdf. Any content that only exists in the inaccessible Online Appendix (e.g., the verbatim instructions handout, Online Tables/Figures referenced as 'Online Appendix X') is labeled 'missing' throughout this package rather than reconstructed." +} diff --git a/studies/study_017/study.json b/studies/study_017/study.json new file mode 100644 index 0000000..0fdcede --- /dev/null +++ b/studies/study_017/study.json @@ -0,0 +1,123 @@ +{ + "_purpose": "Researcher-facing overview of this package. See source/evidence.json for the full provenance record behind every claim below.", + "paper": { + "title": "Analyzing Frictions in Generalized Second-Price Auction Markets", + "authors": [ + "Karthik Kannan", + "Vandith Pamuru", + "Yaroslav Rosokha" + ], + "journal": "Information Systems Research", + "year": 2023, + "volume": "34", + "issue": "4", + "pages": "1437-1454", + "doi": "10.1287/isre.2022.1187", + "source_file": "input/paper.pdf (contributor-uploaded; original filename EBSCO-FullText-08_06_2026.pdf)" + }, + "external_sources_consulted": [ + { + "url": "https://pubsonline.informs.org/doi/suppl/10.1287/isre.2022.1187/suppl_file/isre.2022.1187.sm1.pdf", + "authorized": true, + "authorized_by": "job.json osfUrl field (explicit contributor-supplied URL)", + "outcome": "blocked -- returned HTTP 403 (Cloudflare bot-check challenge) on every attempt; no content retrieved or used", + "detail": "See source/paper_metadata.json:external_sources for full detail" + } + ], + "external_research_used": false, + "empirical_studies_in_paper": [ + { + "study_id": "computational_main", + "human_subject": false, + "included_in_package": false, + "reason_excluded": "Q-learning agent-based simulation (Section 3); not a human-subject study, out of scope for an agent-participant replication package." + }, + { + "study_id": "study_1_main_experiment", + "human_subject": true, + "included_in_package": true, + "paper_section": "Section 4", + "summary": "Laboratory GSP-auction experiment: 138 Purdue participants, 8 sessions of 15-18 each, randomly regrouped into triads every match, 10 matches per session. Between-subjects bid-adjustment friction cost C in {0.0, 0.1}; within-subjects slot-value-similarity alpha in {0.2, 0.5, 0.8}. Continuous-time bidding within each match, ending at a randomly drawn stopping time." + }, + { + "study_id": "study_2_mechanism_experiment", + "human_subject": true, + "included_in_package": true, + "paper_section": "Section 5", + "summary": "Follow-up robustness/mechanism experiment: 183 new Purdue participants (no overlap with Study 1), programmed in oTree, discrete-time (vs. Study 1's continuous time). 2x2 between-subjects factorial: C in {0.0, 0.1} x information cost in {free, $0.5/period}, alpha fixed at 0.5." + }, + { + "study_id": "computational_robustness_section6", + "human_subject": false, + "included_in_package": false, + "reason_excluded": "Two additional Q-learning calibration/robustness studies (Section 6); not human-subject studies." + } + ], + "participant_flow": { + "study_1_main_experiment": "Each agent participant plays num_matches (default 10) sequential matches. At the start of every match, all participants are randomly re-partitioned into triads of 3; each triad's 3 members privately draw a value for good 1 (derived to be Uniform{1,...,10}), submit an initial free bid, then may revise their bid across a random number of decision periods (min 20, then 1%/period hazard) before the match ends and payoffs are realized from each participant's final bid. See task/task.json for full visibility rules.", + "study_2_mechanism_experiment": "Same triad/match structure as Study 1, but discrete periods (hazard 0.1/period, no minimum-period floor stated), independently redrawn values every match, fixed alpha=0.5, and an added per-period choice of whether to view a counterfactual-payoff plot (costly or free depending on condition)." + }, + "conditions": { + "study_1_main_experiment": { + "between_subjects": "C in {0.0, 0.1}", + "within_subjects": "alpha in {0.2, 0.5, 0.8}" + }, + "study_2_mechanism_experiment": { + "between_subjects_factorial": "C in {0.0, 0.1} x information_cost in {0.0, 0.5}", + "fixed": "alpha = 0.5" + } + }, + "outcomes_measured": [ + { + "name": "allocative_efficiency (Psi)", + "definition": "realized welfare / maximum possible welfare across the 2 allocated goods (Section 3.1 eq. 1)" + }, + { + "name": "bid_to_value_ratio (Omega_j)", + "definition": "participant's final bid divided by its private value for good 1 (Section 3.1 eq. 2)" + }, + { + "name": "number_of_bid_adjustments", + "definition": "count of times a participant changed its locked bid during a match (excludes the free initial bid)" + } + ], + "headline_findings_this_package_can_check": { + "evidence_ref": "E20", + "results": [ + "Result 6: Lower costs (C=0.0) lead to more bid adjustments than higher costs (C=0.1).", + "Result 7: The bid-to-value ratio is higher for lower valued agents.", + "Result 8: For the lowest valued agents, the bid-to-value ratio increases as friction (C) decreases.", + "Result 9: The bid-to-value ratio decreases as alpha increases, for medium and highest valued agents.", + "Result 10: Allocative efficiency increases as friction costs (C) increase." + ], + "implemented_by": "evaluation/evaluation.py (each Result becomes 'not_ready' with a stated missing requirement if the supplied session runs don't cover the needed conditions)" + }, + "package_entry_point": { + "runnable_task": "task/adapter.py", + "smoke_test_command": "python3 task/adapter.py --smoke-test", + "real_run_usage": "import task/adapter.py as a library; supply ParticipantAgent subclasses backed by genuine per-participant decision logic; call run_study1_session(agents, Study1Config(...)) or run_study2_session(agents, Study2Config(...)); persist the returned session dict to JSON.", + "evaluation": "python3 evaluation/evaluation.py --session --session " + }, + "readiness_status": { + "overall": "partially_ready", + "ready": [ + "Core GSP payoff mechanics, allocative efficiency and bid-to-value-ratio metrics, and both studies' condition/termination structures are implemented and verified via --smoke-test.", + "evaluation/evaluation.py runs all five checkable Results (6-10) given appropriately paired session runs, using the paper's own permutation-test methodology." + ], + "blocked_or_incomplete": [ + "Verbatim participant instructions/handouts for both studies (Online Appendices L, N.1, N.2) are unavailable; agents are onboarded with the paper's own verbatim rule statements instead of the original handout wording.", + "Points-to-currency exchange rate is unspecified in the accessible text; payoffs are reported in point units.", + "Study 2's exact four duration sequences (Online Table N.1) are unavailable; a matching-hazard-rate substitute is used.", + "Cross-session common-random-number comparability (Study 1 Remark 2) requires explicit seed-sharing across paired runs, which is a researcher setup step, not automated by default." + ], + "see": "audit/missing_information.json for the complete, itemized checklist" + }, + "known_limitations_disclosed_by_original_authors": [ + "Footnote 19: a software bug in the original study's random-seed generation affected the intended value-sharing design for matches 5-10 in sessions 1-4; the authors state the bug was symmetric across treatments and does not bias treatment comparisons." + ], + "contributors": [ + { + "name": "Yuanjun" + } + ] +} diff --git a/studies/study_017/task/__pycache__/adapter.cpython-311.pyc b/studies/study_017/task/__pycache__/adapter.cpython-311.pyc new file mode 100644 index 0000000..cc74344 Binary files /dev/null and b/studies/study_017/task/__pycache__/adapter.cpython-311.pyc differ diff --git a/studies/study_017/task/adapter.py b/studies/study_017/task/adapter.py new file mode 100644 index 0000000..87bbf97 --- /dev/null +++ b/studies/study_017/task/adapter.py @@ -0,0 +1,434 @@ +""" +Runnable-task adapter for Kannan, Pamuru & Rosokha (2023), "Analyzing Frictions +in Generalized Second-Price Auction Markets", Information Systems Research. + +Implements the two human-subject studies (Section 4 "main experiment" and +Section 5 "mechanism experiment") as a multiplayer bidding simulation in which +every original human subject is one agent participant. No network access is +used or required. See task/task.json for the full design contract and +audit/missing_information.json for disclosed departures/assumptions. + +This module is a library first: a harness supplies real agent decision logic +by implementing ParticipantAgent and passing instances into run_study1_session +/ run_study2_session. `python adapter.py --smoke-test` runs a tiny self-test +with a trivial scripted stand-in agent (SmokeTestAgent) purely to verify the +mechanics execute end-to-end -- that stand-in is NOT a valid substitute for a +real participant in a research run. +""" + +import argparse +import json +import random +import sys +from abc import ABC, abstractmethod +from dataclasses import dataclass, field, asdict +from typing import Optional + + +# --------------------------------------------------------------------------- +# Agent interface -- a real run must supply subclasses backed by an LLM or +# other genuine decision-making process. One instance per participant. +# --------------------------------------------------------------------------- + +class ParticipantAgent(ABC): + """One instance represents one human-subject stand-in. Never share state + between agents that were not allowed to communicate in the paper.""" + + @abstractmethod + def initial_bid(self, observation: dict) -> float: + """observation contains: value_good1, value_good2, alpha, cost_c, + match_index, study. Must return a bid in [0, 10].""" + raise NotImplementedError + + @abstractmethod + def decide_bid_update(self, observation: dict) -> Optional[float]: + """Called every period after the initial bid. observation additionally + contains: period, current_own_bid, provisional_rank (1/2/3), + price_if_winning (revealed only if provisional_rank is 1 or 2, else + None), periods_elapsed. Return None to keep the current bid (no cost), + or a new bid in [0, 10] (incurs cost_c).""" + raise NotImplementedError + + def decide_info_access(self, observation: dict) -> bool: + """Study 2 only. observation additionally contains info_cost (0 or + 0.5) and, if available, the counterfactual payoff plot inputs. + Default: never access. Override for real Study-2 participants.""" + return False + + +class SmokeTestAgent(ParticipantAgent): + """Deterministic, minimal stand-in used ONLY by --smoke-test to check that + the simulation mechanics run without error. Bids true value once, never + revises. This is a code self-test fixture, not a research participant.""" + + def initial_bid(self, observation: dict) -> float: + return float(observation["value_good1"]) + + def decide_bid_update(self, observation: dict) -> Optional[float]: + return None + + def decide_info_access(self, observation: dict) -> bool: + return False + + +# --------------------------------------------------------------------------- +# Shared GSP payoff mechanics (Section 3.1 / 4.3, evidence E07/E18) +# --------------------------------------------------------------------------- + +def compute_match_payoffs(values_good1, alpha, final_bids, adjustment_counts, cost_c, + info_costs=None): + """values_good1, final_bids, adjustment_counts: dict participant_id -> value. + Returns dict participant_id -> record with rank, good_won, price_paid, payoff, + allocative efficiency (shared across the match) and bid-to-value ratio. + Ties are broken uniformly at random, matching Section 3.2.1 ('ties broken + randomly').""" + ids = list(values_good1.keys()) + order = sorted(ids, key=lambda i: (-final_bids[i], random.random())) + rank_of = {pid: r + 1 for r, pid in enumerate(order)} + b = {rank_of[pid]: final_bids[pid] for pid in ids} + v = {rank_of[pid]: values_good1[pid] for pid in ids} + + price_paid = {1: b.get(2, 0.0), 2: alpha * b.get(3, 0.0), 3: 0.0} + value_realized = {1: v[1], 2: alpha * v[2], 3: 0.0} + + total_value = sum(value_realized.values()) + max_possible = max(v[1], alpha * v[1]) # best single allocation of good1's top value + # Allocative efficiency per Section 3.1 eq. (1): realized welfare / max possible welfare + # Max possible welfare allocates the two goods to the two highest-value agents. + sorted_vals = sorted(v.values(), reverse=True) + max_possible = sorted_vals[0] * 1.0 + sorted_vals[1] * alpha + efficiency = (total_value / max_possible) if max_possible > 0 else None + + out = {} + for pid in ids: + r = rank_of[pid] + friction_cost = cost_c * adjustment_counts.get(pid, 0) + info_cost = (info_costs or {}).get(pid, 0.0) + payoff = value_realized[r] - price_paid[r] - friction_cost - info_cost + bid_to_value_ratio = (final_bids[pid] / values_good1[pid]) if values_good1[pid] else None + out[pid] = { + "rank": r, + "good_won": 1 if r == 1 else (2 if r == 2 else None), + "final_bid": final_bids[pid], + "value_good1": values_good1[pid], + "price_paid": price_paid[r], + "friction_cost": friction_cost, + "info_cost": info_cost, + "payoff": payoff, + "bid_to_value_ratio": bid_to_value_ratio, + "num_adjustments": adjustment_counts.get(pid, 0), + } + return out, efficiency + + +def draw_termination_period(rng: random.Random, min_periods: int, hazard: float, + max_periods: int) -> int: + """Section 4.2 Remark 1 / Section 5: minimum periods, then i.i.d. per-period + hazard of ending. max_periods is a disclosed practical safety cap (see + task/task.json known_departures_from_original_design).""" + t = min_periods + while t < max_periods: + if rng.random() < hazard: + break + t += 1 + return t + + +def make_triads(participant_ids, rng: random.Random): + """Random partition into groups of three (Section 4.2: 'randomly split into + groups of three ... remained so until the end of the match').""" + if len(participant_ids) % 3 != 0: + raise ValueError("num_agents must be a multiple of 3 (see task/task.json)") + shuffled = list(participant_ids) + rng.shuffle(shuffled) + return [shuffled[i:i + 3] for i in range(0, len(shuffled), 3)] + + +# --------------------------------------------------------------------------- +# Study 1: main experiment (Section 4) +# --------------------------------------------------------------------------- + +@dataclass +class Study1Config: + num_agents: int = 3 + num_matches: int = 10 + condition_c: float = 0.0 # between-subjects: 0.0 or 0.1 + alpha_values: tuple = (0.2, 0.5, 0.8) # within-subjects + min_periods: int = 20 + hazard: float = 0.01 + max_periods: int = 300 # practical cap, see task.json + value_low: int = 1 + value_high: int = 10 + seed: int = 0 + + +def run_study1_match(triad_ids, agents, alpha, cost_c, stop_period, rng, study="study_1_main_experiment"): + values = {pid: rng.randint(1, 10) for pid in triad_ids} + bids = {} + adjustments = {pid: 0 for pid in triad_ids} + period_log = [] + + obs_base = {"study": study, "alpha": alpha, "cost_c": cost_c} + for pid in triad_ids: + obs = dict(obs_base, value_good1=values[pid], value_good2=alpha * values[pid]) + b = agents[pid].initial_bid(obs) + bids[pid] = max(0.0, min(10.0, float(b))) + + for period in range(1, stop_period + 1): + order = sorted(triad_ids, key=lambda i: -bids[i]) + rank_of = {p: r + 1 for r, p in enumerate(order)} + for pid in triad_ids: + r = rank_of[pid] + price_if_winning = None + if r == 1: + price_if_winning = sorted([bids[o] for o in triad_ids if o != pid])[-1] + elif r == 2: + price_if_winning = alpha * min(bids[o] for o in triad_ids if o != pid) + obs = dict(obs_base, value_good1=values[pid], value_good2=alpha * values[pid], + period=period, current_own_bid=bids[pid], provisional_rank=r, + price_if_winning=price_if_winning, periods_elapsed=period) + new_bid = agents[pid].decide_bid_update(obs) + if new_bid is not None: + new_bid = max(0.0, min(10.0, float(new_bid))) + if new_bid != bids[pid]: + adjustments[pid] += 1 + bids[pid] = new_bid + period_log.append({"period": period, "bids": dict(bids)}) + + payoffs, efficiency = compute_match_payoffs(values, alpha, bids, adjustments, cost_c) + return { + "triad": triad_ids, "alpha": alpha, "cost_c": cost_c, + "stop_period": stop_period, "values": values, "final_bids": dict(bids), + "adjustments": adjustments, "payoffs": payoffs, "allocative_efficiency": efficiency, + "period_log": period_log, + } + + +def run_study1_session(agents: dict, config: Study1Config = Study1Config()): + """agents: dict participant_id -> ParticipantAgent, len multiple of 3.""" + rng = random.Random(config.seed) + participant_ids = list(agents.keys()) + if len(participant_ids) != config.num_agents: + raise ValueError("agents dict size must equal config.num_agents") + + matches = [] + for m in range(1, config.num_matches + 1): + triads = make_triads(participant_ids, rng) + alpha = rng.choice(list(config.alpha_values)) + stop_period = draw_termination_period(rng, config.min_periods, config.hazard, config.max_periods) + for triad in triads: + record = run_study1_match(triad, agents, alpha, config.condition_c, stop_period, rng) + record["match_index"] = m + matches.append(record) + + return { + "study_id": "study_1_main_experiment", + "condition_c": config.condition_c, + "num_agents": config.num_agents, + "num_matches": config.num_matches, + "seed": config.seed, + "matches": matches, + "assumptions": [ + "value_good1 drawn Uniform{1,...,10} (evidence_label: derived, see task/task.json)", + "bid domain clamped to [0,10] (evidence_label: derived)", + "no rounding applied to value_good2 = alpha * value_good1 (evidence_label: missing, see E23)", + f"max_periods safety cap = {config.max_periods} applied to the theoretically unbounded random-termination process", + ], + } + + +# --------------------------------------------------------------------------- +# Study 2: mechanism experiment (Section 5) +# --------------------------------------------------------------------------- + +@dataclass +class Study2Config: + num_agents: int = 3 + num_matches: int = 10 + condition_c: float = 0.0 # between-subjects: 0.0 or 0.1 + info_cost: float = 0.0 # between-subjects: 0.0 (free) or 0.5 (costly) + alpha: float = 0.5 # fixed in Study 2 + hazard: float = 0.1 + max_periods: int = 60 # practical cap, see task.json + seed: int = 0 + + +def run_study2_match(triad_ids, agents, alpha, cost_c, info_cost, stop_period, rng): + values = {pid: rng.randint(1, 10) for pid in triad_ids} + bids = {} + adjustments = {pid: 0 for pid in triad_ids} + info_access_counts = {pid: 0 for pid in triad_ids} + period_log = [] + + obs_base = {"study": "study_2_mechanism_experiment", "alpha": alpha, "cost_c": cost_c, + "info_cost": info_cost} + for pid in triad_ids: + obs = dict(obs_base, value_good1=values[pid], value_good2=alpha * values[pid]) + b = agents[pid].initial_bid(obs) + bids[pid] = max(0.0, min(10.0, float(b))) + + prev_bids = dict(bids) + for period in range(1, stop_period + 1): + # Simultaneous decisions within the period; each agent may consult the + # counterfactual-payoff plot (built from the OTHER two agents' previous + # period bids, per evidence E15) before deciding whether to adjust. + order = sorted(triad_ids, key=lambda i: -prev_bids[i]) + rank_of = {p: r + 1 for r, p in enumerate(order)} + new_bids = dict(bids) + for pid in triad_ids: + others_prev = [prev_bids[o] for o in triad_ids if o != pid] + r = rank_of[pid] + price_if_winning = None + if r == 1: + price_if_winning = sorted(others_prev)[-1] + elif r == 2: + price_if_winning = alpha * min(others_prev) + info_obs = dict(obs_base, value_good1=values[pid], value_good2=alpha * values[pid], + period=period, current_own_bid=bids[pid], + counterfactual_grid=[ + {"candidate_bid": cb, + "payoff_if_others_hold_previous_bids": + _counterfactual_payoff(cb, others_prev, values[pid], alpha)} + for cb in [x / 2 for x in range(0, 21)] + ]) + accessed = agents[pid].decide_info_access(info_obs) + if accessed: + info_access_counts[pid] += 1 + + obs = dict(obs_base, value_good1=values[pid], value_good2=alpha * values[pid], + period=period, current_own_bid=bids[pid], provisional_rank=r, + price_if_winning=price_if_winning, periods_elapsed=period, + info_plot=info_obs["counterfactual_grid"] if accessed else None) + new_bid = agents[pid].decide_bid_update(obs) + if new_bid is not None: + new_bid = max(0.0, min(10.0, float(new_bid))) + if new_bid != bids[pid]: + adjustments[pid] += 1 + new_bids[pid] = new_bid + prev_bids = dict(bids) + bids = new_bids + period_log.append({"period": period, "bids": dict(bids), + "info_accessed": {pid: info_access_counts[pid] for pid in triad_ids}}) + + info_costs = {pid: info_cost * info_access_counts[pid] for pid in triad_ids} + payoffs, efficiency = compute_match_payoffs(values, alpha, bids, adjustments, cost_c, + info_costs=info_costs) + return { + "triad": triad_ids, "alpha": alpha, "cost_c": cost_c, "info_cost": info_cost, + "stop_period": stop_period, "values": values, "final_bids": dict(bids), + "adjustments": adjustments, "info_access_counts": info_access_counts, + "payoffs": payoffs, "allocative_efficiency": efficiency, "period_log": period_log, + } + + +def _counterfactual_payoff(candidate_bid, others_prev_bids, own_value, alpha): + all_bids = sorted(others_prev_bids + [candidate_bid], reverse=True) + r = all_bids.index(candidate_bid) + 1 + if r == 1: + price = sorted(others_prev_bids)[-1] + return own_value - price + elif r == 2: + price = alpha * min(others_prev_bids) + return alpha * own_value - price + return 0.0 + + +def run_study2_session(agents: dict, config: Study2Config = Study2Config()): + rng = random.Random(config.seed) + participant_ids = list(agents.keys()) + if len(participant_ids) != config.num_agents: + raise ValueError("agents dict size must equal config.num_agents") + + matches = [] + for m in range(1, config.num_matches + 1): + triads = make_triads(participant_ids, rng) + stop_period = draw_termination_period(rng, min_periods=1, hazard=config.hazard, + max_periods=config.max_periods) + for triad in triads: + record = run_study2_match(triad, agents, config.alpha, config.condition_c, + config.info_cost, stop_period, rng) + record["match_index"] = m + matches.append(record) + + return { + "study_id": "study_2_mechanism_experiment", + "condition_c": config.condition_c, + "info_cost": config.info_cost, + "num_agents": config.num_agents, + "num_matches": config.num_matches, + "seed": config.seed, + "matches": matches, + "assumptions": [ + "value_good1 drawn Uniform{1,...,10} independently every match (evidence_label: derived)", + "termination hazard p=0.1 with no minimum-period floor substitutes for the four unavailable " + "Online Table N.1 duration sequences (evidence_label: missing, see audit/missing_information.json)", + f"max_periods safety cap = {config.max_periods} applied to the theoretically unbounded process", + ], + } + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _run_smoke_test(): + ok = True + messages = [] + + agents1 = {i: SmokeTestAgent() for i in range(3)} + r1 = run_study1_session(agents1, Study1Config(num_agents=3, num_matches=2, + min_periods=2, max_periods=5, seed=1)) + if len(r1["matches"]) != 2: + ok = False + messages.append("study1: expected 2 matches") + for match in r1["matches"]: + payoffs = match["payoffs"] + if len(payoffs) != 3: + ok = False + messages.append("study1: expected 3 payoff records per match") + if match["allocative_efficiency"] is None or not (0 <= match["allocative_efficiency"] <= 1.0001): + ok = False + messages.append("study1: allocative efficiency out of expected [0,1] range") + + agents2 = {i: SmokeTestAgent() for i in range(3)} + r2 = run_study2_session(agents2, Study2Config(num_agents=3, num_matches=2, + max_periods=3, seed=2)) + if len(r2["matches"]) != 2: + ok = False + messages.append("study2: expected 2 matches") + + result = { + "smoke_test": "PASSED" if ok else "FAILED", + "messages": messages, + "study1_sample_match": r1["matches"][0], + "study2_sample_match": r2["matches"][0], + } + print(json.dumps(result, indent=2, default=str)) + return 0 if ok else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--smoke-test", action="store_true", + help="Run a minimal end-to-end self-test with a scripted " + "stand-in agent (not a valid research participant) " + "and print the result as JSON.") + args = parser.parse_args() + + if args.smoke_test: + sys.exit(_run_smoke_test()) + + print(json.dumps({ + "status": "blocked", + "reason": "No agent implementation supplied on the CLI. This adapter is a " + "library: import task.adapter and call run_study1_session / " + "run_study2_session with real ParticipantAgent subclasses backed " + "by genuine per-participant decision logic (see task/task.json " + "for the required observation/action contract). Use --smoke-test " + "to verify the simulation mechanics only.", + }, indent=2)) + sys.exit(2) + + +if __name__ == "__main__": + main() diff --git a/studies/study_017/task/task.json b/studies/study_017/task/task.json new file mode 100644 index 0000000..8f8635c --- /dev/null +++ b/studies/study_017/task/task.json @@ -0,0 +1,145 @@ +{ + "_purpose": "Runnable-task contract: participant structure, condition assignment, visibility, and I/O for the two human-subject studies in Kannan, Pamuru & Rosokha (2023). Every original human subject is represented by one agent participant; there are no confederate/scripted roles because the paper used none.", + "entry_point": "task/adapter.py", + "network_required": false, + "scripted_roles_in_paper": "none", + "scripted_roles_justification": "Sections 4 and 5 describe only unscripted human subjects bidding in their own group; no experimenter or confederate stood in for a subject's decisions in either study. Therefore every agent in this task is a genuine decision-making participant, never a fixed/optimal/sampled stand-in.", + + "studies": [ + { + "study_id": "study_1_main_experiment", + "paper_section": "Section 4 (pp. 1443-1448)", + "adapter_entry": "run_study1_session", + "num_agents": { + "default": 3, + "must_be_multiple_of": 3, + "minimum": 3, + "paper_value": "15-18 human participants per session (138 total across 8 sessions)", + "note": "The paper's mechanism operates on triads of 3; a session was simply many triads run in parallel with random rematching each match. Default num_agents=3 replays a single triad exactly (the minimal faithful unit). Set num_agents to any multiple of 3 (e.g., 15 or 18) to additionally replay the paper's random-rematching-across-a-larger-pool design; the adapter implements the rematching in both cases." + }, + "roles": [ + { + "role_id": "participant", + "count": "num_agents", + "scripted": false, + "description": "Every agent is an independent bidder with private information about only its own valuation. Roles are not fixed identities: at the start of every match each participant is (re)assigned to a random triad, and within a triad its rank (highest/medium/lowest private value) is determined by the random value draw for that match, exactly as in the paper (footnote 13)." + } + ], + "order_of_play": { + "between_matches": "sequential: matches 1..num_matches (default 10) run in order, each participant's cumulative earnings and any within-run learning carries across matches.", + "within_a_match": "simultaneous, iterated: at t=0 all three agents in a triad submit an initial bid at no cost, independently and without seeing each other's value or bid. For t=1..T (T = a randomly drawn stopping period, see match_termination), all three agents simultaneously get the option to revise their bid (paying cost C per revision) before the next live-outcome update; the match ends at the drawn stopping period T and payoffs are computed from each agent's last bid." + }, + "visibility": { + "own_private_value": "always visible to the owning agent only", + "own_current_bid_and_provisional_outcome": "visible live to the owning agent every period (which good it would currently win, if any, and provisional payoff)", + "opponents_raw_bids_during_match": "hidden, EXCEPT: an agent that is currently the highest bidder is told the current second-highest bid (the price it would pay), and an agent that is currently second-highest is told the current third-highest/lowest bid (the price it would pay) -- this mirrors the paper's statement that a winning advertiser 'knows the advertiser's bid, but also knows the next highest bid' (Section 1) and Figure 2's blanked-out opponent scales (#4) until match end.", + "opponents_raw_bids_after_match_ends": "revealed to all three group members together with each participant's value, bid, and payoff for that match, matching Figure 2 note (4): 'these scales remain blank until the match is over, at which point the actions of the other participants are revealed.'", + "cross_triad_visibility": "none -- a participant never observes any information about a triad it is not currently part of, matching the physical-barrier isolation described in Section 4.5 (evidence E02)." + }, + "conditions": { + "between_subjects": { + "name": "C", + "values": [0.0, 0.1], + "assignment": "fixed for all agents and all matches within one run, chosen once at run start (adapter argument condition_c)", + "source": "materials/materials.json:study_1_main_experiment.treatments_table_verbatim" + }, + "within_subjects": { + "name": "alpha", + "values": [0.2, 0.5, 0.8], + "assignment": "drawn per match, shared by all three members of a triad for that match's duration (Section 4.1: 'All participants in a group had the same alpha ... for the duration of each match'). The paper draws matches 1-4 freely and uses common seeds across sessions for matches 5-10 (Remark 2) to allow between-session comparison; a single package run has no other session to compare against, so the adapter draws alpha i.i.d. uniformly from {0.2,0.5,0.8} every match by default and records the draw. See audit/missing_information.json for the cross-session common-seed limitation." + } + }, + "procedure": { + "value_generation": { + "distribution": "v_j ~ discrete Uniform{1,...,10} per participant per match, independent across participants", + "evidence_label": "derived", + "derivation": "Section 4 opens with 'This section describes the experimental design for the auction presented in Section 3.2.1', and Section 3.2.1/footnote 11 states private values vj are drawn from U{1,10}. Section 4 itself does not restate the numeric range, so this is inferred from the explicit cross-reference rather than quoted directly." + }, + "second_good_value": "value_of_good_2_for_participant = alpha * value_of_good_1_for_participant (Section 4.2, evidence E06). Rounding convention is not stated in the accessible text (evidence_label missing, E23); adapter uses the unrounded float and flags this assumption in its output.", + "bid_domain": "bids are real numbers in [0, 10], consistent with the shared auction environment (Section 3.2.1 uses b in U{0,10} for the computational model); Section 4 does not restate an explicit bid ceiling for the human experiment, so this bound is carried over the same way as value_generation above (evidence_label derived).", + "adjustment_cost": "each time a participant's locked bid differs from its immediately preceding locked bid, cost C is charged for that period; the very first (initial) bid is always free (Section 4.1, evidence E03/E07).", + "match_termination": { + "rule": "minimum 20 decision periods; from period 21 onward each additional period has an independent 1% chance of ending the match (Remark 1, evidence E05). Expected match length ~120 periods.", + "reproducibility": "the adapter pre-generates one stopping period per match from this process using the run's seed, and reuses that fixed per-match schedule for every triad in the run, mirroring 'we used the same sequence of seconds across matches in every session.'", + "practical_cap": { + "note": "unbounded random termination is, in principle, without an upper limit. To keep a run computationally tractable for agent participants (each period is an actual agent decision call), the adapter enforces a configurable max_periods safety cap (default 300, i.e. ~2.5x the theoretical expectation). This cap is a runtime safeguard, not a silent redesign of the protocol; see audit/missing_information.json for the disclosed effect." + } + }, + "payoff_rule": "materials/materials.json:study_1_main_experiment.payoff_rule_verbatim; implemented in task/adapter.py:compute_match_payoffs", + "payment_to_participant": "Remark 3 (evidence E08): the researcher-facing payout for a whole session would be based on one randomly chosen match; because this package runs matches independently of any larger paid session, task.json records this rule but evaluation/evaluation.py reports per-match payoffs rather than adjudicating a single paid match, and audit/missing_information.json notes the missing points-to-currency exchange rate (evidence E22)." + }, + "instructions_and_onboarding": { + "operative_rules_given_to_agents": "materials/materials.json:study_1_main_experiment.payoff_rule_verbatim and .design_remarks_verbatim (both verbatim from the paper)", + "verbatim_handout_text": "missing -- see materials/materials.json:study_1_main_experiment.instructions_handout and audit/missing_information.json", + "practice_tasks": "not implemented; paper's six practice tasks and five random examples are structurally described but their exact content is in the inaccessible Online Appendix L (materials/materials.json:study_1_main_experiment.practice_tasks)" + } + }, + + { + "study_id": "study_2_mechanism_experiment", + "paper_section": "Section 5 (pp. 1448-1450)", + "adapter_entry": "run_study2_session", + "num_agents": { + "default": 3, + "must_be_multiple_of": 3, + "minimum": 3, + "paper_value": "183 new Purdue participants, no overlap with Study 1", + "note": "Same rematching logic as Study 1 applies; default replays a single triad." + }, + "roles": [ + { + "role_id": "participant", + "count": "num_agents", + "scripted": false, + "description": "Same as Study 1: unscripted bidders, rank determined by that match's random value draw. Additionally, each participant independently decides every period whether to pay for (or freely view, depending on condition) the counterfactual-payoff information plot." + } + ], + "order_of_play": { + "between_matches": "sequential, matches run in order", + "within_a_match": "discrete-time, simultaneous per period: 'Within each period, they had unlimited time to make a desired bid adjustment (if any). Once all participants in their group submitted their decisions, each saw the outcome for that period.' (evidence E16). Unlike Study 1, there is no continuous real-time feedback between periods -- the outcome is revealed only after every group member has submitted that period's decision." + }, + "visibility": { + "own_private_value": "always visible to the owning agent only, redrawn independently every match (evidence E14: 'independent random draws for each subject in each match', unlike Study 1's shared-seed matches 5-10)", + "own_current_bid_and_outcome": "revealed to the owning agent at the end of each period, once all three group members have submitted", + "opponents_raw_bids_during_match": "same asymmetric rule as Study 1 (current price-relevant bid revealed only to the winner(s) of that period)", + "counterfactual_info_plot": "an optional, agent-requested view showing possible payoffs for candidate own bids, holding the other two participants' PREVIOUS period bids fixed (evidence E15). Available every period; in the InfoCosts condition, requesting it costs $0.5 for that period, subtracted from the match payoff; in the free-information condition it is available at no cost.", + "opponents_raw_bids_after_match_ends": "revealed to the group, as in Study 1", + "cross_triad_visibility": "none" + }, + "conditions": { + "factorial_design": "2x2 between-subjects: C in {0.0, 0.1} (same bid-adjustment-cost dimension as Study 1) crossed with information_cost in {free, costly=$0.5/period} (evidence E16)", + "alpha": "fixed at 0.5 for all matches and all four treatments (evidence E16); there is no within-subjects alpha manipulation in Study 2." + }, + "procedure": { + "value_generation": "v_j ~ discrete Uniform{1,...,10} per participant, drawn independently every match for every participant (no cross-group seed sharing), evidence_label derived using the same cross-reference logic as Study 1 (Section 5 modifies the termination/discreteness but does not state a different valuation distribution).", + "match_termination": { + "rule": "discrete periods with an independent per-period termination probability of 0.1 after any period (Section 5, evidence E14: 'probability of termination was 0.1 instead of 0.01'), drawn from one of four pre-set supergame-length sequences (Online Table N.1).", + "missing_component": "the exact four duration sequences (Online Table N.1) are in the inaccessible Online Appendix; evidence_label missing. The adapter substitutes a geometric-hazard generator with p=0.1 and no minimum-period floor (none is stated for Study 2) as a disclosed, labeled stand-in -- see audit/missing_information.json.", + "practical_cap": "same rationale and mechanism as Study 1, default max_periods=60 (~6x the E[10]-period expectation)." + }, + "info_access_cost": "$0.5 charged for each period in which a participant requests the counterfactual plot, only in the InfoCosts condition (evidence E15)", + "payoff_rule": "identical GSP payoff formula as Study 1 (materials/materials.json:shared_theoretical_material.gsp_payoff_formula_verbatim), minus any info-access costs incurred that match" + }, + "instructions_and_onboarding": { + "verbatim_handout_text": "missing -- Online Appendix N.1 (instructions/decision screens) and N.2 (info-plot screenshots) are inaccessible; see audit/missing_information.json", + "operative_rules_given_to_agents": "materials/materials.json:study_2_mechanism_experiment (verbatim paper text on the design, information feature, and termination redesign)" + } + } + ], + + "outputs": { + "per_period_log": "for every agent, every period: private value(s), current bid, whether/what was revealed to it, whether it changed its bid, whether it accessed the info plot (Study 2 only)", + "per_match_record": "final ranked bids, values, alpha, C, allocative efficiency (Psi), each participant's bid-to-value ratio (Omega), each participant's payoff, number of bid adjustments per participant", + "per_session_record": "list of per_match_record across all matches, plus the condition assignment (C, and for Study 2, information_cost) for the run", + "consumed_by": "evaluation/evaluation.py" + }, + + "known_departures_from_original_design": [ + "Population size compressed from a multi-triad session pool (15-18 or 183 total participants) to a configurable small pool of agents (default one triad of 3); random rematching logic is implemented and can be scaled up by increasing num_agents.", + "Continuous real-time bidding in Study 1 is discretized into fixed decision periods (matching the same hazard-rate mathematics), because agent participants act in discrete API calls rather than continuous mouse movement.", + "An upper bound (max_periods) is imposed on the theoretically unbounded random-termination process, for computational tractability.", + "Cross-session common-random-number comparison (Study 1 Remark 2, matches 5-10) is not meaningful within a single run and is not implemented; alpha is instead drawn i.i.d. per match.", + "Study 2's four exact supergame-length sequences (Online Table N.1) are unavailable; a matching-hazard-rate generator is substituted.", + "The paper's 'silent protocol' (Remark 4) addressed a physical-room information leak specific to co-located human subjects; it has no analogue for non-colocated text-based agents and is therefore not implemented (the information-visibility guarantees it protected are preserved directly by the visibility rules above)." + ] +}