diff --git a/bitcoin/excluded_clients.txt b/bitcoin/excluded_clients.txt new file mode 100644 index 0000000..c9e63f1 --- /dev/null +++ b/bitcoin/excluded_clients.txt @@ -0,0 +1,23 @@ +GhostCore +BitcoinUnlimited +Therealbitcoin.org +CKCoinD +Cooltexture +GrayersDad +Bitcoin +Btcwire +FlamiSatoshi +Aurum +Jwrd.net +Classic +Satoshi-Noderunners +Satushi +Prometheus +Bitcoin SV +Bitcoin ABC +Node +DefcoinCore +B2CCoinCore +USFrancCore +CypherfunkCore +DigitaleuroCore \ No newline at end of file diff --git a/bitcoin/network_decentralization/helper.py b/bitcoin/network_decentralization/helper.py index e2dc9e8..76694ab 100644 --- a/bitcoin/network_decentralization/helper.py +++ b/bitcoin/network_decentralization/helper.py @@ -144,6 +144,23 @@ def get_without_tor_ledgers(): return list(dict.fromkeys(ledgers)) or None +def get_excluded_clients(): + """ + Returns the set of client families to exclude from analyses. + """ + path = ROOT_DIR / "excluded_clients.txt" + + if not path.is_file(): + return set() + + with open(path) as f: + return { + line.strip() + for line in f + if line.strip() and not line.startswith("#") + } + + def get_output_directory(ledger=None, dead=False): """ Reads the config file and retrieves the output directory diff --git a/bitcoin/parse.py b/bitcoin/parse.py index 6a7d90d..ad384f4 100644 --- a/bitcoin/parse.py +++ b/bitcoin/parse.py @@ -341,6 +341,29 @@ def record_versions(reachable_nodes, mode): versions_df.to_csv(f'./output/{name.lower()}_{ledger}.csv', index_label = name) +def filter_nodes_by_client(reachable_nodes): + """ + Removes nodes running excluded clients. + """ + excluded_clients = hlp.get_excluded_clients() + + if not excluded_clients: + return reachable_nodes + + filtered_nodes = [ + node + for node in reachable_nodes + if normalise_client_name(node[2]) not in excluded_clients + ] + + logging.info( + f"Removed {len(reachable_nodes) - len(filtered_nodes)} nodes " + f"running excluded clients." + ) + + return filtered_nodes + + def redistribute_tor_nodes(mode_lower, ledger, df, mode): """ Redistributes Tor node count proportionally across non-Tor rows. @@ -440,6 +463,7 @@ def main(): for ledger in LEDGERS: logging.info(f'parse.py: Getting {ledger} reachable nodes') reachable_nodes[ledger] = hlp.get_reachable_nodes(ledger) + reachable_nodes[ledger] = filter_nodes_by_client(reachable_nodes[ledger]) for mode in MODES: geography(reachable_nodes, ledger, mode) if 'Organizations' in MODES: diff --git a/ethereum/excluded_clients.txt b/ethereum/excluded_clients.txt new file mode 100644 index 0000000..5fea5b7 --- /dev/null +++ b/ethereum/excluded_clients.txt @@ -0,0 +1,24 @@ +Rust-libp2p +Eth-diversity@ +Hermes +Logex +BeraGeth +Bera-reth +Atlas +Qk_node +Naoris +Celo +XDC +Bera1 +World-chain +Ronin +Nimbus-eth1 +Reth_gnosis +Getc +Eth +Gwemix +NimbusExecutionClient +Lighthouse-Pulse +Prysm-Pulse +Bor +Tysm \ No newline at end of file diff --git a/ethereum/helper.py b/ethereum/helper.py index 2d5cdad..0c1d550 100644 --- a/ethereum/helper.py +++ b/ethereum/helper.py @@ -37,6 +37,25 @@ def get_mode(): return get_config_data()['mode'] +def get_excluded_clients(): + """ + Returns the set of client families to exclude from analyses. + """ + path = pathlib.Path(__file__).resolve().parent / "excluded_clients.txt" + + if not path.is_file(): + return set() + + with open(path) as f: + excluded_clients = { + line.strip() + for line in f + if line.strip() and not line.startswith("#") + } + + return excluded_clients + + def get_output_directory(): """ Require the `OUTPUT_DIRECTORY` env var set by the caller diff --git a/ethereum/parse.py b/ethereum/parse.py index b172755..d3afec8 100644 --- a/ethereum/parse.py +++ b/ethereum/parse.py @@ -31,6 +31,77 @@ def normalise_client_name(client_value): return client or 'Unknown' +def filter_nodes_by_client(nodes): + """ + Removes nodes running excluded clients. + """ + excluded_clients = hlp.get_excluded_clients() + + if not excluded_clients: + return nodes + + output_dir = hlp.get_output_directory() + peerfile = output_dir / 'peerstore.csv' + agentsfile = output_dir / 'agents.csv' + + if not peerfile.is_file() or not agentsfile.is_file(): + logging.warning('parse.py: peerstore.csv or agents.csv not found; skipping client exclusion') + return nodes + + peer_df = pd.read_csv( + peerfile, + engine='python', + on_bad_lines='skip', + dtype=str, + keep_default_na=False, + quotechar="'", + ) + + agents_df = pd.read_csv( + agentsfile, + engine='python', + on_bad_lines='skip', + dtype=str, + keep_default_na=False, + ) + + required_peer_columns = {'node_id', 'ip:port'} + required_agent_columns = {'node_id', 'agent_version'} + if not required_peer_columns.issubset(peer_df.columns) or not required_agent_columns.issubset(agents_df.columns): + logging.warning('parse.py: peerstore.csv or agents.csv is missing required columns; skipping client exclusion') + return nodes + + peer_df = peer_df[['node_id', 'ip:port']].copy() + peer_df['node_id'] = peer_df['node_id'].fillna('').astype(str).str.strip() + peer_df['ip:port'] = peer_df['ip:port'].fillna('').astype(str).str.strip() + peer_df = peer_df[(peer_df['node_id'] != '') & (peer_df['ip:port'] != '')].drop_duplicates('node_id') + + agents_df = agents_df[['node_id', 'agent_version']].copy() + agents_df['node_id'] = agents_df['node_id'].fillna('').astype(str).str.strip() + agents_df['agent_version'] = agents_df['agent_version'].fillna('').astype(str).str.strip() + agents_df = agents_df[agents_df['node_id'] != ''].drop_duplicates('node_id') + agents_df['client'] = agents_df['agent_version'].map(normalise_client_name) + + excluded_node_ids = set(agents_df[agents_df['client'].isin(excluded_clients)]['node_id']) + excluded_endpoints = { + row['ip:port'] + for _, row in peer_df.iterrows() + if row['node_id'] in excluded_node_ids + } + + filtered_nodes = { + node + for node in nodes + if f'{node[0]}:{node[1]}' not in excluded_endpoints + } + + logging.info( + f"parse.py: Removed {len(nodes) - len(filtered_nodes)} nodes running excluded clients." + ) + + return filtered_nodes + + def group_nodes(layer, nodes, mode): """ Groups nodes by geolocation information. @@ -252,6 +323,7 @@ def main(): for layer in LAYERS: logging.info(f'parse.py: Getting {layer} nodes') nodes = hlp.get_nodes(layer) + nodes = filter_nodes_by_client(nodes) for mode in MODES: analyse_distribution(nodes, layer, mode) if 'Organizations' in MODES: