Exclude unwanted clients from network and geo

bitcoin/excluded_clients.txt

@@ -0,0 +1,23 @@
+GhostCore
+BitcoinUnlimited
+Therealbitcoin.org
+CKCoinD
+Cooltexture
+GrayersDad
+Bitcoin
+Btcwire
+FlamiSatoshi
+Aurum
+Jwrd.net
+Classic
+Satoshi-Noderunners
+Satushi
+Prometheus
+Bitcoin SV
+Bitcoin ABC
+Node
+DefcoinCore
+B2CCoinCore
+USFrancCore
+CypherfunkCore
+DigitaleuroCore

bitcoin/network_decentralization/helper.py

@@ -144,6 +144,23 @@ def get_without_tor_ledgers():
     return list(dict.fromkeys(ledgers)) or None
 
 
+def get_excluded_clients():
+    """
+    Returns the set of client families to exclude from analyses.
+    """
+    path = ROOT_DIR / "excluded_clients.txt"
+
+    if not path.is_file():
+        return set()
+
+    with open(path) as f:
+        return {
+            line.strip()
+            for line in f
+            if line.strip() and not line.startswith("#")
+        }
+
+
 def get_output_directory(ledger=None, dead=False):
     """
     Reads the config file and retrieves the output directory

bitcoin/parse.py

@@ -341,6 +341,29 @@ def record_versions(reachable_nodes, mode):
             versions_df.to_csv(f'./output/{name.lower()}_{ledger}.csv', index_label = name)
 
 
+def filter_nodes_by_client(reachable_nodes):
+    """
+    Removes nodes running excluded clients.
+    """
+    excluded_clients = hlp.get_excluded_clients()
+
+    if not excluded_clients:
+        return reachable_nodes
+
+    filtered_nodes = [
+        node
+        for node in reachable_nodes
+        if normalise_client_name(node[2]) not in excluded_clients
+    ]
+
+    logging.info(
+        f"Removed {len(reachable_nodes) - len(filtered_nodes)} nodes "
+        f"running excluded clients."
+    )
+
+    return filtered_nodes
+
+
 def redistribute_tor_nodes(mode_lower, ledger, df, mode):
     """
     Redistributes Tor node count proportionally across non-Tor rows.
@@ -440,6 +463,7 @@ def main():
     for ledger in LEDGERS:
         logging.info(f'parse.py: Getting {ledger} reachable nodes')
         reachable_nodes[ledger] = hlp.get_reachable_nodes(ledger)
+        reachable_nodes[ledger] = filter_nodes_by_client(reachable_nodes[ledger])
         for mode in MODES:
             geography(reachable_nodes, ledger, mode)
         if 'Organizations' in MODES:

ethereum/excluded_clients.txt

@@ -0,0 +1,24 @@
+Rust-libp2p
+Eth-diversity@
+Hermes
+Logex
+BeraGeth
+Bera-reth
+Atlas
+Qk_node
+Naoris
+Celo
+XDC
+Bera1
+World-chain
+Ronin
+Nimbus-eth1
+Reth_gnosis
+Getc
+Eth
+Gwemix
+NimbusExecutionClient
+Lighthouse-Pulse
+Prysm-Pulse
+Bor
+Tysm

ethereum/helper.py

@@ -37,6 +37,25 @@ def get_mode():
     return get_config_data()['mode']
 
 
+def get_excluded_clients():
+    """
+    Returns the set of client families to exclude from analyses.
+    """
+    path = pathlib.Path(__file__).resolve().parent / "excluded_clients.txt"
+
+    if not path.is_file():
+        return set()
+
+    with open(path) as f:
+        excluded_clients = {
+            line.strip()
+            for line in f
+            if line.strip() and not line.startswith("#")
+        }
+
+    return excluded_clients
+
+
 def get_output_directory():
     """
     Require the `OUTPUT_DIRECTORY` env var set by the caller

ethereum/parse.py

@@ -31,6 +31,77 @@ def normalise_client_name(client_value):
     return client or 'Unknown'
 
 
+def filter_nodes_by_client(nodes):
+    """
+    Removes nodes running excluded clients.
+    """
+    excluded_clients = hlp.get_excluded_clients()
+
+    if not excluded_clients:
+        return nodes
+
+    output_dir = hlp.get_output_directory()
+    peerfile = output_dir / 'peerstore.csv'
+    agentsfile = output_dir / 'agents.csv'
+
+    if not peerfile.is_file() or not agentsfile.is_file():
+        logging.warning('parse.py: peerstore.csv or agents.csv not found; skipping client exclusion')
+        return nodes
+
+    peer_df = pd.read_csv(
+        peerfile,
+        engine='python',
+        on_bad_lines='skip',
+        dtype=str,
+        keep_default_na=False,
+        quotechar="'",
+    )
+
+    agents_df = pd.read_csv(
+        agentsfile,
+        engine='python',
+        on_bad_lines='skip',
+        dtype=str,
+        keep_default_na=False,
+    )
+
+    required_peer_columns = {'node_id', 'ip:port'}
+    required_agent_columns = {'node_id', 'agent_version'}
+    if not required_peer_columns.issubset(peer_df.columns) or not required_agent_columns.issubset(agents_df.columns):
+        logging.warning('parse.py: peerstore.csv or agents.csv is missing required columns; skipping client exclusion')
+        return nodes
+
+    peer_df = peer_df[['node_id', 'ip:port']].copy()
+    peer_df['node_id'] = peer_df['node_id'].fillna('').astype(str).str.strip()
+    peer_df['ip:port'] = peer_df['ip:port'].fillna('').astype(str).str.strip()
+    peer_df = peer_df[(peer_df['node_id'] != '') & (peer_df['ip:port'] != '')].drop_duplicates('node_id')
+
+    agents_df = agents_df[['node_id', 'agent_version']].copy()
+    agents_df['node_id'] = agents_df['node_id'].fillna('').astype(str).str.strip()
+    agents_df['agent_version'] = agents_df['agent_version'].fillna('').astype(str).str.strip()
+    agents_df = agents_df[agents_df['node_id'] != ''].drop_duplicates('node_id')
+    agents_df['client'] = agents_df['agent_version'].map(normalise_client_name)
+
+    excluded_node_ids = set(agents_df[agents_df['client'].isin(excluded_clients)]['node_id'])
+    excluded_endpoints = {
+        row['ip:port']
+        for _, row in peer_df.iterrows()
+        if row['node_id'] in excluded_node_ids
+    }
+
+    filtered_nodes = {
+        node
+        for node in nodes
+        if f'{node[0]}:{node[1]}' not in excluded_endpoints
+    }
+
+    logging.info(
+        f"parse.py: Removed {len(nodes) - len(filtered_nodes)} nodes running excluded clients."
+    )
+
+    return filtered_nodes
+
+
 def group_nodes(layer, nodes, mode):
     """
     Groups nodes by geolocation information.
@@ -252,6 +323,7 @@ def main():
     for layer in LAYERS:
         logging.info(f'parse.py: Getting {layer} nodes')
         nodes = hlp.get_nodes(layer)
+        nodes = filter_nodes_by_client(nodes)
         for mode in MODES:
             analyse_distribution(nodes, layer, mode)
         if 'Organizations' in MODES: