From 4571220a8b5c54e10eda34364b9a8ff5e83ba9c4 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Wed, 24 Jun 2026 10:31:11 -0400 Subject: [PATCH 1/9] Update "id" schema from number to integer --- tap_github/schemas/commit_comments.json | 4 ++-- tap_github/schemas/commits.json | 2 +- tap_github/schemas/events.json | 8 ++++---- tap_github/schemas/issue_labels.json | 2 +- tap_github/schemas/issue_milestones.json | 4 ++-- tap_github/schemas/pr_commits.json | 2 +- tap_github/schemas/project_cards.json | 6 +++--- tap_github/schemas/project_columns.json | 2 +- tap_github/schemas/projects.json | 6 +++--- tap_github/schemas/pull_requests.json | 2 +- tap_github/schemas/releases.json | 4 ++-- tap_github/schemas/team_members.json | 2 +- tap_github/schemas/teams.json | 2 +- 13 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tap_github/schemas/commit_comments.json b/tap_github/schemas/commit_comments.json index 408448dc..23cccadc 100644 --- a/tap_github/schemas/commit_comments.json +++ b/tap_github/schemas/commit_comments.json @@ -8,7 +8,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/commits.json b/tap_github/schemas/commits.json index 00f0d2c8..ff665dc0 100644 --- a/tap_github/schemas/commits.json +++ b/tap_github/schemas/commits.json @@ -14,7 +14,7 @@ "type": ["null", "integer"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "updated_at": { "type": ["null", "string"], diff --git a/tap_github/schemas/events.json b/tap_github/schemas/events.json index 35255cb2..da61a16f 100644 --- a/tap_github/schemas/events.json +++ b/tap_github/schemas/events.json @@ -17,7 +17,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "login": { "type": ["null", "string"] @@ -31,7 +31,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "org": { "type": ["null", "object"], @@ -43,7 +43,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "login": { "type": ["null", "string"] @@ -1036,7 +1036,7 @@ "type": ["null", "object"], "properties": { "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "name": { "type": ["null", "string"] diff --git a/tap_github/schemas/issue_labels.json b/tap_github/schemas/issue_labels.json index 32a097df..a67607da 100644 --- a/tap_github/schemas/issue_labels.json +++ b/tap_github/schemas/issue_labels.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/issue_milestones.json b/tap_github/schemas/issue_milestones.json index eb14f446..275cc866 100644 --- a/tap_github/schemas/issue_milestones.json +++ b/tap_github/schemas/issue_milestones.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -44,7 +44,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/pr_commits.json b/tap_github/schemas/pr_commits.json index 1108cfb3..a160e7e2 100644 --- a/tap_github/schemas/pr_commits.json +++ b/tap_github/schemas/pr_commits.json @@ -258,7 +258,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "author": { "$ref": "shared/user.json#/" diff --git a/tap_github/schemas/project_cards.json b/tap_github/schemas/project_cards.json index bb62bdf2..eb50e1ce 100644 --- a/tap_github/schemas/project_cards.json +++ b/tap_github/schemas/project_cards.json @@ -20,7 +20,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -117,4 +117,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/project_columns.json b/tap_github/schemas/project_columns.json index 87e72543..b8c1c150 100644 --- a/tap_github/schemas/project_columns.json +++ b/tap_github/schemas/project_columns.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/projects.json b/tap_github/schemas/projects.json index e8d659ad..53f2ea26 100644 --- a/tap_github/schemas/projects.json +++ b/tap_github/schemas/projects.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -44,7 +44,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -114,4 +114,4 @@ "type": ["null", "boolean"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index cbb0128d..1e8f0e8e 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/releases.json b/tap_github/schemas/releases.json index fc97ced2..f3ca0c44 100644 --- a/tap_github/schemas/releases.json +++ b/tap_github/schemas/releases.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -189,4 +189,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/team_members.json b/tap_github/schemas/team_members.json index d872bafa..d507a7e3 100644 --- a/tap_github/schemas/team_members.json +++ b/tap_github/schemas/team_members.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/teams.json b/tap_github/schemas/teams.json index 15a26205..4ad4a45e 100644 --- a/tap_github/schemas/teams.json +++ b/tap_github/schemas/teams.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] From bbb52a8b0d1b86e70d15b0b0700c3025a6e48045 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Wed, 24 Jun 2026 11:14:18 -0400 Subject: [PATCH 2/9] Testing pull_requests id as number --- tap_github/schemas/pull_requests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index 1e8f0e8e..d92752b9 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "node_id": { "type": ["null", "string"] From 1795ac0422a3ab34dffa7cae22803df92cee33d0 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Thu, 9 Jul 2026 10:41:06 -0400 Subject: [PATCH 3/9] Update pull_requests id schema --- tap_github/schemas/pull_requests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index d92752b9..1e8f0e8e 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] From 44c029cb36d4c7082844701a359ab0a722c2943c Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 08:38:32 -0600 Subject: [PATCH 4/9] trigger ci From 3a513013fbccb7b640e3bbab4ce4491c6c8079f4 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 08:58:35 -0600 Subject: [PATCH 5/9] Change PR id to string --- tap_github/schemas/pull_requests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index 1e8f0e8e..cbb0128d 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "integer"] + "type": ["null", "string"] }, "node_id": { "type": ["null", "string"] From 4d274ead36edb2aa981b49625c8e6b0db0539124 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 09:17:16 -0600 Subject: [PATCH 6/9] Testing changes --- tap_github/streams.py | 12 ++++++++++-- tap_github/sync.py | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 7eda4605..2c83cead 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -169,7 +169,11 @@ def get_child_records(self, with singer.Transformer() as transformer: - rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + try: + rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + except Exception e: + print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') + raise e if child_object.tap_stream_id in selected_stream_ids and record.get(child_object.replication_keys, start_date) >= child_bookmark_value: singer.write_record(child_object.tap_stream_id, rec, time_extracted=extraction_time) @@ -191,7 +195,11 @@ def get_child_records(self, with singer.Transformer() as transformer: - rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + try: + rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + except Exception e: + print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') + raise e if child_object.tap_stream_id in selected_stream_ids and records.get(child_object.replication_keys, start_date) >= child_bookmark_value : singer.write_record(child_object.tap_stream_id, rec, time_extracted=extraction_time) diff --git a/tap_github/sync.py b/tap_github/sync.py index 1e4edea7..ca1ae645 100644 --- a/tap_github/sync.py +++ b/tap_github/sync.py @@ -233,6 +233,7 @@ def do_sync(catalog, streams_to_sync, selected_stream_ids, client, start_date, s if stream_id in streams_to_sync and not stream_obj.parent: write_schemas(stream_id, catalog, selected_stream_ids) update_currently_syncing(state, stream_id) + LOGGER.info("Starting sync of stream: %s", stream_id) state = stream_obj.sync_endpoint(client = client, state = state, From e312e952dd8be1c9f7095c11c9e9d4c5933d8f20 Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 09:19:46 -0600 Subject: [PATCH 7/9] wip --- tap_github/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index 2c83cead..ffb95d50 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -171,7 +171,7 @@ def get_child_records(self, try: rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) - except Exception e: + except Exception as e: print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') raise e @@ -197,7 +197,7 @@ def get_child_records(self, try: rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) - except Exception e: + except Exception as e: print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') raise e if child_object.tap_stream_id in selected_stream_ids and records.get(child_object.replication_keys, start_date) >= child_bookmark_value : From e1cc876a2d3d96a01085e576295384a155e0025a Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 09:22:44 -0600 Subject: [PATCH 8/9] wip --- tap_github/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_github/streams.py b/tap_github/streams.py index ffb95d50..7ce68a29 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -172,7 +172,7 @@ def get_child_records(self, try: rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) except Exception as e: - print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') + LOGGER.warning(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') raise e if child_object.tap_stream_id in selected_stream_ids and record.get(child_object.replication_keys, start_date) >= child_bookmark_value: @@ -198,7 +198,7 @@ def get_child_records(self, try: rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) except Exception as e: - print(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') + LOGGER.warning(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') raise e if child_object.tap_stream_id in selected_stream_ids and records.get(child_object.replication_keys, start_date) >= child_bookmark_value : From 77157483551f2e9eb81edb54a700a6d4a341b87a Mon Sep 17 00:00:00 2001 From: Collin Simon Date: Mon, 13 Jul 2026 09:44:08 -0600 Subject: [PATCH 9/9] Fix pr_commits schema and revert testing related changes --- tap_github/schemas/pr_commits.json | 2 +- tap_github/schemas/pull_requests.json | 2 +- tap_github/streams.py | 13 +++---------- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/tap_github/schemas/pr_commits.json b/tap_github/schemas/pr_commits.json index a160e7e2..1108cfb3 100644 --- a/tap_github/schemas/pr_commits.json +++ b/tap_github/schemas/pr_commits.json @@ -258,7 +258,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "integer"] + "type": ["null", "string"] }, "author": { "$ref": "shared/user.json#/" diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index cbb0128d..1e8f0e8e 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/streams.py b/tap_github/streams.py index 7ce68a29..dc004c1f 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -169,11 +169,7 @@ def get_child_records(self, with singer.Transformer() as transformer: - try: - rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) - except Exception as e: - LOGGER.warning(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') - raise e + rec = transformer.transform(record, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) if child_object.tap_stream_id in selected_stream_ids and record.get(child_object.replication_keys, start_date) >= child_bookmark_value: singer.write_record(child_object.tap_stream_id, rec, time_extracted=extraction_time) @@ -195,11 +191,8 @@ def get_child_records(self, with singer.Transformer() as transformer: - try: - rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) - except Exception as e: - LOGGER.warning(f'CHILD STREAM WITH ISSUES: {child_object.tap_stream_id}') - raise e + rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + if child_object.tap_stream_id in selected_stream_ids and records.get(child_object.replication_keys, start_date) >= child_bookmark_value : singer.write_record(child_object.tap_stream_id, rec, time_extracted=extraction_time)