diff --git a/tap_github/schemas/commit_comments.json b/tap_github/schemas/commit_comments.json index 408448dc..23cccadc 100644 --- a/tap_github/schemas/commit_comments.json +++ b/tap_github/schemas/commit_comments.json @@ -8,7 +8,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/commits.json b/tap_github/schemas/commits.json index 00f0d2c8..ff665dc0 100644 --- a/tap_github/schemas/commits.json +++ b/tap_github/schemas/commits.json @@ -14,7 +14,7 @@ "type": ["null", "integer"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "updated_at": { "type": ["null", "string"], diff --git a/tap_github/schemas/events.json b/tap_github/schemas/events.json index 35255cb2..da61a16f 100644 --- a/tap_github/schemas/events.json +++ b/tap_github/schemas/events.json @@ -17,7 +17,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "login": { "type": ["null", "string"] @@ -31,7 +31,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "org": { "type": ["null", "object"], @@ -43,7 +43,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "login": { "type": ["null", "string"] @@ -1036,7 +1036,7 @@ "type": ["null", "object"], "properties": { "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "name": { "type": ["null", "string"] diff --git a/tap_github/schemas/issue_labels.json b/tap_github/schemas/issue_labels.json index 32a097df..a67607da 100644 --- a/tap_github/schemas/issue_labels.json +++ b/tap_github/schemas/issue_labels.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/issue_milestones.json b/tap_github/schemas/issue_milestones.json index eb14f446..275cc866 100644 --- a/tap_github/schemas/issue_milestones.json +++ b/tap_github/schemas/issue_milestones.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -44,7 +44,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/project_cards.json b/tap_github/schemas/project_cards.json index bb62bdf2..eb50e1ce 100644 --- a/tap_github/schemas/project_cards.json +++ b/tap_github/schemas/project_cards.json @@ -20,7 +20,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -117,4 +117,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/project_columns.json b/tap_github/schemas/project_columns.json index 87e72543..b8c1c150 100644 --- a/tap_github/schemas/project_columns.json +++ b/tap_github/schemas/project_columns.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/projects.json b/tap_github/schemas/projects.json index e8d659ad..53f2ea26 100644 --- a/tap_github/schemas/projects.json +++ b/tap_github/schemas/projects.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -44,7 +44,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -114,4 +114,4 @@ "type": ["null", "boolean"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/pull_requests.json b/tap_github/schemas/pull_requests.json index cbb0128d..1e8f0e8e 100644 --- a/tap_github/schemas/pull_requests.json +++ b/tap_github/schemas/pull_requests.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/releases.json b/tap_github/schemas/releases.json index fc97ced2..f3ca0c44 100644 --- a/tap_github/schemas/releases.json +++ b/tap_github/schemas/releases.json @@ -6,7 +6,7 @@ "type": ["string"] }, "id": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] @@ -189,4 +189,4 @@ "type": ["null", "string"] } } -} \ No newline at end of file +} diff --git a/tap_github/schemas/team_members.json b/tap_github/schemas/team_members.json index d872bafa..d507a7e3 100644 --- a/tap_github/schemas/team_members.json +++ b/tap_github/schemas/team_members.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/schemas/teams.json b/tap_github/schemas/teams.json index 15a26205..4ad4a45e 100644 --- a/tap_github/schemas/teams.json +++ b/tap_github/schemas/teams.json @@ -5,7 +5,7 @@ "type": ["null", "string"] }, "id": { - "type": ["null", "number"] + "type": ["null", "integer"] }, "node_id": { "type": ["null", "string"] diff --git a/tap_github/streams.py b/tap_github/streams.py index 7eda4605..dc004c1f 100644 --- a/tap_github/streams.py +++ b/tap_github/streams.py @@ -192,6 +192,7 @@ def get_child_records(self, with singer.Transformer() as transformer: rec = transformer.transform(records, stream_catalog['schema'], metadata=metadata.to_map(stream_catalog['metadata'])) + if child_object.tap_stream_id in selected_stream_ids and records.get(child_object.replication_keys, start_date) >= child_bookmark_value : singer.write_record(child_object.tap_stream_id, rec, time_extracted=extraction_time) diff --git a/tap_github/sync.py b/tap_github/sync.py index 1e4edea7..ca1ae645 100644 --- a/tap_github/sync.py +++ b/tap_github/sync.py @@ -233,6 +233,7 @@ def do_sync(catalog, streams_to_sync, selected_stream_ids, client, start_date, s if stream_id in streams_to_sync and not stream_obj.parent: write_schemas(stream_id, catalog, selected_stream_ids) update_currently_syncing(state, stream_id) + LOGGER.info("Starting sync of stream: %s", stream_id) state = stream_obj.sync_endpoint(client = client, state = state,