Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/cowork-auto-pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Seeded by the repo-improver-rotation Cowork job into cowork/improve-* branches.
# Opens a PR automatically when such a branch is pushed (sandbox cannot reach
# the GitHub API directly; this runs server-side with the repo's GITHUB_TOKEN).
name: cowork-auto-pr
on:
push:
branches: ['cowork/improve-**']
permissions:
contents: read
pull-requests: write
jobs:
ensure-pr:
runs-on: ubuntu-latest
steps:
- name: Open PR for this branch if none exists
env:
GH_TOKEN: ${{ github.token }}
run: |
set -eu
existing=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$GITHUB_REF_NAME" --state open --json number --jq 'length')
if [ "$existing" = "0" ]; then
gh pr create --repo "$GITHUB_REPOSITORY" \
--head "$GITHUB_REF_NAME" \
--title "cowork-bot: automated improvements ($GITHUB_REF_NAME)" \
--body "Automated improvement PR from the Cowork repo-improver rotation (one coherent senior-dev improvement per run; see individual commit messages). Subsequent runs push additional commits to this PR rather than opening new ones."
else
echo "Open PR already exists for $GITHUB_REF_NAME — nothing to do."
fi
4 changes: 4 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ dev = [
"pytest-cov>=7.1.0",
"ruff>=0.15.20",
]
parquet = ["pandas>=2.0.0", "pyarrow>=14.0.0"]
avro = ["fastavro>=1.12.2"]
protobuf = ["protobuf>=7.34.1"]
all = ["datamorph[parquet,avro,protobuf]"]

[project.scripts]
datamorph = "datamorph.cli:cli"
Expand Down
50 changes: 42 additions & 8 deletions src/datamorph/converters.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,12 @@ def write_stream(self, rows: RowStream, path: str | Path) -> int:

class YamlReader(FormatReader):
def read_stream(self, path: str | Path) -> RowStream:
import yaml
try:
import yaml
except ImportError as e:
raise ImportError(
"YAML support requires PyYAML: pip install pyyaml"
) from e

with open(path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
Expand All @@ -298,7 +303,12 @@ def read_stream(self, path: str | Path) -> RowStream:

class YamlWriter(FormatWriter):
def write_stream(self, rows: RowStream, path: str | Path) -> int:
import yaml
try:
import yaml
except ImportError as e:
raise ImportError(
"YAML support requires PyYAML: pip install pyyaml"
) from e

rows_list = list(rows)
with open(path, "w", encoding="utf-8") as f:
Expand All @@ -317,7 +327,13 @@ def write_stream(self, rows: RowStream, path: str | Path) -> int:

class ParquetReader(FormatReader):
def read_stream(self, path: str | Path) -> RowStream:
import pyarrow.parquet as pq
try:
import pyarrow.parquet as pq
except ImportError as e:
raise ImportError(
"Parquet support requires the 'parquet' extra: "
"pip install 'datamorph[parquet]'"
) from e

pf = pq.ParquetFile(path)
for batch in pf.iter_batches():
Expand All @@ -330,9 +346,15 @@ def read_stream(self, path: str | Path) -> RowStream:

class ParquetWriter(FormatWriter):
def write_stream(self, rows: RowStream, path: str | Path) -> int:
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
try:
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError as e:
raise ImportError(
"Parquet support requires the 'parquet' extra: "
"pip install 'datamorph[parquet]'"
) from e

rows_list = list(rows)
if not rows_list:
Expand All @@ -351,7 +373,13 @@ def write_stream(self, rows: RowStream, path: str | Path) -> int:

class AvroReader(FormatReader):
def read_stream(self, path: str | Path) -> RowStream:
import fastavro
try:
import fastavro
except ImportError as e:
raise ImportError(
"Avro support requires the 'avro' extra: "
"pip install 'datamorph[avro]'"
) from e

with open(path, "rb") as f:
reader = fastavro.reader(f)
Expand All @@ -361,7 +389,13 @@ def read_stream(self, path: str | Path) -> RowStream:

class AvroWriter(FormatWriter):
def write_stream(self, rows: RowStream, path: str | Path) -> int:
import fastavro
try:
import fastavro
except ImportError as e:
raise ImportError(
"Avro support requires the 'avro' extra: "
"pip install 'datamorph[avro]'"
) from e

rows_list = list(rows)
if not rows_list:
Expand Down
Loading