diff --git a/.gitignore b/.gitignore index 7d54653d3..b49e146e4 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,11 @@ mssql_python.egg-info/ # The release pipeline still overwrites them per-platform for the full matrix. mssql_python_odbc.egg-info/ +# Rust ODBC provider package (mssql-python-rust-odbc). +# The driver binaries under mssql_python_rust_odbc/libs/ are the committed +# source of truth, synced via eng/scripts/sync-mssql-odbc-native-libs.py. +mssql_python_rust_odbc.egg-info/ + # Python bytecode __pycache__/ *.py[cod] @@ -41,6 +46,8 @@ build/ # C extensions *.so +# ...except the committed Rust ODBC provider driver binaries (see note above). +!mssql_python_rust_odbc/libs/**/*.so *.pyd *.pdb diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index b5b719cef..abde60e3e 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -377,8 +377,12 @@ extends: # ========================= # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages # mssql-python-odbc: 1 ODBC_BuildAll (single host, all 7 wheels) + 1 ConsolidateOdbc = 2 stages - # Total: 21 stages, always built together. The mssql-python build stages depend on + # mssql-python-rust-odbc: 1 RustOdbc_BuildAll (single host, all 7 wheels) + 1 ConsolidateRustOdbc = 2 stages + # Total: 23 stages, always built together. The mssql-python build stages depend on # ConsolidateOdbc so they install the external ODBC wheel before pytest. + # mssql-python-rust-odbc is not installed before pytest: it is an optional, + # runtime-selected provider (mssql_python.native_provider), not required to + # import mssql_python or run the default (classic-provider) test suite. stages: # ========================= # WINDOWS BUILD STAGES @@ -566,3 +570,40 @@ extends: # mssql-python build stages now install the external mssql-python-odbc wheel # (from ConsolidateOdbc) and run the full pytest suite against it — so the # external-package resolution is already validated end-to-end during the build. + + # ========================================================================= + # RUST ODBC PACKAGE BUILD STAGE (mssql-python-rust-odbc) — SINGLE HOST + # ========================================================================= + # ONE Windows stage (RustOdbc_BuildAll) produces ALL 7 data-only wheels + # (2 Windows + 1 macOS universal2 + 4 Linux) — no native compile, no pytest. + # Mirrors ODBC_BuildAll/ConsolidateOdbc above; the package is pure driver + # data (py3-none-), so a single host cross-produces every + # platform's wheel from the committed mssql_python_rust_odbc/libs tree via + # setup_rust_odbc.py's RUST_ODBC_TARGET_* overrides. + # + # Unlike ODBC_BuildAll, the mssql-python build stages do NOT depend on + # ConsolidateRustOdbc: the rust-odbc provider is optional and selected at + # runtime (mssql_python.native_provider), so it is not required to import + # mssql_python or run the default (classic-provider) pytest suite. + - template: /OneBranchPipelines/stages/build-rust-odbc-all-stage.yml@self + parameters: + stageName: RustOdbc_BuildAll + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE RUST ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-rust-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateRustOdbc_ConsolidateArtifacts`. The + # release pipeline selects this artifact when its `releasePackage` + # parameter targets mssql-python-rust-odbc. + - stage: ConsolidateRustOdbc + displayName: 'Consolidate All Rust ODBC Artifacts' + dependsOn: + - RustOdbc_BuildAll + jobs: + - template: /OneBranchPipelines/jobs/consolidate-rust-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index 6176156f2..aa50d5d5c 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,7 +12,7 @@ pr: none # Parameters for DUMMY release pipeline parameters: - # Which package to test-release. Both are produced by the same build pipeline + # Which package to test-release. All three are produced by the same build pipeline # (definition 2199); this switches the consolidated artifact and messaging. - name: releasePackage displayName: '[TEST] Package to Release' @@ -20,6 +20,7 @@ parameters: values: - 'mssql-python' - 'mssql-python-odbc' + - 'mssql-python-rust-odbc' default: 'mssql-python' - name: publishSymbols @@ -43,12 +44,15 @@ variables: - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables # Select which consolidated artifact to download based on the target package. - # Both are produced by the same build pipeline (definition 2199): - # mssql-python -> drop_Consolidate_ConsolidateArtifacts - # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + # All three are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + # mssql-python-rust-odbc -> drop_ConsolidateRustOdbc_ConsolidateArtifacts - name: consolidatedArtifactName ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ elseif eq(parameters.releasePackage, 'mssql-python-rust-odbc') }}: + value: 'drop_ConsolidateRustOdbc_ConsolidateArtifacts' ${{ else }}: value: 'drop_Consolidate_ConsolidateArtifacts' @@ -268,6 +272,60 @@ extends: Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" } + # Guard: same check as above, for the mssql-python-rust-odbc pin. + # $expectedVersion is derived from mssql_python_rust_odbc/__init__.py + # (single source of truth), so it never needs a manual bump. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-rust-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $initFile = "$(Build.SourcesDirectory)/mssql_python_rust_odbc/__init__.py" + $verMatch = Select-String -Path $initFile -Pattern '__version__\s*=\s*["'']([^"'']+)["'']' | Select-Object -First 1 + if (-not $verMatch) { + Write-Error "Could not parse __version__ from $initFile" + exit 1 + } + $expectedVersion = $verMatch.Matches[0].Groups[1].Value + Write-Host "Expected mssql-python-rust-odbc pin (from mssql_python_rust_odbc/__init__.py): $expectedVersion" + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the rust-odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $rustOdbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]rust[-_]odbc' } + if (-not $rustOdbcLines) { + Write-Warning "Wheel declares no mssql-python-rust-odbc dependency (pre-pin build). Skipping rust-odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-rust-odbc==$expectedVersion." + } else { + Write-Host "rust-odbc dependency in metadata:" + $rustOdbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor with a negative lookahead so 0.1.0.1 does not + # satisfy a 0.1.0 pin, while still allowing a trailing + # space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]rust[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $rustOdbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-rust-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct rust-odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-rust-odbc==$expectedVersion" + } + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: '[TEST] Verify Wheel Integrity' diff --git a/OneBranchPipelines/jobs/consolidate-rust-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-rust-odbc-artifacts-job.yml new file mode 100644 index 000000000..6f840c9f8 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-rust-odbc-artifacts-job.yml @@ -0,0 +1,75 @@ +# Consolidate Rust ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-rust-odbc` wheels from the +# RustOdbc_BuildAll stage and consolidates them into a single dist/ folder for +# the release pipeline to publish. Expected: 7 wheels (2 Windows + 1 macOS +# universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All Rust ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download only the mssql-python-rust-odbc platform-stage artifacts + # (drop_RustOdbc_*) from the current build. itemPattern scopes this + # download so only the 6 rust-odbc wheels are consolidated here. + - task: DownloadPipelineArtifact@2 + displayName: 'Download All Rust ODBC Platform Artifacts' + inputs: + buildType: 'current' + itemPattern: 'drop_RustOdbc_*/**' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 + fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} rust-odbc wheels consolidated." + displayName: 'Consolidate rust-odbc wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated Rust ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + # Distinct name so it does not collide with the mssql-python or + # mssql-python-odbc consolidate artifacts in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateRustOdbc`. + artifact: 'drop_ConsolidateRustOdbc_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 3198908bc..4c74250a3 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -12,7 +12,7 @@ pr: none # Parameters for release pipeline parameters: - # Which package to release. Both are produced by the same build pipeline + # Which package to release. All three are produced by the same build pipeline # (definition 2199); this switches the consolidated artifact and messaging. - name: releasePackage displayName: 'Package to Release' @@ -20,6 +20,7 @@ parameters: values: - 'mssql-python' - 'mssql-python-odbc' + - 'mssql-python-rust-odbc' default: 'mssql-python' - name: publishSymbols @@ -43,12 +44,15 @@ variables: - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables # Select which consolidated artifact to download/publish based on the target - # package. Both are produced by the same build pipeline (definition 2199): - # mssql-python -> drop_Consolidate_ConsolidateArtifacts - # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + # package. All three are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + # mssql-python-rust-odbc -> drop_ConsolidateRustOdbc_ConsolidateArtifacts - name: consolidatedArtifactName ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ elseif eq(parameters.releasePackage, 'mssql-python-rust-odbc') }}: + value: 'drop_ConsolidateRustOdbc_ConsolidateArtifacts' ${{ else }}: value: 'drop_Consolidate_ConsolidateArtifacts' @@ -268,6 +272,60 @@ extends: Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" } + # Guard: same check as above, for the mssql-python-rust-odbc pin. + # $expectedVersion is derived from mssql_python_rust_odbc/__init__.py + # (single source of truth), so it never needs a manual bump. + - task: PowerShell@2 + displayName: 'Validate mssql-python-rust-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $initFile = "$(Build.SourcesDirectory)/mssql_python_rust_odbc/__init__.py" + $verMatch = Select-String -Path $initFile -Pattern '__version__\s*=\s*["'']([^"'']+)["'']' | Select-Object -First 1 + if (-not $verMatch) { + Write-Error "Could not parse __version__ from $initFile" + exit 1 + } + $expectedVersion = $verMatch.Matches[0].Groups[1].Value + Write-Host "Expected mssql-python-rust-odbc pin (from mssql_python_rust_odbc/__init__.py): $expectedVersion" + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the rust-odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $rustOdbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]rust[-_]odbc' } + if (-not $rustOdbcLines) { + Write-Warning "Wheel declares no mssql-python-rust-odbc dependency (pre-pin build). Skipping rust-odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-rust-odbc==$expectedVersion." + } else { + Write-Host "rust-odbc dependency in metadata:" + $rustOdbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor with a negative lookahead so 0.1.0.1 does not + # satisfy a 0.1.0 pin, while still allowing a trailing + # space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]rust[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $rustOdbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-rust-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct rust-odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-rust-odbc==$expectedVersion" + } + # Step 4: Verify wheel integrity - task: PowerShell@2 displayName: 'Verify Wheel Integrity' diff --git a/OneBranchPipelines/stages/build-rust-odbc-all-stage.yml b/OneBranchPipelines/stages/build-rust-odbc-all-stage.yml new file mode 100644 index 000000000..5d8b1784e --- /dev/null +++ b/OneBranchPipelines/stages/build-rust-odbc-all-stage.yml @@ -0,0 +1,200 @@ +# Rust ODBC "Build All" Single-Host Stage Template +# ============================================================================ +# Builds ALL platform-specific `mssql-python-rust-odbc` wheels (Windows +# x64/arm64, macOS universal2, Linux manylinux/musllinux x86_64/aarch64 = 7 +# wheels) on ONE Windows agent. +# +# Why this is safe to do on a single host: the `mssql-python-rust-odbc` +# package ships ONLY pre-built mssql-odbc driver binaries (pure data) -- +# there is NO compiled Python extension and NO per-Python-version matrix. +# Every platform's driver binaries are committed under +# `mssql_python_rust_odbc/libs/`, so the build host is irrelevant to the +# wheel CONTENTS: only the wheel's platform TAG and the selected `libs/` +# subtree matter. `setup_rust_odbc.py` honours the +# `RUST_ODBC_TARGET_PLATFORM_TAG` / `RUST_ODBC_TARGET_ARCH` environment +# overrides to cross-produce each target's wheel (see `get_platform_info`), +# and marks the tag as explicitly supplied so `wheel` does not re-derive a +# tag from the host. +parameters: + # Stage identifier (e.g., 'RustOdbc_BuildAll'). + - name: stageName + type: string + default: 'RustOdbc_BuildAll' + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheels. + - name: pythonVersion + type: string + default: '3.12' + # The full release matrix of rust-odbc wheels to produce from this one host. + # Each entry drives RUST_ODBC_TARGET_PLATFORM_TAG (wheel tag) + + # RUST_ODBC_TARGET_ARCH (libs/ subtree selector) for one + # `setup_rust_odbc.py bdist_wheel` invocation. + - name: rustOdbcTargets + type: object + default: + - { tag: 'win_amd64', arch: 'x64' } + - { tag: 'win_arm64', arch: 'arm64' } + # macOS: universal2 data-only wheel (two separate per-arch dylibs, no + # lipo fusion -- see mssql-odbc-native's macos_x64/macos_arm64 tags). + # Tagged macosx_15_0 to match the main mssql-python wheel, same as + # ODBC_BuildAll's macOS entry. + - { tag: 'macosx_15_0_universal2', arch: 'universal2' } + - { tag: 'manylinux_2_28_x86_64', arch: 'x86_64' } + - { tag: 'manylinux_2_28_aarch64', arch: 'aarch64' } + - { tag: 'musllinux_1_2_x86_64', arch: 'x86_64' } + - { tag: 'musllinux_1_2_aarch64', arch: 'aarch64' } + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Rust ODBC Build All Wheels (single host)' + # This stage MUST run first: see ODBC_BuildAll's dependsOn: [] comment for + # why an implicit "depends on the previous stage" link must be broken here. + dependsOn: [] + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build all 7 rust-odbc wheels on one Windows host' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + # Driver binaries are committed under mssql_python_rust_odbc/libs; shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build every target wheel by driving setup_rust_odbc.py's RUST_ODBC_TARGET_* + # overrides. One step per target (compile-time expanded from rustOdbcTargets). + # build/ + egg-info are wiped before each build so a previous target's staged + # libs/ can never leak into the next wheel; all wheels accumulate in dist/. + - ${{ each t in parameters.rustOdbcTargets }}: + - powershell: | + $ErrorActionPreference = "Stop" + Remove-Item -Recurse -Force build, mssql_python_rust_odbc.egg-info -ErrorAction SilentlyContinue + $env:RUST_ODBC_TARGET_PLATFORM_TAG = "${{ t.tag }}" + $env:RUST_ODBC_TARGET_ARCH = "${{ t.arch }}" + python setup_rust_odbc.py bdist_wheel + if ($LASTEXITCODE -ne 0) { Write-Error "Rust ODBC wheel build failed for ${{ t.tag }}"; exit 1 } + displayName: 'Build rust-odbc wheel: ${{ t.tag }}' + + - powershell: | + $ErrorActionPreference = "Stop" + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + python -m twine check dist\*.whl + displayName: 'List + twine check wheels' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifacts' + + # Assert EVERY wheel ships ONLY its own platform's driver binaries -- a data + # wheel that packaged the wrong (or an incomplete) subtree would still pass + # `twine check`. This is the single-host cross-build's safety net: it proves + # the RUST_ODBC_TARGET_* overrides selected the right libs/ subtree per tag + # and that no foreign-platform binaries leaked in. + - powershell: | + $ErrorActionPreference = "Stop" + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheelDir = "$(ob_outputDirectory)\wheels" + + # tag -> @{ Must = ; Forbid = } + $expect = @{ + 'win_amd64' = @{ Must = @('libs/windows/x64/', 'mssqlodbc.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/windows/arm64/') } + 'win_arm64' = @{ Must = @('libs/windows/arm64/', 'mssqlodbc.dll', 'libs/LICENSING'); Forbid = @('libs/linux/', 'libs/windows/x64/') } + 'macosx_15_0_universal2' = @{ Must = @('libs/macos/arm64/', 'libs/macos/x86_64/', 'mssqlodbc.dylib', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/') } + 'manylinux_2_28_x86_64' = @{ Must = @('libs/linux/glibc/x86_64/', 'mssqlodbc.so', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/musl/', '/arm64/') } + 'manylinux_2_28_aarch64' = @{ Must = @('libs/linux/glibc/arm64/', 'mssqlodbc.so', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/musl/', '/x86_64/') } + 'musllinux_1_2_x86_64' = @{ Must = @('libs/linux/musl/x86_64/', 'mssqlodbc.so', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/glibc/', '/arm64/') } + 'musllinux_1_2_aarch64' = @{ Must = @('libs/linux/musl/arm64/', 'mssqlodbc.so', 'libs/LICENSING'); Forbid = @('libs/windows/', 'libs/linux/glibc/', '/x86_64/') } + } + + $failed = $false + foreach ($tag in $expect.Keys) { + $whl = Get-ChildItem -Path $wheelDir -Filter "*$tag.whl" | Select-Object -First 1 + if (-not $whl) { + Write-Error "Missing expected rust-odbc wheel for tag '$tag'" + $failed = $true + continue + } + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $names = $zip.Entries | ForEach-Object { $_.FullName.Replace('\', '/') } + $zip.Dispose() + foreach ($m in $expect[$tag].Must) { + if (-not ($names | Where-Object { $_ -like "*$m*" })) { + Write-Error "$($whl.Name): MISSING required content '$m'" + $failed = $true + } + } + foreach ($f in $expect[$tag].Forbid) { + if ($names | Where-Object { $_ -like "*$f*" }) { + Write-Error "$($whl.Name): contains FOREIGN content '$f' (wrong-platform binaries leaked)" + $failed = $true + } + } + if (-not $failed) { Write-Host "OK: $($whl.Name) ships only its own platform's libs" } + } + if ($failed) { exit 1 } + Write-Host "All 7 rust-odbc wheels verified: each ships only its own platform's driver." + displayName: 'Assert each rust-odbc wheel is platform-correct' + + # OneBranch requires artifact naming: drop__. + # ConsolidateRustOdbc picks this up via its 'drop_RustOdbc_*' item pattern. + - task: PublishPipelineArtifact@1 + displayName: 'Publish Rust ODBC Wheels Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheels for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - Rust ODBC Wheels (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/PyPI_Description_RustODBC.md b/PyPI_Description_RustODBC.md new file mode 100644 index 000000000..d2a38f22b --- /dev/null +++ b/PyPI_Description_RustODBC.md @@ -0,0 +1,19 @@ +# mssql-python-rust-odbc + +Internal implementation package for [mssql-python](https://pypi.org/project/mssql-python/). + +It ships the platform-specific [mssql-odbc](https://github.com/microsoft/mssql-rs) driver +binaries so that `mssql-python` does not have to bundle them in its own wheel. This package +is not intended for direct use. + +To use the `mssql-odbc` driver with `mssql-python`, install `mssql-python` and select the +provider: + +```python +import mssql_python +mssql_python.native_provider = "mssql-odbc" +``` + +or set the `MSSQL_PYTHON_NATIVE_PROVIDER` environment variable to `mssql-odbc` before +connecting. Installing `mssql-python-rust-odbc` directly is only required if +`mssql-python` does not already declare it as a dependency for your platform. diff --git a/eng/scripts/sync-mssql-odbc-native-libs.py b/eng/scripts/sync-mssql-odbc-native-libs.py new file mode 100644 index 000000000..4f2adb870 --- /dev/null +++ b/eng/scripts/sync-mssql-odbc-native-libs.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Sync mssql_python_rust_odbc/libs/ from the mssql-odbc-native NuGet package. + +Unlike eng/scripts/install-mssql-py-core.*, which fetches only the CURRENT +host's platform binary for local dev/CI, this script is a maintainer/pipeline +operation: it downloads every packaged platform's driver binary at once and +lays them out under mssql_python_rust_odbc/libs/, matching exactly what +GetDriverPathCpp() (mssql_python/pybind/ddbc_bindings.cpp) resolves for the +"mssql-odbc" provider. The resulting libs/ tree is the committed source of +truth for setup_rust_odbc.py (mirroring mssql_python_odbc/libs/) and must be +`git add`ed after running this script. + +The mssql-odbc-native package (built by mssql-rs's odbc-native-stages.yml) +ships THREE Linux glibc variants (only two are used here) and two macOS +architecture slices: + - glibc228_* (built on manylinux_2_28 / AlmaLinux 8) -> libs/linux/glibc/ + - musl_* (Alpine) -> libs/linux/musl/ + - linux_* (newer host glibc, for mssql-rs's own distro-container tests) + is NOT manylinux-portable and is intentionally skipped. + - macos_* (per-architecture, no lipo fusion) -> libs/macos/ + +Usage: + python eng/scripts/sync-mssql-odbc-native-libs.py [--feed-url URL] [--version VERSION] +""" + +import argparse +import shutil +import sys +import tempfile +import zipfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parent.parent + +sys.path.insert(0, str(SCRIPT_DIR)) +from resolve_nuget_feed import resolve # noqa: E402 + +DEFAULT_FEED_URL = "https://pkgs.dev.azure.com/sqlclientdrivers/public/_packaging/mssql-rs_Public/nuget/v3/index.json" +PACKAGE_ID = "mssql-odbc-native" + +# NuGet tag inside native// -> destination under mssql_python_rust_odbc/libs/. +# Skips linux_x64 / linux_arm64 (see module docstring). +_TAG_TO_DEST = { + "glibc228_x64": ("mssqlodbc.so", "linux/glibc/x86_64/lib"), + "glibc228_arm64": ("mssqlodbc.so", "linux/glibc/arm64/lib"), + "musl_x64": ("mssqlodbc.so", "linux/musl/x86_64/lib"), + "musl_arm64": ("mssqlodbc.so", "linux/musl/arm64/lib"), + "windows_x64": ("mssqlodbc.dll", "windows/x64"), + "windows_arm64": ("mssqlodbc.dll", "windows/arm64"), + "macos_x64": ("mssqlodbc.dylib", "macos/x86_64/lib"), + "macos_arm64": ("mssqlodbc.dylib", "macos/arm64/lib"), +} + + +def _read_version(version_arg: str) -> str: + if version_arg: + return version_arg + version_file = REPO_ROOT / "eng" / "versions" / "mssql-odbc-native.version" + if not version_file.exists(): + raise SystemExit(f"Version file not found: {version_file}") + version = version_file.read_text(encoding="utf-8").strip() + if not version: + raise SystemExit(f"Version file is empty: {version_file}") + return version + + +def _download_nupkg(feed_url: str, version: str, dest_dir: Path) -> Path: + print(f"Resolving feed: {feed_url}") + package_base_url = resolve(feed_url) + version_lower = version.lower() + nupkg_url = f"{package_base_url}{PACKAGE_ID}/{version_lower}/{PACKAGE_ID}.{version_lower}.nupkg" + nupkg_path = dest_dir / f"{PACKAGE_ID}.{version_lower}.nupkg" + + print(f"Downloading: {nupkg_url}") + import urllib.request + + with urllib.request.urlopen(nupkg_url, timeout=120) as resp, open(nupkg_path, "wb") as out: + shutil.copyfileobj(resp, out) + size_mb = nupkg_path.stat().st_size / (1024 * 1024) + print(f"Downloaded: {nupkg_path} ({size_mb:.2f} MB)") + return nupkg_path + + +def _sync_libs(extract_dir: Path, libs_dir: Path) -> None: + native_dir = extract_dir / "native" + if not native_dir.is_dir(): + raise SystemExit( + f"No 'native' directory found in NuGet package (extracted to {extract_dir})" + ) + + missing = [] + for tag, (filename, dest_subpath) in _TAG_TO_DEST.items(): + src = native_dir / tag / filename + if not src.is_file(): + missing.append(tag) + continue + dest_dir = libs_dir / dest_subpath + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest_dir / filename) + print(f"Synced {tag}: {src} -> {dest_dir / filename}") + + if missing: + raise SystemExit(f"Missing driver binary for tag(s): {', '.join(missing)}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--feed-url", default=DEFAULT_FEED_URL) + parser.add_argument( + "--version", default="", help="Override eng/versions/mssql-odbc-native.version" + ) + args = parser.parse_args() + + version = _read_version(args.version) + print(f"mssql-odbc-native version: {version}") + + libs_dir = REPO_ROOT / "mssql_python_rust_odbc" / "libs" + + with tempfile.TemporaryDirectory(prefix="mssql-odbc-native-") as tmp: + tmp_path = Path(tmp) + nupkg_path = _download_nupkg(args.feed_url, version, tmp_path) + + extract_dir = tmp_path / "extracted" + with zipfile.ZipFile(nupkg_path) as zf: + zf.extractall(extract_dir) + + _sync_libs(extract_dir, libs_dir) + + print("=== mssql_python_rust_odbc/libs/ synced successfully ===") + print("Review and 'git add mssql_python_rust_odbc/libs/' to commit the updated binaries.") + + +if __name__ == "__main__": + main() diff --git a/eng/versions/mssql-odbc-native.version b/eng/versions/mssql-odbc-native.version new file mode 100644 index 000000000..43c5a8b89 --- /dev/null +++ b/eng/versions/mssql-odbc-native.version @@ -0,0 +1 @@ +0.1.0-dev.20260902.171720 diff --git a/mssql_python_rust_odbc/__init__.py b/mssql_python_rust_odbc/__init__.py new file mode 100644 index 000000000..16fb8d260 --- /dev/null +++ b/mssql_python_rust_odbc/__init__.py @@ -0,0 +1,42 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +mssql_python_rust_odbc — mssql-odbc (Rust) native driver binaries. + +Internal implementation package for ``mssql-python``. It ships the +platform-specific ``mssql-odbc`` driver binaries built by ``mssql-rs`` so that +``mssql-python`` does not have to bundle them in its own wheel. It is not +meant for direct consumption — select the ``mssql-odbc`` provider via +``mssql_python.native_provider`` or the ``MSSQL_PYTHON_NATIVE_PROVIDER`` +environment variable instead, which pulls this package in automatically. + +Driver-path resolution lives entirely in the native +``mssql_python.ddbc_bindings`` extension (``GetOdbcLibsBaseDir`` / +``GetDriverPathCpp``): it imports this package purely for its ``__file__`` and +appends ``libs////...`` itself. Keeping a single (C++) +resolver avoids a second copy of the platform/arch/filename logic that could +silently drift out of sync. +""" + +import os + +__all__ = ["get_libs_dir", "__version__"] + +# Version tracks the published mssql-odbc-native NuGet package (built from +# mssql-rs's mssql-odbc/Cargo.toml) and is the single source of truth for the +# driver version. ``setup_rust_odbc.py`` reads it for the wheel version. Bump +# this value when syncing a newer mssql-odbc-native build (see +# eng/scripts/sync-mssql-odbc-native-libs.py and eng/versions/mssql-odbc-native.version). +__version__ = "0.1.0" + + +def get_libs_dir() -> str: + """Return the absolute path to this package's ``libs/`` directory. + + This is the root under which the platform-specific mssql-odbc binaries + live (``libs///...``). The parent of this path + (the package directory) is the base the native loader appends ``libs`` to + when resolving the driver. + """ + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "libs") diff --git a/mssql_python_rust_odbc/libs/LICENSING b/mssql_python_rust_odbc/libs/LICENSING new file mode 100644 index 000000000..375ef37ff --- /dev/null +++ b/mssql_python_rust_odbc/libs/LICENSING @@ -0,0 +1,6 @@ +A. The native libraries in this libs/ tree (mssqlodbc.so / mssqlodbc.dylib / mssqlodbc.dll) +==================================================================================================== +These binaries are the mssql-odbc ODBC driver, built from source in microsoft/mssql-rs and +distributed under the MIT License. + +For the license text, refer to: mssql_python_rust_odbc/licenses/MSSQL_ODBC_LICENSE.txt diff --git a/mssql_python_rust_odbc/libs/linux/glibc/arm64/lib/mssqlodbc.so b/mssql_python_rust_odbc/libs/linux/glibc/arm64/lib/mssqlodbc.so new file mode 100644 index 000000000..21cdda133 Binary files /dev/null and b/mssql_python_rust_odbc/libs/linux/glibc/arm64/lib/mssqlodbc.so differ diff --git a/mssql_python_rust_odbc/libs/linux/glibc/x86_64/lib/mssqlodbc.so b/mssql_python_rust_odbc/libs/linux/glibc/x86_64/lib/mssqlodbc.so new file mode 100644 index 000000000..9b8609951 Binary files /dev/null and b/mssql_python_rust_odbc/libs/linux/glibc/x86_64/lib/mssqlodbc.so differ diff --git a/mssql_python_rust_odbc/libs/linux/musl/arm64/lib/mssqlodbc.so b/mssql_python_rust_odbc/libs/linux/musl/arm64/lib/mssqlodbc.so new file mode 100644 index 000000000..5aa86a644 Binary files /dev/null and b/mssql_python_rust_odbc/libs/linux/musl/arm64/lib/mssqlodbc.so differ diff --git a/mssql_python_rust_odbc/libs/linux/musl/x86_64/lib/mssqlodbc.so b/mssql_python_rust_odbc/libs/linux/musl/x86_64/lib/mssqlodbc.so new file mode 100644 index 000000000..afdee45dd Binary files /dev/null and b/mssql_python_rust_odbc/libs/linux/musl/x86_64/lib/mssqlodbc.so differ diff --git a/mssql_python_rust_odbc/libs/windows/arm64/mssqlodbc.dll b/mssql_python_rust_odbc/libs/windows/arm64/mssqlodbc.dll new file mode 100644 index 000000000..7b6e01cd4 Binary files /dev/null and b/mssql_python_rust_odbc/libs/windows/arm64/mssqlodbc.dll differ diff --git a/mssql_python_rust_odbc/libs/windows/x64/mssqlodbc.dll b/mssql_python_rust_odbc/libs/windows/x64/mssqlodbc.dll new file mode 100644 index 000000000..7fb21bfda Binary files /dev/null and b/mssql_python_rust_odbc/libs/windows/x64/mssqlodbc.dll differ diff --git a/mssql_python_rust_odbc/licenses/MSSQL_ODBC_LICENSE.txt b/mssql_python_rust_odbc/licenses/MSSQL_ODBC_LICENSE.txt new file mode 100644 index 000000000..48ea6616b --- /dev/null +++ b/mssql_python_rust_odbc/licenses/MSSQL_ODBC_LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE diff --git a/setup.py b/setup.py index 921521ff3..847d573f6 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,34 @@ def _read_odbc_version() -> str: ) +def _read_rust_odbc_version() -> str: + """Return the ``mssql-python-rust-odbc`` version -- the single source of truth + for the rust-odbc dependency pin. + + Mirrors ``_read_odbc_version`` exactly (checkout ``__init__.py`` first, then + installed metadata, ``SystemExit`` if neither resolves). No stable + ``mssql-odbc-native`` release exists yet, so this currently pins whatever + dev version is checked in (see ``eng/versions/mssql-odbc-native.version``); + bump it the same way as the classic pin once a stable release exists. + """ + init_file = PROJECT_ROOT / "mssql_python_rust_odbc" / "__init__.py" + if init_file.is_file(): + text = init_file.read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']', text, re.MULTILINE) + if match: + return match.group(1) + + from importlib.metadata import version, PackageNotFoundError + + try: + return version("mssql-python-rust-odbc") + except PackageNotFoundError: + raise SystemExit( + "Could not determine the mssql-python-rust-odbc version: neither " + f"{init_file} exists nor is the mssql-python-rust-odbc package installed." + ) + + # Custom distribution to force platform-specific wheel class BinaryDistribution(Distribution): def has_ext_modules(self): @@ -160,12 +188,20 @@ def run(self): # --------------------------------------------------------------------------- # Find all packages in the current directory. -# Exclude mssql_python_odbc: it is shipped exclusively by the standalone -# mssql-python-odbc distribution (see setup_odbc.py) and pulled in via -# install_requires. Shipping it here too would make two distributions own the -# same import directory (install-order file overwrites; uninstall of one can -# remove files the other needs). -packages = find_packages(exclude=["mssql_python_odbc", "mssql_python_odbc.*"]) +# Exclude mssql_python_odbc and mssql_python_rust_odbc: each is shipped +# exclusively by its own standalone distribution (see setup_odbc.py / +# setup_rust_odbc.py) and pulled in via install_requires/extras_require. +# Shipping either here too would make two distributions own the same import +# directory (install-order file overwrites; uninstall of one can remove files +# the other needs). +packages = find_packages( + exclude=[ + "mssql_python_odbc", + "mssql_python_odbc.*", + "mssql_python_rust_odbc", + "mssql_python_rust_odbc.*", + ] +) # Get platform info using consolidated function arch, platform_tag = get_platform_info() @@ -195,6 +231,10 @@ def run(self): ], } +extras_require = { + "pyarrow": ["pyarrow>=14.0.0"], +} + setup( name="mssql-python", version="1.14.0", @@ -216,10 +256,14 @@ def run(self): # mssql_python_odbc.__version__ (single source of truth) so it can never # drift from the published mssql-python-odbc package. f"mssql-python-odbc=={_read_odbc_version()}", + # mssql-odbc (Rust) provider binaries (standalone package). Same pin + # pattern as above, derived from mssql_python_rust_odbc.__version__. + # Installed by default but not loaded unless selected at runtime via + # mssql_python.native_provider = "mssql-odbc" (or the + # MSSQL_PYTHON_NATIVE_PROVIDER env var) -- see mssql_python/odbc_provider.py. + f"mssql-python-rust-odbc=={_read_rust_odbc_version()}", ], - extras_require={ - "pyarrow": ["pyarrow>=14.0.0"], - }, + extras_require=extras_require, classifiers=[ "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", diff --git a/setup_rust_odbc.py b/setup_rust_odbc.py new file mode 100644 index 000000000..751103503 --- /dev/null +++ b/setup_rust_odbc.py @@ -0,0 +1,249 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Build script for the ``mssql-python-rust-odbc`` package. + +This packages the mssql-odbc (Rust) driver binaries, built by +`microsoft/mssql-rs `_ and published as +the ``mssql-odbc-native`` NuGet package, into a standalone, platform-specific +wheel. ``mssql-python`` depends on this package only when the ``mssql-odbc`` +provider is selected (see ``mssql_python.native_provider`` / +``MSSQL_PYTHON_NATIVE_PROVIDER``). Build it with:: + + python setup_rust_odbc.py bdist_wheel + +The driver binaries live under ``mssql_python_rust_odbc/libs/`` (the committed +source of truth, populated by +``eng/scripts/sync-mssql-odbc-native-libs.py``). Each wheel ships ONLY its own +platform's ``libs/`` subtree (see ``_target_libs_globs``); a single build host +can produce every platform's wheel via ``RUST_ODBC_TARGET_PLATFORM_TAG`` / +``RUST_ODBC_TARGET_ARCH`` (see ``get_platform_info``). +""" + +import os +import re +import sys +from pathlib import Path + +import setuptools +from setuptools import setup +from setuptools.dist import Distribution +from wheel.bdist_wheel import bdist_wheel + +PROJECT_ROOT = Path(__file__).resolve().parent +PACKAGE_NAME = "mssql_python_rust_odbc" +PACKAGE_DIR = PROJECT_ROOT / PACKAGE_NAME + +# See setup_odbc.py for why this is required: recursive ``libs/**/*`` globs in +# ``package_data`` need setuptools >= 62.3.0. +MIN_SETUPTOOLS = (62, 3, 0) + + +def _read_version() -> str: + """Return ``__version__`` from ``mssql_python_rust_odbc/__init__.py``. + + Single source of truth for the package version: the value is defined once + in the package's ``__init__.py`` and read here (by regex, without + importing the package) so the wheel version can never drift from what the + package reports at runtime. + """ + init_file = PACKAGE_DIR / "__init__.py" + text = init_file.read_text(encoding="utf-8") + match = re.search(r'^__version__\s*=\s*["\']([^"\']+)["\']', text, re.MULTILINE) + if not match: + raise SystemExit(f"Could not find __version__ in {init_file}") + return match.group(1) + + +def _require_min_setuptools() -> None: + raw = setuptools.__version__ + parts = tuple(int(m) for m in re.findall(r"\d+", raw)[:3]) + parts += (0,) * (3 - len(parts)) + if parts < MIN_SETUPTOOLS: + raise SystemExit( + "setup_rust_odbc.py requires setuptools >= " + f"{'.'.join(map(str, MIN_SETUPTOOLS))} to package the driver " + "binaries via the recursive 'libs/**/*' glob; found setuptools " + f"{raw}. Upgrade with:\n" + ' python -m pip install --upgrade "setuptools>=62.3.0"' + ) + + +class BinaryDistribution(Distribution): + """Force a platform-specific wheel (the package ships native binaries).""" + + def has_ext_modules(self): + return True + + +def get_platform_info(): + """Get platform-specific architecture and platform tag information. + + Kept in sync with ``setup.py`` / ``setup_odbc.py`` so this wheel carries + the same platform tags as the rest of the mssql-python distribution. + """ + # Explicit target override for single-host cross-building (see + # get_platform_info in setup_odbc.py for the same pattern). Distinct env + # var names so this script can run alongside setup_odbc.py without either + # one picking up the other's target. + explicit_tag = os.environ.get("RUST_ODBC_TARGET_PLATFORM_TAG") + if explicit_tag: + target_arch = os.environ.get("RUST_ODBC_TARGET_ARCH", "").strip() + if not target_arch: + raise OSError( + "RUST_ODBC_TARGET_ARCH must be set (non-empty) when " + "RUST_ODBC_TARGET_PLATFORM_TAG is provided: an empty arch would expand " + "the libs/ package_data globs to EVERY architecture's subtree and leak " + "foreign-platform driver binaries into the wheel." + ) + return target_arch, explicit_tag + + if sys.platform.startswith("win"): + arch = os.environ.get("ARCHITECTURE", "x64") + if isinstance(arch, str): + arch = arch.strip("\"'") + if arch == "arm64": + return "arm64", "win_arm64" + elif arch in ["x86", "win32"]: + raise OSError( + "mssql-odbc has no Windows x86 build lane; only x64 and arm64 are " "supported." + ) + else: + return "x64", "win_amd64" + + elif sys.platform.startswith("darwin"): + return "universal2", "macosx_15_0_universal2" + + elif sys.platform.startswith("linux"): + import platform + + target_arch = os.environ.get("targetArch", platform.machine()) + libc_name, _ = platform.libc_ver() + is_musl = libc_name == "" or "musl" in libc_name.lower() + manylinux_tag = os.environ.get("MANYLINUX_TAG", "manylinux_2_28") + + if target_arch == "x86_64": + return "x86_64", "musllinux_1_2_x86_64" if is_musl else f"{manylinux_tag}_x86_64" + elif target_arch in ["aarch64", "arm64"]: + return "aarch64", "musllinux_1_2_aarch64" if is_musl else f"{manylinux_tag}_aarch64" + else: + raise OSError( + f"Unsupported architecture '{target_arch}' for Linux; " + f"expected 'x86_64' or 'aarch64'." + ) + + raise OSError(f"Unsupported platform: {sys.platform!r}") + + +def _target_libs_globs(platform_tag: str, arch: str) -> list: + """Return the ``package_data`` globs for exactly ONE target platform's libs. + + The committed ``mssql_python_rust_odbc/libs/`` tree holds every supported + platform's driver binary. A wheel must ship only its own platform's + subtree, so we translate the (``platform_tag``, ``arch``) of the wheel + being built into the minimal set of ``libs/`` globs -- matching exactly + what ``GetDriverPathCpp`` (mssql_python/pybind/ddbc_bindings.cpp) resolves + for the ``mssql-odbc`` provider. Combined with ``include_package_data=False`` + this guarantees a Windows wheel never carries Linux binaries (and vice + versa), whether the build runs on the native OS or is cross-built on a + single host via the ``RUST_ODBC_TARGET_*`` overrides. + """ + globs = ["libs/LICENSING"] + tag = platform_tag.lower() + + def _subtree(root: str) -> None: + globs.append(f"{root}/*") + globs.append(f"{root}/**/*") + + if tag.startswith("win"): + # arch is already the libs dir name on Windows: x64 / arm64. + _subtree(f"libs/windows/{arch}") + elif tag.startswith("macos"): + # The universal2 wheel serves both slices (arm64 + x86_64). + _subtree("libs/macos") + elif "musllinux" in tag: + libs_arch = "arm64" if arch in ("aarch64", "arm64") else "x86_64" + _subtree(f"libs/linux/musl/{libs_arch}") + elif "manylinux" in tag: + # manylinux_2_28 is the broadest-compatibility glibc build (see + # mssql-rs's build-odbc-glibc228-template.yml); it is the ONLY glibc + # variant packaged here. mssql-rs also builds a plain 'linux_*' + # variant against a newer host glibc for its own distro-container + # testing -- that one is not manylinux-portable and is intentionally + # not packaged into this wheel. + libs_arch = "arm64" if arch in ("aarch64", "arm64") else "x86_64" + _subtree(f"libs/linux/glibc/{libs_arch}") + else: + raise OSError(f"Cannot determine libs subtree for platform tag {platform_tag!r}") + return globs + + +class CustomBdistWheel(bdist_wheel): + """Force a platform-specific but Python-agnostic tag. + + The package ships only pre-built driver binaries (data), not a compiled + Python extension, so one ``py3-none-`` wheel serves every + supported Python version (3.10+). See ``setup_odbc.py``'s + ``CustomBdistWheel`` for the full rationale -- identical here. + """ + + def finalize_options(self): + bdist_wheel.finalize_options(self) + arch, platform_tag = get_platform_info() + self.plat_name = platform_tag + self.plat_name_supplied = True + self.root_is_pure = False + print(f"Setting wheel platform tag to: {self.plat_name} (arch: {arch})") + + def get_tag(self): + _python, _abi, plat = bdist_wheel.get_tag(self) + return "py3", "none", plat + + +_require_min_setuptools() + +_TARGET_ARCH, _TARGET_PLATFORM_TAG = get_platform_info() +_LIBS_GLOBS = _target_libs_globs(_TARGET_PLATFORM_TAG, _TARGET_ARCH) +print(f"Rust ODBC wheel target: tag={_TARGET_PLATFORM_TAG} arch={_TARGET_ARCH!r}") +print(f"Rust ODBC libs globs: {_LIBS_GLOBS}") + +_LONG_DESCRIPTION = (PROJECT_ROOT / "PyPI_Description_RustODBC.md").read_text(encoding="utf-8") + +setup( + name="mssql-python-rust-odbc", + version=_read_version(), + description=( + "Internal implementation package for mssql-python: mssql-odbc (Rust) " + "driver binaries. Not intended for direct use." + ), + long_description=_LONG_DESCRIPTION, + long_description_content_type="text/markdown", + author="Microsoft Corporation", + author_email="mssql-python@microsoft.com", + url="https://github.com/microsoft/mssql-python", + license="MIT", + license_files=[ + "mssql_python_rust_odbc/licenses/MSSQL_ODBC_LICENSE.txt", + ], + packages=[PACKAGE_NAME], + package_data={ + PACKAGE_NAME: _LIBS_GLOBS, + }, + # include_package_data MUST stay False: the committed libs/ tree holds + # EVERY platform, so SCM-based inclusion would sweep them all into every + # wheel. We rely solely on the target-specific _LIBS_GLOBS above. + include_package_data=False, + python_requires=">=3.10", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + ], + zip_safe=False, + distclass=BinaryDistribution, + cmdclass={ + "bdist_wheel": CustomBdistWheel, + }, +) diff --git a/tests/test_026_odbc_provider.py b/tests/test_026_odbc_provider.py index 5b5b73e34..c64fe2f76 100644 --- a/tests/test_026_odbc_provider.py +++ b/tests/test_026_odbc_provider.py @@ -134,6 +134,58 @@ def test_rust_provider_driver_path_matches_packaging_layout(): assert Path(proc.stdout.strip()) == expected +def test_rust_provider_driver_loads_real_binary(): + # Smoke test: the packaged mssql-odbc binary for this host is a genuine, + # loadable ODBC driver -- not just a path-string match. No live SQL Server + # is needed: LoadDriverOrThrowException() resolves every required ODBC + # function pointer by name before any network I/O happens, so even a + # connection attempt against an unreachable server proves the load and + # symbol resolution succeeded, as long as the failure is a connect-time + # error rather than one of its "driver not found"/"failed to load" errors. + try: + importlib.import_module("mssql_python_rust_odbc") + except ImportError: + pytest.skip("mssql-python-rust-odbc not installed; build via setup_rust_odbc.py") + + if sys.platform == "darwin": + pytest.skip("mssql-odbc has no macOS build yet") + + script = """ +from mssql_python import ddbc_bindings + +ddbc_bindings._set_odbc_provider("mssql-odbc") +try: + ddbc_bindings.Connection( + "Driver=x;Server=127.0.0.1,1;Database=x;UID=x;PWD=x;Encrypt=no;", + False, + {}, + "", + None, + ) + print("CONNECTED") +except RuntimeError as e: + print(f"ERROR: {e}") +""" + proc = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr + + output = proc.stdout.strip() + load_failure_markers = ( + "driver not found", + "failed to load the driver", + "failed to load required function pointers", + "failed to load library", + ) + assert not any(marker in output.lower() for marker in load_failure_markers), ( + f"expected a connect-time failure (proving the driver loaded), got: " + f"{output}\n{proc.stderr}" + ) + + def test_direct_native_default_load_accepts_later_explicit_default(): # A direct native connection bypasses ProviderManager and therefore loads # the classic default without first pushing it into native selection state.