From 8ebba3f5659e00994effc7803c05b2e92dff1434 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 18 Mar 2026 23:31:14 +0000 Subject: [PATCH 1/4] Added synthetic commit history creation using gogit --- go.mod | 14 + go.sum | 52 ++ .../componentbuilder/componentbuilder_test.go | 8 +- .../app/azldev/core/sources/sourceprep.go | 281 ++++++--- .../azldev/core/sources/sourceprep_test.go | 21 +- .../app/azldev/core/sources/synthistory.go | 566 ++++++++++++++++++ .../azldev/core/sources/synthistory_test.go | 549 +++++++++++++++++ internal/projectconfig/configfile.go | 11 + .../sourceproviders/fedorasourceprovider.go | 23 +- .../sourceproviders/rpmcontentsprovider.go | 2 +- .../sourceproviders/sourcemanager.go | 51 +- .../sourcemanager_mocks.go | 14 +- 12 files changed, 1473 insertions(+), 119 deletions(-) create mode 100644 internal/app/azldev/core/sources/synthistory.go create mode 100644 internal/app/azldev/core/sources/synthistory_test.go diff --git a/go.mod b/go.mod index b60ca3b..b989b4d 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,8 @@ require ( github.com/docker/docker v28.5.2+incompatible github.com/fatih/color v1.18.0 github.com/gkampitakis/go-snaps v0.5.21 + github.com/go-git/go-billy/v5 v5.8.0 + github.com/go-git/go-git/v5 v5.17.0 github.com/go-playground/validator/v10 v10.30.1 github.com/google/renameio v1.0.1 github.com/google/uuid v1.6.0 @@ -59,6 +61,7 @@ require ( cyphar.com/go-pathrs v0.2.4 // indirect github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/buger/jsonparser v1.1.2 // indirect @@ -72,6 +75,7 @@ require ( github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -84,19 +88,24 @@ require ( github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gkampitakis/ciinfo v0.3.4 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/goccy/go-yaml v1.19.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -120,6 +129,7 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -129,6 +139,7 @@ require ( github.com/sergi/go-diff v1.4.0 // indirect github.com/shirou/gopsutil/v4 v4.26.2 // indirect github.com/sirupsen/logrus v1.9.4 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect @@ -137,6 +148,7 @@ require ( github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -150,8 +162,10 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/term v0.41.0 // indirect golang.org/x/text v0.35.0 // indirect google.golang.org/protobuf v1.36.8 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 5626a37..0529d58 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,19 @@ github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8af github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= github.com/acobaugh/osrelease v0.1.0/go.mod h1:4bFEs0MtgHNHBrmHCt67gNisnabCRAlzdVasCEGHTWY= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/auribuo/stylishcobra v1.0.1 h1:bKGeCcLCW1Pzk5ZiT2Ubijdccn+unGrB5SBf3TQBGeA= github.com/auribuo/stylishcobra v1.0.1/go.mod h1:NRNkmflkRM1078o68o3pFpPFZeb7wtD6EscbqLZmDsU= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -69,6 +76,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= @@ -103,6 +112,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= @@ -120,6 +133,16 @@ github.com/gkampitakis/ciinfo v0.3.4 h1:5eBSibVuSMbb/H6Elc0IIEFbkzCJi3lm94n0+U7Z github.com/gkampitakis/ciinfo v0.3.4/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-snaps v0.5.21 h1:SvhSFeZviQXwlT+dnGyAIATVehkhqRVW6qfQZhCZH+Y= github.com/gkampitakis/go-snaps v0.5.21/go.mod h1:gC3YqxQTPyIXvQrw/Vpt3a8VqR1MO8sVpZFWN4DGwNs= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= +github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.17.0 h1:AbyI4xf+7DsjINHMu35quAh4wJygKBKBuXVjV/pxesM= +github.com/go-git/go-git/v5 v5.17.0/go.mod h1:f82C4YiLx+Lhi8eHxltLeGC5uBTXSFa6PC5WW9o4SjI= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -138,6 +161,8 @@ github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy0 github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= @@ -154,10 +179,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E= github.com/invopop/jsonschema v0.13.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty/v6 v6.7.8 h1:BVYrDy5DPBA3Qn9ICT+PokP9cvCv1KaHv2i+Hc8sr5o= github.com/jedib0t/go-pretty/v6 v6.7.8/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/knqyf263/go-rpm-version v0.0.0-20240918084003-2afd7dc6a38f h1:xt29M2T6STgldg+WEP51gGePQCsQvklmP2eIhPIBK3g= @@ -229,6 +258,8 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -237,6 +268,8 @@ github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22 github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -264,8 +297,11 @@ github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKl github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -280,6 +316,7 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -307,6 +344,8 @@ github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= @@ -339,16 +378,24 @@ go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b h1:DXr+pvt3nC887026GRP39Ej11UATqWDmWuS99x26cD0= golang.org/x/exp v0.0.0-20250819193227-8b4c13bb791b/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -357,12 +404,15 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= @@ -377,6 +427,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go index 253e9de..7fdf7ec 100644 --- a/internal/app/azldev/core/componentbuilder/componentbuilder_test.go +++ b/internal/app/azldev/core/componentbuilder/componentbuilder_test.go @@ -21,6 +21,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/buildenv" "github.com/microsoft/azure-linux-dev-tools/internal/buildenv/buildenv_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" @@ -50,8 +51,11 @@ func setupBuilder(t *testing.T) *componentBuilderTestParams { sourceManager := sourceproviders_test.NewMockSourceManager(ctrl) // Configure the source manager to create a spec file when FetchComponent is called. - sourceManager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( - func(ctx context.Context, component components.Component, outputDir string) error { + sourceManager.EXPECT().FetchComponent(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().DoAndReturn( + func( + _ context.Context, component components.Component, + outputDir string, _ ...sourceproviders.FetchComponentOption, + ) error { // Create the expected spec file. specPath := filepath.Join(outputDir, component.GetName()+".spec") diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 7bc4c56..f5f9870 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -7,11 +7,13 @@ import ( "context" "errors" "fmt" + "log/slog" "path/filepath" "slices" "strings" "unicode" + gogit "github.com/go-git/go-git/v5" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" @@ -80,12 +82,14 @@ func NewPreparer( return nil, errors.New("dry runnable interface cannot be nil") } - return &sourcePreparerImpl{ + impl := &sourcePreparerImpl{ sourceManager: sourceManager, fs: fs, eventListener: eventListener, dryRunnable: dryRunnable, - }, nil + } + + return impl, nil } // PrepareSources implements the [SourcePreparer] interface. @@ -99,40 +103,170 @@ func (p *sourcePreparerImpl) PrepareSources( component.GetName(), err) } + // Preserve the upstream .git directory when overlays will be applied. This is + // required so that overlay commits can be appended on top of the upstream commit + // log during synthetic history generation. + var fetchOpts []sourceproviders.FetchComponentOption + if applyOverlays { + fetchOpts = append(fetchOpts, sourceproviders.WithPreserveGitDir()) + } + // Use the source manager to fetch the component (spec file and sidecar files). - err = p.sourceManager.FetchComponent(ctx, component, outputDir) + err = p.sourceManager.FetchComponent(ctx, component, outputDir, fetchOpts...) if err != nil { return fmt.Errorf("failed to fetch sources for component %#q:\n%w", component.GetName(), err) } - if applyOverlays { - // Emit computed macros to a macros file in the output directory. - // If the build configuration produces no macros, no file is written and - // macrosFileName will be empty, signaling postProcessSources to skip - // injecting the macro load directive and Source9999 tag. - var macrosFileName string - - macrosFilePath, err := p.writeMacrosFile(component, outputDir) - if err != nil { - return fmt.Errorf("failed to write macros file for component %#q:\n%w", - component.GetName(), err) - } + if !applyOverlays { + return nil + } - if macrosFilePath != "" { - macrosFileName = filepath.Base(macrosFilePath) - } + return p.applyOverlaysToSources(ctx, component, outputDir) +} - // Apply any postprocessing to the sources in-place, in the output directory. - err = p.postProcessSources(component, outputDir, macrosFileName) - if err != nil { - return fmt.Errorf("failed to post-process sources for component %q:\n%w", component.GetName(), err) - } +// applyOverlaysToSources writes the macros file and then applies overlays using either the +// synthetic history path or the standard postProcessSources path. +func (p *sourcePreparerImpl) applyOverlaysToSources( + ctx context.Context, component components.Component, outputDir string, +) error { + // Emit computed macros to a macros file in the output directory. + // If the build configuration produces no macros, no file is written and + // macrosFileName will be empty, signaling postProcessSources to skip + // injecting the macro load directive and Source9999 tag. + var macrosFileName string + + macrosFilePath, err := p.writeMacrosFile(component, outputDir) + if err != nil { + return fmt.Errorf("failed to write macros file for component %#q:\n%w", + component.GetName(), err) + } + + if macrosFilePath != "" { + macrosFileName = filepath.Base(macrosFilePath) + } + + // Apply user-defined overlays as attributed synthetic git commits, then apply + // system overlays separately. If synthetic history cannot be generated (e.g. no + // git repo available), applyOverlaysWithHistory falls back to the standard path. + err = p.applyOverlaysWithHistory(ctx, component, outputDir, macrosFileName) + if err != nil { + return fmt.Errorf("failed to post-process sources for component %q:\n%w", component.GetName(), err) + } + + return nil +} + +// applyOverlaysWithHistory applies user-defined overlays as attributed synthetic git +// commits in the upstream repository, then applies system overlays (macros, check-skip, +// header) outside of the synthetic history so they don't pollute blame attribution. +// +// When the sources directory does not contain a git repository (e.g. local or unspecified +// spec sources), the function falls back to applying overlays sequentially without history. +func (p *sourcePreparerImpl) applyOverlaysWithHistory( + _ context.Context, component components.Component, sourcesDirPath, macrosFileName string, +) error { + event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) + defer event.End() + + // Resolve the spec path once for all overlay operations in this call. + absSpecPath, err := p.resolveSpecPath(component, sourcesDirPath) + if err != nil { + return err + } + + config := component.GetConfig() + if len(config.Overlays) == 0 { + slog.Debug("No overlays defined; skipping synthetic history generation", + "component", component.GetName()) + + return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) + } + + // Try the synthetic history path. If it succeeds, apply system overlays and return. + if err := p.trySyntheticHistory(component, config, sourcesDirPath, absSpecPath); err == nil { + return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) + } + + // Synthetic history was not possible; apply user overlays sequentially. + if err := p.applyOverlayList(config.Overlays, sourcesDirPath, absSpecPath); err != nil { + return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) + } + + return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) +} + +// trySyntheticHistory attempts to apply user-defined overlays as synthetic git commits. +// Returns nil on success, or a non-nil error if the synthetic history path is not available +// (e.g. no .git directory, no resolvable blame groups). The caller should fall back to +// sequential overlay application when this returns an error. +func (p *sourcePreparerImpl) trySyntheticHistory( + component components.Component, + config *projectconfig.ComponentConfig, + sourcesDirPath, absSpecPath string, +) error { + // Check for an upstream git repository in the sources directory. Local and + // unspecified spec sources won't have a .git directory. + gitDirPath := filepath.Join(sourcesDirPath, ".git") + if _, err := p.fs.Stat(gitDirPath); err != nil { + slog.Debug("No .git directory in sources; falling back to standard overlay path", + "component", component.GetName(), + "sourcesDirPath", sourcesDirPath) + + return fmt.Errorf("no .git directory in sources dir %#q:\n%w", sourcesDirPath, err) + } + + // Resolve the project repository and blame the config file to produce overlay groups. + groups, err := buildOverlayGroups(p.fs, config, component.GetName()) + if err != nil { + return fmt.Errorf("failed to build overlay groups:\n%w", err) + } + + if len(groups) == 0 { + return errors.New("no overlay groups resolved") + } + + // Open the upstream git repository where synthetic commits will be recorded. + sourcesRepo, err := gogit.PlainOpen(sourcesDirPath) + if err != nil { + return fmt.Errorf("failed to open upstream repository at %#q:\n%w", sourcesDirPath, err) + } + + // Build the overlay-application callback using the shared helper. + applyFn := func(overlays []projectconfig.ComponentOverlay) error { + return p.applyOverlayList(overlays, sourcesDirPath, absSpecPath) + } + + if err := CommitSyntheticHistory(sourcesRepo, groups, applyFn); err != nil { + return fmt.Errorf("failed to commit synthetic history:\n%w", err) } return nil } +// applySystemOverlays applies the macros-load, check-skip, and file-header overlays that +// are synthesized at build time rather than defined in the project TOML config. +func (p *sourcePreparerImpl) applySystemOverlays( + component components.Component, sourcesDirPath, absSpecPath, macrosFileName string, +) error { + // Collect all system overlays in application order: macros, check-skip, file header. + var systemOverlays []projectconfig.ComponentOverlay + + if macrosFileName != "" { + macroOverlays, macroErr := synthesizeMacroLoadOverlays(macrosFileName) + if macroErr != nil { + return fmt.Errorf("failed to compute macros load overlays:\n%w", macroErr) + } + + systemOverlays = append(systemOverlays, macroOverlays...) + } + + systemOverlays = append(systemOverlays, synthesizeCheckSkipOverlays(component.GetConfig().Build.Check)...) + systemOverlays = append(systemOverlays, generateFileHeaderOverlay()...) + + return p.applyOverlayList(systemOverlays, sourcesDirPath, absSpecPath) +} + // DiffSources implements the [SourcePreparer] interface. // It fetches the component's sources once, copies them to a second directory, applies overlays // to the copy, then diffs the two trees. This avoids fetching the sources twice. @@ -286,74 +420,6 @@ func GenerateMacrosFileContents(buildConfig projectconfig.ComponentBuildConfig) return strings.Join(lines, "\n") + "\n" } -func (p *sourcePreparerImpl) postProcessSources( - component components.Component, sourcesDirPath, macrosFileName string, -) error { - event := p.eventListener.StartEvent("Applying overlays to sources", "component", component.GetName()) - defer event.End() - - // Find the spec. - specPath, err := findSpecInDir(p.fs, component, sourcesDirPath) - if err != nil { - return fmt.Errorf("failed to find spec in acquired sources dir %#q:\n%w", sourcesDirPath, err) - } - - // Get an absolute path to the spec for better errors. - absSpecPath, err := filepath.Abs(specPath) - if err != nil { - return fmt.Errorf("failed to get absolute path for %#q:\n%w", specPath, err) - } - - // Compute any synthetic overlays required to load the macros file, if one was written. - if macrosFileName != "" { - macroOverlays, macroErr := synthesizeMacroLoadOverlays(macrosFileName) - if macroErr != nil { - return fmt.Errorf("failed to compute macros load overlays:\n%w", macroErr) - } - - // Apply those overlays *first*, in sequence. - for _, overlay := range macroOverlays { - err = ApplyOverlayToSources(p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath) - if err != nil { - return fmt.Errorf("failed to apply system overlay to sources for component %#q:\n%w", component.GetName(), err) - } - } - } - - // Get the file header overlay. - headerOverlay := generateFileHeaderOverlay() - - // Apply all overlays in sequence. - for _, overlay := range component.GetConfig().Overlays { - err = ApplyOverlayToSources(p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath) - if err != nil { - return fmt.Errorf( - "failed to apply %#q overlay to sources for component %#q:\n%w", - overlay.Type, component.GetName(), err, - ) - } - } - - // Apply check skip overlay if configured. - checkSkipOverlays := synthesizeCheckSkipOverlays(component.GetConfig().Build.Check) - for _, overlay := range checkSkipOverlays { - err = ApplyOverlayToSources(p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath) - if err != nil { - return fmt.Errorf("failed to apply check skip overlay to sources for component %#q:\n%w", component.GetName(), err) - } - } - - // Finally, apply the file header overlay. - for _, overlay := range headerOverlay { - err = ApplyOverlayToSources(p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath) - if err != nil { - return fmt.Errorf("failed to apply file header overlay to sources for component %#q:\n%w", component.GetName(), err) - } - } - - return nil -} - func synthesizeMacroLoadOverlays(macrosFileName string) ([]projectconfig.ComponentOverlay, error) { // Basic check that the macros file name is valid and doesn't require escaping. if strings.ContainsFunc(macrosFileName, func(r rune) bool { @@ -427,6 +493,39 @@ func synthesizeCheckSkipOverlays(checkConfig projectconfig.CheckConfig) []projec } } +// resolveSpecPath locates and returns the absolute path to the component's spec file +// within the given sources directory. +func (p *sourcePreparerImpl) resolveSpecPath( + component components.Component, sourcesDirPath string, +) (string, error) { + specPath, err := findSpecInDir(p.fs, component, sourcesDirPath) + if err != nil { + return "", fmt.Errorf("failed to find spec in sources dir %#q:\n%w", sourcesDirPath, err) + } + + absSpecPath, err := filepath.Abs(specPath) + if err != nil { + return "", fmt.Errorf("failed to get absolute path for spec %#q:\n%w", specPath, err) + } + + return absSpecPath, nil +} + +// applyOverlayList applies a list of overlays to the component sources sequentially. +func (p *sourcePreparerImpl) applyOverlayList( + overlays []projectconfig.ComponentOverlay, sourcesDirPath, absSpecPath string, +) error { + for _, overlay := range overlays { + if err := ApplyOverlayToSources( + p.dryRunnable, p.fs, overlay, sourcesDirPath, absSpecPath, + ); err != nil { + return fmt.Errorf("failed to apply %#q overlay:\n%w", overlay.Type, err) + } + } + + return nil +} + func findSpecInDir( fs opctx.FS, component components.Component, dirPath string, ) (string, error) { diff --git a/internal/app/azldev/core/sources/sourceprep_test.go b/internal/app/azldev/core/sources/sourceprep_test.go index 4e5f2f1..edb4bf1 100644 --- a/internal/app/azldev/core/sources/sourceprep_test.go +++ b/internal/app/azldev/core/sources/sourceprep_test.go @@ -13,6 +13,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/stretchr/testify/assert" @@ -56,8 +57,8 @@ func TestPrepareSources_Success(t *testing.T) { component.EXPECT().GetName().AnyTimes().Return("test-component") component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{}) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) - sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { // Create the expected spec file. return fileutils.WriteFile(ctx.FS(), outputSpecPath, []byte("# test spec"), 0o644) }, @@ -115,8 +116,8 @@ func TestPrepareSources_WritesMacrosFile(t *testing.T) { }, }) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) - sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { // Create the expected spec file. specPath := filepath.Join(outputDir, "my-package.spec") @@ -365,8 +366,8 @@ func TestPrepareSources_CheckSkip(t *testing.T) { }, }) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) - sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { // Create the expected spec file with a %check section. specContent := `Name: test-component Version: 1.0 @@ -422,8 +423,8 @@ func TestPrepareSources_CheckSkipDisabled(t *testing.T) { }, }) sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil) - sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { // Create the expected spec file with a %check section. specContent := `Name: test-component Version: 1.0 @@ -469,7 +470,7 @@ func TestDiffSources_NoOverlays(t *testing.T) { // DiffSources fetches sources once, then copies them for overlay application. sourceManager.EXPECT().FetchFiles(gomock.Any(), component, gomock.Any()).Times(1).Return(nil) sourceManager.EXPECT().FetchComponent(gomock.Any(), component, gomock.Any()).Times(1).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { specPath := filepath.Join(outputDir, "test-component.spec") return fileutils.WriteFile(ctx.FS(), specPath, []byte("Name: test-component\nVersion: 1.0\n"), 0o644) @@ -512,7 +513,7 @@ func TestDiffSources_WithOverlays(t *testing.T) { // DiffSources fetches sources once, then copies them for overlay application. sourceManager.EXPECT().FetchFiles(gomock.Any(), component, gomock.Any()).Times(1).Return(nil) sourceManager.EXPECT().FetchComponent(gomock.Any(), component, gomock.Any()).Times(1).DoAndReturn( - func(_ interface{}, _ interface{}, outputDir string) error { + func(_ interface{}, _ interface{}, outputDir string, _ ...sourceproviders.FetchComponentOption) error { specPath := filepath.Join(outputDir, "test-component.spec") return fileutils.WriteFile(ctx.FS(), specPath, []byte("Name: test-component\nVersion: 1.0\n"), 0o644) diff --git a/internal/app/azldev/core/sources/synthistory.go b/internal/app/azldev/core/sources/synthistory.go new file mode 100644 index 0000000..d641c7c --- /dev/null +++ b/internal/app/azldev/core/sources/synthistory.go @@ -0,0 +1,566 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +var ( + // ErrNoGitRepository is returned when no enclosing git repository can be found. + ErrNoGitRepository = errors.New("no git repository found") + + // ErrNoOverlaysToCommit is returned when there are no overlay groups to commit. + ErrNoOverlaysToCommit = errors.New("no overlays to commit") + + // ErrLineRangeOverlayMismatch is returned when the number of located overlay line ranges + // does not match the number of overlays on the component. + ErrLineRangeOverlayMismatch = errors.New("line range count does not match overlay count") + + // sectionHeaderRegexp matches any TOML table or array-of-tables header line. + sectionHeaderRegexp = regexp.MustCompile(`^\s*\[{1,2}[^\]]+\]{1,2}\s*$`) +) + +// BlameEntry represents a single line's blame information from a git repository. +type BlameEntry struct { + // CommitHash is the hash of the commit that last modified this line. + CommitHash string + // Author is the name of the author who last modified this line. + Author string + // Timestamp is when the line was last modified. + Timestamp int64 + // Line is the 1-based line number. + Line int + // Content is the text content of the line. + Content string +} + +// CommitMetadata holds full metadata for a commit in the project repository. +type CommitMetadata struct { + Hash string + Author string + AuthorEmail string + Timestamp int64 + Message string +} + +// OverlayCommitGroup groups overlays that originate from the same git commit in the project +// configuration repository. During synthetic history generation, all overlays in a group are +// applied together and recorded as a single commit. +type OverlayCommitGroup struct { + // Commit holds metadata from the originating commit in the project repository. + Commit CommitMetadata + // Overlays contains the overlay definitions to apply as part of this synthetic commit. + Overlays []projectconfig.ComponentOverlay +} + +// OverlayApplyFunc is a callback that applies a batch of overlays to the component sources. +// It is called once per [OverlayCommitGroup] during synthetic history generation. +type OverlayApplyFunc func(overlays []projectconfig.ComponentOverlay) error + +// ConfigBlameResult holds the per-line blame entries for a configuration file. +type ConfigBlameResult struct { + // Entries contains one [BlameEntry] per line in the blamed file. + Entries []BlameEntry +} + +// OverlayLineRange tracks the line range of a single [[components.X.overlays]] block +// in a TOML config file. +type OverlayLineRange struct { + StartLine int // 1-based, inclusive (the [[...]] header line) + EndLine int // 1-based, inclusive + Index int // positional index in the component's overlays slice +} + +// BlameFile performs git blame on the specified file within the provided go-git repository. +// The filePath must be relative to the repository root. +func BlameFile(repo *gogit.Repository, filePath string) (*ConfigBlameResult, error) { + head, err := repo.Head() + if err != nil { + return nil, fmt.Errorf("failed to get HEAD reference:\n%w", err) + } + + commit, err := repo.CommitObject(head.Hash()) + if err != nil { + return nil, fmt.Errorf("failed to get HEAD commit:\n%w", err) + } + + blameResult, err := gogit.Blame(commit, filePath) + if err != nil { + return nil, fmt.Errorf("failed to blame file %#q:\n%w", filePath, err) + } + + entries := make([]BlameEntry, len(blameResult.Lines)) + for i, line := range blameResult.Lines { + entries[i] = BlameEntry{ + CommitHash: line.Hash.String(), + Author: line.AuthorName, + Timestamp: line.Date.Unix(), + Line: i + 1, + Content: line.Text, + } + } + + return &ConfigBlameResult{Entries: entries}, nil +} + +// FindOverlayLineRanges parses raw TOML content to locate the line ranges of all overlay +// definitions for the named component. It supports two TOML styles: +// +// 1. Array-of-tables: [[components..overlays]] blocks. +// 2. Inline array: overlays = [ { ... }, { ... } ] under a [components.] section. +// +// The returned ranges are ordered by their position in the file, matching the +// serialization order of the component's overlay slice. +func FindOverlayLineRanges(configContent string, componentName string) []OverlayLineRange { + lines := strings.Split(configContent, "\n") + + ranges := findArrayOfTablesOverlays(lines, componentName) + if len(ranges) > 0 { + return ranges + } + + return findInlineArrayOverlays(lines, componentName) +} + +// findArrayOfTablesOverlays locates overlays declared as [[components..overlays]] blocks. +func findArrayOfTablesOverlays(lines []string, componentName string) []OverlayLineRange { + expectedHeaders := []string{ + fmt.Sprintf("[[components.%s.overlays]]", componentName), + fmt.Sprintf(`[[components."%s".overlays]]`, componentName), + } + + var ranges []OverlayLineRange + + overlayIndex := 0 + + for lineIdx := 0; lineIdx < len(lines); lineIdx++ { + trimmed := strings.TrimSpace(lines[lineIdx]) + + if !slices.Contains(expectedHeaders, trimmed) { + continue + } + + startLine := lineIdx + 1 // convert to 1-based + + // Find the end of this overlay block: the line before the next section header, or EOF. + endLineExclusive := len(lines) + for j := lineIdx + 1; j < len(lines); j++ { + if sectionHeaderRegexp.MatchString(lines[j]) { + endLineExclusive = j + + break + } + } + + for endLineExclusive > lineIdx+1 && strings.TrimSpace(lines[endLineExclusive-1]) == "" { + endLineExclusive-- + } + + ranges = append(ranges, OverlayLineRange{ + StartLine: startLine, + EndLine: endLineExclusive, + Index: overlayIndex, + }) + + overlayIndex++ + lineIdx = endLineExclusive - 1 // advance past this block (loop increments) + } + + return ranges +} + +// findInlineArrayOverlays locates overlays declared as an inline array under a +// [components.] section (e.g. overlays = [ { type = "patch-add", ... }, ... ]). +func findInlineArrayOverlays(lines []string, componentName string) []OverlayLineRange { + sectionHeaders := []string{ + fmt.Sprintf("[components.%s]", componentName), + fmt.Sprintf(`[components."%s"]`, componentName), + } + + // Locate the section header for this component. + sectionStart := -1 + + for i, line := range lines { + if slices.Contains(sectionHeaders, strings.TrimSpace(line)) { + sectionStart = i + + break + } + } + + if sectionStart < 0 { + return nil + } + + // Scan forward from the section header to find "overlays = [", stopping at the next + // section header. + overlaysStart := -1 + + for lineIdx := sectionStart + 1; lineIdx < len(lines); lineIdx++ { + if sectionHeaderRegexp.MatchString(lines[lineIdx]) { + break + } + + trimmed := strings.TrimSpace(lines[lineIdx]) + if strings.HasPrefix(trimmed, "overlays") && strings.Contains(trimmed, "=") && strings.Contains(trimmed, "[") { + overlaysStart = lineIdx + + break + } + } + + if overlaysStart < 0 { + return nil + } + + return parseInlineOverlayEntries(lines, overlaysStart) +} + +// parseInlineOverlayEntries parses individual { ... } entries from an inline overlay array +// starting at the line containing "overlays = [". Each top-level brace pair is one overlay. +func parseInlineOverlayEntries(lines []string, overlaysStart int) []OverlayLineRange { + var ranges []OverlayLineRange + + overlayIndex := 0 + braceDepth := 0 + entryStartLine := -1 + + for lineIdx := overlaysStart; lineIdx < len(lines); lineIdx++ { + line := lines[lineIdx] + + for _, ch := range line { + switch ch { + case '{': + if braceDepth == 0 { + entryStartLine = lineIdx + 1 // 1-based + } + + braceDepth++ + case '}': + braceDepth-- + + if braceDepth == 0 && entryStartLine > 0 { + ranges = append(ranges, OverlayLineRange{ + StartLine: entryStartLine, + EndLine: lineIdx + 1, // 1-based + Index: overlayIndex, + }) + + overlayIndex++ + entryStartLine = -1 + } + } + } + + // Stop scanning when we hit the closing ']' of the array (outside any braces). + trimmed := strings.TrimSpace(line) + if braceDepth == 0 && lineIdx > overlaysStart && (trimmed == "]" || strings.HasSuffix(trimmed, "]")) { + break + } + } + + return ranges +} + +// MapOverlaysToCommits groups overlays by their originating commit hash using blame data +// and overlay line ranges. It retrieves full commit metadata (author email, message) from +// the project repository for each unique commit. Groups are returned sorted chronologically. +func MapOverlaysToCommits( + repo *gogit.Repository, + overlays []projectconfig.ComponentOverlay, + lineRanges []OverlayLineRange, + blame *ConfigBlameResult, +) ([]OverlayCommitGroup, error) { + if len(overlays) == 0 { + return nil, nil + } + + if blame == nil { + return nil, errors.New("blame result cannot be nil") + } + + if len(lineRanges) != len(overlays) { + return nil, fmt.Errorf( + "%w: found %d line ranges but component has %d overlays", + ErrLineRangeOverlayMismatch, len(lineRanges), len(overlays), + ) + } + + // Map each overlay to a blame commit hash derived from the full TOML block range + // (StartLine..EndLine) + commitOverlays := make(map[string][]projectconfig.ComponentOverlay) + + for _, lineRange := range lineRanges { + if lineRange.StartLine < 1 || + lineRange.EndLine < 1 || + lineRange.StartLine > lineRange.EndLine || + lineRange.StartLine > len(blame.Entries) { + return nil, fmt.Errorf( + "overlay at index %d has line range [%d, %d], but blame has only %d lines", + lineRange.Index, lineRange.StartLine, lineRange.EndLine, len(blame.Entries), + ) + } + + // Clamp EndLine to the blame length. TOML blocks at EOF may extend past the + // last blamed line when the file has a trailing newline that git blame omits. + endLine := min(lineRange.EndLine, len(blame.Entries)) + + // Attribute the overlay to the most recent commit that touched any line in + // the block. + var selectedHash string + + var latestTimestamp int64 + + for i := lineRange.StartLine; i <= endLine; i++ { + entry := blame.Entries[i-1] + if entry.Timestamp > latestTimestamp { + latestTimestamp = entry.Timestamp + selectedHash = entry.CommitHash + } + } + + commitOverlays[selectedHash] = append(commitOverlays[selectedHash], overlays[lineRange.Index]) + } + + // Build groups with full commit metadata from the project repository. + commitCache := make(map[string]*CommitMetadata) + + groups := make([]OverlayCommitGroup, 0, len(commitOverlays)) + + for hash, overlayList := range commitOverlays { + meta, err := resolveCommitMetadata(repo, hash, commitCache) + if err != nil { + return nil, fmt.Errorf("failed to resolve commit metadata for %#q:\n%w", hash, err) + } + + groups = append(groups, OverlayCommitGroup{ + Commit: *meta, + Overlays: overlayList, + }) + } + + // Sort groups chronologically so synthetic commits preserve temporal ordering. + sort.Slice(groups, func(i, j int) bool { + return groups[i].Commit.Timestamp < groups[j].Commit.Timestamp + }) + + return groups, nil +} + +// CommitSyntheticHistory creates synthetic commits in the provided git repository, one per +// [OverlayCommitGroup]. For each group the applyFn callback is invoked to mutate the working +// tree, then all changes are staged and committed with the group's metadata. +func CommitSyntheticHistory( + repo *gogit.Repository, + groups []OverlayCommitGroup, + applyFn OverlayApplyFunc, +) error { + if len(groups) == 0 { + return ErrNoOverlaysToCommit + } + + if applyFn == nil { + return errors.New("applyFn callback is required") + } + + worktree, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree:\n%w", err) + } + + for groupIdx, group := range groups { + slog.Info("Creating synthetic commit", + "commit", groupIdx+1, + "total", len(groups), + "originalHash", group.Commit.Hash, + "overlayCount", len(group.Overlays), + ) + + // Apply the overlay batch to the working tree. + if err := applyFn(group.Overlays); err != nil { + return fmt.Errorf("failed to apply overlays for synthetic commit %d (original %s):\n%w", + groupIdx+1, group.Commit.Hash, err) + } + + // Stage all changes (modified, added, and deleted files). + if err := worktree.AddWithOptions(&gogit.AddOptions{All: true}); err != nil { + return fmt.Errorf("failed to stage changes for synthetic commit %d:\n%w", groupIdx+1, err) + } + + // Create the synthetic commit preserving author attribution from the project repo. + message := fmt.Sprintf("[azldev] %s\n\nOriginal commit: %s", + group.Commit.Message, group.Commit.Hash) + + _, err := worktree.Commit(message, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: group.Commit.Author, + Email: group.Commit.AuthorEmail, + When: unixToTime(group.Commit.Timestamp), + }, + }) + if err != nil { + return fmt.Errorf("failed to create synthetic commit %d:\n%w", groupIdx+1, err) + } + } + + slog.Info("Synthetic history generation complete", + "commitsCreated", len(groups)) + + return nil +} + +// buildOverlayGroups resolves the project repository from the component's config file, blames +// the config to attribute lines to commits, and maps overlays to [OverlayCommitGroup] values +// sorted chronologically. Returns nil groups when overlay line ranges cannot be located. +func buildOverlayGroups( + fs opctx.FS, config *projectconfig.ComponentConfig, componentName string, +) ([]OverlayCommitGroup, error) { + configFilePath, err := resolveConfigFilePath(config, componentName) + if err != nil { + return nil, err + } + + projectRepo, relConfigPath, err := openProjectRepo(configFilePath) + if err != nil { + return nil, err + } + + blame, err := BlameFile(projectRepo, relConfigPath) + if err != nil { + return nil, fmt.Errorf("failed to blame config file %#q:\n%w", relConfigPath, err) + } + + configContent, err := fileutils.ReadFile(fs, configFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read config file %#q:\n%w", configFilePath, err) + } + + lineRanges := FindOverlayLineRanges(string(configContent), config.Name) + if len(lineRanges) == 0 { + slog.Warn("Could not locate overlay definitions in config file; "+ + "falling back to standard overlay processing", + "component", componentName, "configFile", configFilePath) + + return nil, nil + } + + return MapOverlaysToCommits(projectRepo, config.Overlays, lineRanges, blame) +} + +// resolveConfigFilePath extracts and validates the source config file path from the component config. +func resolveConfigFilePath(config *projectconfig.ComponentConfig, componentName string) (string, error) { + configFile := config.SourceConfigFile + if configFile == nil { + return "", fmt.Errorf("component %#q has no source config file reference", componentName) + } + + configFilePath := configFile.SourcePath() + if configFilePath == "" { + return "", fmt.Errorf("component %#q source config file has no path", componentName) + } + + return configFilePath, nil +} + +// openProjectRepo finds the git repository root containing configFilePath, opens it, and +// returns the repository handle along with the config file path relative to the repo root. +func openProjectRepo(configFilePath string) (*gogit.Repository, string, error) { + projectRepoPath, err := findRepoRoot(filepath.Dir(configFilePath)) + if err != nil { + return nil, "", fmt.Errorf("failed to find project repository for config file %#q:\n%w", + configFilePath, err) + } + + projectRepo, err := gogit.PlainOpen(projectRepoPath) + if err != nil { + return nil, "", fmt.Errorf("failed to open project repository at %#q:\n%w", projectRepoPath, err) + } + + relConfigPath, err := filepath.Rel(projectRepoPath, configFilePath) + if err != nil { + return nil, "", fmt.Errorf("failed to compute relative config path:\n%w", err) + } + + return projectRepo, relConfigPath, nil +} + +// resolveCommitMetadata retrieves full commit metadata from the repository, using a cache +// to avoid redundant lookups for the same commit hash. +func resolveCommitMetadata( + repo *gogit.Repository, + hash string, + cache map[string]*CommitMetadata, +) (*CommitMetadata, error) { + if meta, ok := cache[hash]; ok { + return meta, nil + } + + commitHash := plumbing.NewHash(hash) + + commit, err := repo.CommitObject(commitHash) + if err != nil { + return nil, fmt.Errorf("failed to get commit %#q:\n%w", hash, err) + } + + meta := &CommitMetadata{ + Hash: hash, + Author: commit.Author.Name, + AuthorEmail: commit.Author.Email, + Timestamp: commit.Author.When.Unix(), + Message: strings.TrimSpace(commit.Message), + } + + cache[hash] = meta + + return meta, nil +} + +// findRepoRoot walks up the directory tree from startDir to find a directory containing +// a .git directory or file (for worktrees). +func findRepoRoot(startDir string) (string, error) { + dir, err := filepath.Abs(startDir) + if err != nil { + return "", fmt.Errorf("failed to get absolute path for %#q:\n%w", startDir, err) + } + + for { + gitPath := filepath.Join(dir, ".git") + + if info, statErr := os.Stat(gitPath); statErr == nil { + // Accept both .git directories and .git files (for git worktrees). + if info.IsDir() || info.Mode().IsRegular() { + return dir, nil + } + } + + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("%w: searched from %#q to filesystem root", ErrNoGitRepository, startDir) + } + + dir = parent + } +} + +// unixToTime converts a Unix timestamp to a [time.Time] in UTC. +func unixToTime(unix int64) time.Time { + return time.Unix(unix, 0).UTC() +} diff --git a/internal/app/azldev/core/sources/synthistory_test.go b/internal/app/azldev/core/sources/synthistory_test.go new file mode 100644 index 0000000..7161d28 --- /dev/null +++ b/internal/app/azldev/core/sources/synthistory_test.go @@ -0,0 +1,549 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources_test + +import ( + "fmt" + "testing" + "time" + + memfs "github.com/go-git/go-billy/v5/memfs" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/storage/memory" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestRepo creates an in-memory git repository with a single file committed. +// Returns the repo, the commit hash, and the billy filesystem. +func createTestRepo(t *testing.T, fileName, fileContent, commitMsg string) (*gogit.Repository, plumbing.Hash) { + t.Helper() + + memFS := memfs.New() + storer := memory.NewStorage() + + repo, err := gogit.Init(storer, memFS) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + // Create the file. + file, err := memFS.Create(fileName) + require.NoError(t, err) + + _, err = file.Write([]byte(fileContent)) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add(fileName) + require.NoError(t, err) + + hash, err := worktree.Commit(commitMsg, &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Test Author", + Email: "test@example.com", + When: time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + return repo, hash +} + +func TestBlameFile(t *testing.T) { + const ( + fileName = "config.toml" + fileContent = "[project]\ndescription = \"test\"\n" + commitMsg = "initial commit" + ) + + repo, commitHash := createTestRepo(t, fileName, fileContent, commitMsg) + + result, err := sources.BlameFile(repo, fileName) + require.NoError(t, err) + require.NotNil(t, result) + + assert.Len(t, result.Entries, 2, "should have one entry per line") + assert.Equal(t, commitHash.String(), result.Entries[0].CommitHash) + assert.Equal(t, "Test Author", result.Entries[0].Author) + assert.Equal(t, 1, result.Entries[0].Line) + assert.Equal(t, "[project]", result.Entries[0].Content) + assert.Equal(t, 2, result.Entries[1].Line) + assert.Contains(t, result.Entries[1].Content, "description") +} + +func TestBlameFile_NonexistentFile(t *testing.T) { + repo, _ := createTestRepo(t, "other.toml", "content", "init") + + result, err := sources.BlameFile(repo, "missing.toml") + require.Error(t, err) + assert.Nil(t, result) +} + +func TestFindOverlayLineRanges(t *testing.T) { + tests := []struct { + name string + content string + componentName string + expected []struct { + startLine int + index int + } + }{ + { + name: "single overlay", + content: `[components.curl] +spec = { type = "upstream" } + +[[components.curl.overlays]] +type = "patch-add" +source = "patches/fix.patch" +`, + componentName: "curl", + expected: []struct { + startLine int + index int + }{ + {startLine: 4, index: 0}, + }, + }, + { + name: "multiple overlays", + content: `[components.curl] +spec = { type = "upstream" } + +[[components.curl.overlays]] +type = "patch-add" +source = "patches/fix.patch" + +[[components.curl.overlays]] +type = "spec-set-tag" +tag = "Release" +value = "2%{?dist}" +`, + componentName: "curl", + expected: []struct { + startLine int + index int + }{ + {startLine: 4, index: 0}, + {startLine: 8, index: 1}, + }, + }, + { + name: "no overlays for component", + content: `[components.curl] +spec = { type = "upstream" } +`, + componentName: "curl", + expected: nil, + }, + { + name: "wrong component name", + content: `[[components.wget.overlays]] +type = "patch-add" +`, + componentName: "curl", + expected: nil, + }, + { + name: "mixed components", + content: `[[components.curl.overlays]] +type = "patch-add" +source = "a.patch" + +[[components.wget.overlays]] +type = "patch-add" +source = "b.patch" + +[[components.curl.overlays]] +type = "spec-set-tag" +tag = "Release" +`, + componentName: "curl", + expected: []struct { + startLine int + index int + }{ + {startLine: 1, index: 0}, + {startLine: 9, index: 1}, + }, + }, + { + name: "quoted component name", + content: `[[components."my-pkg".overlays]] +type = "patch-add" +source = "fix.patch" +`, + componentName: "my-pkg", + expected: []struct { + startLine int + index int + }{ + {startLine: 1, index: 0}, + }, + }, + { + name: "inline array single entry", + content: `[components.shim] +spec = { type = "upstream" } +overlays = [ + { type = "spec-search-replace", regex = 'foo', replacement = "bar" }, +] +`, + componentName: "shim", + expected: []struct { + startLine int + index int + }{ + {startLine: 4, index: 0}, + }, + }, + { + name: "inline array multiple entries", + content: `[components.shim] +spec = { type = "upstream" } +overlays = [ + { type = "spec-search-replace", regex = 'foo', replacement = "bar" }, + { type = "spec-append-lines", section = "%prep", lines = ["echo hello"] }, + { type = "patch-add", source = "patches/fix.patch" }, +] +`, + componentName: "shim", + expected: []struct { + startLine int + index int + }{ + {startLine: 4, index: 0}, + {startLine: 5, index: 1}, + {startLine: 6, index: 2}, + }, + }, + { + name: "inline array multiline entries", + content: `[components.shim] +overlays = [ + { + type = "spec-search-replace", + regex = 'foo', + replacement = "bar", + }, + { + type = "patch-add", + source = "fix.patch", + }, +] +`, + componentName: "shim", + expected: []struct { + startLine int + index int + }{ + {startLine: 3, index: 0}, + {startLine: 8, index: 1}, + }, + }, + { + name: "inline array wrong component", + content: `[components.curl] +overlays = [ + { type = "patch-add", source = "fix.patch" }, +] +`, + componentName: "shim", + expected: nil, + }, + { + name: "inline array with no overlays key", + content: `[components.shim] +spec = { type = "upstream" } +`, + componentName: "shim", + expected: nil, + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + ranges := sources.FindOverlayLineRanges(testCase.content, testCase.componentName) + + if testCase.expected == nil { + assert.Empty(t, ranges) + + return + } + + require.Len(t, ranges, len(testCase.expected)) + + for i, exp := range testCase.expected { + assert.Equal(t, exp.startLine, ranges[i].StartLine, "range %d startLine", i) + assert.Equal(t, exp.index, ranges[i].Index, "range %d index", i) + } + }) + } +} + +func TestMapOverlaysToCommits(t *testing.T) { + // Create an in-memory repo with two commits to different sections of a TOML file. + memFS := memfs.New() + storer := memory.NewStorage() + + repo, err := gogit.Init(storer, memFS) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + // First commit: add the component and first overlay. + firstContent := `[components.curl] +spec = { type = "upstream" } + +[[components.curl.overlays]] +type = "patch-add" +source = "fix.patch" +` + file, err := memFS.Create("azldev.toml") + require.NoError(t, err) + + _, err = file.Write([]byte(firstContent)) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add("azldev.toml") + require.NoError(t, err) + + firstHash, err := worktree.Commit("Add curl with first overlay", &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Alice", + Email: "alice@example.com", + When: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + // Second commit: add a second overlay. + secondContent := `[components.curl] +spec = { type = "upstream" } + +[[components.curl.overlays]] +type = "patch-add" +source = "fix.patch" + +[[components.curl.overlays]] +type = "spec-set-tag" +tag = "Release" +value = "2%{?dist}" +` + file, err = memFS.Create("azldev.toml") + require.NoError(t, err) + + _, err = file.Write([]byte(secondContent)) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add("azldev.toml") + require.NoError(t, err) + + secondHash, err := worktree.Commit("Add second overlay to curl", &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Bob", + Email: "bob@example.com", + When: time.Date(2025, 2, 20, 14, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + // Now blame and map. + blame, err := sources.BlameFile(repo, "azldev.toml") + require.NoError(t, err) + + lineRanges := sources.FindOverlayLineRanges(secondContent, "curl") + require.Len(t, lineRanges, 2) + + overlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayAddPatch, Source: "fix.patch"}, + {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2%{?dist}"}, + } + + groups, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, blame) + require.NoError(t, err) + + // Expect two groups: one from Alice (first overlay), one from Bob (second overlay). + // Groups should be sorted chronologically (Alice first). + require.Len(t, groups, 2) + + assert.Equal(t, firstHash.String(), groups[0].Commit.Hash) + assert.Equal(t, "Alice", groups[0].Commit.Author) + assert.Equal(t, "alice@example.com", groups[0].Commit.AuthorEmail) + assert.Len(t, groups[0].Overlays, 1) + assert.Equal(t, projectconfig.ComponentOverlayAddPatch, groups[0].Overlays[0].Type) + + assert.Equal(t, secondHash.String(), groups[1].Commit.Hash) + assert.Equal(t, "Bob", groups[1].Commit.Author) + assert.Equal(t, "bob@example.com", groups[1].Commit.AuthorEmail) + assert.Len(t, groups[1].Overlays, 1) + assert.Equal(t, projectconfig.ComponentOverlaySetSpecTag, groups[1].Overlays[0].Type) +} + +func TestMapOverlaysToCommits_MismatchedCounts(t *testing.T) { + repo, _ := createTestRepo(t, "config.toml", "content", "init") + + overlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayAddPatch}, + {Type: projectconfig.ComponentOverlaySetSpecTag}, + } + + // Only one line range for two overlays. + lineRanges := []sources.OverlayLineRange{ + {StartLine: 1, EndLine: 3, Index: 0}, + } + + _, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, &sources.ConfigBlameResult{}) + require.Error(t, err) + assert.ErrorIs(t, err, sources.ErrLineRangeOverlayMismatch) +} + +func TestMapOverlaysToCommits_NilBlame(t *testing.T) { + repo, _ := createTestRepo(t, "config.toml", "content", "init") + + overlays := []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayAddPatch}, + } + + lineRanges := []sources.OverlayLineRange{ + {StartLine: 1, EndLine: 1, Index: 0}, + } + + _, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "blame result cannot be nil") +} + +func TestCommitSyntheticHistory(t *testing.T) { + // Create an in-memory repo with an initial commit (simulating upstream). + memFS := memfs.New() + storer := memory.NewStorage() + + repo, err := gogit.Init(storer, memFS) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + // Create an initial file. + file, err := memFS.Create("package.spec") + require.NoError(t, err) + + _, err = file.Write([]byte("Name: package\nVersion: 1.0\n")) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add("package.spec") + require.NoError(t, err) + + _, err = worktree.Commit("upstream: initial", &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Upstream", + Email: "upstream@fedora.org", + When: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + // Define overlay groups. + groups := []sources.OverlayCommitGroup{ + { + Commit: sources.CommitMetadata{ + Hash: "abc123def456", + Author: "Alice", + AuthorEmail: "alice@example.com", + Timestamp: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC).Unix(), + Message: "Apply patch fix", + }, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlayAddPatch}, + }, + }, + { + Commit: sources.CommitMetadata{ + Hash: "789abc012def", + Author: "Bob", + AuthorEmail: "bob@example.com", + Timestamp: time.Date(2025, 2, 20, 14, 0, 0, 0, time.UTC).Unix(), + Message: "Bump release", + }, + Overlays: []projectconfig.ComponentOverlay{ + {Type: projectconfig.ComponentOverlaySetSpecTag}, + }, + }, + } + + // applyFn simulates overlay application by modifying the spec file. + callCount := 0 + applyFn := func(overlays []projectconfig.ComponentOverlay) error { + callCount++ + + specFile, createErr := memFS.Create("package.spec") + if createErr != nil { + return createErr + } + + // Write different content each call so the worktree has changes to commit. + content := fmt.Sprintf("Name: package\nVersion: 1.0\n# overlay applied (call %d)\n", callCount) + _, createErr = specFile.Write([]byte(content)) + + closeErr := specFile.Close() + + if createErr != nil { + return createErr + } + + return closeErr + } + + err = sources.CommitSyntheticHistory(repo, groups, applyFn) + require.NoError(t, err) + assert.Equal(t, 2, callCount, "applyFn should be called once per group") + + // Verify the commit log has 3 commits: upstream + 2 synthetic. + head, err := repo.Head() + require.NoError(t, err) + + commitIter, err := repo.Log(&gogit.LogOptions{From: head.Hash()}) + require.NoError(t, err) + + var commits []*object.Commit + + err = commitIter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + + return nil + }) + require.NoError(t, err) + + assert.Len(t, commits, 3, "should have upstream + 2 synthetic commits") + + // Most recent commit (Bob's). + assert.Contains(t, commits[0].Message, "Bump release") + assert.Equal(t, "Bob", commits[0].Author.Name) + assert.Equal(t, "bob@example.com", commits[0].Author.Email) + + // Second commit (Alice's). + assert.Contains(t, commits[1].Message, "Apply patch fix") + assert.Equal(t, "Alice", commits[1].Author.Name) + + // Original upstream commit. + assert.Equal(t, "upstream: initial", commits[2].Message) +} + +func TestCommitSyntheticHistory_EmptyGroups(t *testing.T) { + repo, _ := createTestRepo(t, "file.txt", "content", "init") + err := sources.CommitSyntheticHistory(repo, nil, nil) + assert.ErrorIs(t, err, sources.ErrNoOverlaysToCommit) +} diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index 9274472..1c15346 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -49,6 +49,17 @@ type ConfigFile struct { dir string `toml:"-" validate:"dir"` } +// SourcePath returns the absolute path to the config file on disk. +func (f ConfigFile) SourcePath() string { + return f.sourcePath +} + +// Dir returns the directory containing the config file; relative paths within the config +// are resolved against this directory. +func (f ConfigFile) Dir() string { + return f.dir +} + // Validates the format and internal consistency of the config file. Semantic errors are reported. func (f ConfigFile) Validate() error { err := validator.New().Struct(f) diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index 9dddbc4..b66bb14 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -85,8 +85,10 @@ func NewFedoraSourcesProviderImpl( } func (g *FedoraSourcesProviderImpl) GetComponent( - ctx context.Context, component components.Component, destDirPath string, + ctx context.Context, component components.Component, destDirPath string, opts ...FetchComponentOption, ) (err error) { + resolved := resolveFetchComponentOptions(opts) + componentName := component.GetName() if componentName == "" { return errors.New("component name cannot be empty") @@ -144,7 +146,11 @@ func (g *FedoraSourcesProviderImpl) GetComponent( // Process the cloned repo: checkout target commit, extract sources, copy to destination. return g.processClonedRepo(ctx, component.GetConfig().Spec.UpstreamCommit, +<<<<<<< HEAD + tempDir, upstreamNameToUse, componentName, destDirPath, skipFileNames, resolved) +======= tempDir, upstreamNameToUse, componentName, destDirPath, skipFileNames) +>>>>>>> d9f58c4 (fix(source preparation): Source Files Download Ordering (#501)) } // processClonedRepo handles the post-clone steps: checking out the target commit, @@ -154,16 +160,23 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( upstreamCommit string, tempDir, upstreamName, componentName, destDirPath string, skipFilenames []string, +<<<<<<< HEAD + opts FetchComponentOptions, +======= +>>>>>>> d9f58c4 (fix(source preparation): Source Files Download Ordering (#501)) ) error { // Checkout the appropriate commit based on component/distro config if err := g.checkoutTargetCommit(ctx, upstreamCommit, tempDir); err != nil { return fmt.Errorf("failed to checkout target commit:\n%w", err) } - // Delete the .git directory so it's not copied to destination. - if err := g.fs.RemoveAll(filepath.Join(tempDir, ".git")); err != nil { - return fmt.Errorf("failed to remove .git directory from cloned repository at %#q:\n%w", - tempDir, err) + // Delete the .git directory so it's not copied to destination, unless the caller + // requested that it be preserved (e.g., for synthetic history generation). + if !opts.PreserveGitDir { + if err := g.fs.RemoveAll(filepath.Join(tempDir, ".git")); err != nil { + return fmt.Errorf("failed to remove .git directory from cloned repository at %#q:\n%w", + tempDir, err) + } } // Extract sources from repo (downloads lookaside files into the temp dir). diff --git a/internal/providers/sourceproviders/rpmcontentsprovider.go b/internal/providers/sourceproviders/rpmcontentsprovider.go index 4240d5f..a02525c 100644 --- a/internal/providers/sourceproviders/rpmcontentsprovider.go +++ b/internal/providers/sourceproviders/rpmcontentsprovider.go @@ -46,7 +46,7 @@ func NewRPMContentsProviderImpl( // GetComponent downloads the source RPM for a component and extracts its contents // in the provided destination path. func (r *RPMContentsProviderImpl) GetComponent( - ctx context.Context, component components.Component, destDirPath string, + ctx context.Context, component components.Component, destDirPath string, _ ...FetchComponentOption, ) (err error) { if component.GetName() == "" { return errors.New("component name cannot be empty") diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 8bb54c8..4cb89f9 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -37,6 +37,35 @@ type FileSourceProvider interface { GetFiles(ctx context.Context, fileRefs []projectconfig.SourceFileReference, destDirPath string) error } +// FetchComponentOptions holds optional parameters for component fetching operations. +type FetchComponentOptions struct { + // PreserveGitDir, when true, instructs the provider to keep the upstream .git directory + // in the fetched component sources instead of deleting it. This is required for building + // synthetic git history from overlay blame metadata. + PreserveGitDir bool +} + +// FetchComponentOption is a functional option for configuring component fetch behavior. +type FetchComponentOption func(*FetchComponentOptions) + +// WithPreserveGitDir returns a [FetchComponentOption] that instructs the provider to preserve +// the upstream .git directory in the fetched component sources. +func WithPreserveGitDir() FetchComponentOption { + return func(o *FetchComponentOptions) { + o.PreserveGitDir = true + } +} + +// resolveFetchComponentOptions applies all functional options and returns the resolved options. +func resolveFetchComponentOptions(opts []FetchComponentOption) FetchComponentOptions { + var resolved FetchComponentOptions + for _, opt := range opts { + opt(&resolved) + } + + return resolved +} + // ComponentSourceProvider is an abstract interface implemented by a source provider that can retrieve the // full file contents of a given component. type ComponentSourceProvider interface { @@ -44,7 +73,10 @@ type ComponentSourceProvider interface { // GetComponent retrieves the `.spec` for the specified component along with any sidecar // files stored along with it, placing the fetched files in the provided directory. - GetComponent(ctx context.Context, component components.Component, destDirPath string) error + GetComponent( + ctx context.Context, component components.Component, destDirPath string, + opts ...FetchComponentOption, + ) error } // SourceManager is an abstract interface for a facility that can fetch arbitrary component sources. @@ -53,7 +85,12 @@ type SourceManager interface { FetchFiles(ctx context.Context, component components.Component, destDirPath string) error // FetchComponent fetches an entire upstream component, including its `.spec` file and any sidecar files. - FetchComponent(ctx context.Context, component components.Component, destDirPath string) error + // Optional [FetchComponentOption] values may be passed to control provider behavior (e.g., preserving + // the upstream .git directory). + FetchComponent( + ctx context.Context, component components.Component, destDirPath string, + opts ...FetchComponentOption, + ) error } // ResolvedDistro holds the fully resolved distro configuration for a component. @@ -380,7 +417,9 @@ func resolvePackageName(component components.Component) string { return component.GetName() } -func (m *sourceManager) FetchComponent(ctx context.Context, component components.Component, destDirPath string) error { +func (m *sourceManager) FetchComponent( + ctx context.Context, component components.Component, destDirPath string, opts ...FetchComponentOption, +) error { if component.GetName() == "" { return errors.New("component name is empty") } @@ -392,7 +431,7 @@ func (m *sourceManager) FetchComponent(ctx context.Context, component components return m.fetchLocalComponent(ctx, component, destDirPath) case projectconfig.SpecSourceTypeUpstream: - return m.fetchUpstreamComponent(ctx, component, destDirPath) + return m.fetchUpstreamComponent(ctx, component, destDirPath, opts...) } return fmt.Errorf("spec for component %#q not found in any configured provider", @@ -444,7 +483,7 @@ func (m *sourceManager) downloadLookasideSources( } func (m *sourceManager) fetchUpstreamComponent( - ctx context.Context, component components.Component, destDirPath string, + ctx context.Context, component components.Component, destDirPath string, opts ...FetchComponentOption, ) error { if len(m.upstreamComponentProviders) == 0 { return fmt.Errorf("no upstream component origins configured for component %#q", @@ -455,7 +494,7 @@ func (m *sourceManager) fetchUpstreamComponent( // Try each upstream component provider, until one succeeds for _, provider := range m.upstreamComponentProviders { - err := provider.GetComponent(ctx, component, destDirPath) + err := provider.GetComponent(ctx, component, destDirPath, opts...) if err == nil { slog.Debug("Successfully fetched upstream component", "component", component.GetName(), diff --git a/internal/providers/sourceproviders/sourceproviders_test/sourcemanager_mocks.go b/internal/providers/sourceproviders/sourceproviders_test/sourcemanager_mocks.go index e6a6a23..db09ac8 100644 --- a/internal/providers/sourceproviders/sourceproviders_test/sourcemanager_mocks.go +++ b/internal/providers/sourceproviders/sourceproviders_test/sourcemanager_mocks.go @@ -14,6 +14,7 @@ import ( reflect "reflect" components "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + sourceproviders "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" gomock "go.uber.org/mock/gomock" ) @@ -42,17 +43,22 @@ func (m *MockSourceManager) EXPECT() *MockSourceManagerMockRecorder { } // FetchComponent mocks base method. -func (m *MockSourceManager) FetchComponent(ctx context.Context, component components.Component, destDirPath string) error { +func (m *MockSourceManager) FetchComponent(ctx context.Context, component components.Component, destDirPath string, opts ...sourceproviders.FetchComponentOption) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FetchComponent", ctx, component, destDirPath) + varargs := []any{ctx, component, destDirPath} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "FetchComponent", varargs...) ret0, _ := ret[0].(error) return ret0 } // FetchComponent indicates an expected call of FetchComponent. -func (mr *MockSourceManagerMockRecorder) FetchComponent(ctx, component, destDirPath any) *gomock.Call { +func (mr *MockSourceManagerMockRecorder) FetchComponent(ctx, component, destDirPath any, opts ...any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchComponent", reflect.TypeOf((*MockSourceManager)(nil).FetchComponent), ctx, component, destDirPath) + varargs := append([]any{ctx, component, destDirPath}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchComponent", reflect.TypeOf((*MockSourceManager)(nil).FetchComponent), varargs...) } // FetchFiles mocks base method. From 7bdc9599b39332ab77cda07c9e10aad02b049f15 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Wed, 18 Mar 2026 23:53:27 +0000 Subject: [PATCH 2/4] Refactored git syth to detect 'Affects:' pattern in commit logs --- .../app/azldev/core/sources/sourceprep.go | 169 ++--- .../app/azldev/core/sources/synthistory.go | 532 ++++--------- .../azldev/core/sources/synthistory_test.go | 699 ++++++++---------- .../sourceproviders/fedorasourceprovider.go | 7 - .../sourceproviders/sourcemanager.go | 5 + 5 files changed, 541 insertions(+), 871 deletions(-) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index f5f9870..7550418 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -125,15 +125,14 @@ func (p *sourcePreparerImpl) PrepareSources( return p.applyOverlaysToSources(ctx, component, outputDir) } -// applyOverlaysToSources writes the macros file and then applies overlays using either the -// synthetic history path or the standard postProcessSources path. +// applyOverlaysToSources writes the macros file and then applies all overlays and +// records synthetic git history. func (p *sourcePreparerImpl) applyOverlaysToSources( ctx context.Context, component components.Component, outputDir string, ) error { // Emit computed macros to a macros file in the output directory. // If the build configuration produces no macros, no file is written and - // macrosFileName will be empty, signaling postProcessSources to skip - // injecting the macro load directive and Source9999 tag. + // macrosFileName will be empty. var macrosFileName string macrosFilePath, err := p.writeMacrosFile(component, outputDir) @@ -146,23 +145,22 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( macrosFileName = filepath.Base(macrosFilePath) } - // Apply user-defined overlays as attributed synthetic git commits, then apply - // system overlays separately. If synthetic history cannot be generated (e.g. no - // git repo available), applyOverlaysWithHistory falls back to the standard path. + // Apply all overlays and record synthetic git history. err = p.applyOverlaysWithHistory(ctx, component, outputDir, macrosFileName) if err != nil { - return fmt.Errorf("failed to post-process sources for component %q:\n%w", component.GetName(), err) + return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } return nil } -// applyOverlaysWithHistory applies user-defined overlays as attributed synthetic git -// commits in the upstream repository, then applies system overlays (macros, check-skip, -// header) outside of the synthetic history so they don't pollute blame attribution. +// applyOverlaysWithHistory applies all overlays (user-defined and system-generated) to the +// component sources, then attempts to record the changes as synthetic git history. // -// When the sources directory does not contain a git repository (e.g. local or unspecified -// spec sources), the function falls back to applying overlays sequentially without history. +// Overlay application is fully decoupled from git history generation: overlays are always +// applied first, then the changes are optionally committed. If the sources directory does +// not contain a git repository (e.g. local or unspecified spec sources), overlays are still +// applied but no synthetic history is created. func (p *sourcePreparerImpl) applyOverlaysWithHistory( _ context.Context, component components.Component, sourcesDirPath, macrosFileName string, ) error { @@ -175,55 +173,97 @@ func (p *sourcePreparerImpl) applyOverlaysWithHistory( return err } - config := component.GetConfig() - if len(config.Overlays) == 0 { - slog.Debug("No overlays defined; skipping synthetic history generation", - "component", component.GetName()) + // Collect all overlays in application order. This ensures every change is + // captured in the synthetic history, including build configuration changes. + allOverlays := p.collectOverlays(component, macrosFileName) - return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) + if len(allOverlays) == 0 { + return nil } - // Try the synthetic history path. If it succeeds, apply system overlays and return. - if err := p.trySyntheticHistory(component, config, sourcesDirPath, absSpecPath); err == nil { - return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) + // Apply all overlays to the working tree. + if err := p.applyOverlayList(allOverlays, sourcesDirPath, absSpecPath); err != nil { + return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - // Synthetic history was not possible; apply user overlays sequentially. - if err := p.applyOverlayList(config.Overlays, sourcesDirPath, absSpecPath); err != nil { - return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) + // Record the changes as synthetic git history. This is required for rpmautospec + // release numbering and delta builds, so failure here is fatal. + if err := p.trySyntheticHistory(component, sourcesDirPath); err != nil { + return fmt.Errorf("failed to generate synthetic history for component %#q:\n%w", + component.GetName(), err) } - return p.applySystemOverlays(component, sourcesDirPath, absSpecPath, macrosFileName) + return nil } -// trySyntheticHistory attempts to apply user-defined overlays as synthetic git commits. -// Returns nil on success, or a non-nil error if the synthetic history path is not available -// (e.g. no .git directory, no resolvable blame groups). The caller should fall back to -// sequential overlay application when this returns an error. +// collectOverlays gathers all overlays for a component into a single ordered slice: +// component overlays first, followed by macros-load, check-skip, and file-header overlays. +func (p *sourcePreparerImpl) collectOverlays( + component components.Component, macrosFileName string, +) []projectconfig.ComponentOverlay { + config := component.GetConfig() + + var allOverlays []projectconfig.ComponentOverlay + + allOverlays = append(allOverlays, config.Overlays...) + + if macrosFileName != "" { + macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName) + if err != nil { + slog.Error("Failed to compute macros load overlays", + "component", component.GetName(), "error", err) + + panic(fmt.Sprintf("failed to compute macros load overlays for component %q: %v", component.GetName(), err)) + } + + allOverlays = append(allOverlays, macroOverlays...) + } + + allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...) + allOverlays = append(allOverlays, generateFileHeaderOverlay()...) + + return allOverlays +} + +// trySyntheticHistory attempts to create synthetic git commits on top of the upstream +// repository. All file changes must already be present in the working tree before calling +// this function — it only handles staging and committing. +// +// Returns nil when there is no .git directory (legitimate for local/unspecified specs). +// Returns a non-nil error if a .git directory exists but history generation fails. func (p *sourcePreparerImpl) trySyntheticHistory( component components.Component, - config *projectconfig.ComponentConfig, - sourcesDirPath, absSpecPath string, + sourcesDirPath string, ) error { // Check for an upstream git repository in the sources directory. Local and - // unspecified spec sources won't have a .git directory. + // unspecified spec sources won't have a .git directory — that's expected. gitDirPath := filepath.Join(sourcesDirPath, ".git") - if _, err := p.fs.Stat(gitDirPath); err != nil { - slog.Debug("No .git directory in sources; falling back to standard overlay path", - "component", component.GetName(), - "sourcesDirPath", sourcesDirPath) - return fmt.Errorf("no .git directory in sources dir %#q:\n%w", sourcesDirPath, err) + hasGitDir, err := fileutils.Exists(p.fs, gitDirPath) + if err != nil { + return fmt.Errorf("failed to check for .git directory at %#q:\n%w", gitDirPath, err) + } + + if !hasGitDir { + slog.Debug("No .git directory in sources; skipping synthetic history", + "component", component.GetName()) + + return nil } - // Resolve the project repository and blame the config file to produce overlay groups. - groups, err := buildOverlayGroups(p.fs, config, component.GetName()) + config := component.GetConfig() + + // Build commit metadata from Affects commits and dirty state. + commits, err := buildSyntheticCommits(config, component.GetName()) if err != nil { - return fmt.Errorf("failed to build overlay groups:\n%w", err) + return fmt.Errorf("failed to build synthetic commits:\n%w", err) } - if len(groups) == 0 { - return errors.New("no overlay groups resolved") + if len(commits) == 0 { + slog.Debug("No synthetic commits to create; skipping history generation", + "component", component.GetName()) + + return nil } // Open the upstream git repository where synthetic commits will be recorded. @@ -232,41 +272,13 @@ func (p *sourcePreparerImpl) trySyntheticHistory( return fmt.Errorf("failed to open upstream repository at %#q:\n%w", sourcesDirPath, err) } - // Build the overlay-application callback using the shared helper. - applyFn := func(overlays []projectconfig.ComponentOverlay) error { - return p.applyOverlayList(overlays, sourcesDirPath, absSpecPath) - } - - if err := CommitSyntheticHistory(sourcesRepo, groups, applyFn); err != nil { + if err := CommitSyntheticHistory(sourcesRepo, commits); err != nil { return fmt.Errorf("failed to commit synthetic history:\n%w", err) } return nil } -// applySystemOverlays applies the macros-load, check-skip, and file-header overlays that -// are synthesized at build time rather than defined in the project TOML config. -func (p *sourcePreparerImpl) applySystemOverlays( - component components.Component, sourcesDirPath, absSpecPath, macrosFileName string, -) error { - // Collect all system overlays in application order: macros, check-skip, file header. - var systemOverlays []projectconfig.ComponentOverlay - - if macrosFileName != "" { - macroOverlays, macroErr := synthesizeMacroLoadOverlays(macrosFileName) - if macroErr != nil { - return fmt.Errorf("failed to compute macros load overlays:\n%w", macroErr) - } - - systemOverlays = append(systemOverlays, macroOverlays...) - } - - systemOverlays = append(systemOverlays, synthesizeCheckSkipOverlays(component.GetConfig().Build.Check)...) - systemOverlays = append(systemOverlays, generateFileHeaderOverlay()...) - - return p.applyOverlayList(systemOverlays, sourcesDirPath, absSpecPath) -} - // DiffSources implements the [SourcePreparer] interface. // It fetches the component's sources once, copies them to a second directory, applies overlays // to the copy, then diffs the two trees. This avoids fetching the sources twice. @@ -306,19 +318,8 @@ func (p *sourcePreparerImpl) DiffSources( } // Apply overlays in-place to the copied directory only. - var macrosFileName string - - macrosFilePath, err := p.writeMacrosFile(component, overlaidDir) - if err != nil { - return nil, fmt.Errorf("failed to write macros file for component %#q:\n%w", component.GetName(), err) - } - - if macrosFilePath != "" { - macrosFileName = filepath.Base(macrosFilePath) - } - - if err := p.postProcessSources(component, overlaidDir, macrosFileName); err != nil { - return nil, fmt.Errorf("failed to post-process sources for component %#q:\n%w", component.GetName(), err) + if err := p.applyOverlaysToSources(ctx, component, overlaidDir); err != nil { + return nil, fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } // Diff the original tree against the overlaid tree. diff --git a/internal/app/azldev/core/sources/synthistory.go b/internal/app/azldev/core/sources/synthistory.go index d641c7c..e93c295 100644 --- a/internal/app/azldev/core/sources/synthistory.go +++ b/internal/app/azldev/core/sources/synthistory.go @@ -9,47 +9,50 @@ import ( "log/slog" "os" "path/filepath" - "regexp" "slices" - "sort" "strings" "time" gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" - "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" - "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) +// AffectsPrefix is the commit message marker used to associate a project repository +// commit with a component. Developers include "Affects: " anywhere +// in a commit message to indicate the commit pertains to that component. +const AffectsPrefix = "Affects: " + var ( // ErrNoGitRepository is returned when no enclosing git repository can be found. ErrNoGitRepository = errors.New("no git repository found") - // ErrNoOverlaysToCommit is returned when there are no overlay groups to commit. - ErrNoOverlaysToCommit = errors.New("no overlays to commit") + // ErrNoOverlaysToCommit is returned when there are no synthetic commits to create. + ErrNoOverlaysToCommit = errors.New("no synthetic commits to create") +) - // ErrLineRangeOverlayMismatch is returned when the number of located overlay line ranges - // does not match the number of overlays on the component. - ErrLineRangeOverlayMismatch = errors.New("line range count does not match overlay count") +// IsRepoDirty reports whether the given go-git repository has staged changes +// in its index. Unstaged modifications and untracked files are intentionally +// ignored so the developer must explicitly stage changes to trigger an extra +// synthetic commit. +func IsRepoDirty(repo *gogit.Repository) (bool, error) { + worktree, err := repo.Worktree() + if err != nil { + return false, fmt.Errorf("failed to get worktree:\n%w", err) + } - // sectionHeaderRegexp matches any TOML table or array-of-tables header line. - sectionHeaderRegexp = regexp.MustCompile(`^\s*\[{1,2}[^\]]+\]{1,2}\s*$`) -) + status, err := worktree.Status() + if err != nil { + return false, fmt.Errorf("failed to get worktree status:\n%w", err) + } -// BlameEntry represents a single line's blame information from a git repository. -type BlameEntry struct { - // CommitHash is the hash of the commit that last modified this line. - CommitHash string - // Author is the name of the author who last modified this line. - Author string - // Timestamp is when the line was last modified. - Timestamp int64 - // Line is the 1-based line number. - Line int - // Content is the text content of the line. - Content string + for _, fileStatus := range status { + if fileStatus.Staging != gogit.Unmodified && fileStatus.Staging != gogit.Untracked { + return true, nil + } + } + + return false, nil } // CommitMetadata holds full metadata for a commit in the project repository. @@ -61,408 +64,196 @@ type CommitMetadata struct { Message string } -// OverlayCommitGroup groups overlays that originate from the same git commit in the project -// configuration repository. During synthetic history generation, all overlays in a group are -// applied together and recorded as a single commit. -type OverlayCommitGroup struct { - // Commit holds metadata from the originating commit in the project repository. - Commit CommitMetadata - // Overlays contains the overlay definitions to apply as part of this synthetic commit. - Overlays []projectconfig.ComponentOverlay -} - -// OverlayApplyFunc is a callback that applies a batch of overlays to the component sources. -// It is called once per [OverlayCommitGroup] during synthetic history generation. -type OverlayApplyFunc func(overlays []projectconfig.ComponentOverlay) error - -// ConfigBlameResult holds the per-line blame entries for a configuration file. -type ConfigBlameResult struct { - // Entries contains one [BlameEntry] per line in the blamed file. - Entries []BlameEntry -} - -// OverlayLineRange tracks the line range of a single [[components.X.overlays]] block -// in a TOML config file. -type OverlayLineRange struct { - StartLine int // 1-based, inclusive (the [[...]] header line) - EndLine int // 1-based, inclusive - Index int // positional index in the component's overlays slice -} - -// BlameFile performs git blame on the specified file within the provided go-git repository. -// The filePath must be relative to the repository root. -func BlameFile(repo *gogit.Repository, filePath string) (*ConfigBlameResult, error) { +// FindAffectsCommits walks the git log from HEAD and returns metadata for all commits +// whose message contains "Affects: ". Results are sorted chronologically +// (oldest first). +func FindAffectsCommits(repo *gogit.Repository, componentName string) ([]CommitMetadata, error) { head, err := repo.Head() if err != nil { return nil, fmt.Errorf("failed to get HEAD reference:\n%w", err) } - commit, err := repo.CommitObject(head.Hash()) - if err != nil { - return nil, fmt.Errorf("failed to get HEAD commit:\n%w", err) - } - - blameResult, err := gogit.Blame(commit, filePath) + commitIter, err := repo.Log(&gogit.LogOptions{From: head.Hash()}) if err != nil { - return nil, fmt.Errorf("failed to blame file %#q:\n%w", filePath, err) - } - - entries := make([]BlameEntry, len(blameResult.Lines)) - for i, line := range blameResult.Lines { - entries[i] = BlameEntry{ - CommitHash: line.Hash.String(), - Author: line.AuthorName, - Timestamp: line.Date.Unix(), - Line: i + 1, - Content: line.Text, - } - } - - return &ConfigBlameResult{Entries: entries}, nil -} - -// FindOverlayLineRanges parses raw TOML content to locate the line ranges of all overlay -// definitions for the named component. It supports two TOML styles: -// -// 1. Array-of-tables: [[components..overlays]] blocks. -// 2. Inline array: overlays = [ { ... }, { ... } ] under a [components.] section. -// -// The returned ranges are ordered by their position in the file, matching the -// serialization order of the component's overlay slice. -func FindOverlayLineRanges(configContent string, componentName string) []OverlayLineRange { - lines := strings.Split(configContent, "\n") - - ranges := findArrayOfTablesOverlays(lines, componentName) - if len(ranges) > 0 { - return ranges - } - - return findInlineArrayOverlays(lines, componentName) -} - -// findArrayOfTablesOverlays locates overlays declared as [[components..overlays]] blocks. -func findArrayOfTablesOverlays(lines []string, componentName string) []OverlayLineRange { - expectedHeaders := []string{ - fmt.Sprintf("[[components.%s.overlays]]", componentName), - fmt.Sprintf(`[[components."%s".overlays]]`, componentName), + return nil, fmt.Errorf("failed to iterate commit log:\n%w", err) } - var ranges []OverlayLineRange + var matches []CommitMetadata - overlayIndex := 0 + err = commitIter.ForEach(func(commit *object.Commit) error { + found := false - for lineIdx := 0; lineIdx < len(lines); lineIdx++ { - trimmed := strings.TrimSpace(lines[lineIdx]) + for _, line := range strings.Split(commit.Message, "\n") { + trimmed := strings.TrimSpace(line) + lowerTrimmed := strings.ToLower(trimmed) - if !slices.Contains(expectedHeaders, trimmed) { - continue - } - - startLine := lineIdx + 1 // convert to 1-based - - // Find the end of this overlay block: the line before the next section header, or EOF. - endLineExclusive := len(lines) - for j := lineIdx + 1; j < len(lines); j++ { - if sectionHeaderRegexp.MatchString(lines[j]) { - endLineExclusive = j - - break + lowerPrefix := strings.ToLower(AffectsPrefix) + if !strings.HasPrefix(lowerTrimmed, lowerPrefix) { + continue } - } - - for endLineExclusive > lineIdx+1 && strings.TrimSpace(lines[endLineExclusive-1]) == "" { - endLineExclusive-- - } - - ranges = append(ranges, OverlayLineRange{ - StartLine: startLine, - EndLine: endLineExclusive, - Index: overlayIndex, - }) - - overlayIndex++ - lineIdx = endLineExclusive - 1 // advance past this block (loop increments) - } - - return ranges -} - -// findInlineArrayOverlays locates overlays declared as an inline array under a -// [components.] section (e.g. overlays = [ { type = "patch-add", ... }, ... ]). -func findInlineArrayOverlays(lines []string, componentName string) []OverlayLineRange { - sectionHeaders := []string{ - fmt.Sprintf("[components.%s]", componentName), - fmt.Sprintf(`[components."%s"]`, componentName), - } - - // Locate the section header for this component. - sectionStart := -1 - - for i, line := range lines { - if slices.Contains(sectionHeaders, strings.TrimSpace(line)) { - sectionStart = i - - break - } - } - - if sectionStart < 0 { - return nil - } - - // Scan forward from the section header to find "overlays = [", stopping at the next - // section header. - overlaysStart := -1 - - for lineIdx := sectionStart + 1; lineIdx < len(lines); lineIdx++ { - if sectionHeaderRegexp.MatchString(lines[lineIdx]) { - break - } - - trimmed := strings.TrimSpace(lines[lineIdx]) - if strings.HasPrefix(trimmed, "overlays") && strings.Contains(trimmed, "=") && strings.Contains(trimmed, "[") { - overlaysStart = lineIdx - - break - } - } - - if overlaysStart < 0 { - return nil - } - - return parseInlineOverlayEntries(lines, overlaysStart) -} - -// parseInlineOverlayEntries parses individual { ... } entries from an inline overlay array -// starting at the line containing "overlays = [". Each top-level brace pair is one overlay. -func parseInlineOverlayEntries(lines []string, overlaysStart int) []OverlayLineRange { - var ranges []OverlayLineRange - - overlayIndex := 0 - braceDepth := 0 - entryStartLine := -1 - - for lineIdx := overlaysStart; lineIdx < len(lines); lineIdx++ { - line := lines[lineIdx] - - for _, ch := range line { - switch ch { - case '{': - if braceDepth == 0 { - entryStartLine = lineIdx + 1 // 1-based - } - - braceDepth++ - case '}': - braceDepth-- - - if braceDepth == 0 && entryStartLine > 0 { - ranges = append(ranges, OverlayLineRange{ - StartLine: entryStartLine, - EndLine: lineIdx + 1, // 1-based - Index: overlayIndex, - }) - - overlayIndex++ - entryStartLine = -1 - } + // Extract the component name after the "Affects: " prefix, preserving original + // casing but trimming surrounding whitespace, and compare case-insensitively. + if len(trimmed) < len(AffectsPrefix) { + continue } - } - - // Stop scanning when we hit the closing ']' of the array (outside any braces). - trimmed := strings.TrimSpace(line) - if braceDepth == 0 && lineIdx > overlaysStart && (trimmed == "]" || strings.HasSuffix(trimmed, "]")) { - break - } - } - - return ranges -} - -// MapOverlaysToCommits groups overlays by their originating commit hash using blame data -// and overlay line ranges. It retrieves full commit metadata (author email, message) from -// the project repository for each unique commit. Groups are returned sorted chronologically. -func MapOverlaysToCommits( - repo *gogit.Repository, - overlays []projectconfig.ComponentOverlay, - lineRanges []OverlayLineRange, - blame *ConfigBlameResult, -) ([]OverlayCommitGroup, error) { - if len(overlays) == 0 { - return nil, nil - } - - if blame == nil { - return nil, errors.New("blame result cannot be nil") - } - if len(lineRanges) != len(overlays) { - return nil, fmt.Errorf( - "%w: found %d line ranges but component has %d overlays", - ErrLineRangeOverlayMismatch, len(lineRanges), len(overlays), - ) - } - - // Map each overlay to a blame commit hash derived from the full TOML block range - // (StartLine..EndLine) - commitOverlays := make(map[string][]projectconfig.ComponentOverlay) - - for _, lineRange := range lineRanges { - if lineRange.StartLine < 1 || - lineRange.EndLine < 1 || - lineRange.StartLine > lineRange.EndLine || - lineRange.StartLine > len(blame.Entries) { - return nil, fmt.Errorf( - "overlay at index %d has line range [%d, %d], but blame has only %d lines", - lineRange.Index, lineRange.StartLine, lineRange.EndLine, len(blame.Entries), - ) - } + component := strings.TrimSpace(trimmed[len(AffectsPrefix):]) + if strings.HasPrefix(strings.ToLower(component), strings.ToLower(componentName)) { + found = true - // Clamp EndLine to the blame length. TOML blocks at EOF may extend past the - // last blamed line when the file has a trailing newline that git blame omits. - endLine := min(lineRange.EndLine, len(blame.Entries)) - - // Attribute the overlay to the most recent commit that touched any line in - // the block. - var selectedHash string - - var latestTimestamp int64 - - for i := lineRange.StartLine; i <= endLine; i++ { - entry := blame.Entries[i-1] - if entry.Timestamp > latestTimestamp { - latestTimestamp = entry.Timestamp - selectedHash = entry.CommitHash + break } } - commitOverlays[selectedHash] = append(commitOverlays[selectedHash], overlays[lineRange.Index]) - } - - // Build groups with full commit metadata from the project repository. - commitCache := make(map[string]*CommitMetadata) - - groups := make([]OverlayCommitGroup, 0, len(commitOverlays)) - - for hash, overlayList := range commitOverlays { - meta, err := resolveCommitMetadata(repo, hash, commitCache) - if err != nil { - return nil, fmt.Errorf("failed to resolve commit metadata for %#q:\n%w", hash, err) + if found { + matches = append(matches, CommitMetadata{ + Hash: commit.Hash.String(), + Author: commit.Author.Name, + AuthorEmail: commit.Author.Email, + Timestamp: commit.Author.When.Unix(), + Message: strings.TrimSpace(commit.Message), + }) } - groups = append(groups, OverlayCommitGroup{ - Commit: *meta, - Overlays: overlayList, - }) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to walk commit log:\n%w", err) } - // Sort groups chronologically so synthetic commits preserve temporal ordering. - sort.Slice(groups, func(i, j int) bool { - return groups[i].Commit.Timestamp < groups[j].Commit.Timestamp - }) + // Log iteration returns newest-first; reverse to get chronological order. + slices.Reverse(matches) - return groups, nil + return matches, nil } -// CommitSyntheticHistory creates synthetic commits in the provided git repository, one per -// [OverlayCommitGroup]. For each group the applyFn callback is invoked to mutate the working -// tree, then all changes are staged and committed with the group's metadata. +// CommitSyntheticHistory stages all pending working tree changes and creates synthetic +// commits in the provided git repository. The first commit captures all file changes; +// subsequent commits are created as empty commits to preserve the commit count for +// rpmautospec release numbering. Overlay application must happen before calling this +// function — it only handles the git history. func CommitSyntheticHistory( repo *gogit.Repository, - groups []OverlayCommitGroup, - applyFn OverlayApplyFunc, + commits []CommitMetadata, ) error { - if len(groups) == 0 { + if len(commits) == 0 { return ErrNoOverlaysToCommit } - if applyFn == nil { - return errors.New("applyFn callback is required") - } - worktree, err := repo.Worktree() if err != nil { return fmt.Errorf("failed to get worktree:\n%w", err) } - for groupIdx, group := range groups { + // Stage all working tree changes once — overlays have already been applied. + if err := worktree.AddWithOptions(&gogit.AddOptions{All: true}); err != nil { + return fmt.Errorf("failed to stage changes:\n%w", err) + } + + for commitIdx, commitMeta := range commits { slog.Info("Creating synthetic commit", - "commit", groupIdx+1, - "total", len(groups), - "originalHash", group.Commit.Hash, - "overlayCount", len(group.Overlays), + "commit", commitIdx+1, + "total", len(commits), + "projectHash", commitMeta.Hash, ) - // Apply the overlay batch to the working tree. - if err := applyFn(group.Overlays); err != nil { - return fmt.Errorf("failed to apply overlays for synthetic commit %d (original %s):\n%w", - groupIdx+1, group.Commit.Hash, err) - } - - // Stage all changes (modified, added, and deleted files). - if err := worktree.AddWithOptions(&gogit.AddOptions{All: true}); err != nil { - return fmt.Errorf("failed to stage changes for synthetic commit %d:\n%w", groupIdx+1, err) - } - - // Create the synthetic commit preserving author attribution from the project repo. - message := fmt.Sprintf("[azldev] %s\n\nOriginal commit: %s", - group.Commit.Message, group.Commit.Hash) + message := fmt.Sprintf("[azldev] %s\n\nProject commit: %s", + commitMeta.Message, commitMeta.Hash) _, err := worktree.Commit(message, &gogit.CommitOptions{ + AllowEmptyCommits: commitIdx > 0, Author: &object.Signature{ - Name: group.Commit.Author, - Email: group.Commit.AuthorEmail, - When: unixToTime(group.Commit.Timestamp), + Name: commitMeta.Author, + Email: commitMeta.AuthorEmail, + When: unixToTime(commitMeta.Timestamp), }, }) if err != nil { - return fmt.Errorf("failed to create synthetic commit %d:\n%w", groupIdx+1, err) + return fmt.Errorf("failed to create synthetic commit %d:\n%w", commitIdx+1, err) } } slog.Info("Synthetic history generation complete", - "commitsCreated", len(groups)) + "commitsCreated", len(commits)) return nil } -// buildOverlayGroups resolves the project repository from the component's config file, blames -// the config to attribute lines to commits, and maps overlays to [OverlayCommitGroup] values -// sorted chronologically. Returns nil groups when overlay line ranges cannot be located. -func buildOverlayGroups( - fs opctx.FS, config *projectconfig.ComponentConfig, componentName string, -) ([]OverlayCommitGroup, error) { +// buildSyntheticCommits resolves the project repository from the component's config file, +// walks the git log for commits containing "Affects: ", and returns the +// matching commit metadata sorted chronologically. An additional local-changes entry is +// appended when the project repo has staged changes. Returns nil when no matching commits +// are found and the repo is clean. +func buildSyntheticCommits( + config *projectconfig.ComponentConfig, componentName string, +) ([]CommitMetadata, error) { configFilePath, err := resolveConfigFilePath(config, componentName) if err != nil { - return nil, err + // No config file reference means this component can't have Affects commits. + slog.Debug("Cannot resolve config file for synthetic commits; skipping", + "component", componentName, "error", err) + + return nil, nil } - projectRepo, relConfigPath, err := openProjectRepo(configFilePath) + projectRepo, _, err := openProjectRepo(configFilePath) if err != nil { + // Project config may not live inside a git repo (e.g. scenario tests, + // CI environments). This is expected — skip synthetic history gracefully. + if errors.Is(err, ErrNoGitRepository) { + slog.Debug("Project config is not inside a git repository; skipping synthetic commits", + "component", componentName) + + return nil, nil + } + return nil, err } - blame, err := BlameFile(projectRepo, relConfigPath) + affectsCommits, err := FindAffectsCommits(projectRepo, componentName) if err != nil { - return nil, fmt.Errorf("failed to blame config file %#q:\n%w", relConfigPath, err) + return nil, fmt.Errorf("failed to find Affects commits for component %#q:\n%w", componentName, err) } - configContent, err := fileutils.ReadFile(fs, configFilePath) + slog.Info("Found commits affecting component", + "component", componentName, + "commitCount", len(affectsCommits)) + + commits := make([]CommitMetadata, 0, len(affectsCommits)+1) + + // Create one synthetic commit per Affects commit, preserving each commit's + // original message and author attribution in the upstream history. + commits = append(commits, affectsCommits...) + + // When the project repo has staged changes the developer is iterating + // locally. Append an extra commit so rpmautospec sees a new commit + // and assigns a fresh release number instead of colliding with the last build. + dirty, err := IsRepoDirty(projectRepo) if err != nil { - return nil, fmt.Errorf("failed to read config file %#q:\n%w", configFilePath, err) + slog.Warn("Could not determine project repo dirty state; skipping local-changes commit", + "error", err) + } else if dirty { + slog.Info("Project repo has staged changes; adding local-changes synthetic commit", + "component", componentName) + + commits = append(commits, CommitMetadata{ + Hash: "local", + Author: "local", + AuthorEmail: "local@dev", + Timestamp: time.Now().Unix(), + Message: "Local uncommitted changes for " + componentName, + }) } - lineRanges := FindOverlayLineRanges(string(configContent), config.Name) - if len(lineRanges) == 0 { - slog.Warn("Could not locate overlay definitions in config file; "+ + if len(commits) == 0 { + slog.Warn("No commits with Affects marker found and repo is clean; "+ "falling back to standard overlay processing", - "component", componentName, "configFile", configFilePath) + "component", componentName) return nil, nil } - return MapOverlaysToCommits(projectRepo, config.Overlays, lineRanges, blame) + return commits, nil } // resolveConfigFilePath extracts and validates the source config file path from the component config. @@ -502,37 +293,6 @@ func openProjectRepo(configFilePath string) (*gogit.Repository, string, error) { return projectRepo, relConfigPath, nil } -// resolveCommitMetadata retrieves full commit metadata from the repository, using a cache -// to avoid redundant lookups for the same commit hash. -func resolveCommitMetadata( - repo *gogit.Repository, - hash string, - cache map[string]*CommitMetadata, -) (*CommitMetadata, error) { - if meta, ok := cache[hash]; ok { - return meta, nil - } - - commitHash := plumbing.NewHash(hash) - - commit, err := repo.CommitObject(commitHash) - if err != nil { - return nil, fmt.Errorf("failed to get commit %#q:\n%w", hash, err) - } - - meta := &CommitMetadata{ - Hash: hash, - Author: commit.Author.Name, - AuthorEmail: commit.Author.Email, - Timestamp: commit.Author.When.Unix(), - Message: strings.TrimSpace(commit.Message), - } - - cache[hash] = meta - - return meta, nil -} - // findRepoRoot walks up the directory tree from startDir to find a directory containing // a .git directory or file (for worktrees). func findRepoRoot(startDir string) (string, error) { diff --git a/internal/app/azldev/core/sources/synthistory_test.go b/internal/app/azldev/core/sources/synthistory_test.go index 7161d28..225211e 100644 --- a/internal/app/azldev/core/sources/synthistory_test.go +++ b/internal/app/azldev/core/sources/synthistory_test.go @@ -10,419 +10,280 @@ import ( memfs "github.com/go-git/go-billy/v5/memfs" gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/storage/memory" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" - "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// createTestRepo creates an in-memory git repository with a single file committed. -// Returns the repo, the commit hash, and the billy filesystem. -func createTestRepo(t *testing.T, fileName, fileContent, commitMsg string) (*gogit.Repository, plumbing.Hash) { +// createInMemoryRepo creates an empty in-memory git repository. +func createInMemoryRepo(t *testing.T) *gogit.Repository { t.Helper() - memFS := memfs.New() - storer := memory.NewStorage() - - repo, err := gogit.Init(storer, memFS) + repo, err := gogit.Init(memory.NewStorage(), memfs.New()) require.NoError(t, err) + return repo +} + +// addCommit creates a commit in the in-memory repository with the given message, author name, +// email, and timestamp. A dummy file change is added to ensure the commit is non-empty. +func addCommit( + t *testing.T, repo *gogit.Repository, message, authorName, authorEmail string, when time.Time, +) { + t.Helper() + worktree, err := repo.Worktree() require.NoError(t, err) - // Create the file. - file, err := memFS.Create(fileName) + fs := worktree.Filesystem + + // Write a unique file per commit to guarantee a non-empty diff. + fileName := fmt.Sprintf("file-%d.txt", when.UnixNano()) + + f, err := fs.Create(fileName) require.NoError(t, err) - _, err = file.Write([]byte(fileContent)) + _, err = f.Write([]byte(message)) require.NoError(t, err) - require.NoError(t, file.Close()) + require.NoError(t, f.Close()) _, err = worktree.Add(fileName) require.NoError(t, err) - hash, err := worktree.Commit(commitMsg, &gogit.CommitOptions{ + _, err = worktree.Commit(message, &gogit.CommitOptions{ Author: &object.Signature{ - Name: "Test Author", - Email: "test@example.com", - When: time.Date(2025, 1, 15, 10, 0, 0, 0, time.UTC), + Name: authorName, + Email: authorEmail, + When: when, }, }) require.NoError(t, err) - - return repo, hash } -func TestBlameFile(t *testing.T) { - const ( - fileName = "config.toml" - fileContent = "[project]\ndescription = \"test\"\n" - commitMsg = "initial commit" - ) +func TestFindAffectsCommits(t *testing.T) { + repo := createInMemoryRepo(t) + + // Three commits: two mention curl, one does not. + addCommit(t, repo, + "Initial setup", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - repo, commitHash := createTestRepo(t, fileName, fileContent, commitMsg) + addCommit(t, repo, + "Fix CVE-2025-1234\n\nAffects: curl", + "Bob", "bob@example.com", + time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) - result, err := sources.BlameFile(repo, fileName) + addCommit(t, repo, + "Bump release\n\nAffects: curl", + "Charlie", "charlie@example.com", + time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC)) + + results, err := sources.FindAffectsCommits(repo, "curl") require.NoError(t, err) - require.NotNil(t, result) - assert.Len(t, result.Entries, 2, "should have one entry per line") - assert.Equal(t, commitHash.String(), result.Entries[0].CommitHash) - assert.Equal(t, "Test Author", result.Entries[0].Author) - assert.Equal(t, 1, result.Entries[0].Line) - assert.Equal(t, "[project]", result.Entries[0].Content) - assert.Equal(t, 2, result.Entries[1].Line) - assert.Contains(t, result.Entries[1].Content, "description") -} + // Expect 2 matching commits, oldest first. + require.Len(t, results, 2) -func TestBlameFile_NonexistentFile(t *testing.T) { - repo, _ := createTestRepo(t, "other.toml", "content", "init") + assert.Equal(t, "Bob", results[0].Author) + assert.Equal(t, "bob@example.com", results[0].AuthorEmail) + assert.Contains(t, results[0].Message, "Fix CVE-2025-1234") - result, err := sources.BlameFile(repo, "missing.toml") - require.Error(t, err) - assert.Nil(t, result) + assert.Equal(t, "Charlie", results[1].Author) + assert.Equal(t, "charlie@example.com", results[1].AuthorEmail) + assert.Contains(t, results[1].Message, "Bump release") + + // Chronological order: Bob's timestamp < Charlie's timestamp. + assert.Less(t, results[0].Timestamp, results[1].Timestamp) } -func TestFindOverlayLineRanges(t *testing.T) { - tests := []struct { - name string - content string - componentName string - expected []struct { - startLine int - index int - } - }{ - { - name: "single overlay", - content: `[components.curl] -spec = { type = "upstream" } - -[[components.curl.overlays]] -type = "patch-add" -source = "patches/fix.patch" -`, - componentName: "curl", - expected: []struct { - startLine int - index int - }{ - {startLine: 4, index: 0}, - }, - }, - { - name: "multiple overlays", - content: `[components.curl] -spec = { type = "upstream" } - -[[components.curl.overlays]] -type = "patch-add" -source = "patches/fix.patch" - -[[components.curl.overlays]] -type = "spec-set-tag" -tag = "Release" -value = "2%{?dist}" -`, - componentName: "curl", - expected: []struct { - startLine int - index int - }{ - {startLine: 4, index: 0}, - {startLine: 8, index: 1}, - }, - }, - { - name: "no overlays for component", - content: `[components.curl] -spec = { type = "upstream" } -`, - componentName: "curl", - expected: nil, - }, - { - name: "wrong component name", - content: `[[components.wget.overlays]] -type = "patch-add" -`, - componentName: "curl", - expected: nil, - }, - { - name: "mixed components", - content: `[[components.curl.overlays]] -type = "patch-add" -source = "a.patch" - -[[components.wget.overlays]] -type = "patch-add" -source = "b.patch" - -[[components.curl.overlays]] -type = "spec-set-tag" -tag = "Release" -`, - componentName: "curl", - expected: []struct { - startLine int - index int - }{ - {startLine: 1, index: 0}, - {startLine: 9, index: 1}, - }, - }, - { - name: "quoted component name", - content: `[[components."my-pkg".overlays]] -type = "patch-add" -source = "fix.patch" -`, - componentName: "my-pkg", - expected: []struct { - startLine int - index int - }{ - {startLine: 1, index: 0}, - }, - }, - { - name: "inline array single entry", - content: `[components.shim] -spec = { type = "upstream" } -overlays = [ - { type = "spec-search-replace", regex = 'foo', replacement = "bar" }, -] -`, - componentName: "shim", - expected: []struct { - startLine int - index int - }{ - {startLine: 4, index: 0}, - }, - }, - { - name: "inline array multiple entries", - content: `[components.shim] -spec = { type = "upstream" } -overlays = [ - { type = "spec-search-replace", regex = 'foo', replacement = "bar" }, - { type = "spec-append-lines", section = "%prep", lines = ["echo hello"] }, - { type = "patch-add", source = "patches/fix.patch" }, -] -`, - componentName: "shim", - expected: []struct { - startLine int - index int - }{ - {startLine: 4, index: 0}, - {startLine: 5, index: 1}, - {startLine: 6, index: 2}, - }, - }, - { - name: "inline array multiline entries", - content: `[components.shim] -overlays = [ - { - type = "spec-search-replace", - regex = 'foo', - replacement = "bar", - }, - { - type = "patch-add", - source = "fix.patch", - }, -] -`, - componentName: "shim", - expected: []struct { - startLine int - index int - }{ - {startLine: 3, index: 0}, - {startLine: 8, index: 1}, - }, - }, - { - name: "inline array wrong component", - content: `[components.curl] -overlays = [ - { type = "patch-add", source = "fix.patch" }, -] -`, - componentName: "shim", - expected: nil, - }, - { - name: "inline array with no overlays key", - content: `[components.shim] -spec = { type = "upstream" } -`, - componentName: "shim", - expected: nil, - }, - } +func TestFindAffectsCommits_NoMatches(t *testing.T) { + repo := createInMemoryRepo(t) - for _, testCase := range tests { - t.Run(testCase.name, func(t *testing.T) { - ranges := sources.FindOverlayLineRanges(testCase.content, testCase.componentName) + addCommit(t, repo, + "Unrelated change", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - if testCase.expected == nil { - assert.Empty(t, ranges) + results, err := sources.FindAffectsCommits(repo, "curl") + require.NoError(t, err) + assert.Empty(t, results) +} - return - } +func TestFindAffectsCommits_MultipleComponents(t *testing.T) { + repo := createInMemoryRepo(t) - require.Len(t, ranges, len(testCase.expected)) + addCommit(t, repo, + "Fix curl issue\n\nAffects: curl", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - for i, exp := range testCase.expected { - assert.Equal(t, exp.startLine, ranges[i].StartLine, "range %d startLine", i) - assert.Equal(t, exp.index, ranges[i].Index, "range %d index", i) - } - }) - } -} + addCommit(t, repo, + "Fix wget issue\n\nAffects: wget", + "Bob", "bob@example.com", + time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) -func TestMapOverlaysToCommits(t *testing.T) { - // Create an in-memory repo with two commits to different sections of a TOML file. - memFS := memfs.New() - storer := memory.NewStorage() + addCommit(t, repo, + "Fix both\n\nAffects: curl\nAffects: wget", + "Charlie", "charlie@example.com", + time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC)) - repo, err := gogit.Init(storer, memFS) + curlResults, err := sources.FindAffectsCommits(repo, "curl") require.NoError(t, err) + require.Len(t, curlResults, 2, "curl should match 2 commits") + assert.Equal(t, "Alice", curlResults[0].Author) + assert.Equal(t, "Charlie", curlResults[1].Author) - worktree, err := repo.Worktree() + wgetResults, err := sources.FindAffectsCommits(repo, "wget") require.NoError(t, err) + require.Len(t, wgetResults, 2, "wget should match 2 commits") + assert.Equal(t, "Bob", wgetResults[0].Author) + assert.Equal(t, "Charlie", wgetResults[1].Author) +} - // First commit: add the component and first overlay. - firstContent := `[components.curl] -spec = { type = "upstream" } +func TestFindAffectsCommits_SubstringMatch(t *testing.T) { + repo := createInMemoryRepo(t) -[[components.curl.overlays]] -type = "patch-add" -source = "fix.patch" -` - file, err := memFS.Create("azldev.toml") + // "Affects: curl-minimal" contains "Affects: curl" as a substring. + addCommit(t, repo, + "Update curl-minimal\n\nAffects: curl-minimal", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) + + addCommit(t, repo, + "Update curl itself\n\nAffects: curl", + "Bob", "bob@example.com", + time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) + + // Searching for "curl" matches both because "Affects: curl-minimal" contains "Affects: curl". + curlResults, err := sources.FindAffectsCommits(repo, "curl") require.NoError(t, err) + assert.Len(t, curlResults, 2, "substring match includes curl-minimal commit") - _, err = file.Write([]byte(firstContent)) + // Searching for "curl-minimal" matches only the first commit. + minimalResults, err := sources.FindAffectsCommits(repo, "curl-minimal") require.NoError(t, err) - require.NoError(t, file.Close()) + require.Len(t, minimalResults, 1) + assert.Equal(t, "Alice", minimalResults[0].Author) +} + +func TestFindAffectsCommits_AffectsInSubject(t *testing.T) { + repo := createInMemoryRepo(t) - _, err = worktree.Add("azldev.toml") + // Affects marker in the subject line (not just the body). + addCommit(t, repo, + "Affects: curl - fix build failure", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) + + results, err := sources.FindAffectsCommits(repo, "curl") require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "Alice", results[0].Author) +} - firstHash, err := worktree.Commit("Add curl with first overlay", &gogit.CommitOptions{ - Author: &object.Signature{ - Name: "Alice", - Email: "alice@example.com", - When: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC), - }, - }) +func TestFindAffectsCommits_CaseInsensitive(t *testing.T) { + repo := createInMemoryRepo(t) + + addCommit(t, repo, + "Bump release\n\nAffects: Kernel", + "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) + + addCommit(t, repo, + "Fix CVE\n\nAFFECTS: KERNEL", + "Bob", "bob@example.com", + time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) + + // Search with lowercase component name should match both. + results, err := sources.FindAffectsCommits(repo, "kernel") require.NoError(t, err) + require.Len(t, results, 2) + assert.Equal(t, "Alice", results[0].Author) + assert.Equal(t, "Bob", results[1].Author) +} - // Second commit: add a second overlay. - secondContent := `[components.curl] -spec = { type = "upstream" } +func TestIsRepoDirty_CleanRepo(t *testing.T) { + repo := createInMemoryRepo(t) -[[components.curl.overlays]] -type = "patch-add" -source = "fix.patch" + addCommit(t, repo, "initial", "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) -[[components.curl.overlays]] -type = "spec-set-tag" -tag = "Release" -value = "2%{?dist}" -` - file, err = memFS.Create("azldev.toml") + dirty, err := sources.IsRepoDirty(repo) require.NoError(t, err) + assert.False(t, dirty, "repo with no staged changes should be clean") +} + +func TestIsRepoDirty_UnstagedModification(t *testing.T) { + repo := createInMemoryRepo(t) + + when := time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC) + addCommit(t, repo, "initial", "Alice", "alice@example.com", when) - _, err = file.Write([]byte(secondContent)) + // Modify a tracked file without staging. + worktree, err := repo.Worktree() require.NoError(t, err) - require.NoError(t, file.Close()) - _, err = worktree.Add("azldev.toml") + fileName := fmt.Sprintf("file-%d.txt", when.UnixNano()) + f, err := worktree.Filesystem.Create(fileName) require.NoError(t, err) - secondHash, err := worktree.Commit("Add second overlay to curl", &gogit.CommitOptions{ - Author: &object.Signature{ - Name: "Bob", - Email: "bob@example.com", - When: time.Date(2025, 2, 20, 14, 0, 0, 0, time.UTC), - }, - }) + _, err = f.Write([]byte("modified content")) require.NoError(t, err) + require.NoError(t, f.Close()) - // Now blame and map. - blame, err := sources.BlameFile(repo, "azldev.toml") + dirty, err := sources.IsRepoDirty(repo) require.NoError(t, err) + assert.False(t, dirty, "unstaged modifications should not count as dirty") +} - lineRanges := sources.FindOverlayLineRanges(secondContent, "curl") - require.Len(t, lineRanges, 2) +func TestIsRepoDirty_UntrackedFile(t *testing.T) { + repo := createInMemoryRepo(t) - overlays := []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayAddPatch, Source: "fix.patch"}, - {Type: projectconfig.ComponentOverlaySetSpecTag, Tag: "Release", Value: "2%{?dist}"}, - } + addCommit(t, repo, "initial", "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - groups, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, blame) + // Create an untracked file (not staged). + worktree, err := repo.Worktree() require.NoError(t, err) - // Expect two groups: one from Alice (first overlay), one from Bob (second overlay). - // Groups should be sorted chronologically (Alice first). - require.Len(t, groups, 2) + f, err := worktree.Filesystem.Create("untracked-new-file.txt") + require.NoError(t, err) - assert.Equal(t, firstHash.String(), groups[0].Commit.Hash) - assert.Equal(t, "Alice", groups[0].Commit.Author) - assert.Equal(t, "alice@example.com", groups[0].Commit.AuthorEmail) - assert.Len(t, groups[0].Overlays, 1) - assert.Equal(t, projectconfig.ComponentOverlayAddPatch, groups[0].Overlays[0].Type) + _, err = f.Write([]byte("new")) + require.NoError(t, err) + require.NoError(t, f.Close()) - assert.Equal(t, secondHash.String(), groups[1].Commit.Hash) - assert.Equal(t, "Bob", groups[1].Commit.Author) - assert.Equal(t, "bob@example.com", groups[1].Commit.AuthorEmail) - assert.Len(t, groups[1].Overlays, 1) - assert.Equal(t, projectconfig.ComponentOverlaySetSpecTag, groups[1].Overlays[0].Type) + dirty, err := sources.IsRepoDirty(repo) + require.NoError(t, err) + assert.False(t, dirty, "untracked files should not count as dirty") } -func TestMapOverlaysToCommits_MismatchedCounts(t *testing.T) { - repo, _ := createTestRepo(t, "config.toml", "content", "init") +func TestIsRepoDirty_StagedChanges(t *testing.T) { + repo := createInMemoryRepo(t) - overlays := []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayAddPatch}, - {Type: projectconfig.ComponentOverlaySetSpecTag}, - } - - // Only one line range for two overlays. - lineRanges := []sources.OverlayLineRange{ - {StartLine: 1, EndLine: 3, Index: 0}, - } + addCommit(t, repo, "initial", "Alice", "alice@example.com", + time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - _, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, &sources.ConfigBlameResult{}) - require.Error(t, err) - assert.ErrorIs(t, err, sources.ErrLineRangeOverlayMismatch) -} + // Modify a file and stage it. + worktree, err := repo.Worktree() + require.NoError(t, err) -func TestMapOverlaysToCommits_NilBlame(t *testing.T) { - repo, _ := createTestRepo(t, "config.toml", "content", "init") + f, err := worktree.Filesystem.Create("file-946684800000000000.txt") + require.NoError(t, err) - overlays := []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayAddPatch}, - } + _, err = f.Write([]byte("staged content")) + require.NoError(t, err) + require.NoError(t, f.Close()) - lineRanges := []sources.OverlayLineRange{ - {StartLine: 1, EndLine: 1, Index: 0}, - } + _, err = worktree.Add("file-946684800000000000.txt") + require.NoError(t, err) - _, err := sources.MapOverlaysToCommits(repo, overlays, lineRanges, nil) - require.Error(t, err) - assert.Contains(t, err.Error(), "blame result cannot be nil") + dirty, err := sources.IsRepoDirty(repo) + require.NoError(t, err) + assert.True(t, dirty, "repo with staged changes should be dirty") } func TestCommitSyntheticHistory(t *testing.T) { @@ -436,7 +297,7 @@ func TestCommitSyntheticHistory(t *testing.T) { worktree, err := repo.Worktree() require.NoError(t, err) - // Create an initial file. + // Create an initial file (upstream). file, err := memFS.Create("package.spec") require.NoError(t, err) @@ -456,60 +317,34 @@ func TestCommitSyntheticHistory(t *testing.T) { }) require.NoError(t, err) - // Define overlay groups. - groups := []sources.OverlayCommitGroup{ + // Simulate overlay application by modifying the working tree before committing. + specFile, err := memFS.Create("package.spec") + require.NoError(t, err) + + _, err = specFile.Write([]byte("Name: package\nVersion: 1.0\n# overlays applied\n")) + require.NoError(t, err) + require.NoError(t, specFile.Close()) + + // Define synthetic commits. + commits := []sources.CommitMetadata{ { - Commit: sources.CommitMetadata{ - Hash: "abc123def456", - Author: "Alice", - AuthorEmail: "alice@example.com", - Timestamp: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC).Unix(), - Message: "Apply patch fix", - }, - Overlays: []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlayAddPatch}, - }, + Hash: "abc123def456", + Author: "Alice", + AuthorEmail: "alice@example.com", + Timestamp: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC).Unix(), + Message: "Apply patch fix", }, { - Commit: sources.CommitMetadata{ - Hash: "789abc012def", - Author: "Bob", - AuthorEmail: "bob@example.com", - Timestamp: time.Date(2025, 2, 20, 14, 0, 0, 0, time.UTC).Unix(), - Message: "Bump release", - }, - Overlays: []projectconfig.ComponentOverlay{ - {Type: projectconfig.ComponentOverlaySetSpecTag}, - }, + Hash: "789abc012def", + Author: "Bob", + AuthorEmail: "bob@example.com", + Timestamp: time.Date(2025, 2, 20, 14, 0, 0, 0, time.UTC).Unix(), + Message: "Bump release", }, } - // applyFn simulates overlay application by modifying the spec file. - callCount := 0 - applyFn := func(overlays []projectconfig.ComponentOverlay) error { - callCount++ - - specFile, createErr := memFS.Create("package.spec") - if createErr != nil { - return createErr - } - - // Write different content each call so the worktree has changes to commit. - content := fmt.Sprintf("Name: package\nVersion: 1.0\n# overlay applied (call %d)\n", callCount) - _, createErr = specFile.Write([]byte(content)) - - closeErr := specFile.Close() - - if createErr != nil { - return createErr - } - - return closeErr - } - - err = sources.CommitSyntheticHistory(repo, groups, applyFn) + err = sources.CommitSyntheticHistory(repo, commits) require.NoError(t, err) - assert.Equal(t, 2, callCount, "applyFn should be called once per group") // Verify the commit log has 3 commits: upstream + 2 synthetic. head, err := repo.Head() @@ -518,32 +353,108 @@ func TestCommitSyntheticHistory(t *testing.T) { commitIter, err := repo.Log(&gogit.LogOptions{From: head.Hash()}) require.NoError(t, err) - var commits []*object.Commit + var logCommits []*object.Commit err = commitIter.ForEach(func(c *object.Commit) error { - commits = append(commits, c) + logCommits = append(logCommits, c) return nil }) require.NoError(t, err) - assert.Len(t, commits, 3, "should have upstream + 2 synthetic commits") + require.Len(t, logCommits, 3, "should have upstream + 2 synthetic commits") - // Most recent commit (Bob's). - assert.Contains(t, commits[0].Message, "Bump release") - assert.Equal(t, "Bob", commits[0].Author.Name) - assert.Equal(t, "bob@example.com", commits[0].Author.Email) + // Most recent commit (Bob's) — empty commit. + assert.Contains(t, logCommits[0].Message, "Bump release") + assert.Equal(t, "Bob", logCommits[0].Author.Name) + assert.Equal(t, "bob@example.com", logCommits[0].Author.Email) - // Second commit (Alice's). - assert.Contains(t, commits[1].Message, "Apply patch fix") - assert.Equal(t, "Alice", commits[1].Author.Name) + // Second commit (Alice's) — has the actual file changes. + assert.Contains(t, logCommits[1].Message, "Apply patch fix") + assert.Equal(t, "Alice", logCommits[1].Author.Name) // Original upstream commit. - assert.Equal(t, "upstream: initial", commits[2].Message) + assert.Equal(t, "upstream: initial", logCommits[2].Message) } -func TestCommitSyntheticHistory_EmptyGroups(t *testing.T) { - repo, _ := createTestRepo(t, "file.txt", "content", "init") - err := sources.CommitSyntheticHistory(repo, nil, nil) +func TestCommitSyntheticHistory_SingleCommit(t *testing.T) { + memFS := memfs.New() + storer := memory.NewStorage() + + repo, err := gogit.Init(storer, memFS) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + file, err := memFS.Create("package.spec") + require.NoError(t, err) + + _, err = file.Write([]byte("Name: package\n")) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, err = worktree.Add("package.spec") + require.NoError(t, err) + + _, err = worktree.Commit("upstream: initial", &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "Upstream", + Email: "upstream@fedora.org", + When: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + }) + require.NoError(t, err) + + // Modify working tree (simulates overlay application). + specFile, err := memFS.Create("package.spec") + require.NoError(t, err) + + _, err = specFile.Write([]byte("Name: package\n# modified\n")) + require.NoError(t, err) + require.NoError(t, specFile.Close()) + + commits := []sources.CommitMetadata{ + { + Hash: "abc123", + Author: "Alice", + AuthorEmail: "alice@example.com", + Timestamp: time.Date(2025, 1, 10, 10, 0, 0, 0, time.UTC).Unix(), + Message: "Fix build", + }, + } + + err = sources.CommitSyntheticHistory(repo, commits) + require.NoError(t, err) + + // Verify working tree changes are in the single synthetic commit. + head, err := repo.Head() + require.NoError(t, err) + + headCommit, err := repo.CommitObject(head.Hash()) + require.NoError(t, err) + + assert.Contains(t, headCommit.Message, "Fix build") + assert.Equal(t, "Alice", headCommit.Author.Name) + + // Verify file content was committed. + tree, err := headCommit.Tree() + require.NoError(t, err) + + entry, err := tree.File("package.spec") + require.NoError(t, err) + + content, err := entry.Contents() + require.NoError(t, err) + assert.Contains(t, content, "# modified") +} + +func TestCommitSyntheticHistory_EmptyCommits(t *testing.T) { + repo := createInMemoryRepo(t) + + addCommit(t, repo, "initial", "Test", "test@example.com", + time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) + + err := sources.CommitSyntheticHistory(repo, nil) assert.ErrorIs(t, err, sources.ErrNoOverlaysToCommit) } diff --git a/internal/providers/sourceproviders/fedorasourceprovider.go b/internal/providers/sourceproviders/fedorasourceprovider.go index b66bb14..d86117f 100644 --- a/internal/providers/sourceproviders/fedorasourceprovider.go +++ b/internal/providers/sourceproviders/fedorasourceprovider.go @@ -146,11 +146,7 @@ func (g *FedoraSourcesProviderImpl) GetComponent( // Process the cloned repo: checkout target commit, extract sources, copy to destination. return g.processClonedRepo(ctx, component.GetConfig().Spec.UpstreamCommit, -<<<<<<< HEAD tempDir, upstreamNameToUse, componentName, destDirPath, skipFileNames, resolved) -======= - tempDir, upstreamNameToUse, componentName, destDirPath, skipFileNames) ->>>>>>> d9f58c4 (fix(source preparation): Source Files Download Ordering (#501)) } // processClonedRepo handles the post-clone steps: checking out the target commit, @@ -160,10 +156,7 @@ func (g *FedoraSourcesProviderImpl) processClonedRepo( upstreamCommit string, tempDir, upstreamName, componentName, destDirPath string, skipFilenames []string, -<<<<<<< HEAD opts FetchComponentOptions, -======= ->>>>>>> d9f58c4 (fix(source preparation): Source Files Download Ordering (#501)) ) error { // Checkout the appropriate commit based on component/distro config if err := g.checkoutTargetCommit(ctx, upstreamCommit, tempDir); err != nil { diff --git a/internal/providers/sourceproviders/sourcemanager.go b/internal/providers/sourceproviders/sourcemanager.go index 4cb89f9..51fe503 100644 --- a/internal/providers/sourceproviders/sourcemanager.go +++ b/internal/providers/sourceproviders/sourcemanager.go @@ -59,7 +59,12 @@ func WithPreserveGitDir() FetchComponentOption { // resolveFetchComponentOptions applies all functional options and returns the resolved options. func resolveFetchComponentOptions(opts []FetchComponentOption) FetchComponentOptions { var resolved FetchComponentOptions + for _, opt := range opts { + if opt == nil { + continue + } + opt(&resolved) } From aa085ed8c2c85c051d6243fed3e39fd7d4c17351 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 20 Mar 2026 22:38:35 +0000 Subject: [PATCH 3/4] Added --with-git flag for opt-in dist-git creation --- .../reference/cli/azldev_component_build.md | 1 + .../cli/azldev_component_prepare-sources.md | 1 + internal/app/azldev/cmds/component/build.go | 10 +- .../app/azldev/cmds/component/diffsources.go | 1 + .../azldev/cmds/component/preparesources.go | 10 +- .../app/azldev/core/sources/sourceprep.go | 181 ++++++++++++------ .../app/azldev/core/sources/synthistory.go | 100 ++-------- .../azldev/core/sources/synthistory_test.go | 108 ++--------- scenario/internal/buildtest/buildtest.go | 4 +- 9 files changed, 181 insertions(+), 235 deletions(-) diff --git a/docs/user/reference/cli/azldev_component_build.md b/docs/user/reference/cli/azldev_component_build.md index f7ffafc..06c437d 100644 --- a/docs/user/reference/cli/azldev_component_build.md +++ b/docs/user/reference/cli/azldev_component_build.md @@ -51,6 +51,7 @@ azldev component build [flags] --preserve-buildenv policy Preserve build environment {on-failure, always, never} (default on-failure) -s, --spec-path stringArray Spec path --srpm-only Build SRPM (source RPM) *only* + --with-git Create a dist-git repository with synthetic commit history (requires a project git repository) ``` ### Options inherited from parent commands diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 0b99f71..8810dd0 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -40,6 +40,7 @@ azldev component prepare-sources [flags] -o, --output-dir string output directory --skip-overlays skip applying overlays to prepared sources -s, --spec-path stringArray Spec path + --with-git Create a dist-git repository with synthetic commit history (requires a project git repository) ``` ### Options inherited from parent commands diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index d472687..f8aa76f 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -26,6 +26,7 @@ type ComponentBuildOptions struct { ContinueOnError bool NoCheck bool + WithGitRepo bool SourcePackageOnly bool BuildEnvPolicy BuildEnvPreservePolicy @@ -94,6 +95,8 @@ builds can consume.`, cmd.Flags().BoolVarP(&options.ContinueOnError, "continue-on-error", "k", false, "Continue building when some components fail") cmd.Flags().BoolVar(&options.NoCheck, "no-check", false, "Skip package %check tests") + cmd.Flags().BoolVar(&options.WithGitRepo, "with-git", false, + "Create a dist-git repository with synthetic commit history (requires a project git repository)") cmd.Flags().BoolVar(&options.SourcePackageOnly, "srpm-only", false, "Build SRPM (source RPM) *only*") cmd.Flags().Var(&options.BuildEnvPolicy, "preserve-buildenv", fmt.Sprintf("Preserve build environment {%s, %s, %s}", @@ -212,7 +215,12 @@ func BuildComponent( return nil }, &err) - sourcePreparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env) + var preparerOpts []sources.PreparerOption + if options.WithGitRepo { + preparerOpts = append(preparerOpts, sources.WithGitRepo()) + } + + sourcePreparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { return ComponentBuildResults{}, fmt.Errorf("failed to create source preparer for component %q:\n%w", component.GetName(), err) diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 4444f8b..9877654 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -63,6 +63,7 @@ overlays to the copy and displays the resulting diff between the two trees.`, // DiffComponentSources computes the diff between original and overlaid sources for a single component. // When color is enabled and the output format is not JSON, the returned value is a pre-colorized // string. Otherwise it is [*dirdiff.DiffResult] for structured output. + func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interface{}, error) { resolver := components.NewResolver(env) diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index b5dfb52..6bff06b 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -20,6 +20,7 @@ type PrepareSourcesOptions struct { OutputDir string SkipOverlays bool + WithGitRepo bool Force bool } @@ -65,6 +66,8 @@ Only one component may be selected at a time.`, _ = cmd.MarkFlagDirname("output-dir") cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, "skip applying overlays to prepared sources") + cmd.Flags().BoolVar(&options.WithGitRepo, "with-git", false, + "Create a dist-git repository with synthetic commit history (requires a project git repository)") cmd.Flags().BoolVar(&options.Force, "force", false, "delete and recreate the output directory if it already exists") return cmd @@ -114,7 +117,12 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er return err } - preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env) + var preparerOpts []sources.PreparerOption + if options.WithGitRepo { + preparerOpts = append(preparerOpts, sources.WithGitRepo()) + } + + preparer, err := sources.NewPreparer(sourceManager, env.FS(), env, env, preparerOpts...) if err != nil { return fmt.Errorf("failed to create source preparer:\n%w", err) } diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 7550418..bc29412 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -8,12 +8,15 @@ import ( "errors" "fmt" "log/slog" + "os" "path/filepath" "slices" "strings" + "time" "unicode" gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" @@ -51,20 +54,40 @@ type SourcePreparer interface { DiffSources(ctx context.Context, component components.Component, baseDir string) (*dirdiff.DiffResult, error) } +// PreparerOption is a functional option for configuring a [SourcePreparer]. +type PreparerOption func(*sourcePreparerImpl) + +// WithGitRepo returns a [PreparerOption] that enables dist-git repository +// creation during source preparation. When set, the upstream .git directory +// is preserved and synthetic commit history is generated on top of it. This +// requires the project configuration to reside inside a git repository. +// Without this option, no dist-git is created and synthetic history is skipped. +func WithGitRepo() PreparerOption { + return func(p *sourcePreparerImpl) { + p.withGitRepo = true + } +} + // Standard implementation of the [SourcePreparer] interface. type sourcePreparerImpl struct { sourceManager sourceproviders.SourceManager fs opctx.FS eventListener opctx.EventListener dryRunnable opctx.DryRunnable + + // withGitRepo, when true, enables dist-git creation by preserving the + // upstream .git directory and generating synthetic commit history. + withGitRepo bool } -// NewPreparer creates a new [SourcePreparer] instance. All arguments are required. +// NewPreparer creates a new [SourcePreparer] instance. All positional arguments +// are required. Optional behavior can be configured via [PreparerOption] values. func NewPreparer( sourceManager sourceproviders.SourceManager, fs opctx.FS, eventListener opctx.EventListener, dryRunnable opctx.DryRunnable, + opts ...PreparerOption, ) (SourcePreparer, error) { if sourceManager == nil { return nil, errors.New("source manager cannot be nil") @@ -89,6 +112,12 @@ func NewPreparer( dryRunnable: dryRunnable, } + for _, opt := range opts { + if opt != nil { + opt(impl) + } + } + return impl, nil } @@ -103,11 +132,11 @@ func (p *sourcePreparerImpl) PrepareSources( component.GetName(), err) } - // Preserve the upstream .git directory when overlays will be applied. This is - // required so that overlay commits can be appended on top of the upstream commit - // log during synthetic history generation. + // Preserve the upstream .git directory only when dist-git creation is + // requested via --with-git. This is required so that overlay commits can be + // appended on top of the upstream commit log during synthetic history generation. var fetchOpts []sourceproviders.FetchComponentOption - if applyOverlays { + if applyOverlays && p.withGitRepo { fetchOpts = append(fetchOpts, sourceproviders.WithPreserveGitDir()) } @@ -122,11 +151,22 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } - return p.applyOverlaysToSources(ctx, component, outputDir) + if err := p.applyOverlaysToSources(ctx, component, outputDir); err != nil { + return err + } + + // Record the changes as synthetic git history when dist-git creation is enabled. + if p.withGitRepo { + if err := p.trySyntheticHistory(component, outputDir); err != nil { + return fmt.Errorf("failed to generate synthetic history for component %#q:\n%w", + component.GetName(), err) + } + } + + return nil } -// applyOverlaysToSources writes the macros file and then applies all overlays and -// records synthetic git history. +// applyOverlaysToSources writes the macros file and then applies all overlays. func (p *sourcePreparerImpl) applyOverlaysToSources( ctx context.Context, component components.Component, outputDir string, ) error { @@ -145,23 +185,19 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( macrosFileName = filepath.Base(macrosFilePath) } - // Apply all overlays and record synthetic git history. - err = p.applyOverlaysWithHistory(ctx, component, outputDir, macrosFileName) - if err != nil { + // Apply all overlays to prepared sources. + if err := p.applyOverlays(ctx, component, outputDir, macrosFileName); err != nil { return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } return nil } -// applyOverlaysWithHistory applies all overlays (user-defined and system-generated) to the -// component sources, then attempts to record the changes as synthetic git history. -// -// Overlay application is fully decoupled from git history generation: overlays are always -// applied first, then the changes are optionally committed. If the sources directory does -// not contain a git repository (e.g. local or unspecified spec sources), overlays are still -// applied but no synthetic history is created. -func (p *sourcePreparerImpl) applyOverlaysWithHistory( +// applyOverlays applies all overlays (user-defined and system-generated) to the +// component sources. Overlay application is decoupled from git history generation: +// overlays modify the working tree; synthetic history is recorded separately by +// [trySyntheticHistory]. +func (p *sourcePreparerImpl) applyOverlays( _ context.Context, component components.Component, sourcesDirPath, macrosFileName string, ) error { event := p.eventListener.StartEvent("Applying overlays", "component", component.GetName()) @@ -175,7 +211,10 @@ func (p *sourcePreparerImpl) applyOverlaysWithHistory( // Collect all overlays in application order. This ensures every change is // captured in the synthetic history, including build configuration changes. - allOverlays := p.collectOverlays(component, macrosFileName) + allOverlays, err := p.collectOverlays(component, macrosFileName) + if err != nil { + return fmt.Errorf("failed to collect overlays for component %#q:\n%w", component.GetName(), err) + } if len(allOverlays) == 0 { return nil @@ -186,21 +225,14 @@ func (p *sourcePreparerImpl) applyOverlaysWithHistory( return fmt.Errorf("failed to apply overlays for component %#q:\n%w", component.GetName(), err) } - // Record the changes as synthetic git history. This is required for rpmautospec - // release numbering and delta builds, so failure here is fatal. - if err := p.trySyntheticHistory(component, sourcesDirPath); err != nil { - return fmt.Errorf("failed to generate synthetic history for component %#q:\n%w", - component.GetName(), err) - } - return nil } // collectOverlays gathers all overlays for a component into a single ordered slice: -// component overlays first, followed by macros-load, check-skip, and file-header overlays. +// user overlays first, followed by macros-load, check-skip, and file-header overlays. func (p *sourcePreparerImpl) collectOverlays( component components.Component, macrosFileName string, -) []projectconfig.ComponentOverlay { +) ([]projectconfig.ComponentOverlay, error) { config := component.GetConfig() var allOverlays []projectconfig.ComponentOverlay @@ -210,10 +242,7 @@ func (p *sourcePreparerImpl) collectOverlays( if macrosFileName != "" { macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName) if err != nil { - slog.Error("Failed to compute macros load overlays", - "component", component.GetName(), "error", err) - - panic(fmt.Sprintf("failed to compute macros load overlays for component %q: %v", component.GetName(), err)) + return nil, fmt.Errorf("failed to compute macros load overlays:\n%w", err) } allOverlays = append(allOverlays, macroOverlays...) @@ -222,38 +251,57 @@ func (p *sourcePreparerImpl) collectOverlays( allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...) allOverlays = append(allOverlays, generateFileHeaderOverlay()...) - return allOverlays + return allOverlays, nil } -// trySyntheticHistory attempts to create synthetic git commits on top of the upstream -// repository. All file changes must already be present in the working tree before calling -// this function — it only handles staging and committing. -// -// Returns nil when there is no .git directory (legitimate for local/unspecified specs). -// Returns a non-nil error if a .git directory exists but history generation fails. -func (p *sourcePreparerImpl) trySyntheticHistory( - component components.Component, - sourcesDirPath string, -) error { - // Check for an upstream git repository in the sources directory. Local and - // unspecified spec sources won't have a .git directory — that's expected. - gitDirPath := filepath.Join(sourcesDirPath, ".git") +// initSourcesRepo initializes a new git repository in sourcesDirPath, stages all files, +// and creates an initial commit. This is used for components that don't have an upstream +// dist-git so that Affects commits can still be layered on top. +func initSourcesRepo(sourcesDirPath string) (*gogit.Repository, error) { + slog.Info("Initializing git repository for sources", "path", sourcesDirPath) - hasGitDir, err := fileutils.Exists(p.fs, gitDirPath) + repo, err := gogit.PlainInit(sourcesDirPath, false) if err != nil { - return fmt.Errorf("failed to check for .git directory at %#q:\n%w", gitDirPath, err) + return nil, fmt.Errorf("failed to initialize git repository at %#q:\n%w", sourcesDirPath, err) } - if !hasGitDir { - slog.Debug("No .git directory in sources; skipping synthetic history", - "component", component.GetName()) + worktree, err := repo.Worktree() + if err != nil { + return nil, fmt.Errorf("failed to get worktree:\n%w", err) + } - return nil + if err := worktree.AddWithOptions(&gogit.AddOptions{All: true}); err != nil { + return nil, fmt.Errorf("failed to stage files:\n%w", err) + } + + _, err = worktree.Commit("Initial sources", &gogit.CommitOptions{ + Author: &object.Signature{ + Name: "azldev", + Email: "azldev@microsoft.com", + When: time.Unix(0, 0).UTC(), + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to create initial commit:\n%w", err) } + return repo, nil +} + +// trySyntheticHistory attempts to create synthetic git commits on top of the +// component's sources directory. If no .git directory exists, one is initialized +// with an initial commit so Affects commits can be layered on uniformly for all +// component types. +// +// Returns nil when there are no Affects commits to apply. +// Returns a non-nil error if history generation fails. +func (p *sourcePreparerImpl) trySyntheticHistory( + component components.Component, + sourcesDirPath string, +) error { config := component.GetConfig() - // Build commit metadata from Affects commits and dirty state. + // Build commit metadata from Affects commits. commits, err := buildSyntheticCommits(config, component.GetName()) if err != nil { return fmt.Errorf("failed to build synthetic commits:\n%w", err) @@ -266,10 +314,31 @@ func (p *sourcePreparerImpl) trySyntheticHistory( return nil } - // Open the upstream git repository where synthetic commits will be recorded. + // Check for an existing git repository in the sources directory. + // Use os.Stat rather than p.fs because go-git's PlainInit/PlainOpen always + // operate on the real OS filesystem — the check must use the same source of + // truth to avoid disagreement when p.fs is an in-memory FS (e.g. unit tests). + gitDirPath := filepath.Join(sourcesDirPath, ".git") + + _, statErr := os.Stat(gitDirPath) + + if statErr != nil && !os.IsNotExist(statErr) { + return fmt.Errorf("failed to check for .git directory at %#q:\n%w", gitDirPath, statErr) + } + + if os.IsNotExist(statErr) { + slog.Info("No .git directory in sources; initializing repository", + "component", component.GetName()) + + if _, err := initSourcesRepo(sourcesDirPath); err != nil { + return fmt.Errorf("failed to initialize sources repository:\n%w", err) + } + } + + // Open the git repository where synthetic commits will be recorded. sourcesRepo, err := gogit.PlainOpen(sourcesDirPath) if err != nil { - return fmt.Errorf("failed to open upstream repository at %#q:\n%w", sourcesDirPath, err) + return fmt.Errorf("failed to open sources repository at %#q:\n%w", sourcesDirPath, err) } if err := CommitSyntheticHistory(sourcesRepo, commits); err != nil { diff --git a/internal/app/azldev/core/sources/synthistory.go b/internal/app/azldev/core/sources/synthistory.go index e93c295..e4e13ab 100644 --- a/internal/app/azldev/core/sources/synthistory.go +++ b/internal/app/azldev/core/sources/synthistory.go @@ -9,6 +9,7 @@ import ( "log/slog" "os" "path/filepath" + "regexp" "slices" "strings" "time" @@ -18,10 +19,10 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" ) -// AffectsPrefix is the commit message marker used to associate a project repository -// commit with a component. Developers include "Affects: " anywhere -// in a commit message to indicate the commit pertains to that component. -const AffectsPrefix = "Affects: " +// affectsRegexPattern is the regex pattern prefix used to match an [AffectsPrefix] marker +// line in a commit message. It is combined with a quoted component name to form the full +// match expression. +const affectsRegexPattern = `(?m)^\s*Affects:\s*` var ( // ErrNoGitRepository is returned when no enclosing git repository can be found. @@ -31,30 +32,6 @@ var ( ErrNoOverlaysToCommit = errors.New("no synthetic commits to create") ) -// IsRepoDirty reports whether the given go-git repository has staged changes -// in its index. Unstaged modifications and untracked files are intentionally -// ignored so the developer must explicitly stage changes to trigger an extra -// synthetic commit. -func IsRepoDirty(repo *gogit.Repository) (bool, error) { - worktree, err := repo.Worktree() - if err != nil { - return false, fmt.Errorf("failed to get worktree:\n%w", err) - } - - status, err := worktree.Status() - if err != nil { - return false, fmt.Errorf("failed to get worktree status:\n%w", err) - } - - for _, fileStatus := range status { - if fileStatus.Staging != gogit.Unmodified && fileStatus.Staging != gogit.Untracked { - return true, nil - } - } - - return false, nil -} - // CommitMetadata holds full metadata for a commit in the project repository. type CommitMetadata struct { Hash string @@ -80,32 +57,10 @@ func FindAffectsCommits(repo *gogit.Repository, componentName string) ([]CommitM var matches []CommitMetadata - err = commitIter.ForEach(func(commit *object.Commit) error { - found := false - - for _, line := range strings.Split(commit.Message, "\n") { - trimmed := strings.TrimSpace(line) - lowerTrimmed := strings.ToLower(trimmed) + re := regexp.MustCompile(affectsRegexPattern + regexp.QuoteMeta(componentName) + `(?:$|\s|[,;])`) - lowerPrefix := strings.ToLower(AffectsPrefix) - if !strings.HasPrefix(lowerTrimmed, lowerPrefix) { - continue - } - // Extract the component name after the "Affects: " prefix, preserving original - // casing but trimming surrounding whitespace, and compare case-insensitively. - if len(trimmed) < len(AffectsPrefix) { - continue - } - - component := strings.TrimSpace(trimmed[len(AffectsPrefix):]) - if strings.HasPrefix(strings.ToLower(component), strings.ToLower(componentName)) { - found = true - - break - } - } - - if found { + err = commitIter.ForEach(func(commit *object.Commit) error { + if re.MatchString(commit.Message) { matches = append(matches, CommitMetadata{ Hash: commit.Hash.String(), Author: commit.Author.Name, @@ -181,9 +136,9 @@ func CommitSyntheticHistory( // buildSyntheticCommits resolves the project repository from the component's config file, // walks the git log for commits containing "Affects: ", and returns the -// matching commit metadata sorted chronologically. An additional local-changes entry is -// appended when the project repo has staged changes. Returns nil when no matching commits -// are found and the repo is clean. +// matching commit metadata sorted chronologically. +// +// Returns nil when no matching commits are found. func buildSyntheticCommits( config *projectconfig.ComponentConfig, componentName string, ) ([]CommitMetadata, error) { @@ -198,15 +153,6 @@ func buildSyntheticCommits( projectRepo, _, err := openProjectRepo(configFilePath) if err != nil { - // Project config may not live inside a git repo (e.g. scenario tests, - // CI environments). This is expected — skip synthetic history gracefully. - if errors.Is(err, ErrNoGitRepository) { - slog.Debug("Project config is not inside a git repository; skipping synthetic commits", - "component", componentName) - - return nil, nil - } - return nil, err } @@ -219,34 +165,14 @@ func buildSyntheticCommits( "component", componentName, "commitCount", len(affectsCommits)) - commits := make([]CommitMetadata, 0, len(affectsCommits)+1) + commits := make([]CommitMetadata, 0, len(affectsCommits)) // Create one synthetic commit per Affects commit, preserving each commit's // original message and author attribution in the upstream history. commits = append(commits, affectsCommits...) - // When the project repo has staged changes the developer is iterating - // locally. Append an extra commit so rpmautospec sees a new commit - // and assigns a fresh release number instead of colliding with the last build. - dirty, err := IsRepoDirty(projectRepo) - if err != nil { - slog.Warn("Could not determine project repo dirty state; skipping local-changes commit", - "error", err) - } else if dirty { - slog.Info("Project repo has staged changes; adding local-changes synthetic commit", - "component", componentName) - - commits = append(commits, CommitMetadata{ - Hash: "local", - Author: "local", - AuthorEmail: "local@dev", - Timestamp: time.Now().Unix(), - Message: "Local uncommitted changes for " + componentName, - }) - } - if len(commits) == 0 { - slog.Warn("No commits with Affects marker found and repo is clean; "+ + slog.Warn("No commits with Affects marker found; "+ "falling back to standard overlay processing", "component", componentName) diff --git a/internal/app/azldev/core/sources/synthistory_test.go b/internal/app/azldev/core/sources/synthistory_test.go index 225211e..acc320f 100644 --- a/internal/app/azldev/core/sources/synthistory_test.go +++ b/internal/app/azldev/core/sources/synthistory_test.go @@ -143,10 +143,10 @@ func TestFindAffectsCommits_MultipleComponents(t *testing.T) { assert.Equal(t, "Charlie", wgetResults[1].Author) } -func TestFindAffectsCommits_SubstringMatch(t *testing.T) { +func TestFindAffectsCommits_NoSubstringMatch(t *testing.T) { repo := createInMemoryRepo(t) - // "Affects: curl-minimal" contains "Affects: curl" as a substring. + // "Affects: curl-minimal" should NOT match when searching for "curl". addCommit(t, repo, "Update curl-minimal\n\nAffects: curl-minimal", "Alice", "alice@example.com", @@ -157,12 +157,13 @@ func TestFindAffectsCommits_SubstringMatch(t *testing.T) { "Bob", "bob@example.com", time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) - // Searching for "curl" matches both because "Affects: curl-minimal" contains "Affects: curl". + // Searching for "curl" matches only Bob's commit (exact component name). curlResults, err := sources.FindAffectsCommits(repo, "curl") require.NoError(t, err) - assert.Len(t, curlResults, 2, "substring match includes curl-minimal commit") + require.Len(t, curlResults, 1, "exact match should not include curl-minimal commit") + assert.Equal(t, "Bob", curlResults[0].Author) - // Searching for "curl-minimal" matches only the first commit. + // Searching for "curl-minimal" matches only Alice's commit. minimalResults, err := sources.FindAffectsCommits(repo, "curl-minimal") require.NoError(t, err) require.Len(t, minimalResults, 1) @@ -184,7 +185,7 @@ func TestFindAffectsCommits_AffectsInSubject(t *testing.T) { assert.Equal(t, "Alice", results[0].Author) } -func TestFindAffectsCommits_CaseInsensitive(t *testing.T) { +func TestFindAffectsCommits_CaseSensitive(t *testing.T) { repo := createInMemoryRepo(t) addCommit(t, repo, @@ -197,93 +198,22 @@ func TestFindAffectsCommits_CaseInsensitive(t *testing.T) { "Bob", "bob@example.com", time.Date(2025, 2, 1, 10, 0, 0, 0, time.UTC)) - // Search with lowercase component name should match both. - results, err := sources.FindAffectsCommits(repo, "kernel") - require.NoError(t, err) - require.Len(t, results, 2) - assert.Equal(t, "Alice", results[0].Author) - assert.Equal(t, "Bob", results[1].Author) -} - -func TestIsRepoDirty_CleanRepo(t *testing.T) { - repo := createInMemoryRepo(t) - - addCommit(t, repo, "initial", "Alice", "alice@example.com", - time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - - dirty, err := sources.IsRepoDirty(repo) - require.NoError(t, err) - assert.False(t, dirty, "repo with no staged changes should be clean") -} - -func TestIsRepoDirty_UnstagedModification(t *testing.T) { - repo := createInMemoryRepo(t) - - when := time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC) - addCommit(t, repo, "initial", "Alice", "alice@example.com", when) - - // Modify a tracked file without staging. - worktree, err := repo.Worktree() - require.NoError(t, err) - - fileName := fmt.Sprintf("file-%d.txt", when.UnixNano()) - f, err := worktree.Filesystem.Create(fileName) - require.NoError(t, err) - - _, err = f.Write([]byte("modified content")) - require.NoError(t, err) - require.NoError(t, f.Close()) - - dirty, err := sources.IsRepoDirty(repo) - require.NoError(t, err) - assert.False(t, dirty, "unstaged modifications should not count as dirty") -} - -func TestIsRepoDirty_UntrackedFile(t *testing.T) { - repo := createInMemoryRepo(t) - - addCommit(t, repo, "initial", "Alice", "alice@example.com", - time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - - // Create an untracked file (not staged). - worktree, err := repo.Worktree() - require.NoError(t, err) - - f, err := worktree.Filesystem.Create("untracked-new-file.txt") - require.NoError(t, err) - - _, err = f.Write([]byte("new")) - require.NoError(t, err) - require.NoError(t, f.Close()) - - dirty, err := sources.IsRepoDirty(repo) - require.NoError(t, err) - assert.False(t, dirty, "untracked files should not count as dirty") -} - -func TestIsRepoDirty_StagedChanges(t *testing.T) { - repo := createInMemoryRepo(t) - - addCommit(t, repo, "initial", "Alice", "alice@example.com", - time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) - - // Modify a file and stage it. - worktree, err := repo.Worktree() - require.NoError(t, err) - - f, err := worktree.Filesystem.Create("file-946684800000000000.txt") - require.NoError(t, err) - - _, err = f.Write([]byte("staged content")) - require.NoError(t, err) - require.NoError(t, f.Close()) + addCommit(t, repo, + "Upstream fix\n\nAffects: kernel", + "Charlie", "charlie@example.com", + time.Date(2025, 3, 1, 10, 0, 0, 0, time.UTC)) - _, err = worktree.Add("file-946684800000000000.txt") + // Matching is case-sensitive: searching for "kernel" only matches the exact-case commit. + results, err := sources.FindAffectsCommits(repo, "kernel") require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "Charlie", results[0].Author) - dirty, err := sources.IsRepoDirty(repo) + // Searching for "Kernel" matches only Alice's commit (exact case on component name). + results, err = sources.FindAffectsCommits(repo, "Kernel") require.NoError(t, err) - assert.True(t, dirty, "repo with staged changes should be dirty") + require.Len(t, results, 1) + assert.Equal(t, "Alice", results[0].Author) } func TestCommitSyntheticHistory(t *testing.T) { diff --git a/scenario/internal/buildtest/buildtest.go b/scenario/internal/buildtest/buildtest.go index 9982666..b8f6f98 100644 --- a/scenario/internal/buildtest/buildtest.go +++ b/scenario/internal/buildtest/buildtest.go @@ -34,7 +34,9 @@ func NewBuildTest( componentName string, options ...projecttest.ProjectTestOption, ) *BuildTest { - projectTest := projecttest.NewProjectTest(project, []string{"component", "build", componentName}, options...) + projectTest := projecttest.NewProjectTest( + project, []string{"component", "build", componentName}, options..., + ) return &BuildTest{ inner: projectTest, From afa3c6640626c97a20843a78ee2e02454fd0db88 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Thu, 26 Mar 2026 22:44:19 +0000 Subject: [PATCH 4/4] Create default commit if no Affects project commit --- .../app/azldev/cmds/component/diffsources.go | 1 - .../app/azldev/core/sources/sourceprep.go | 22 ++- .../app/azldev/core/sources/synthistory.go | 142 +++++++----------- .../azldev/core/sources/synthistory_test.go | 42 ++++-- 4 files changed, 98 insertions(+), 109 deletions(-) diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 9877654..4444f8b 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -63,7 +63,6 @@ overlays to the copy and displays the resulting diff between the two trees.`, // DiffComponentSources computes the diff between original and overlaid sources for a single component. // When color is enabled and the output format is not JSON, the returned value is a pre-colorized // string. Otherwise it is [*dirdiff.DiffResult] for structured output. - func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interface{}, error) { resolver := components.NewResolver(env) diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index bc29412..79c99a7 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -147,12 +147,11 @@ func (p *sourcePreparerImpl) PrepareSources( component.GetName(), err) } - if !applyOverlays { - return nil - } - - if err := p.applyOverlaysToSources(ctx, component, outputDir); err != nil { - return err + if applyOverlays { + err := p.applyOverlaysToSources(ctx, component, outputDir) + if err != nil { + return err + } } // Record the changes as synthetic git history when dist-git creation is enabled. @@ -229,7 +228,7 @@ func (p *sourcePreparerImpl) applyOverlays( } // collectOverlays gathers all overlays for a component into a single ordered slice: -// user overlays first, followed by macros-load, check-skip, and file-header overlays. +// macros-load first, then user overlays, followed by check-skip and file-header overlays. func (p *sourcePreparerImpl) collectOverlays( component components.Component, macrosFileName string, ) ([]projectconfig.ComponentOverlay, error) { @@ -237,8 +236,6 @@ func (p *sourcePreparerImpl) collectOverlays( var allOverlays []projectconfig.ComponentOverlay - allOverlays = append(allOverlays, config.Overlays...) - if macrosFileName != "" { macroOverlays, err := synthesizeMacroLoadOverlays(macrosFileName) if err != nil { @@ -248,6 +245,7 @@ func (p *sourcePreparerImpl) collectOverlays( allOverlays = append(allOverlays, macroOverlays...) } + allOverlays = append(allOverlays, config.Overlays...) allOverlays = append(allOverlays, synthesizeCheckSkipOverlays(config.Build.Check)...) allOverlays = append(allOverlays, generateFileHeaderOverlay()...) @@ -276,9 +274,8 @@ func initSourcesRepo(sourcesDirPath string) (*gogit.Repository, error) { _, err = worktree.Commit("Initial sources", &gogit.CommitOptions{ Author: &object.Signature{ - Name: "azldev", - Email: "azldev@microsoft.com", - When: time.Unix(0, 0).UTC(), + Name: "azldev", + When: time.Unix(0, 0).UTC(), }, }) if err != nil { @@ -293,7 +290,6 @@ func initSourcesRepo(sourcesDirPath string) (*gogit.Repository, error) { // with an initial commit so Affects commits can be layered on uniformly for all // component types. // -// Returns nil when there are no Affects commits to apply. // Returns a non-nil error if history generation fails. func (p *sourcePreparerImpl) trySyntheticHistory( component components.Component, diff --git a/internal/app/azldev/core/sources/synthistory.go b/internal/app/azldev/core/sources/synthistory.go index e4e13ab..0c41f2e 100644 --- a/internal/app/azldev/core/sources/synthistory.go +++ b/internal/app/azldev/core/sources/synthistory.go @@ -4,10 +4,8 @@ package sources import ( - "errors" "fmt" "log/slog" - "os" "path/filepath" "regexp" "slices" @@ -19,18 +17,9 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" ) -// affectsRegexPattern is the regex pattern prefix used to match an [AffectsPrefix] marker -// line in a commit message. It is combined with a quoted component name to form the full -// match expression. -const affectsRegexPattern = `(?m)^\s*Affects:\s*` - -var ( - // ErrNoGitRepository is returned when no enclosing git repository can be found. - ErrNoGitRepository = errors.New("no git repository found") - - // ErrNoOverlaysToCommit is returned when there are no synthetic commits to create. - ErrNoOverlaysToCommit = errors.New("no synthetic commits to create") -) +// affectsRegexPattern is the regex pattern prefix used to match an "Affects:" trailer +// line in a commit message. Each line must contain exactly one component name. +const affectsRegexPattern = `(?m)^[ \t]*Affects:[ \t]*` // CommitMetadata holds full metadata for a commit in the project repository. type CommitMetadata struct { @@ -41,9 +30,17 @@ type CommitMetadata struct { Message string } +// MessageAffectsComponent reports whether a commit message contains an "Affects:" +// trailer line naming the given component. +func MessageAffectsComponent(message, componentName string) bool { + re := regexp.MustCompile(affectsRegexPattern + regexp.QuoteMeta(componentName) + `[ \t]*$`) + + return re.MatchString(message) +} + // FindAffectsCommits walks the git log from HEAD and returns metadata for all commits -// whose message contains "Affects: ". Results are sorted chronologically -// (oldest first). +// whose message contains an "Affects: " trailer line. Results are sorted +// chronologically (oldest first). func FindAffectsCommits(repo *gogit.Repository, componentName string) ([]CommitMetadata, error) { head, err := repo.Head() if err != nil { @@ -57,10 +54,8 @@ func FindAffectsCommits(repo *gogit.Repository, componentName string) ([]CommitM var matches []CommitMetadata - re := regexp.MustCompile(affectsRegexPattern + regexp.QuoteMeta(componentName) + `(?:$|\s|[,;])`) - err = commitIter.ForEach(func(commit *object.Commit) error { - if re.MatchString(commit.Message) { + if MessageAffectsComponent(commit.Message, componentName) { matches = append(matches, CommitMetadata{ Hash: commit.Hash.String(), Author: commit.Author.Name, @@ -85,16 +80,11 @@ func FindAffectsCommits(repo *gogit.Repository, componentName string) ([]CommitM // CommitSyntheticHistory stages all pending working tree changes and creates synthetic // commits in the provided git repository. The first commit captures all file changes; // subsequent commits are created as empty commits to preserve the commit count for -// rpmautospec release numbering. Overlay application must happen before calling this -// function — it only handles the git history. +// rpmautospec release numbering. func CommitSyntheticHistory( repo *gogit.Repository, commits []CommitMetadata, ) error { - if len(commits) == 0 { - return ErrNoOverlaysToCommit - } - worktree, err := repo.Worktree() if err != nil { return fmt.Errorf("failed to get worktree:\n%w", err) @@ -112,11 +102,11 @@ func CommitSyntheticHistory( "projectHash", commitMeta.Hash, ) - message := fmt.Sprintf("[azldev] %s\n\nProject commit: %s", + message := fmt.Sprintf("%s\n\nProject commit: %s", commitMeta.Message, commitMeta.Hash) _, err := worktree.Commit(message, &gogit.CommitOptions{ - AllowEmptyCommits: commitIdx > 0, + AllowEmptyCommits: true, Author: &object.Signature{ Name: commitMeta.Author, Email: commitMeta.AuthorEmail, @@ -136,9 +126,8 @@ func CommitSyntheticHistory( // buildSyntheticCommits resolves the project repository from the component's config file, // walks the git log for commits containing "Affects: ", and returns the -// matching commit metadata sorted chronologically. -// -// Returns nil when no matching commits are found. +// matching commit metadata sorted chronologically. If no Affects commits are found, a +// single default overlay commit is returned instead. func buildSyntheticCommits( config *projectconfig.ComponentConfig, componentName string, ) ([]CommitMetadata, error) { @@ -151,7 +140,7 @@ func buildSyntheticCommits( return nil, nil } - projectRepo, _, err := openProjectRepo(configFilePath) + projectRepo, err := openProjectRepo(configFilePath) if err != nil { return nil, err } @@ -165,21 +154,41 @@ func buildSyntheticCommits( "component", componentName, "commitCount", len(affectsCommits)) - commits := make([]CommitMetadata, 0, len(affectsCommits)) + if len(affectsCommits) == 0 { + slog.Info("No commits with Affects marker found; "+ + "creating default commit", + "component", componentName) - // Create one synthetic commit per Affects commit, preserving each commit's - // original message and author attribution in the upstream history. - commits = append(commits, affectsCommits...) + return []CommitMetadata{ + defaultOverlayCommit(projectRepo, componentName), + }, nil + } - if len(commits) == 0 { - slog.Warn("No commits with Affects marker found; "+ - "falling back to standard overlay processing", - "component", componentName) + return affectsCommits, nil +} - return nil, nil +// defaultOverlayCommit returns a single [CommitMetadata] entry that represents a generic +// commit when no Affects commits exist in the project history. The commit hash is +// set to the current HEAD of the project repository. +func defaultOverlayCommit(repo *gogit.Repository, componentName string) CommitMetadata { + var ( + timestamp int64 + hash string + ) + + if head, err := repo.Head(); err == nil { + hash = head.Hash().String() + if commit, commitErr := repo.CommitObject(head.Hash()); commitErr == nil { + timestamp = commit.Author.When.Unix() + } } - return commits, nil + return CommitMetadata{ + Hash: hash, + Author: "azldev", + Timestamp: timestamp, + Message: "Latest state for " + componentName, + } } // resolveConfigFilePath extracts and validates the source config file path from the component config. @@ -197,53 +206,18 @@ func resolveConfigFilePath(config *projectconfig.ComponentConfig, componentName return configFilePath, nil } -// openProjectRepo finds the git repository root containing configFilePath, opens it, and -// returns the repository handle along with the config file path relative to the repo root. -func openProjectRepo(configFilePath string) (*gogit.Repository, string, error) { - projectRepoPath, err := findRepoRoot(filepath.Dir(configFilePath)) +// openProjectRepo finds and opens the git repository containing configFilePath by +// walking up the directory tree. +func openProjectRepo(configFilePath string) (*gogit.Repository, error) { + repo, err := gogit.PlainOpenWithOptions(filepath.Dir(configFilePath), &gogit.PlainOpenOptions{ + DetectDotGit: true, + }) if err != nil { - return nil, "", fmt.Errorf("failed to find project repository for config file %#q:\n%w", + return nil, fmt.Errorf("failed to find project repository for config file %#q:\n%w", configFilePath, err) } - projectRepo, err := gogit.PlainOpen(projectRepoPath) - if err != nil { - return nil, "", fmt.Errorf("failed to open project repository at %#q:\n%w", projectRepoPath, err) - } - - relConfigPath, err := filepath.Rel(projectRepoPath, configFilePath) - if err != nil { - return nil, "", fmt.Errorf("failed to compute relative config path:\n%w", err) - } - - return projectRepo, relConfigPath, nil -} - -// findRepoRoot walks up the directory tree from startDir to find a directory containing -// a .git directory or file (for worktrees). -func findRepoRoot(startDir string) (string, error) { - dir, err := filepath.Abs(startDir) - if err != nil { - return "", fmt.Errorf("failed to get absolute path for %#q:\n%w", startDir, err) - } - - for { - gitPath := filepath.Join(dir, ".git") - - if info, statErr := os.Stat(gitPath); statErr == nil { - // Accept both .git directories and .git files (for git worktrees). - if info.IsDir() || info.Mode().IsRegular() { - return dir, nil - } - } - - parent := filepath.Dir(dir) - if parent == dir { - return "", fmt.Errorf("%w: searched from %#q to filesystem root", ErrNoGitRepository, startDir) - } - - dir = parent - } + return repo, nil } // unixToTime converts a Unix timestamp to a [time.Time] in UTC. diff --git a/internal/app/azldev/core/sources/synthistory_test.go b/internal/app/azldev/core/sources/synthistory_test.go index acc320f..24ef820 100644 --- a/internal/app/azldev/core/sources/synthistory_test.go +++ b/internal/app/azldev/core/sources/synthistory_test.go @@ -175,7 +175,7 @@ func TestFindAffectsCommits_AffectsInSubject(t *testing.T) { // Affects marker in the subject line (not just the body). addCommit(t, repo, - "Affects: curl - fix build failure", + "Affects: curl", "Alice", "alice@example.com", time.Date(2025, 1, 1, 10, 0, 0, 0, time.UTC)) @@ -216,6 +216,36 @@ func TestFindAffectsCommits_CaseSensitive(t *testing.T) { assert.Equal(t, "Alice", results[0].Author) } +func TestMessageAffectsComponent(t *testing.T) { + tests := []struct { + name string + message string + component string + want bool + }{ + // Positive matches. + {"exact match in body", "Fix bug\n\nAffects: curl", "curl", true}, + {"trailing whitespace", "Fix bug\n\nAffects: curl ", "curl", true}, + {"leading whitespace on line", "Fix bug\n\n Affects: curl", "curl", true}, + {"subject line only", "Affects: curl", "curl", true}, + + // Negative matches. + {"different component", "Fix bug\n\nAffects: wget", "curl", false}, + {"no substring match", "Fix bug\n\nAffects: curl-minimal", "curl", false}, + {"comma separated", "Fix bug\n\nAffects: curl, wget", "curl", false}, + {"extra text after name", "Affects: curl - fix build failure", "curl", false}, + {"case sensitive", "Fix bug\n\nAffects: Curl", "curl", false}, + {"no match across newlines", "Fix bug\n\nAffects:\ncurl", "curl", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := sources.MessageAffectsComponent(tt.message, tt.component) + assert.Equal(t, tt.want, got) + }) + } +} + func TestCommitSyntheticHistory(t *testing.T) { // Create an in-memory repo with an initial commit (simulating upstream). memFS := memfs.New() @@ -378,13 +408,3 @@ func TestCommitSyntheticHistory_SingleCommit(t *testing.T) { require.NoError(t, err) assert.Contains(t, content, "# modified") } - -func TestCommitSyntheticHistory_EmptyCommits(t *testing.T) { - repo := createInMemoryRepo(t) - - addCommit(t, repo, "initial", "Test", "test@example.com", - time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) - - err := sources.CommitSyntheticHistory(repo, nil) - assert.ErrorIs(t, err, sources.ErrNoOverlaysToCommit) -}