chore: Add options to support v1 logging accurately for v1-to-v2 compatibility

cmd/options.go

@@ -76,6 +76,38 @@ func WithAutoIP() Option {
 	}
 }
 
+// WithProxyV1Compatibility enables legacy behavior of v1 and will print
+// out "Ready for new connections" when the proxy starts.
+func WithProxyV1Compatibility() Option {
+	return func(c *Command) {
+		c.conf.ProxyV1Compatibility = true
+	}
+}
+
+// WithProxyV1LogDebugStdout enables legacy behavior of v1 and will print
+// debug/info logs to stdout instead of stderr.
+func WithProxyV1LogDebugStdout() Option {
+	return func(c *Command) {
+		c.conf.LogDebugStdout = true
+	}
+}
+
+// WithProxyV1Projects enables legacy behavior of v1 and will connect to
+// all Second Generation instances in the provided projects.
+func WithProxyV1Projects(projects []string) Option {
+	return func(c *Command) {
+		c.conf.Projects = projects
+	}
+}
+
+// WithProxyV1Verbose enables legacy behavior of v1 and will set
+// the debug-logs flag on the v2 proxy.
+func WithProxyV1Verbose(v bool) Option {
+	return func(c *Command) {
+		c.conf.DebugLogs = v
+	}
+}
+
 // WithQuietLogging configures the Proxy to log error messages only.
 func WithQuietLogging() Option {
 	return func(c *Command) {

cmd/options_test.go

@@ -146,6 +146,36 @@ func TestCommandOptions(t *testing.T) {
 			},
 			option: WithLazyRefresh(),
 		},
+		{
+			desc: "with proxy v1 compatibility",
+			isValid: func(c *Command) error {
+				if !c.conf.ProxyV1Compatibility {
+					return errors.New("compatibility was false, but should be true")
+				}
+				return nil
+			},
+			option: WithProxyV1Compatibility(),
+		},
+		{
+			desc: "with proxy v1 verbose true",
+			isValid: func(c *Command) error {
+				if !c.conf.DebugLogs {
+					return errors.New("DebugLogs was false, but should be true")
+				}
+				return nil
+			},
+			option: WithProxyV1Verbose(true),
+		},
+		{
+			desc: "with proxy v1 verbose false",
+			isValid: func(c *Command) error {
+				if c.conf.DebugLogs {
+					return errors.New("DebugLogs was true, but should be false")
+				}
+				return nil
+			},
+			option: WithProxyV1Verbose(false),
+		},
 	}
 
 	for _, tc := range tcs {

cmd/root.go

@@ -33,6 +33,7 @@
 	"syscall"
 	"time"
 
+	"cloud.google.com/go/compute/metadata"
 	"contrib.go.opencensus.io/exporter/prometheus"
 	"contrib.go.opencensus.io/exporter/stackdriver"
 	"github.com/GoogleCloudPlatform/cloud-sql-proxy/v2/cloudsql"
@@ -44,6 +45,10 @@
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"go.opencensus.io/trace"
+	"golang.org/x/oauth2"
+	"google.golang.org/api/impersonate"
+	"google.golang.org/api/option"
+	sqladmin "google.golang.org/api/sqladmin/v1"
 )
 
 var (
@@ -73,7 +78,12 @@
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	if err := NewCommand().Execute(); err != nil {
+	ExecuteCommand(NewCommand())
+}
+
+// ExecuteCommand runs a preconfigured command, and exits the appropriate error code.
+func ExecuteCommand(c *Command) {
+	if err := c.Execute(); err != nil {
 		exit := 1
 		var terr *exitError
 		if errors.As(err, &terr) {
@@ -593,6 +603,11 @@
 		`If set, the Proxy will skip any instances that are invalid/unreachable (
 only applicable to Unix sockets)`)
 
+	localFlags.StringVar(&c.conf.InstancesMetadata, "instances-metadata", "",
+		`If provided, it is treated as a path to a metadata value which
+contains a comma-separated list of instance connection names.
+Only supported when running on Google Compute Engine.`)
+
 	// Global and per instance flags
 	localFlags.StringVarP(&c.conf.Addr, "address", "a", "127.0.0.1",
 		"(*) Address to bind Cloud SQL instance listeners.")
@@ -639,13 +654,33 @@
 		o(c)
 	}
 
+	// If instances-metadata is set, fetch instances from GCE metadata.
+	if c.conf.InstancesMetadata != "" {
+		mArgs, err := instanceFromMetadata(c.conf.InstancesMetadata)
+		if err != nil {
+			return err
+		}
+		args = append(args, mArgs...)
+	}
+
+	// If projects is set, fetch instances from those projects.
+	if len(c.conf.Projects) > 0 {
+		pArgs, err := instanceFromProjects(c.Context(), c.conf)
+		if err != nil {
+			return err
+		}
+		args = append(args, pArgs...)
+	}
+
 	// Handle logger separately from config
 	if c.conf.StructuredLogs {
 		c.logger = log.NewStructuredLogger(c.conf.Quiet)
 	}
 
 	if c.conf.Quiet {
 		c.logger = log.NewStdLogger(io.Discard, os.Stderr)
+	} else if c.conf.LogDebugStdout {
+		c.logger = log.NewStdLogger(os.Stdout, os.Stderr)
 	}
 
 	err = parseConfig(c, c.conf, args)
@@ -1105,6 +1140,9 @@
 		return err
 	case p = <-startCh:
 		cmd.logger.Infof("The proxy has started successfully and is ready for new connections!")
+		if cmd.conf.ProxyV1Compatibility {
+			cmd.logger.Infof("Ready for new connections")
+		}
 		// If running under systemd with Type=notify, it will send a message to the
 		// service manager that it is ready to handle connections now.
 		go func() {
@@ -1260,3 +1298,129 @@
 		l.Errorf("failed to shutdown HTTP server: %v\n", err)
 	}
 }
+
+func instanceFromMetadata(path string) ([]string, error) {
+	if !metadata.OnGCE() {
+		return nil, newBadCommandError("instances-metadata unsupported outside of Google Compute Engine")
+	}
+	val, err := metadata.Get(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch metadata from %q: %v", path, err)
+	}
+
+	var args []string
+	for _, inst := range strings.Split(val, ",") {
+		inst = strings.TrimSpace(inst)
+		if inst != "" {
+			args = append(args, inst)
+		}
+	}
+	return args, nil
+}
+
+func instanceFromProjects(ctx context.Context, conf *proxy.Config) ([]string, error) {
+	var opts []option.ClientOption
+	if conf.APIEndpointURL != "" {
+		opts = append(opts, option.WithEndpoint(conf.APIEndpointURL))
+	}
+	if conf.QuotaProject != "" {
+		opts = append(opts, option.WithQuotaProject(conf.QuotaProject))
+	}
+
+	// Handle credentials
+	switch {
+	case conf.ImpersonationChain != "":
+		target, delegates := parseImpersonationChain(conf.ImpersonationChain)
+		var iopts []option.ClientOption
+		switch {
+		case conf.Token != "":
+			ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: conf.Token})
+			iopts = append(iopts, option.WithTokenSource(ts))
+		case conf.CredentialsFile != "":
+			iopts = append(iopts, option.WithCredentialsFile(conf.CredentialsFile))
+		case conf.CredentialsJSON != "":
+			iopts = append(iopts, option.WithCredentialsJSON([]byte(conf.CredentialsJSON)))
+		}
+		ts, err := impersonate.CredentialsTokenSource(ctx, impersonate.CredentialsConfig{
+			TargetPrincipal: target,
+			Delegates:       delegates,
+			Scopes:          []string{sqladmin.SqlserviceAdminScope},
+		}, iopts...)
+		if err != nil {
+			return nil, err
+		}
+		opts = append(opts, option.WithTokenSource(ts))
+	case conf.Token != "":
+		ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: conf.Token})
+		opts = append(opts, option.WithTokenSource(ts))
+	case conf.CredentialsFile != "":
+		opts = append(opts, option.WithCredentialsFile(conf.CredentialsFile))
+	case conf.CredentialsJSON != "":
+		opts = append(opts, option.WithCredentialsJSON([]byte(conf.CredentialsJSON)))
+	}
+
+	sql, err := sqladmin.NewService(ctx, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	ch := make(chan string)
+	errCh := make(chan error, len(conf.Projects))
+	var wg sync.WaitGroup
+	wg.Add(len(conf.Projects))
+	for _, proj := range conf.Projects {
+		proj := proj
+		go func() {
+			defer wg.Done()
+			err := sql.Instances.List(proj).Pages(ctx, func(r *sqladmin.InstancesListResponse) error {
+				for _, in := range r.Items {
+					// The Proxy only supports Second Gen
+					if in.BackendType == "SECOND_GEN" {
+						ch <- in.ConnectionName
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				errCh <- fmt.Errorf("failed to list instances in %q: %v", proj, err)
+			}
+		}()
+	}
+	go func() {
+		wg.Wait()
+		close(ch)
+		close(errCh)
+	}()
+
+	var args []string
+	for x := range ch {
+		args = append(args, x)
+	}
+
+	// Check for any errors
+	for err := range errCh {
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if len(args) == 0 {
+		return nil, fmt.Errorf("no Cloud SQL Instances found in projects: %v", conf.Projects)
+	}
+	return args, nil
+}
+
+func parseImpersonationChain(chain string) (string, []string) {
+	accts := strings.Split(chain, ",")
+	target := accts[0]
+	// Assign delegates if the chain is more than one account. Delegation
+	// goes from last back towards target, e.g., With sa1,sa2,sa3, sa3
+	// delegates to sa2, which impersonates the target sa1.
+	var delegates []string
+	if l := len(accts); l > 1 {
+		for i := l - 1; i > 0; i-- {
+			delegates = append(delegates, accts[i])
+		}
+	}
+	return target, delegates
+}

internal/proxy/proxy.go

@@ -188,6 +188,14 @@ type Config struct {
 	// endpoint for all instances
 	PSC bool
 
+	// ProxyV1Compatibility supports a legacy behavior where the Proxy will
+	// print out "Ready for new connections" when the proxy starts.
+	ProxyV1Compatibility bool
+
+	// LogDebugStdout supports a legacy behavior where the Proxy will
+	// print debug/info logs to stdout instead of stderr.
+	LogDebugStdout bool
+
 	// AutoIP supports a legacy behavior where the Proxy will connect to
 	// the first IP address returned from the SQL ADmin API response. This
 	// setting should be avoided and used only to support legacy Proxy
@@ -200,6 +208,14 @@ type Config struct {
 	// of a request context, e.g., Cloud Run.
 	LazyRefresh bool
 
+	// InstancesMetadata is the GCE metadata path to a value that contains a
+	// comma-separated list of instance connection names.
+	InstancesMetadata string
+
+	// Projects is a list of projects from which to connect to all
+	// Second Generation instances.
+	Projects []string
+
 	// Instances are configuration for individual instances. Instance
 	// configuration takes precedence over global configuration.
 	Instances []InstanceConnConfig

migration-guide.md

@@ -77,6 +77,16 @@ vs
 ./cloud-sql-proxy --unix-socket /cloudsql <INSTANCE_CONNECTION_NAME>
 ```
 
+### Automatic instance discovery from metadata
+
+```shell
+# v1
+./cloud_sql_proxy -dir /cloudsql -instances_metadata=instance/attributes/cloud-sql-instances
+
+# v2
+./cloud-sql-proxy --unix-socket /cloudsql --instances-metadata instance/attributes/cloud-sql-instances
+```
+
 ### Listen on multiple TCP sockets with incrementing ports
 
 ```shell
@@ -155,11 +165,11 @@ The following table lists in alphabetical order v1 flags and their v2 version.
 | fuse_tmp                    | fuse-temp-dir               |                                                                                      |
 | health_check_port           | http-port                   |  Use --http-address=0.0.0.0 when using a health check in Kubernetes                  |
 | host                        | sqladmin-api-endpoint       |                                                                                      |
-| instances_metadata          | 🤔                          | [Feature Request](https://github.com/GoogleCloudPlatform/cloudsql-proxy/issues/1259) |
+| instances_metadata          | instances-metadata          |                                                                                      |
 | ip_address_types            | private-ip                  | Defaults to public. To connect to a private IP, you must add the --private-ip flag   |
 | log_debug_stdout            | ❌                          | v2 logs to stdout, errors to stderr by default                                       |
 | max_connections             | max-connections             |                                                                                      |
-| projects                    | ❌                          | v2 prefers explicit connection configuration to avoid user error                     |
+| projects                    | ❌                          | Not supported as a v2 flag. v2 prefers explicit configuration. v1 -projects is supported in v2 compatibility mode. |
 | quiet                       | quiet                       | quiet disables all logging except errors                                             |
 | quota_project               | quota-project               |                                                                                      |
 | refresh_config_throttle     | ❌                          |                                                                                      |