Make map() keep its column types, ENUMs included

src/include/duckdb_python/numpy/numpy_result_conversion.hpp

@@ -38,4 +38,7 @@ class NumpyResultConversion {
 	bool pandas;
 };
 
+//! The dtype df() gives an ENUM column: a pandas CategoricalDtype over the enum's values in order, ordered
+nb::object EnumCategoricalDtype(const LogicalType &enum_type);
+
 } // namespace duckdb

src/include/duckdb_python/numpy/numpy_scan.hpp

@@ -2,19 +2,33 @@
 
 #include "duckdb_python/nb/casters.hpp"
 #include "duckdb/common/common.hpp"
+#include "duckdb/common/types.hpp"
 
 namespace duckdb {
 
 struct PandasColumnBindData;
 
+//! A cell of a declared column whose type was implied is not of that type's kind; source is the type the cell
+//! converted to, and the caller reports it in the words of its own type check
+struct ImpliedTypeRefused : public std::exception {
+	explicit ImpliedTypeRefused(LogicalType source_p) : source(std::move(source_p)) {
+	}
+	const char *what() const noexcept override {
+		return "a value is not of the kind of its column's implied type";
+	}
+	LogicalType source;
+};
+
 struct NumpyScan {
 	static void Scan(ClientContext &context, PandasColumnBindData &bind_data, idx_t count, idx_t offset, Vector &out);
 	//! masked holds one numpy bool per row of the whole column, read as a byte, or is null when the column has no mask
 	static void ScanObjectColumn(ClientContext &context, PyObject **col, idx_t stride, const uint8_t *masked,
 	                             idx_t count, idx_t offset, Vector &out);
-	//! Converts each cell into the declared type of the column as the appender converts a value
+	//! Converts each cell into the declared type of the column as the appender converts a value; with implied_type,
+	//! a cell that is not of the type's kind raises ImpliedTypeRefused, and with bit_storage_refused so does one
+	//! holding bytes where the type holds a BIT
 	static void ScanDeclaredColumn(ClientContext &context, PyObject **col, idx_t stride, const uint8_t *masked,
-	                               idx_t count, idx_t offset, Vector &out);
+	                               idx_t count, idx_t offset, Vector &out, bool implied_type, bool bit_storage_refused);
 };
 
 } // namespace duckdb

src/include/duckdb_python/pandas/pandas_bind.hpp

@@ -40,13 +40,27 @@ struct PandasColumnBindData {
 	PythonObjectContainer object_str_val;
 	//! The caller declared the column's type: each value converts into it as the appender converts a row
 	bool declared = false;
+	//! Of a declared column: the caller implied the type rather than stating it, so a cell is read only when it is of
+	//! the type's kind, since a cast could change what it was
+	bool implied_type = false;
+	//! Of a declared column map() reads: a cell holding bytes where the type holds a BIT is refused, since df() hands
+	//! a BIT value to the function as its storage bytes
+	bool bit_storage_refused = false;
+};
+
+//! The categories of a CategoricalDtype the caller built from an ENUM's values, in order
+struct PandasEnumDtype {
+	nb::handle categories;
+	LogicalType type;
 };
 
 struct Pandas {
-	//! Returns the frame's row count; declared_types, when given, hold one type per column, INVALID where none
+	//! Returns the frame's row count; declared_types, when given, hold one type per column, INVALID where none; a
+	//! categorical column whose categories are those of one of enum_dtypes, that very object, reads its codes as that
+	//! ENUM
 	static idx_t Bind(ClientContext &config, nb::handle df, vector<PandasColumnBindData> &out,
 	                  vector<LogicalType> &return_types, vector<Identifier> &names,
-	                  const vector<LogicalType> &declared_types = {});
+	                  const vector<LogicalType> &declared_types = {}, const vector<PandasEnumDtype> &enum_dtypes = {});
 };
 
 } // namespace duckdb

src/include/duckdb_python/pyresult.hpp

@@ -136,8 +136,6 @@ struct DuckDBPyResult {
 	//! and whose originals the caller carries here. Empty means the result's own names.
 	vector<Identifier> names_override;
 	unique_ptr<DataChunk> current_chunk;
-	// Holds the categories of Categorical/ENUM types
-	unordered_map<idx_t, nb::list> categories;
 	// Holds the categorical type of Categorical/ENUM types
 	unordered_map<idx_t, nb::object> categories_type;
 };

src/include/duckdb_python/python_conversion.hpp

@@ -53,6 +53,38 @@ LogicalType SniffPythonIntegerType(nb::handle ele);
 //! The type of a duckdb.Value, or InvalidInputException when its 'type' attribute is not a DuckDB type
 LogicalType PythonValueType(nb::handle value);
 bool DictionaryHasMapFormat(const PyDictionary &dict);
+//! A dict of exactly 'key' and 'value' with a list on one side and a plain value on the other: a STRUCT of those
+//! fields without a target, and refused as a MAP's key and value lists for a MAP target
+bool MixesAListAndAValue(const PyDictionary &dict);
+//! Keeps the field indexes the conversion of shaped STRUCT dicts builds on this thread while a declared scan runs; the
+//! outermost scope drops them when it ends, so no index outlives the shapes it was built for
+class StructFieldIndexScope {
+public:
+	StructFieldIndexScope();
+	~StructFieldIndexScope();
+	StructFieldIndexScope(const StructFieldIndexScope &) = delete;
+	StructFieldIndexScope &operator=(const StructFieldIndexScope &) = delete;
+};
+//! A key or value of a MAP whose target is a type without children converts without a target and is then cast with
+//! the connection's settings, as a value of a column is, so each keeps its own type until the cast instead of meeting
+//! its siblings' common type (2**60 + 1 beside 1.0 would read as a DOUBLE, rounded)
+Value TransformMapEntry(optional_ptr<ClientContext> context, nb::handle entry, const LogicalType &target_type,
+                        bool nan_as_null);
+//! A target the declared scan builds from a column's type (a shape), as opposed to a column's own type: a dict for a
+//! shaped STRUCT takes its own keys, as without a target; a shaped MAP casts each key and value whose type has no
+//! children with the connection's settings; a shaped naive timestamp reads a cell as TransformTimestamp does
+LogicalType AsShape(const LogicalType &type);
+bool IsShape(const LogicalType &type);
+//! The nanosecond a datetime subclass such as a pandas Timestamp gives as an int, or -1 when it gives none
+int64_t NanosecondOf(nb::handle cell);
+//! Whether a cell is a pandas Timestamp with nanoseconds, which no Python datetime holds, naive or, when zoned, zone
+//! aware; they are then the nanoseconds since the epoch, in UTC for a zone aware one
+bool HoldsNanoseconds(nb::handle cell, timestamp_ns_t &nanoseconds, bool zoned);
+//! A value for a timestamp type converted without a target, but keeping the nanoseconds of a pandas Timestamp for a
+//! TIMESTAMP_NS or a TIMESTAMPTZ_NS, and reading a numpy scalar as PythonValueOfNumpyScalar does, so a datetime64 finer
+//! than a microsecond is a TIMESTAMP_NS that a coarser unit takes through the engine's cast
+Value TransformTimestamp(optional_ptr<ClientContext> context, nb::handle ele, const LogicalType &target_type,
+                         bool nan_as_null);
 void TransformPythonObject(optional_ptr<ClientContext> context, nb::handle ele, Vector &vector, idx_t result_offset,
                            bool nan_as_null = true);
 Value TransformPythonValue(optional_ptr<ClientContext> context, nb::handle ele,