diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f6cfbb5126158e1..bd2407949db90dc 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -211,8 +211,8 @@ than one MIME-type database; it provides an interface similar to the one of the .. class:: MimeTypes(filenames=(), strict=True) This class represents a MIME-types database. By default, it provides access to - the same database as the rest of this module. The initial database is a copy of - that provided by the module, and may be extended by loading additional + the same database as the rest of this module. The initial database is created from + Python's built-in MIME type tables. It may be extended by loading additional :file:`mime.types`\ -style files into the database using the :meth:`read` or :meth:`readfp` methods. The mapping dictionaries may also be cleared before loading additional data if the default data is not desired. @@ -226,30 +226,32 @@ than one MIME-type database; it provides an interface similar to the one of the Dictionary mapping suffixes to suffixes. This is used to allow recognition of encoded files for which the encoding and the type are indicated by the same extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + to allow the encoding and type to be recognized separately. + This is initialized with some predefined values. .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. + This is initialized with some predefined values. .. attribute:: MimeTypes.types_map Tuple containing two dictionaries, mapping filename extensions to MIME types: the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + the standard types. + They are initialized with some predefined values and MIME type + information loaded from files specified by the *filenames* argument. .. attribute:: MimeTypes.types_map_inv Tuple containing two dictionaries, mapping MIME types to a list of filename extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + second one is for the standard types. + They are initialized with some predefined values and MIME type + information loaded from files specified by the *filenames* argument. .. method:: MimeTypes.guess_extension(type, strict=True) diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index c88411ce0b7b91f..abd68c5006bd475 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -172,6 +172,20 @@ XMLParser Objects or ``None`` if :meth:`SetBase` hasn't been called. +.. method:: xmlparser.GetSpecifiedAttributeCount() + + Return the index just past the attributes given in the start tag. + Attributes defaulted from the DTD follow the specified ones, + so attributes at lower indices in the list + passed to :attr:`StartElementHandler` were given in the start tag. + Each attribute takes two items in that list, + its name and its value. + Only meaningful inside a :attr:`StartElementHandler` call, + and only if :attr:`ordered_attributes` is true. + + .. versionadded:: next + + .. method:: xmlparser.GetInputContext() Returns the input data that generated the current event as a string. The data is diff --git a/Doc/library/site.rst b/Doc/library/site.rst index fe8718280b2a015..090c4ef3ba03fc2 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -453,9 +453,10 @@ Module contents Return a list containing all global site-packages directories. - For each directory present in *prefixes* (or :data:`PREFIXES` if *prefixes* - is ``None``), this function will find its site-packages subdirectory - depending on the system environment, and will return a list of full paths. + For each directory given in *prefixes* (or :data:`PREFIXES` if *prefixes* + is ``None``), this function will compute its site-packages subdirectory + depending on the system environment, and will return a list of full paths, + which are not checked for existence. .. versionadded:: 3.2 diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 1a5291d018ac702..efc81f31e36a5be 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -245,21 +245,47 @@ rules apply: Instead, :mod:`!xml.dom.minidom` uses standard Python exceptions such as :exc:`TypeError` and :exc:`AttributeError`. -* :class:`NodeList` objects are implemented using Python's built-in list type. - These objects provide the interface defined in the DOM specification, but with - earlier versions of Python they do not support the official API. They are, - however, much more "Pythonic" than the interface defined in the W3C - recommendations. +* Each of the :class:`~xml.dom.NodeList` and :class:`~xml.dom.NamedNodeMap` + interfaces has two implementations, which provide additional methods and + operations. + + :attr:`~xml.dom.Node.childNodes` is a subclass of :class:`list`, or, for + nodes which cannot have children, a subclass of :class:`tuple`. + It supports iteration, concatenation, indexing and slicing. + + :attr:`~xml.dom.Node.attributes` supports ``len()``, the :keyword:`in` + operator, subscription by a name or by a ``(namespaceURI, localName)`` + tuple, assignment and deletion, and the methods :meth:`!get`, :meth:`!keys`, + :meth:`!keysNS`, :meth:`!values`, :meth:`!items` and :meth:`!itemsNS`. + :attr:`~xml.dom.DocumentType.entities` and + :attr:`~xml.dom.DocumentType.notations` are read-only and support only + ``len()`` and subscription by a name. + +* :attr:`~xml.dom.Document.strictErrorChecking` is always ``False``. + + .. versionchanged:: next + Previously, :attr:`~xml.dom.Attr.specified` was always ``False``. + +* The constraints of the DOM are now enforced, + and the corresponding exceptions are raised. + + .. versionchanged:: next + Previously, many invalid operations silently succeeded + and produced an invalid document, + but removing an absent attribute raised :exc:`~xml.dom.NotFoundErr`. The following interfaces have no implementation in :mod:`!xml.dom.minidom`: * :class:`DOMTimeStamp` -* :class:`EntityReference` - -Most of these reflect information in the XML document that is not of general +This reflects information in the XML document that is not of general utility to most DOM users. +.. versionchanged:: next + :class:`~xml.dom.EntityReference` is now implemented. + Note that the parser expands entity references, + so they only occur in a document if created explicitly. + .. rubric:: Footnotes .. [1] The encoding name included in the XML output should conform to diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 34e58dcad930125..6f5904fdf35e808 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -95,13 +95,15 @@ The :mod:`!xml.dom` contains the following functions: module name of a DOM implementation, or ``None``. If it is not ``None``, imports the corresponding module and returns a :class:`DOMImplementation` object if the import succeeds. If no name is given, and if the environment variable - :envvar:`PYTHON_DOM` is set, this variable is used to find the implementation. + :envvar:`!PYTHON_DOM` is set, this variable is used to find the implementation. + The only well-known name in the standard library is ``'minidom'``, + for :mod:`xml.dom.minidom`. If name is not given, this examines the available implementations to find one with the required feature set. If no implementation can be found, raise an :exc:`ImportError`. The features list must be a sequence of ``(feature, - version)`` pairs which are passed to the :meth:`hasFeature` method on available - :class:`DOMImplementation` objects. + version)`` pairs which are passed to the :meth:`~DOMImplementation.hasFeature` + method on available :class:`DOMImplementation` objects. Some convenience constants are also provided: @@ -109,8 +111,8 @@ Some convenience constants are also provided: .. data:: EMPTY_NAMESPACE The value used to indicate that no namespace is associated with a node in the - DOM. This is typically found as the :attr:`namespaceURI` of a node, or used as - the *namespaceURI* parameter to a namespaces-specific method. + DOM. This is typically found as the :attr:`~Node.namespaceURI` of a node, or + used as the *namespaceURI* parameter to a namespaces-specific method. .. data:: XML_NAMESPACE @@ -137,7 +139,7 @@ exception classes. The :class:`Node` class provided by this module does not implement any of the methods or attributes defined by the DOM specification; concrete DOM implementations must provide those. The :class:`Node` class provided as part of this module does provide the constants used for the -:attr:`nodeType` attribute on concrete :class:`Node` objects; they are located +:attr:`~Node.nodeType` attribute on concrete :class:`Node` objects; they are located within the class rather than at the module level to conform with the DOM specifications. @@ -151,6 +153,11 @@ Objects in the DOM The definitive documentation for the DOM is the DOM specification from the W3C. +The names documented in this section are DOM interfaces. +With the exception of :class:`Node` and the exception classes, +they are not provided by the :mod:`!xml.dom` module itself, +but by concrete DOM implementations, such as :mod:`xml.dom.minidom`. + Note that DOM attributes may also be manipulated as nodes instead of as simple strings. It is fairly rare that you must do this, however, so this usage is not yet documented. @@ -199,6 +206,9 @@ in Python. DOMImplementation Objects ^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: DOMImplementation + :no-typesetting: + The :class:`DOMImplementation` interface provides a way for applications to determine the availability of particular features in the DOM they are using. DOM Level 2 added the ability to create new :class:`Document` and @@ -227,25 +237,61 @@ DOM Level 2 added the ability to create new :class:`Document` and *qualifiedName*, *publicId*, and *systemId* strings, representing the information contained in an XML document type declaration. + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + .. _dom-node-objects: Node Objects ^^^^^^^^^^^^ +.. class:: Node + :no-typesetting: + All of the components of an XML document are subclasses of :class:`Node`. +Only nodes of the following types can have children, +and only children of the listed types: + +:class:`Document` + at most one :class:`Element`, at most one :class:`DocumentType`, + :class:`ProcessingInstruction` and :class:`Comment` + +:class:`DocumentFragment` and :class:`Element` + :class:`Element`, :class:`Text`, :class:`CDATASection`, + :class:`ProcessingInstruction` and :class:`Comment` + +:class:`Attr` + :class:`Text` + +Nodes of other types cannot have children. +Inserting a child of a not allowed type raises :exc:`HierarchyRequestErr`. + .. attribute:: Node.nodeType An integer representing the node type. Symbolic constants for the types are on - the :class:`Node` object: :const:`ELEMENT_NODE`, :const:`ATTRIBUTE_NODE`, - :const:`TEXT_NODE`, :const:`CDATA_SECTION_NODE`, :const:`ENTITY_NODE`, - :const:`PROCESSING_INSTRUCTION_NODE`, :const:`COMMENT_NODE`, - :const:`DOCUMENT_NODE`, :const:`DOCUMENT_TYPE_NODE`, :const:`NOTATION_NODE`. + the :class:`Node` object. This is a read-only attribute. +.. data:: Node.ELEMENT_NODE + Node.ATTRIBUTE_NODE + Node.TEXT_NODE + Node.CDATA_SECTION_NODE + Node.ENTITY_REFERENCE_NODE + Node.ENTITY_NODE + Node.PROCESSING_INSTRUCTION_NODE + Node.COMMENT_NODE + Node.DOCUMENT_NODE + Node.DOCUMENT_TYPE_NODE + Node.DOCUMENT_FRAGMENT_NODE + Node.NOTATION_NODE + + Integer constants for the possible values + of the :attr:`~Node.nodeType` attribute. + + .. attribute:: Node.parentNode The parent of the current node, or ``None`` for the document node. The value is @@ -281,7 +327,9 @@ All of the components of an XML document are subclasses of :class:`Node`. .. attribute:: Node.childNodes - A list of nodes contained within this node. This is a read-only attribute. + A :class:`NodeList` of the children of this node. + If the node has no children, the list is empty. + This is a read-only attribute. .. attribute:: Node.firstChild @@ -298,14 +346,14 @@ All of the components of an XML document are subclasses of :class:`Node`. .. attribute:: Node.localName - The part of the :attr:`tagName` following the colon if there is one, else the - entire :attr:`tagName`. The value is a string. + The part of the :attr:`~Element.tagName` following the colon if there is one, + else the entire :attr:`~Element.tagName`. The value is a string. .. attribute:: Node.prefix - The part of the :attr:`tagName` preceding the colon if there is one, else the - empty string. The value is a string, or ``None``. + The part of the :attr:`~Element.tagName` preceding the colon if there is one, + else the empty string. The value is a string, or ``None``. .. attribute:: Node.namespaceURI @@ -314,21 +362,77 @@ All of the components of an XML document are subclasses of :class:`Node`. ``None``. This is a read-only attribute. +.. attribute:: Node.ownerDocument + + The :class:`Document` object to which this node belongs, or ``None`` + for a document itself. + This is a read-only attribute. + + +.. method:: Node.isSupported(feature, version) + + Return whether the DOM implementation supports a particular *feature*, + as :meth:`DOMImplementation.hasFeature` does. + + +.. method:: Node.setUserData(key, data, handler) + + Associate *data* with *key* on this node and return the data previously + associated with *key*, or ``None``. + If *data* is ``None``, the association is removed. + *handler* is called when the node is cloned, imported, renamed or deleted; + pass ``None`` if no notification is needed. + + +.. method:: Node.getUserData(key) + + Return the data associated with *key* on this node + by :meth:`~Node.setUserData`, or ``None``. + + .. attribute:: Node.nodeName - This has a different meaning for each node type; see the DOM specification for - details. You can always get the information you would get here from another - property such as the :attr:`tagName` property for elements or the :attr:`name` - property for attributes. For all node types, the value of this attribute will be - either a string or ``None``. This is a read-only attribute. + The name of this node, depending on its type; see the table below. + You can always get the information you would get here from another + property such as the :attr:`~Element.tagName` property for elements or the + :attr:`~Attr.name` property for attributes. + This is a read-only attribute. .. attribute:: Node.nodeValue - This has a different meaning for each node type; see the DOM specification for - details. The situation is similar to that with :attr:`nodeName`. The value is - a string or ``None``. - + The value of this node, depending on its type; see the table below. + The value is a string or ``None``. + + +The values of :attr:`~Node.nodeName` and :attr:`~Node.nodeValue` +for each node type are: + ++--------------------------------+---------------------------------------+-------------------------------------+ +| Node type | nodeName | nodeValue | ++================================+=======================================+=====================================+ +| :class:`Attr` | :attr:`~Attr.name` | :attr:`~Attr.value` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`CDATASection` | ``'#cdata-section'`` | the content | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Comment` | ``'#comment'`` | the content | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Document` | ``'#document'`` | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`DocumentFragment` | ``'#document-fragment'`` | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`DocumentType` | :attr:`~DocumentType.name` | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Element` | :attr:`~Element.tagName` | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Entity` | the name of the entity | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Notation` | the name of the notation | ``None`` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`ProcessingInstruction` | :attr:`~ProcessingInstruction.target` | :attr:`~ProcessingInstruction.data` | ++--------------------------------+---------------------------------------+-------------------------------------+ +| :class:`Text` | ``'#text'`` | the content | ++--------------------------------+---------------------------------------+-------------------------------------+ .. method:: Node.hasAttributes() @@ -360,26 +464,39 @@ All of the components of an XML document are subclasses of :class:`Node`. children, returning *newChild*. If the node was already in the tree, it is removed first. + Raise :exc:`WrongDocumentErr` if *newChild* was created by another + document, and :exc:`HierarchyRequestErr` if it is this node itself or + its ancestor. + .. method:: Node.insertBefore(newChild, refChild) Insert a new child node before an existing child. It must be the case that - *refChild* is a child of this node; if not, :exc:`ValueError` is raised. + *refChild* is a child of this node; if not, :exc:`NotFoundErr` is raised. *newChild* is returned. If *refChild* is ``None``, it inserts *newChild* at the end of the children's list. + Raise :exc:`WrongDocumentErr` if *newChild* was created by another + document, and :exc:`HierarchyRequestErr` if it is this node itself or + its ancestor. + .. method:: Node.removeChild(oldChild) Remove a child node. *oldChild* must be a child of this node; if not, - :exc:`ValueError` is raised. *oldChild* is returned on success. If *oldChild* - will not be used further, its :meth:`unlink` method should be called. + :exc:`NotFoundErr` is raised. *oldChild* is returned on success. If *oldChild* + will not be used further, its :meth:`~xml.dom.minidom.Node.unlink` method + should be called. .. method:: Node.replaceChild(newChild, oldChild) Replace an existing node with a new node. It must be the case that *oldChild* - is a child of this node; if not, :exc:`ValueError` is raised. + is a child of this node; if not, :exc:`NotFoundErr` is raised. + + Raise :exc:`WrongDocumentErr` if *newChild* was created by another + document, and :exc:`HierarchyRequestErr` if it is this node itself or + its ancestor. .. method:: Node.normalize() @@ -400,11 +517,16 @@ All of the components of an XML document are subclasses of :class:`Node`. NodeList Objects ^^^^^^^^^^^^^^^^ +.. class:: NodeList + :no-typesetting: + A :class:`NodeList` represents a sequence of nodes. These objects are used in two ways in the DOM Core recommendation: an :class:`Element` object provides -one as its list of child nodes, and the :meth:`getElementsByTagName` and -:meth:`getElementsByTagNameNS` methods of :class:`Node` return objects with this -interface to represent query results. +one as its list of child nodes, and the :meth:`~Element.getElementsByTagName` +and :meth:`~Element.getElementsByTagNameNS` methods of :class:`Node` return +objects with this interface to represent query results. + +:class:`NodeList` does *not* inherit from :class:`Node`. The DOM Level 2 recommendation defines one method and one attribute for these objects: @@ -412,9 +534,9 @@ objects: .. method:: NodeList.item(i) - Return the *i*'th item from the sequence, if there is one, or ``None``. The - index *i* is not allowed to be less than zero or greater than or equal to the - length of the sequence. + Return the *i*'th item from the sequence, + or ``None`` if *i* is out of range. + Negative indices are not supported. .. attribute:: NodeList.length @@ -439,12 +561,15 @@ If a DOM implementation supports modification of the document, the DocumentType Objects ^^^^^^^^^^^^^^^^^^^^ +.. class:: DocumentType + :no-typesetting: + Information about the notations and entities declared by a document (including the external subset if the parser uses it and can provide the information) is available from a :class:`DocumentType` object. The :class:`DocumentType` for a -document is available from the :class:`Document` object's :attr:`doctype` +document is available from the :class:`Document` object's :attr:`~Document.doctype` attribute; if there is no ``DOCTYPE`` declaration for the document, the -document's :attr:`doctype` attribute will be set to ``None`` instead of an +document's :attr:`~Document.doctype` attribute will be set to ``None`` instead of an instance of this interface. :class:`DocumentType` is a specialization of :class:`Node`, and adds the @@ -453,14 +578,14 @@ following attributes: .. attribute:: DocumentType.publicId - The public identifier for the external subset of the document type definition. - This will be a string or ``None``. + The public identifier for the external subset of the document type definition, + or ``None`` if the ``DOCTYPE`` declaration does not specify it. .. attribute:: DocumentType.systemId - The system identifier for the external subset of the document type definition. - This will be a URI as a string, or ``None``. + The system identifier, a URI, for the external subset of the document type + definition, or ``None`` if the ``DOCTYPE`` declaration does not specify it. .. attribute:: DocumentType.internalSubset @@ -478,7 +603,8 @@ following attributes: .. attribute:: DocumentType.entities - This is a :class:`NamedNodeMap` giving the definitions of external entities. + This is a :class:`NamedNodeMap` of :class:`Entity` nodes + giving the definitions of external entities. For entity names defined more than once, only the first definition is provided (others are ignored as required by the XML recommendation). This may be ``None`` if the information is not provided by the parser, or if no entities are @@ -487,7 +613,8 @@ following attributes: .. attribute:: DocumentType.notations - This is a :class:`NamedNodeMap` giving the definitions of notations. For + This is a :class:`NamedNodeMap` of :class:`Notation` nodes + giving the definitions of notations. For notation names defined more than once, only the first definition is provided (others are ignored as required by the XML recommendation). This may be ``None`` if the information is not provided by the parser, or if no notations @@ -499,6 +626,9 @@ following attributes: Document Objects ^^^^^^^^^^^^^^^^ +.. class:: Document + :no-typesetting: + A :class:`Document` represents an entire XML document, including its constituent elements, attributes, processing instructions, comments etc. Remember that it inherits properties from :class:`Node`. @@ -509,11 +639,52 @@ inherits properties from :class:`Node`. The one and only root element of the document. +.. attribute:: Document.doctype + + The :class:`DocumentType` node of the document, or ``None``. + This is a read-only attribute. + + +.. attribute:: Document.implementation + + The :class:`DOMImplementation` object which created this document. + This is a read-only attribute. + + +.. attribute:: Document.strictErrorChecking + + Whether error checking is enforced. + + +.. attribute:: Document.documentURI + + The location of the document, or ``None`` if it is unknown. + + +.. method:: Document.createDocumentFragment() + + Create and return an empty :class:`DocumentFragment` node. + + +.. method:: Document.createCDATASection(data) + + Create and return a :class:`CDATASection` node containing *data*. + + +.. method:: Document.importNode(importedNode, deep) + + Return a copy of *importedNode* which belongs to this document. + The original node is not removed from its document. + If *deep* is true, the descendants of the node are copied too. + + .. method:: Document.createElement(tagName) Create and return a new element node. The element is not inserted into the document when it is created. You need to explicitly insert it with one of the - other methods such as :meth:`insertBefore` or :meth:`appendChild`. + other methods such as :meth:`~Node.insertBefore` or :meth:`~Node.appendChild`. + + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. .. method:: Document.createElementNS(namespaceURI, tagName) @@ -521,7 +692,9 @@ inherits properties from :class:`Node`. Create and return a new element with a namespace. The *tagName* may have a prefix. The element is not inserted into the document when it is created. You need to explicitly insert it with one of the other methods such as - :meth:`insertBefore` or :meth:`appendChild`. + :meth:`~Node.insertBefore` or :meth:`~Node.appendChild`. + + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. .. method:: Document.createTextNode(data) @@ -531,6 +704,16 @@ inherits properties from :class:`Node`. tree. +.. method:: Document.createEntityReference(name) + + Create and return a new entity reference node. + The node is not inserted into the document when it is created. + + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + + .. versionadded:: next + + .. method:: Document.createComment(data) Create and return a comment node containing the data passed as a parameter. As @@ -544,22 +727,35 @@ inherits properties from :class:`Node`. *data* passed as parameters. As with the other creation methods, this one does not insert the node into the tree. + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + .. method:: Document.createAttribute(name) Create and return an attribute node. This method does not associate the attribute node with any particular element. You must use - :meth:`setAttributeNode` on the appropriate :class:`Element` object to use the - newly created attribute instance. + :meth:`~Element.setAttributeNode` on the appropriate :class:`Element` object + to use the newly created attribute instance. + + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. .. method:: Document.createAttributeNS(namespaceURI, qualifiedName) Create and return an attribute node with a namespace. The *tagName* may have a prefix. This method does not associate the attribute node with any particular - element. You must use :meth:`setAttributeNode` on the appropriate + element. You must use :meth:`~Element.setAttributeNode` on the appropriate :class:`Element` object to use the newly created attribute instance. + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + + +.. method:: Document.getElementById(id) + + Return the element with the given ID, or ``None``. + Only attributes declared as being of type ID in the DTD + or by :meth:`Element.setIdAttribute` are searched. + .. method:: Document.getElementsByTagName(tagName) @@ -574,11 +770,26 @@ inherits properties from :class:`Node`. namespace after the prefix. +.. method:: Document.renameNode(n, namespaceURI, name) + + Rename the element or attribute node *n* + and return it. + *namespaceURI* is the new namespace URI, or + :data:`~xml.dom.EMPTY_NAMESPACE` if the node does not belong to a namespace. + *name* is the new qualified name. + + Raise :exc:`WrongDocumentErr` if *n* was created by another document, + and :exc:`NotSupportedErr` if it is neither an element nor an attribute. + + .. _dom-element-objects: Element Objects ^^^^^^^^^^^^^^^ +.. class:: Element + :no-typesetting: + :class:`Element` is a subclass of :class:`Node`, so inherits all the attributes of that class. @@ -589,6 +800,25 @@ of that class. The value is a string. +.. method:: Element.setIdAttribute(name) + + Declare that the attribute *name* is of type ID, + so that the element is found by :meth:`Document.getElementById`. + Raise :exc:`NotFoundErr` if the element has no such attribute. + + +.. method:: Element.setIdAttributeNS(namespaceURI, localName) + + The same as :meth:`~Element.setIdAttribute`, + but for an attribute specified by its namespace URI and local name. + + +.. method:: Element.setIdAttributeNode(idAttr) + + The same as :meth:`~Element.setIdAttribute`, + but for an already retrieved attribute node. + + .. method:: Element.getElementsByTagName(tagName) Same as equivalent method in the :class:`Document` class. @@ -635,8 +865,7 @@ of that class. .. method:: Element.removeAttribute(name) - Remove an attribute by name. If there is no matching attribute, a - :exc:`NotFoundErr` is raised. + Remove an attribute by name. .. method:: Element.removeAttributeNode(oldAttr) @@ -647,29 +876,35 @@ of that class. .. method:: Element.removeAttributeNS(namespaceURI, localName) - Remove an attribute by name. Note that it uses a localName, not a qname. No - exception is raised if there is no matching attribute. + Remove an attribute by name. Note that it uses a localName, not a qname. .. method:: Element.setAttribute(name, value) Set an attribute value from a string. + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + .. method:: Element.setAttributeNode(newAttr) Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`name` attribute matches. If a replacement occurs, the - old attribute node will be returned. If *newAttr* is already in use, + necessary if the :attr:`~Attr.name` attribute matches. If a replacement + occurs, the old attribute node will be returned. If *newAttr* is already in use, :exc:`InuseAttributeErr` will be raised. + Raise :exc:`WrongDocumentErr` if *newAttr* was created by another document. + .. method:: Element.setAttributeNodeNS(newAttr) Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`namespaceURI` and :attr:`localName` attributes match. - If a replacement occurs, the old attribute node will be returned. If *newAttr* - is already in use, :exc:`InuseAttributeErr` will be raised. + necessary if the :attr:`~Node.namespaceURI` and :attr:`~Attr.localName` + attributes match. If a replacement occurs, the old attribute node will be + returned. If *newAttr* is already in use, :exc:`InuseAttributeErr` will be + raised. + + Raise :exc:`WrongDocumentErr` if *newAttr* was created by another document. .. method:: Element.setAttributeNS(namespaceURI, qname, value) @@ -677,14 +912,25 @@ of that class. Set an attribute value from a string, given a *namespaceURI* and a *qname*. Note that a qname is the whole attribute name. This is different than above. + Raise :exc:`InvalidCharacterErr` if the name is not a valid XML name. + .. _dom-attr-objects: Attr Objects ^^^^^^^^^^^^ +.. class:: Attr + :no-typesetting: + :class:`Attr` inherits from :class:`Node`, so inherits all its attributes. +Attribute nodes are not part of the document tree. +They are contained in the :attr:`~Node.attributes` map of an element, +not in its children, +and their :attr:`~Node.parentNode`, :attr:`~Node.previousSibling` +and :attr:`~Node.nextSibling` are always ``None``. + .. attribute:: Attr.name @@ -705,10 +951,32 @@ Attr Objects empty string. +.. attribute:: Attr.isId + + Whether this attribute is of type ID, + either because it is declared as such in the DTD + or because :meth:`Element.setIdAttribute` was used. + This is a read-only attribute. + + +.. attribute:: Attr.ownerElement + + The :class:`Element` node to which this attribute belongs, + or ``None`` if it is not used. + This is a read-only attribute. + + +.. attribute:: Attr.specified + + Whether the value of the attribute was explicitly set in the document, + as opposed to being defaulted from the DTD. + This is a read-only attribute. + + .. attribute:: Attr.value The text value of the attribute. This is a synonym for the - :attr:`nodeValue` attribute. + :attr:`~Node.nodeValue` attribute. .. _dom-attributelist-objects: @@ -716,6 +984,9 @@ Attr Objects NamedNodeMap Objects ^^^^^^^^^^^^^^^^^^^^ +.. class:: NamedNodeMap + :no-typesetting: + :class:`NamedNodeMap` does *not* inherit from :class:`Node`. @@ -728,11 +999,121 @@ NamedNodeMap Objects Return an attribute with a particular index. The order you get the attributes in is arbitrary but will be consistent for the life of a DOM. Each item is an - attribute node. Get its value with the :attr:`value` attribute. + attribute node. Get its value with the :attr:`~Attr.value` attribute. + -There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`!getAttribute\*` family -of methods on the :class:`Element` objects. +.. method:: NamedNodeMap.getNamedItem(name) + + Return the node with the given :attr:`~Attr.name`, + or ``None`` if there is no such node. + + +.. method:: NamedNodeMap.getNamedItemNS(namespaceURI, localName) + + Return the node with the given namespace URI and local name, + or ``None`` if there is no such node. + + +.. method:: NamedNodeMap.setNamedItem(node) + + Add *node* to the map, using its :attr:`~Attr.name` as the key. + Return the node which it replaces, or ``None`` if it replaces no node. + + Raise :exc:`WrongDocumentErr` if *node* was created by another document, + and :exc:`InuseAttributeErr` if it belongs to another element. + + +.. method:: NamedNodeMap.setNamedItemNS(node) + + Add *node* to the map, + using its namespace URI and local name as the key. + Return the node which it replaces, or ``None`` if it replaces no node. + + Raise :exc:`WrongDocumentErr` if *node* was created by another document, + and :exc:`InuseAttributeErr` if it belongs to another element. + + +.. method:: NamedNodeMap.removeNamedItem(name) + + Remove and return the node with the given :attr:`~Attr.name`. + Raise :exc:`NotFoundErr` if there is no such node. + + +.. method:: NamedNodeMap.removeNamedItemNS(namespaceURI, localName) + + Remove and return the node with the given namespace URI and local name. + Raise :exc:`NotFoundErr` if there is no such node. + +You can also use the standardized :meth:`!getAttribute\*` family of methods +on the :class:`Element` objects. + + +.. _dom-documentfragment-objects: + +DocumentFragment Objects +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: DocumentFragment + :no-typesetting: + +:class:`DocumentFragment` is a lightweight container of nodes. +It is a subclass of :class:`Node`. +When it is inserted into the document tree, +its children are inserted instead of it, +and it becomes empty. + + +.. _dom-characterdata-objects: + +CharacterData Objects +^^^^^^^^^^^^^^^^^^^^^ + +.. class:: CharacterData + :no-typesetting: + +:class:`CharacterData` represents text-like data in the XML document. +It is a subclass of :class:`Node`, and the base class +of :class:`Text`, :class:`CDATASection` and :class:`Comment`. +Such nodes cannot have child nodes. + + +.. attribute:: CharacterData.data + + The content of the node as a string. + + +.. attribute:: CharacterData.length + + The number of characters in :attr:`~CharacterData.data`. + This is a read-only attribute. + + +.. method:: CharacterData.substringData(offset, count) + + Return the substring of :attr:`~CharacterData.data` + of *count* characters starting at *offset*. + + +.. method:: CharacterData.appendData(arg) + + Append the string *arg* to :attr:`~CharacterData.data`. + + +.. method:: CharacterData.insertData(offset, arg) + + Insert the string *arg* into :attr:`~CharacterData.data` at *offset*. + + +.. method:: CharacterData.deleteData(offset, count) + + Remove *count* characters from :attr:`~CharacterData.data` + starting at *offset*. + + +.. method:: CharacterData.replaceData(offset, count, arg) + + Replace *count* characters of :attr:`~CharacterData.data` + starting at *offset* with the string *arg*. .. _dom-comment-objects: @@ -740,8 +1121,11 @@ of methods on the :class:`Element` objects. Comment Objects ^^^^^^^^^^^^^^^ -:class:`Comment` represents a comment in the XML document. It is a subclass of -:class:`Node`, but cannot have child nodes. +.. class:: Comment + :no-typesetting: + +:class:`Comment` represents a comment in the XML document. +It is a subclass of :class:`CharacterData`. .. attribute:: Comment.data @@ -756,20 +1140,47 @@ Comment Objects Text and CDATASection Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: Text + :no-typesetting: + +.. class:: CDATASection + :no-typesetting: + The :class:`Text` interface represents text in the XML document. If the parser and DOM implementation support the DOM's XML extension, portions of the text enclosed in CDATA marked sections are stored in :class:`CDATASection` objects. These two interfaces are identical, but provide different values for the -:attr:`nodeType` attribute. +:attr:`~Node.nodeType` attribute. -These interfaces extend the :class:`Node` interface. They cannot have child -nodes. +:class:`Text` extends the :class:`CharacterData` interface, +and :class:`CDATASection` extends :class:`Text`. .. attribute:: Text.data The content of the text node as a string. + +.. attribute:: Text.wholeText + + The text of all :class:`Text` nodes logically adjacent to this node, + concatenated in document order. + This is a read-only attribute. + + +.. method:: Text.replaceWholeText(content) + + Replace the text of all :class:`Text` nodes logically adjacent + to this node with *content*, removing the other nodes. + Return this node, or ``None`` if *content* is empty. + + +.. method:: Text.splitText(offset) + + Split this node into two nodes at *offset*, + keeping the first part in this node + and returning a new sibling node with the rest. + .. note:: The use of a :class:`CDATASection` node does not indicate that the node @@ -784,6 +1195,9 @@ nodes. ProcessingInstruction Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: ProcessingInstruction + :no-typesetting: + Represents a processing instruction in the XML document; this inherits from the :class:`Node` interface and cannot have child nodes. @@ -800,6 +1214,88 @@ Represents a processing instruction in the XML document; this inherits from the character. +.. _dom-entity-objects: + +Entity Objects +^^^^^^^^^^^^^^ + +.. class:: Entity + :no-typesetting: + +:class:`Entity` represents a parsed or unparsed entity declared in the DTD. +It is a subclass of :class:`Node`. +Entity nodes are contained in :attr:`DocumentType.entities` +and cannot be inserted into the document tree. +The name of the entity is its :attr:`~Node.nodeName`. + + +.. attribute:: Entity.publicId + + The public identifier of the entity, + or ``None`` if it is not specified. + This is a read-only attribute. + + +.. attribute:: Entity.systemId + + The system identifier of the entity, + or ``None`` if it is not specified. + This is a read-only attribute. + + +.. attribute:: Entity.notationName + + The name of the notation for an unparsed entity, + or ``None`` for a parsed entity. + This is a read-only attribute. + + +.. _dom-notation-objects: + +Notation Objects +^^^^^^^^^^^^^^^^ + +.. class:: Notation + :no-typesetting: + +:class:`Notation` represents a notation declared in the DTD. +It is a subclass of :class:`Node` and cannot have child nodes. +Notation nodes are contained in :attr:`DocumentType.notations` +and cannot be inserted into the document tree. +The name of the notation is its :attr:`~Node.nodeName`. + + +.. attribute:: Notation.publicId + + The public identifier of the notation, + or ``None`` if it is not specified. + This is a read-only attribute. + + +.. attribute:: Notation.systemId + + The system identifier of the notation, + or ``None`` if it is not specified. + This is a read-only attribute. +.. _dom-entityreference-objects: + +EntityReference Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: EntityReference + :no-typesetting: + +:class:`EntityReference` represents an entity reference in the XML document. +It is a subclass of :class:`Node`. +The name of the referenced entity is its :attr:`~Node.nodeName`. +Its children are the replacement text of the entity, and are read-only. + +Parsers may expand entity references, +so such a node only occurs in a document if it was created explicitly. + +.. versionadded:: next + + .. _dom-exceptions: Exceptions @@ -912,48 +1408,59 @@ attribute. .. XXX how is this different from InvalidCharacterErr? +.. exception:: ValidationErr + + Raised when an operation would make the document invalid + with respect to partial validity. + This is not known to be used in the Python DOM implementations, + but may be received from DOM implementations not written in Python. + + .. exception:: WrongDocumentErr Raised when a node is inserted in a different document than it currently belongs to, and the implementation does not support migrating the node from one document to the other. + The exception codes defined in the DOM recommendation map to the exceptions described above according to this table: -+--------------------------------------+---------------------------------+ -| Constant | Exception | -+======================================+=================================+ -| :const:`DOMSTRING_SIZE_ERR` | :exc:`DomstringSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`HIERARCHY_REQUEST_ERR` | :exc:`HierarchyRequestErr` | -+--------------------------------------+---------------------------------+ -| :const:`INDEX_SIZE_ERR` | :exc:`IndexSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INUSE_ATTRIBUTE_ERR` | :exc:`InuseAttributeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_ACCESS_ERR` | :exc:`InvalidAccessErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_CHARACTER_ERR` | :exc:`InvalidCharacterErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_MODIFICATION_ERR` | :exc:`InvalidModificationErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_STATE_ERR` | :exc:`InvalidStateErr` | -+--------------------------------------+---------------------------------+ -| :const:`NAMESPACE_ERR` | :exc:`NamespaceErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_FOUND_ERR` | :exc:`NotFoundErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_SUPPORTED_ERR` | :exc:`NotSupportedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_DATA_ALLOWED_ERR` | :exc:`NoDataAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_MODIFICATION_ALLOWED_ERR` | :exc:`NoModificationAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`SYNTAX_ERR` | :exc:`SyntaxErr` | -+--------------------------------------+---------------------------------+ -| :const:`WRONG_DOCUMENT_ERR` | :exc:`WrongDocumentErr` | -+--------------------------------------+---------------------------------+ ++---------------------------------------+---------------------------------+ +| Constant | Exception | ++=======================================+=================================+ +| .. data:: DOMSTRING_SIZE_ERR | :exc:`DomstringSizeErr` | ++---------------------------------------+---------------------------------+ +| .. data:: HIERARCHY_REQUEST_ERR | :exc:`HierarchyRequestErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INDEX_SIZE_ERR | :exc:`IndexSizeErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INUSE_ATTRIBUTE_ERR | :exc:`InuseAttributeErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INVALID_ACCESS_ERR | :exc:`InvalidAccessErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INVALID_CHARACTER_ERR | :exc:`InvalidCharacterErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INVALID_MODIFICATION_ERR | :exc:`InvalidModificationErr` | ++---------------------------------------+---------------------------------+ +| .. data:: INVALID_STATE_ERR | :exc:`InvalidStateErr` | ++---------------------------------------+---------------------------------+ +| .. data:: NAMESPACE_ERR | :exc:`NamespaceErr` | ++---------------------------------------+---------------------------------+ +| .. data:: NOT_FOUND_ERR | :exc:`NotFoundErr` | ++---------------------------------------+---------------------------------+ +| .. data:: NOT_SUPPORTED_ERR | :exc:`NotSupportedErr` | ++---------------------------------------+---------------------------------+ +| .. data:: NO_DATA_ALLOWED_ERR | :exc:`NoDataAllowedErr` | ++---------------------------------------+---------------------------------+ +| .. data:: NO_MODIFICATION_ALLOWED_ERR | :exc:`NoModificationAllowedErr` | ++---------------------------------------+---------------------------------+ +| .. data:: SYNTAX_ERR | :exc:`SyntaxErr` | ++---------------------------------------+---------------------------------+ +| .. data:: VALIDATION_ERR | :exc:`ValidationErr` | ++---------------------------------------+---------------------------------+ +| .. data:: WRONG_DOCUMENT_ERR | :exc:`WrongDocumentErr` | ++---------------------------------------+---------------------------------+ .. _dom-conformance: @@ -1002,9 +1509,9 @@ Mapping the IDL declarations :: readonly attribute string someValue; attribute string anotherValue; -yields three accessor functions: a "get" method for :attr:`someValue` -(:meth:`_get_someValue`), and "get" and "set" methods for :attr:`anotherValue` -(:meth:`_get_anotherValue` and :meth:`_set_anotherValue`). The mapping, in +yields three accessor functions: a "get" method for :attr:`!someValue` +(:meth:`!_get_someValue`), and "get" and "set" methods for :attr:`!anotherValue` +(:meth:`!_get_anotherValue` and :meth:`!_set_anotherValue`). The mapping, in particular, does not require that the IDL attributes are accessible as normal Python attributes: ``object.someValue`` is *not* required to work, and may raise an :exc:`AttributeError`. @@ -1025,6 +1532,6 @@ considered unnecessary since the attributes are accessible directly from Python. The IDL definitions do not fully embody the requirements of the W3C DOM API, such as the notion of certain objects, such as the return value of -:meth:`getElementsByTagName`, being "live". The Python DOM API does not require -implementations to enforce such requirements. +:meth:`~Element.getElementsByTagName`, being "live". The Python DOM API does +not require implementations to enforce such requirements. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 976cc3b2a5282d8..4e7ec83723b344c 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -24,7 +24,6 @@ Doc/library/urllib.request.rst Doc/library/wsgiref.rst Doc/library/xml.dom.minidom.rst Doc/library/xml.dom.pulldom.rst -Doc/library/xml.dom.rst Doc/library/xml.sax.reader.rst Doc/library/xml.sax.rst Doc/library/xmlrpc.client.rst diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index a1a8415482b97aa..f3ddae7a2b2fdd1 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -630,6 +630,24 @@ xml instead of failing later, when encountering non-ASCII data. (Contributed by Serhiy Storchaka in :gh:`62259`.) +* :mod:`xml.dom.minidom` now conforms closer to the DOM Level 1 specification. + It checks names passed to the factory methods, + rejects inserting or setting a node created by another document + or making a node a descendant of itself, + rejects a second element or document type child of a document + and a notation child of a document fragment, + reports attributes defaulted in the DTD + and whether an attribute was given in the start tag, + and implements :class:`!EntityReference` nodes + and :meth:`!Document.createEntityReference`. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + +* Add :meth:`!GetSpecifiedAttributeCount` method + to the :mod:`XML parser ` objects. + It tells how many of the reported attributes were given in the start tag + rather than defaulted from the DTD. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + zipfile ------- @@ -845,6 +863,24 @@ that may require changes to your code. :exc:`TypeError`. (Contributed by Serhiy Storchaka in :gh:`152587`.) +* :mod:`xml.dom.minidom` now raises :exc:`~xml.dom.InvalidCharacterErr` + for a name which is not a valid XML name, + :exc:`~xml.dom.WrongDocumentErr` + for inserting or setting a node created by another document, + :exc:`~xml.dom.InuseAttributeErr` for setting an attribute node + which belongs to another element, + and :exc:`~xml.dom.HierarchyRequestErr` for inserting a node into itself + or its descendant. + It also rejects a second element or document type child of a document + and a notation child of a document fragment. + Such operations formerly succeeded + and produced an invalid document or an endless loop. + On the other hand, :meth:`!Element.removeAttribute` and + :meth:`!Element.removeAttributeNS` no longer raise + :exc:`~xml.dom.NotFoundErr` if there is no matching attribute. + Attributes defaulted in the DTD are no longer omitted when parsing. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + * On Windows, seeking a pipe now fails instead of silently appearing to succeed: :func:`os.lseek` and :meth:`~io.IOBase.seek` raise :exc:`OSError`, and :meth:`~io.IOBase.seekable` returns ``False``. As a consequence, diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index ca6819d2cd44730..253d26fe3a3cd8b 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -359,6 +359,10 @@ struct PyInterpreterView { PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); +extern int _PyInterpreterGuard_TryAcquire(PyInterpreterState *interp, + PyInterpreterGuard *guard); +extern void _PyInterpreterGuard_Release(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 9974ea3c4143fa6..0522b24cd8fc67d 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -44,9 +44,10 @@ struct __attribute__((packed)) #endif tracemalloc_frame { - /* filename cannot be NULL: "" is used if the Python frame - filename is NULL */ - PyObject *filename; + /* Interned NUL terminated UTF-8 (surrogatepass) string. + Cannot be NULL: "" is used if the Python frame filename + cannot be captured. */ + const char *filename; unsigned int lineno; }; @@ -85,7 +86,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ size_t peak_traced_memory; /* Hash table used as a set to intern filenames: - PyObject* => PyObject*. + char* (NUL terminated UTF-8 string) => NULL. Protected by the TABLES_LOCK(). */ _Py_hashtable_t *filenames; /* Buffer to store a new traceback in traceback_new(). diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 44ecd62d6735def..1b0724c2b1c99dc 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -244,7 +244,7 @@ def __eq__(self, other): list1 = [X()] list2 = [Y()] - self.assertTrue(list1 == list2) + self.assertFalse(list1 == list2) list3 = [Z()] list4 = [1] @@ -261,6 +261,54 @@ def __lt__(self, other): with self.assertRaises(TypeError): a[0] < a + def test_richcompare_stale_element_list_vitem(self): + # gh-148442: list_richcompare_impl must use the captured vitem for + # the final ordering comparison, not re-read list1's slot after __eq__ + # may have mutated it. + # + # x.__eq__(0) puts AlwaysLT() into list1[0] and returns False. + class AlwaysLT: + def __eq__(self, other: object) -> bool: + return False + + def __gt__(self, other: object) -> bool: + return False + + class Mutating: + def __eq__(self, other: object) -> bool: + list1[0] = AlwaysLT() + return False + + def __gt__(self, other: object) -> bool: + return True + + list1 = [Mutating(), 0] + list2 = [0, 0] + self.assertTrue(list1 > list2) + + def test_richcompare_stale_element_list_witem(self): + # gh-148442: list_richcompare_impl must use the captured witem for + # the final ordering comparison, not re-read list2's slot after __eq__ + # may have mutated it. + # + # x.__eq__(0) puts AlwaysGT() into list2[0] and returns False. + class AlwaysGT: + pass + + class Mutating: + def __eq__(self, other: object) -> bool: + list2[0] = AlwaysGT() + return False + + def __gt__(self, other: object) -> bool: + if isinstance(other, AlwaysGT): + return False # pretend AlwaysGT beats us + return True # beat everything else (including 0) + + list1 = [Mutating(), 0] + list2 = [0, 0] + self.assertTrue(list1 > list2) + def test_list_index_modifing_operand(self): # See gh-120384 class evil: diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 46249e5138aed52..3735a6046891ea9 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -354,7 +354,9 @@ def testRemoveAttr(self): child.setAttribute("def", "ghi") self.assertEqual(len(child.attributes), 1) - self.assertRaises(xml.dom.NotFoundErr, child.removeAttribute, "foo") + # removing an absent attribute has no effect + child.removeAttribute("foo") + self.assertEqual(len(child.attributes), 1) child.removeAttribute("def") self.assertEqual(len(child.attributes), 0) dom.unlink() @@ -366,8 +368,8 @@ def testRemoveAttrNS(self): child.setAttributeNS("http://www.w3.org", "xmlns:python", "http://www.python.org") child.setAttributeNS("http://www.python.org", "python:abcattr", "foo") - self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNS, - "foo", "http://www.python.org") + # removing an absent attribute has no effect + child.removeAttributeNS("foo", "http://www.python.org") self.assertEqual(len(child.attributes), 2) child.removeAttributeNS("http://www.python.org", "abcattr") self.assertEqual(len(child.attributes), 1) @@ -1784,5 +1786,190 @@ def test_cdata_parsing(self): dom2 = parseString(dom1.toprettyxml()) self.checkWholeText(dom2.getElementsByTagName('node')[0].firstChild, '') + def testInvalidCharacterErr(self): + doc = parseString("") + impl = getDOMImplementation() + for name in ("", "bad name", "1st", "-x", ".x", "a") + other = parseString("") + elem = doc.documentElement + alien = other.createElement("alien") + self.assertRaises(xml.dom.WrongDocumentErr, elem.appendChild, alien) + self.assertRaises(xml.dom.WrongDocumentErr, elem.insertBefore, + alien, elem.firstChild) + self.assertRaises(xml.dom.WrongDocumentErr, elem.replaceChild, + alien, elem.firstChild) + self.assertRaises(xml.dom.WrongDocumentErr, doc.appendChild, alien) + # the rejected node is left alone + self.assertIs(alien.ownerDocument, other) + self.assertIsNone(alien.parentNode) + # importNode() is the supported way to do this + elem.appendChild(doc.importNode(alien, True)) + self.assertEqual(elem.lastChild.tagName, "alien") + doc.unlink() + other.unlink() + + def testAncestorLoops(self): + doc = parseString("") + elem = doc.documentElement + child = elem.firstChild + grandchild = child.firstChild + for node in elem, child, grandchild: + self.assertRaises(xml.dom.HierarchyRequestErr, + node.appendChild, node) + self.assertRaises(xml.dom.HierarchyRequestErr, child.appendChild, elem) + self.assertRaises(xml.dom.HierarchyRequestErr, + grandchild.appendChild, elem) + self.assertRaises(xml.dom.HierarchyRequestErr, + grandchild.insertBefore, child, None) + self.assertRaises(xml.dom.HierarchyRequestErr, + grandchild.replaceChild, elem, None) + # the tree is unchanged + self.assertIs(child.parentNode, elem) + self.assertIs(grandchild.parentNode, child) + doc.unlink() + + def testOnlyOneElementAndDocumentType(self): + impl = getDOMImplementation() + doc = impl.createDocument(None, "root", None) + elem = doc.documentElement + comment = doc.appendChild(doc.createComment("c")) + self.assertRaises(xml.dom.HierarchyRequestErr, + doc.appendChild, doc.createElement("x")) + self.assertRaises(xml.dom.HierarchyRequestErr, + doc.insertBefore, doc.createElement("x"), comment) + self.assertRaises(xml.dom.HierarchyRequestErr, + doc.replaceChild, doc.createElement("x"), comment) + # replacing the document element and reordering it are allowed + other = doc.createElement("other") + doc.replaceChild(other, elem) + self.assertIs(doc.documentElement, other) + doc.appendChild(other) + self.assertEqual([n.nodeName for n in doc.childNodes], + ["#comment", "other"]) + + doc2 = impl.createDocument(None, None, None) + doctype = impl.createDocumentType("a", None, None) + doc2.appendChild(doctype) + self.assertRaises(xml.dom.HierarchyRequestErr, doc2.appendChild, + impl.createDocumentType("b", None, None)) + # re-adding the same node and replacing it are allowed + doc2.appendChild(doctype) + doc2.replaceChild(impl.createDocumentType("b", None, None), doctype) + doc.unlink() + doc2.unlink() + + def testDocumentFragmentChildren(self): + doc = parseString(']>') + frag = doc.createDocumentFragment() + notation = doc.doctype.notations.item(0) + self.assertRaises(xml.dom.HierarchyRequestErr, + frag.appendChild, notation) + frag.appendChild(doc.createElement("e")) + frag.appendChild(doc.createTextNode("t")) + frag.appendChild(doc.createComment("c")) + self.assertEqual(len(frag.childNodes), 3) + doc.unlink() + + def testSetAttributeNodeWrongDocument(self): + doc = parseString("") + other = parseString("") + elem = doc.documentElement + self.assertRaises(xml.dom.WrongDocumentErr, elem.setAttributeNode, + other.createAttribute("z")) + self.assertRaises(xml.dom.WrongDocumentErr, elem.setAttributeNodeNS, + other.createAttributeNS(None, "z")) + # an imported attribute belongs to this document and is accepted + elem.setAttributeNode(doc.importNode(other.createAttribute("z"), True)) + self.assertTrue(elem.hasAttribute("z")) + # re-setting an own attribute is not an error + elem.setAttributeNode(elem.getAttributeNode("a")) + + # the same checks are applied in NamedNodeMap.setNamedItem() + attrs = elem.attributes + self.assertRaises(xml.dom.WrongDocumentErr, attrs.setNamedItem, + other.createAttribute("y")) + self.assertRaises(xml.dom.WrongDocumentErr, attrs.setNamedItemNS, + other.createAttributeNS(None, "y")) + attrs.setNamedItem(doc.createAttribute("y")) + self.assertTrue(elem.hasAttribute("y")) + + # an attribute of another element of the same document is in use + doc2 = parseString("") + a, b = doc2.documentElement.childNodes + self.assertRaises(xml.dom.InuseAttributeErr, b.attributes.setNamedItem, + a.getAttributeNode("x")) + doc2.unlink() + doc.unlink() + other.unlink() + + def testAttrSpecified(self): + doc = parseString("" + " " + "]>") + elem = doc.documentElement + # attributes defaulted from the DTD are reported too + self.assertEqual(sorted(elem.attributes.keys()), ["a", "b"]) + self.assertEqual(elem.getAttribute("a"), "default") + self.assertFalse(elem.getAttributeNode("a").specified) + self.assertEqual(elem.getAttribute("b"), "given") + self.assertTrue(elem.getAttributeNode("b").specified) + doc.unlink() + + def testEntityReference(self): + doc = parseString("") + ref = doc.createEntityReference("ent") + self.assertEqual(ref.nodeType, Node.ENTITY_REFERENCE_NODE) + self.assertEqual(ref.nodeName, "ent") + self.assertIsNone(ref.nodeValue) + self.assertIs(ref.ownerDocument, doc) + doc.documentElement.appendChild(ref) + self.assertEqual(doc.documentElement.toxml(), "&ent;") + # entity reference nodes are read-only + text = doc.createTextNode("x") + self.assertRaises(xml.dom.NoModificationAllowedErr, + ref.appendChild, text) + self.assertRaises(xml.dom.NoModificationAllowedErr, + ref.insertBefore, text, None) + self.assertRaises(xml.dom.NoModificationAllowedErr, + ref.removeChild, text) + self.assertRaises(xml.dom.NoModificationAllowedErr, + ref.replaceChild, text, None) + self.assertEqual(ref.cloneNode(True).nodeName, "ent") + other = parseString("") + self.assertEqual(other.importNode(ref, True).nodeName, "ent") + doc.unlink() + other.unlink() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_os/test_windows.py b/Lib/test/test_os/test_windows.py index 3fb8454cddfde93..b0be69cc64b94b3 100644 --- a/Lib/test/test_os/test_windows.py +++ b/Lib/test/test_os/test_windows.py @@ -646,5 +646,43 @@ def test_not_a_console(self): self.assertIsNone(encoding) +class Win32StatExecutableTests(unittest.TestCase): + # gh-84419: Windows strips trailing dots and spaces from the last + # component of the path, so they should be ignored when guessing + # the execute permissions from the file extension. + + SUFFIXES = ['', ' ', ' ', '.', '..', ' . .'] + + def check(self, ext, mask): + filename = os_helper.TESTFN + ext + create_file(filename) + try: + for suffix in self.SUFFIXES: + with self.subTest(suffix=suffix): + mode = os.stat(filename + suffix).st_mode + self.assertEqual(mode & 0o111, mask) + finally: + os_helper.unlink(filename) + + def test_executable_extension(self): + for ext in '.exe', '.bat', '.cmd', '.com', '.EXE', '.Bat': + with self.subTest(ext=ext): + self.check(ext, 0o111) + + def test_not_executable_extension(self): + for ext in '.txt', '.py', '.exe.txt', '': + with self.subTest(ext=ext): + self.check(ext, 0) + + def test_extended_path(self): + # The \\?\ prefix disables normalization: trailing spaces and dots + # are part of the file name. + filename = os.path.abspath(os_helper.TESTFN + '.exe') + create_file(filename) + self.addCleanup(os_helper.unlink, filename) + self.assertEqual(os.stat('\\\\?\\' + filename).st_mode & 0o111, 0o111) + self.assertRaises(OSError, os.stat, '\\\\?\\' + filename + ' ') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 9d3ff8a620b6f26..ee02fbd16af0c4e 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1056,8 +1056,8 @@ def test_track(self): self.check_track(False) def test_track_without_gil(self): - # check that calling _PyTraceMalloc_Track() without holding the GIL - # works too + # check that calling PyTraceMalloc_Track() without the GIL + # (detached thread state) still captures the Python traceback self.check_track(True) def test_track_already_tracked(self): diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py index 7dd667bf3fbe042..d56b2ddfdb25698 100644 --- a/Lib/xml/dom/expatbuilder.py +++ b/Lib/xml/dom/expatbuilder.py @@ -159,7 +159,6 @@ def getParser(self): self._intern_setdefault = self._parser.intern.setdefault self._parser.buffer_text = True self._parser.ordered_attributes = True - self._parser.specified_attributes = True self.install(self._parser) return self._parser @@ -352,11 +351,13 @@ def start_element_handler(self, name, attributes): self.curNode = node if attributes: + specified = self.getParser().GetSpecifiedAttributeCount() for i in range(0, len(attributes), 2): a = minidom.Attr(attributes[i], EMPTY_NAMESPACE, None, EMPTY_PREFIX) value = attributes[i+1] a.value = value + a.specified = i < specified a.ownerDocument = self.document _set_attribute_node(node, a) @@ -760,6 +761,7 @@ def start_element_handler(self, name, attributes): node._ensure_attributes() _attrs = node._attrs _attrsNS = node._attrsNS + specified = self.getParser().GetSpecifiedAttributeCount() for i in range(0, len(attributes), 2): aname = attributes[i] value = attributes[i+1] @@ -775,6 +777,7 @@ def start_element_handler(self, name, attributes): _attrsNS[(EMPTY_NAMESPACE, aname)] = a a.ownerDocument = self.document a.value = value + a.specified = i < specified a.ownerElement = node if __debug__: diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index 16b33b90184dc59..5fd3911bd3c9ebb 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -16,6 +16,7 @@ """ import io +import xml import xml.dom from xml.dom import EMPTY_NAMESPACE, EMPTY_PREFIX, XMLNS_NAMESPACE, domreg @@ -79,15 +80,32 @@ def _get_lastChild(self): if self.childNodes: return self.childNodes[-1] + def _check_new_child(self, newChild, oldChild=None): + # Common checks for insertBefore(), appendChild() and replaceChild(). + doc = self.ownerDocument or self + newChildDoc = newChild.ownerDocument + if newChildDoc is not doc and newChildDoc is not None: + raise xml.dom.WrongDocumentErr( + "%s was created by a different document" % repr(newChild)) + if newChild.nodeType not in self._child_node_types: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of %s" % (repr(newChild), repr(self))) + if newChild is self: + raise xml.dom.HierarchyRequestErr( + "%s cannot be child of itself" % repr(self)) + # A node without children cannot be an ancestor, and testing this + # first keeps appending leaf nodes linear in the depth of the tree. + if newChild.childNodes and _is_ancestor(newChild, self): + raise xml.dom.HierarchyRequestErr( + "%s is an ancestor of %s" % (repr(newChild), repr(self))) + def insertBefore(self, newChild, refChild): if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: for c in tuple(newChild.childNodes): self.insertBefore(c, refChild) ### The DOM does not clearly specify what to return in this case return newChild - if newChild.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(newChild), repr(self))) + self._check_new_child(newChild) if newChild.parentNode is not None: newChild.parentNode.removeChild(newChild) if refChild is None: @@ -117,10 +135,8 @@ def appendChild(self, node): self.appendChild(c) ### The DOM does not clearly specify what to return in this case return node - if node.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(node), repr(self))) - elif node.nodeType in _nodeTypes_with_children: + self._check_new_child(node) + if node.nodeType in _nodeTypes_with_children: _clear_id_cache(self) if node.parentNode is not None: node.parentNode.removeChild(node) @@ -133,11 +149,9 @@ def replaceChild(self, newChild, oldChild): refChild = oldChild.nextSibling self.removeChild(oldChild) return self.insertBefore(newChild, refChild) - if newChild.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(newChild), repr(self))) if newChild is oldChild: return + self._check_new_child(newChild, oldChild) if newChild.parentNode is not None: newChild.parentNode.removeChild(newChild) try: @@ -282,6 +296,22 @@ def __exit__(self, et, ev, tb): defproperty(Node, "localName", doc="Namespace-local name of this node.") +def _check_name(name): + if not xml.is_valid_name(name): + raise xml.dom.InvalidCharacterErr( + "%r is not a valid XML name" % (name,)) + + +def _is_ancestor(node, other): + "Returns true iff node is an ancestor of other." + other = other.parentNode + while other is not None: + if other is node: + return True + other = other.parentNode + return False + + def _append_child(self, node): # fast path with less checks; usable by DOM builders if careful childNodes = self.childNodes @@ -344,8 +374,7 @@ class DocumentFragment(Node): Node.CDATA_SECTION_NODE, Node.ENTITY_REFERENCE_NODE, Node.PROCESSING_INSTRUCTION_NODE, - Node.COMMENT_NODE, - Node.NOTATION_NODE) + Node.COMMENT_NODE) def __init__(self): self.childNodes = NodeList() @@ -620,6 +649,12 @@ def setNamedItem(self, node): if not isinstance(node, Attr): raise xml.dom.HierarchyRequestErr( "%s cannot be child of %s" % (repr(node), repr(self))) + owner = self._ownerElement + if node.ownerDocument not in (None, owner.ownerDocument): + raise xml.dom.WrongDocumentErr( + "%s was created by another document" % repr(node)) + if node.ownerElement not in (None, owner): + raise xml.dom.InuseAttributeErr("attribute node already owned") old = self._attrs.get(node.name) if old: old.unlink() @@ -758,6 +793,7 @@ def getAttributeNS(self, namespaceURI, localName): def setAttribute(self, attname, value): attr = self.getAttributeNode(attname) if attr is None: + _check_name(attname) attr = Attr(attname) attr.value = value # also sets nodeValue attr.ownerDocument = self.ownerDocument @@ -771,6 +807,7 @@ def setAttributeNS(self, namespaceURI, qualifiedName, value): prefix, localname = _nssplit(qualifiedName) attr = self.getAttributeNodeNS(namespaceURI, localname) if attr is None: + _check_name(qualifiedName) attr = Attr(qualifiedName, namespaceURI, localname, prefix) attr.value = value attr.ownerDocument = self.ownerDocument @@ -795,6 +832,9 @@ def getAttributeNodeNS(self, namespaceURI, localName): return self._attrsNS.get((namespaceURI, localName)) def setAttributeNode(self, attr): + if attr.ownerDocument not in (None, self.ownerDocument): + raise xml.dom.WrongDocumentErr( + "%s was created by another document" % repr(attr)) if attr.ownerElement not in (None, self): raise xml.dom.InuseAttributeErr("attribute node already owned") self._ensure_attributes() @@ -816,22 +856,17 @@ def setAttributeNode(self, attr): setAttributeNodeNS = setAttributeNode def removeAttribute(self, name): - if self._attrsNS is None: - raise xml.dom.NotFoundErr() - try: - attr = self._attrs[name] - except KeyError: - raise xml.dom.NotFoundErr() - self.removeAttributeNode(attr) + # The DOM specifies that removing an absent attribute has no effect. + if self._attrs is not None: + attr = self._attrs.get(name) + if attr is not None: + self.removeAttributeNode(attr) def removeAttributeNS(self, namespaceURI, localName): - if self._attrsNS is None: - raise xml.dom.NotFoundErr() - try: - attr = self._attrsNS[(namespaceURI, localName)] - except KeyError: - raise xml.dom.NotFoundErr() - self.removeAttributeNode(attr) + if self._attrsNS is not None: + attr = self._attrsNS.get((namespaceURI, localName)) + if attr is not None: + self.removeAttributeNode(attr) def removeAttributeNode(self, node): if node is None: @@ -1416,6 +1451,45 @@ def replaceChild(self, newChild, oldChild): raise xml.dom.HierarchyRequestErr( "cannot replace children of an entity node") +class EntityReference(Node): + nodeType = Node.ENTITY_REFERENCE_NODE + nodeValue = None + attributes = None + + _child_node_types = (Node.ELEMENT_NODE, + Node.PROCESSING_INSTRUCTION_NODE, + Node.COMMENT_NODE, + Node.TEXT_NODE, + Node.CDATA_SECTION_NODE, + Node.ENTITY_REFERENCE_NODE) + + def __init__(self, name): + self.nodeName = name + self.childNodes = NodeList() + + def _get_nodeName(self): + return self.nodeName + + def appendChild(self, newChild): + raise xml.dom.NoModificationAllowedErr( + "entity reference nodes are read-only") + + def insertBefore(self, newChild, refChild): + raise xml.dom.NoModificationAllowedErr( + "entity reference nodes are read-only") + + def removeChild(self, oldChild): + raise xml.dom.NoModificationAllowedErr( + "entity reference nodes are read-only") + + def replaceChild(self, newChild, oldChild): + raise xml.dom.NoModificationAllowedErr( + "entity reference nodes are read-only") + + def writexml(self, writer, indent="", addindent="", newl=""): + writer.write("&%s;" % self.nodeName) + + class Notation(Identified, Childless, Node): nodeType = Node.NOTATION_NODE nodeValue = None @@ -1487,6 +1561,7 @@ def createDocument(self, namespaceURI, qualifiedName, doctype): return doc def createDocumentType(self, qualifiedName, publicId, systemId): + _check_name(qualifiedName) doctype = DocumentType(qualifiedName) doctype.publicId = publicId doctype.systemId = systemId @@ -1621,20 +1696,26 @@ def _get_strictErrorChecking(self): def _get_version(self): return self.version + def _check_new_child(self, newChild, oldChild=None): + Node._check_new_child(self, newChild, oldChild) + # A document can have only one element and only one document type. + if newChild.nodeType == Node.ELEMENT_NODE: + what = "document elements" + elif newChild.nodeType == Node.DOCUMENT_TYPE_NODE: + what = "document types" + else: + return + for node in self.childNodes: + if (node.nodeType == newChild.nodeType + and node is not newChild and node is not oldChild): + raise xml.dom.HierarchyRequestErr("two %s disallowed" % what) + def appendChild(self, node): - if node.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(node), repr(self))) + self._check_new_child(node) if node.parentNode is not None: - # This needs to be done before the next test since this - # may *be* the document element, in which case it should + # This may *be* the document element, in which case it should # end up re-ordered to the end. node.parentNode.removeChild(node) - - if node.nodeType == Node.ELEMENT_NODE \ - and self._get_documentElement(): - raise xml.dom.HierarchyRequestErr( - "two document elements disallowed") return Node.appendChild(self, node) def removeChild(self, oldChild): @@ -1687,6 +1768,7 @@ def createDocumentFragment(self): return d def createElement(self, tagName): + _check_name(tagName) e = Element(tagName) e.ownerDocument = self return e @@ -1707,29 +1789,39 @@ def createCDATASection(self, data): c.ownerDocument = self return c + def createEntityReference(self, name): + _check_name(name) + e = EntityReference(name) + e.ownerDocument = self + return e + def createComment(self, data): c = Comment(data) c.ownerDocument = self return c def createProcessingInstruction(self, target, data): + _check_name(target) p = ProcessingInstruction(target, data) p.ownerDocument = self return p def createAttribute(self, qName): + _check_name(qName) a = Attr(qName) a.ownerDocument = self a.value = "" return a def createElementNS(self, namespaceURI, qualifiedName): + _check_name(qualifiedName) prefix, localName = _nssplit(qualifiedName) e = Element(qualifiedName, namespaceURI, prefix) e.ownerDocument = self return e def createAttributeNS(self, namespaceURI, qualifiedName): + _check_name(qualifiedName) prefix, localName = _nssplit(qualifiedName) a = Attr(qualifiedName, namespaceURI, localName, prefix) a.ownerDocument = self @@ -1935,6 +2027,8 @@ def _clone_node(node, deep, newOwnerDocument): node.data) elif node.nodeType == Node.COMMENT_NODE: clone = newOwnerDocument.createComment(node.data) + elif node.nodeType == Node.ENTITY_REFERENCE_NODE: + clone = newOwnerDocument.createEntityReference(node.nodeName) elif node.nodeType == Node.ATTRIBUTE_NODE: clone = newOwnerDocument.createAttributeNS(node.namespaceURI, node.nodeName) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-00-00-00.gh-issue-148442.aBcDeF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-00-00-00.gh-issue-148442.aBcDeF.rst new file mode 100644 index 000000000000000..e30a3dee6f54dd7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-13-00-00-00.gh-issue-148442.aBcDeF.rst @@ -0,0 +1,4 @@ +Fix a race condition in ``list.__lt__``, ``list.__le__``, ``list.__gt__``, +and ``list.__ge__``. Previously, under concurrent list mutation, the items +being compared could be replaced with other objects before the final comparison, +causing the final comparison result to be incorrect. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst new file mode 100644 index 000000000000000..3e157446954f61e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst @@ -0,0 +1,6 @@ +:mod:`tracemalloc` no longer acquires the :term:`GIL` nor creates a +temporary thread state to trace memory allocations: traceback frames now +store plain UTF-8 strings instead of Python str objects, and tracebacks are +captured using the Python thread state already associated with the calling +thread, even if it is not attached. Threads without a Python thread state +record the traceback as ````. diff --git a/Misc/NEWS.d/next/Library/2026-08-08-21-40-00.gh-issue-84419.statexe.rst b/Misc/NEWS.d/next/Library/2026-08-08-21-40-00.gh-issue-84419.statexe.rst new file mode 100644 index 000000000000000..03b90b42e8c3e63 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-08-21-40-00.gh-issue-84419.statexe.rst @@ -0,0 +1,3 @@ +Fix :func:`os.stat` on Windows: trailing dots and spaces, which are ignored +by the operating system, are no longer taken into account when the execute +permissions are guessed from the file extension. diff --git a/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Qw3Lm8.rst b/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Qw3Lm8.rst new file mode 100644 index 000000000000000..d3adb49db69620f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Qw3Lm8.rst @@ -0,0 +1,19 @@ +:mod:`xml.dom.minidom` now conforms closer to the DOM Level 1 specification. +The factory methods now raise :exc:`~xml.dom.InvalidCharacterErr` +if the name is not a valid XML name. +Removing an absent attribute with :meth:`!Element.removeAttribute` or +:meth:`!Element.removeAttributeNS` no longer raises +:exc:`~xml.dom.NotFoundErr`, it has no effect as the DOM specifies. +Inserting or setting a node created by another document +now raises :exc:`~xml.dom.WrongDocumentErr`, +setting an attribute node which belongs to another element +now raises :exc:`~xml.dom.InuseAttributeErr`, +and inserting a node into itself or its descendant +now raises :exc:`~xml.dom.HierarchyRequestErr`. +A document can no longer have two element or two document type children, +and a document fragment can no longer have notation children. +Attributes defaulted in the DTD are no longer omitted when parsing, +and :attr:`!Attr.specified` now reports +whether the attribute was given in the start tag. +:class:`!EntityReference` nodes +and :meth:`!Document.createEntityReference` are now implemented. diff --git a/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Rt7Nv2.rst b/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Rt7Nv2.rst new file mode 100644 index 000000000000000..1d81ecea11c5d0a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-12-21-34-12.gh-issue-44871.Rt7Nv2.rst @@ -0,0 +1,4 @@ +Parser objects of the :mod:`XML parser ` have a new method +:meth:`!GetSpecifiedAttributeCount`. +It tells how many of the attributes reported to :attr:`!StartElementHandler` +were given in the start tag rather than defaulted from the DTD. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index ec71a89a2b04ec6..9a771cd0ad5fbf0 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6761,6 +6761,9 @@ local_timezone_from_timestamp(time_t timestamp) struct tm local_time_tm; PyObject *nameo = NULL; const char *zone = NULL; +#ifndef HAVE_STRUCT_TM_TM_ZONE + char buf[100]; // for zone, which is used after the block below +#endif if (_PyTime_localtime(timestamp, &local_time_tm) != 0) return NULL; @@ -6771,9 +6774,9 @@ local_timezone_from_timestamp(time_t timestamp) { PyObject *local_time, *utc_time; struct tm utc_time_tm; - char buf[100]; - strftime(buf, sizeof(buf), "%Z", &local_time_tm); - zone = buf; + if (strftime(buf, sizeof(buf), "%Z", &local_time_tm) != 0) { + zone = buf; + } local_time = new_datetime(local_time_tm.tm_year + 1900, local_time_tm.tm_mon + 1, local_time_tm.tm_mday, @@ -6783,8 +6786,10 @@ local_timezone_from_timestamp(time_t timestamp) if (local_time == NULL) { return NULL; } - if (_PyTime_gmtime(timestamp, &utc_time_tm) != 0) + if (_PyTime_gmtime(timestamp, &utc_time_tm) != 0) { + Py_DECREF(local_time); return NULL; + } utc_time = new_datetime(utc_time_tm.tm_year + 1900, utc_time_tm.tm_mon + 1, utc_time_tm.tm_mday, diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 1a07726d303ecad..9e32bb079c1ea68 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -212,6 +212,30 @@ pyexpat_xmlparser_GetBase(PyObject *self, PyObject *Py_UNUSED(ignored)) return pyexpat_xmlparser_GetBase_impl((xmlparseobject *)self); } +PyDoc_STRVAR(pyexpat_xmlparser_GetSpecifiedAttributeCount__doc__, +"GetSpecifiedAttributeCount($self, /)\n" +"--\n" +"\n" +"Return the index just past the attributes given in the start tag.\n" +"\n" +"Attributes defaulted from the DTD follow the specified ones, so\n" +"attributes at lower indices in the list passed to\n" +"StartElementHandler were given in the start tag. Each attribute\n" +"takes two items in that list, its name and its value. Only\n" +"meaningful inside a StartElementHandler call."); + +#define PYEXPAT_XMLPARSER_GETSPECIFIEDATTRIBUTECOUNT_METHODDEF \ + {"GetSpecifiedAttributeCount", (PyCFunction)pyexpat_xmlparser_GetSpecifiedAttributeCount, METH_NOARGS, pyexpat_xmlparser_GetSpecifiedAttributeCount__doc__}, + +static PyObject * +pyexpat_xmlparser_GetSpecifiedAttributeCount_impl(xmlparseobject *self); + +static PyObject * +pyexpat_xmlparser_GetSpecifiedAttributeCount(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return pyexpat_xmlparser_GetSpecifiedAttributeCount_impl((xmlparseobject *)self); +} + PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, "GetInputContext($self, /)\n" "--\n" @@ -839,4 +863,4 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ -/*[clinic end generated code: output=270a0bfe3300e8a1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d7e59d139fe45487 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a114a617918f7ce..847237142854da1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2021,6 +2021,14 @@ win32_wchdir(LPCWSTR path) #define HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES 1 #define HAVE_STRUCT_STAT_ST_REPARSE_TAG 1 +/* The \\?\ prefix disables the path normalization, in particular + stripping of trailing dots and spaces. */ +static int +is_extended_path(const wchar_t *path) +{ + return wcsncmp(path, L"\\\\?\\", 4) == 0; +} + static void find_data_to_file_info(WIN32_FIND_DATAW *pFileData, FILE_BASIC_INFO* basic_info, @@ -2094,12 +2102,20 @@ update_st_mode_from_path(const wchar_t *path, DWORD attr, GetSecurityInfo, OpenThreadToken/OpenProcessToken, and AccessCheck to check for generic read, write, and execute access. */ - const wchar_t *fileExtension = wcsrchr(path, '.'); - if (fileExtension) { - if (_wcsicmp(fileExtension, L".exe") == 0 || - _wcsicmp(fileExtension, L".bat") == 0 || - _wcsicmp(fileExtension, L".cmd") == 0 || - _wcsicmp(fileExtension, L".com") == 0) { + size_t len = wcslen(path); + if (!is_extended_path(path)) { + /* Trailing dots and spaces are stripped from the last component + of the path. */ + while (len > 0 && (path[len - 1] == L'.' || path[len - 1] == L' ')) { + len--; + } + } + if (len >= 4) { + const wchar_t *fileExtension = path + len - 4; + if (_wcsnicmp(fileExtension, L".exe", 4) == 0 || + _wcsnicmp(fileExtension, L".bat", 4) == 0 || + _wcsnicmp(fileExtension, L".cmd", 4) == 0 || + _wcsnicmp(fileExtension, L".com", 4) == 0) { result->st_mode |= 0111; } } @@ -16718,12 +16734,6 @@ static PyType_Spec DirEntryType_spec = { #ifdef MS_WINDOWS -static int -is_extended_path(const wchar_t *path) -{ - return wcsncmp(path, L"\\\\?\\", 4) == 0; -} - static wchar_t * join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename, int normalize) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 397a441f574fe45..fa8b0db60806233 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1054,6 +1054,25 @@ pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) return conv_string_to_unicode(XML_GetBase(self->itself)); } +/*[clinic input] +pyexpat.xmlparser.GetSpecifiedAttributeCount + +Return the index just past the attributes given in the start tag. + +Attributes defaulted from the DTD follow the specified ones, so +attributes at lower indices in the list passed to +StartElementHandler were given in the start tag. Each attribute +takes two items in that list, its name and its value. Only +meaningful inside a StartElementHandler call. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_GetSpecifiedAttributeCount_impl(xmlparseobject *self) +/*[clinic end generated code: output=f96b627de9393c0c input=4981c36cf99ebe9f]*/ +{ + return PyLong_FromLong(XML_GetSpecifiedAttributeCount(self->itself)); +} + /*[clinic input] @permit_long_summary pyexpat.xmlparser.GetInputContext @@ -1419,6 +1438,7 @@ static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_PARSEFILE_METHODDEF PYEXPAT_XMLPARSER_SETBASE_METHODDEF PYEXPAT_XMLPARSER_GETBASE_METHODDEF + PYEXPAT_XMLPARSER_GETSPECIFIEDATTRIBUTECOUNT_METHODDEF PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF PYEXPAT_XMLPARSER_EXTERNALENTITYPARSERCREATE_METHODDEF PYEXPAT_XMLPARSER_SETPARAMENTITYPARSING_METHODDEF diff --git a/Objects/listobject.c b/Objects/listobject.c index 8a9c9bda68269b8..81eb3e1102159b5 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3454,7 +3454,10 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) Py_RETURN_TRUE; } - /* Search for the first index where items are different */ + /* Search for the first index where items are different. + * We incref vitem/witem before calling PyObject_RichCompareBool, which may + * release the GIL and allow the list to be mutated in the meantime. + */ for (i = 0; i < Py_SIZE(vl) && i < Py_SIZE(wl); i++) { PyObject *vitem = vl->ob_item[i]; PyObject *witem = wl->ob_item[i]; @@ -3465,36 +3468,36 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) Py_INCREF(vitem); Py_INCREF(witem); int k = PyObject_RichCompareBool(vitem, witem, Py_EQ); - Py_DECREF(vitem); - Py_DECREF(witem); - if (k < 0) + if (k < 0) { + Py_DECREF(vitem); + Py_DECREF(witem); return NULL; - if (!k) - break; - } - - if (i >= Py_SIZE(vl) || i >= Py_SIZE(wl)) { - /* No more items to compare -- compare sizes */ - Py_RETURN_RICHCOMPARE(Py_SIZE(vl), Py_SIZE(wl), op); - } + } + if (!k) { + /* We have a differing item -- shortcuts for EQ/NE */ + if (op == Py_EQ) { + Py_DECREF(vitem); + Py_DECREF(witem); + Py_RETURN_FALSE; + } + if (op == Py_NE) { + Py_DECREF(vitem); + Py_DECREF(witem); + Py_RETURN_TRUE; + } + /* Compare the differing items using the proper operator */ + PyObject *result = PyObject_RichCompare(vitem, witem, op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; + } - /* We have an item that differs -- shortcuts for EQ/NE */ - if (op == Py_EQ) { - Py_RETURN_FALSE; - } - if (op == Py_NE) { - Py_RETURN_TRUE; + Py_DECREF(vitem); + Py_DECREF(witem); } - /* Compare the final item again using the proper operator */ - PyObject *vitem = vl->ob_item[i]; - PyObject *witem = wl->ob_item[i]; - Py_INCREF(vitem); - Py_INCREF(witem); - PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); - Py_DECREF(vitem); - Py_DECREF(witem); - return result; + /* All compared elements were equal -- compare sizes */ + Py_RETURN_RICHCOMPARE(Py_SIZE(vl), Py_SIZE(wl), op); } static PyObject * diff --git a/Python/pystate.c b/Python/pystate.c index 6b0d6a7a372c926..9a2dc9431f8bb24 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -3419,8 +3419,8 @@ PyInterpreterGuard_FromCurrent(void) return guard; } -void -PyInterpreterGuard_Close(PyInterpreterGuard *guard) +static void +release_interp_guard(PyInterpreterGuard *guard) { PyInterpreterState *interp = guard->interp; assert(interp != NULL); @@ -3432,9 +3432,28 @@ PyInterpreterGuard_Close(PyInterpreterGuard *guard) } assert(old_value > 0); +} + +void +PyInterpreterGuard_Close(PyInterpreterGuard *guard) +{ + release_interp_guard(guard); PyMem_RawFree(guard); } +int +_PyInterpreterGuard_TryAcquire(PyInterpreterState *interp, + PyInterpreterGuard *guard) +{ + return try_acquire_interp_guard(interp, guard); +} + +void +_PyInterpreterGuard_Release(PyInterpreterGuard *guard) +{ + release_interp_guard(guard); +} + PyInterpreterView * PyInterpreterView_FromCurrent(void) { diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 0afc84e021817cf..8ca2134921ff913 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -7,8 +7,9 @@ #include "pycore_lock.h" // PyMutex_LockFlags() #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config +#include "pycore_pystate.h" // _PyInterpreterGuard_TryAcquire() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_traceback.h" // _Py_DumpASCII() +#include "pycore_traceback.h" // _Py_DumpHexadecimal() #include // malloc() @@ -32,9 +33,10 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, #define allocators _PyRuntime.tracemalloc.allocators -/* This lock is needed because tracemalloc_free() is called without - the GIL held from PyMem_RawFree(). It cannot acquire the lock because it - would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ +/* This lock protects the trace tables. It is acquired by threads which may + not have an attached thread state, such as tracemalloc_free() called from + PyMem_RawFree(): tracing never acquires the GIL nor attaches a thread + state. */ #define tables_lock _PyRuntime.tracemalloc.tables_lock #define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) #define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) @@ -45,6 +47,10 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, typedef struct tracemalloc_frame frame_t; typedef struct tracemalloc_traceback traceback_t; +/* Filename used when the Python frame filename cannot be captured */ +static const char tracemalloc_unknown_filename[] = ""; +#define UNKNOWN_FILENAME tracemalloc_unknown_filename + #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME)) @@ -126,24 +132,19 @@ set_reentrant(int reentrant) static Py_uhash_t -hashtable_hash_pyobject(const void *key) +hashtable_hash_filename(const void *key) { - PyObject *obj = (PyObject *)key; - return PyObject_Hash(obj); + const char *filename = (const char *)key; + return (Py_uhash_t)Py_HashBuffer(filename, (Py_ssize_t)strlen(filename)); } static int -hashtable_compare_unicode(const void *key1, const void *key2) +hashtable_compare_filename(const void *key1, const void *key2) { - PyObject *obj1 = (PyObject *)key1; - PyObject *obj2 = (PyObject *)key2; - if (obj1 != NULL && obj2 != NULL) { - return (PyUnicode_Compare(obj1, obj2) == 0); - } - else { - return obj1 == obj2; - } + const char *filename1 = (const char *)key1; + const char *filename2 = (const char *)key2; + return (strcmp(filename1, filename2) == 0); } @@ -181,6 +182,100 @@ raw_free(void *ptr) } +/* Encode a str object to a NUL terminated UTF-8 (surrogatepass) string, + without using the Python C API. Return NULL on allocation failure. */ +static char * +tracemalloc_encode_filename(PyObject *obj) +{ + int kind = PyUnicode_KIND(obj); + const void *data = PyUnicode_DATA(obj); + Py_ssize_t length = PyUnicode_GET_LENGTH(obj); + + // worst case: 4 UTF-8 bytes per code point, plus the NUL terminator + if ((size_t)length > (SIZE_MAX - 1) / 4) { + return NULL; + } + char *buffer = raw_malloc((size_t)length * 4 + 1); + if (buffer == NULL) { + return NULL; + } + + char *p = buffer; + for (Py_ssize_t i = 0; i < length; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch < 0x80) { + *p++ = (char)ch; + } + else if (ch < 0x800) { + *p++ = (char)(0xc0 | (ch >> 6)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + else if (ch < 0x10000) { + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + else { + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + } + *p = '\0'; + return buffer; +} + + +/* Intern a str object in the tracemalloc_filenames hash table as a NUL + terminated UTF-8 string. Return NULL on allocation failure. + The caller must hold the TABLES_LOCK(). */ +static const char * +tracemalloc_intern_filename(PyObject *obj) +{ + assert(PyUnicode_Check(obj)); + + const char *utf8; + char *encoded = NULL; + if (PyUnicode_IS_COMPACT_ASCII(obj)) { + // ASCII string data is valid UTF-8 and is NUL terminated + utf8 = (const char *)PyUnicode_DATA(obj); + } + else { + encoded = tracemalloc_encode_filename(obj); + if (encoded == NULL) { + return NULL; + } + utf8 = encoded; + } + + const char *result; + _Py_hashtable_entry_t *entry; + entry = _Py_hashtable_get_entry(tracemalloc_filenames, utf8); + if (entry != NULL) { + result = (const char *)entry->key; + } + else { + size_t size = strlen(utf8) + 1; + char *filename = raw_malloc(size); + if (filename == NULL) { + raw_free(encoded); + return NULL; + } + memcpy(filename, utf8, size); + + if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL) < 0) { + raw_free(filename); + raw_free(encoded); + return NULL; + } + result = filename; + } + raw_free(encoded); + return result; +} + + static Py_uhash_t hashtable_hash_traceback(const void *key) { @@ -209,8 +304,8 @@ hashtable_compare_traceback(const void *key1, const void *key2) if (frame1->lineno != frame2->lineno) { return 0; } + // Filenames are interned: compare by pointer if (frame1->filename != frame2->filename) { - assert(PyUnicode_Compare(frame1->filename, frame2->filename) != 0); return 0; } } @@ -222,7 +317,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { assert(PyStackRef_CodeCheck(pyframe->f_executable)); - frame->filename = &_Py_STR(anon_unknown); + frame->filename = UNKNOWN_FILENAME; int lineno = -1; PyCodeObject *code = _PyFrame_GetCode(pyframe); @@ -253,26 +348,15 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } /* intern the filename */ - _Py_hashtable_entry_t *entry; - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); - if (entry != NULL) { - filename = (PyObject *)entry->key; - } - else { - /* tracemalloc_filenames is responsible to keep a reference - to the filename */ - if (_Py_hashtable_set(tracemalloc_filenames, Py_NewRef(filename), - NULL) < 0) { - Py_DECREF(filename); + const char *filename_copy = tracemalloc_intern_filename(filename); + if (filename_copy == NULL) { #ifdef TRACE_DEBUG - tracemalloc_error("failed to intern the filename"); + tracemalloc_error("failed to intern the filename"); #endif - return; - } + return; } - /* the tracemalloc_filenames table keeps a reference to the filename */ - frame->filename = filename; + frame->filename = filename_copy; } @@ -288,7 +372,8 @@ traceback_hash(traceback_t *traceback) x = 0x345678UL; frame = traceback->frames; while (--len >= 0) { - y = (Py_uhash_t)PyObject_Hash(frame->filename); + // Filenames are interned: hash the pointer + y = (Py_uhash_t)Py_HashPointer(frame->filename); y ^= (Py_uhash_t)frame->lineno; frame++; @@ -303,11 +388,8 @@ traceback_hash(traceback_t *traceback) static void -traceback_get_frames(traceback_t *traceback) +traceback_get_frames(traceback_t *traceback, PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate != NULL); - _PyInterpreterFrame *pyframe = _PyThreadState_GetFrame(tstate); while (pyframe) { if (traceback->nframe < tracemalloc_config.max_nframe) { @@ -329,13 +411,36 @@ traceback_new(void) traceback_t *traceback; _Py_hashtable_entry_t *entry; - _Py_AssertHoldsTstate(); + // Capturing a traceback needs a thread state to walk the frame stack, + // but the thread state doesn't need to be attached: only the thread + // itself pushes and pops its own frames, and no Python object is used + // or modified. If not attached (e.g. the GIL was released), fall back + // to the thread state most recently bound to the thread, if any. + int detached = 0; + PyInterpreterGuard guard = {NULL}; + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + if (_PyInterpreterGuard_TryAcquire(_PyInterpreterState_Main(), + &guard) < 0) { + return tracemalloc_empty_traceback; + } + detached = 1; + tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { + // the thread never had a thread state: no frames to capture + _PyInterpreterGuard_Release(&guard); + return tracemalloc_empty_traceback; + } + } /* get frames */ traceback = tracemalloc_traceback; traceback->nframe = 0; traceback->total_nframe = 0; - traceback_get_frames(traceback); + traceback_get_frames(traceback, tstate); + if (detached) { + _PyInterpreterGuard_Release(&guard); + } if (traceback->nframe == 0) { return tracemalloc_empty_traceback; } @@ -497,8 +602,7 @@ tracemalloc_add_trace_unlocked(unsigned int domain, uintptr_t ptr, static void* -tracemalloc_alloc(int need_gil, int use_calloc, - void *ctx, size_t nelem, size_t elsize) +tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) { assert(elsize == 0 || nelem <= SIZE_MAX / elsize); @@ -506,12 +610,9 @@ tracemalloc_alloc(int need_gil, int use_calloc, // Ignore reentrant call. // - // For example, PyObjet_Malloc() calls + // For example, PyObject_Malloc() calls // PyMem_Malloc() for allocations larger than 512 bytes: don't trace the // same memory allocation twice. - // - // If reentrant calls are not ignored, PyGILState_Ensure() can call - // PyMem_RawMalloc() which would call PyGILState_Ensure() again in a loop. if (!reentrant) { set_reentrant(1); } @@ -532,10 +633,6 @@ tracemalloc_alloc(int need_gil, int use_calloc, goto done; } - PyGILState_STATE gil_state; - if (need_gil) { - gil_state = PyGILState_Ensure(); - } TABLES_LOCK(); if (tracemalloc_config.tracing) { @@ -548,9 +645,6 @@ tracemalloc_alloc(int need_gil, int use_calloc, // else: gh-128679: tracemalloc.stop() was called by another thread TABLES_UNLOCK(); - if (need_gil) { - PyGILState_Release(gil_state); - } done: if (!reentrant) { @@ -561,7 +655,7 @@ tracemalloc_alloc(int need_gil, int use_calloc, static void* -tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) +tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) { int reentrant = get_reentrant(); @@ -582,10 +676,6 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) goto done; } - PyGILState_STATE gil_state; - if (need_gil) { - gil_state = PyGILState_Ensure(); - } TABLES_LOCK(); if (!tracemalloc_config.tracing) { @@ -610,8 +700,8 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) // This case is very unlikely: a hash entry has just been released, // so the hash table should have at least one free entry. // - // The GIL and the table lock ensures that only one thread is - // allocating memory. + // The table lock ensures that no other thread touched the trace + // tables in the meantime. Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } } @@ -627,9 +717,6 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) unlock: TABLES_UNLOCK(); - if (need_gil) { - PyGILState_Release(gil_state); - } done: if (!reentrant) { @@ -665,61 +752,22 @@ tracemalloc_free(void *ctx, void *ptr) static void* -tracemalloc_malloc_gil(void *ctx, size_t size) -{ - return tracemalloc_alloc(0, 0, ctx, 1, size); -} - - -static void* -tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) -{ - return tracemalloc_alloc(0, 1, ctx, nelem, elsize); -} - - -static void* -tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) -{ - return tracemalloc_realloc(0, ctx, ptr, new_size); -} - - -static void* -tracemalloc_raw_malloc(void *ctx, size_t size) +tracemalloc_malloc(void *ctx, size_t size) { - return tracemalloc_alloc(1, 0, ctx, 1, size); + return tracemalloc_alloc(0, ctx, 1, size); } static void* -tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) +tracemalloc_calloc(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_alloc(1, 1, ctx, nelem, elsize); -} - - -static void* -tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) -{ - return tracemalloc_realloc(1, ctx, ptr, new_size); -} - - -static void -tracemalloc_clear_filename(void *value) -{ - PyObject *filename = (PyObject *)value; - Py_DECREF(filename); + return tracemalloc_alloc(1, ctx, nelem, elsize); } static void tracemalloc_clear_traces_unlocked(void) { - // Clearing tracemalloc_filenames requires the GIL to call Py_DECREF() - _Py_AssertHoldsTstate(); - set_reentrant(1); _Py_hashtable_clear(tracemalloc_traces); @@ -745,9 +793,9 @@ _PyTraceMalloc_Init(void) return _PyStatus_NO_MEMORY(); } - tracemalloc_filenames = hashtable_new(hashtable_hash_pyobject, - hashtable_compare_unicode, - tracemalloc_clear_filename, NULL); + tracemalloc_filenames = hashtable_new(hashtable_hash_filename, + hashtable_compare_filename, + raw_free, NULL); tracemalloc_tracebacks = hashtable_new(hashtable_hash_traceback, hashtable_compare_traceback, @@ -770,8 +818,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback->nframe = 1; tracemalloc_empty_traceback->total_nframe = 1; - /* borrowed reference */ - tracemalloc_empty_traceback->frames[0].filename = &_Py_STR(anon_unknown); + tracemalloc_empty_traceback->frames[0].filename = UNKNOWN_FILENAME; tracemalloc_empty_traceback->frames[0].lineno = 0; tracemalloc_empty_traceback->hash = traceback_hash(tracemalloc_empty_traceback); @@ -829,20 +876,15 @@ _PyTraceMalloc_Start(int max_nframe) } PyMemAllocatorEx alloc; - alloc.malloc = tracemalloc_raw_malloc; - alloc.calloc = tracemalloc_raw_calloc; - alloc.realloc = tracemalloc_raw_realloc; + alloc.malloc = tracemalloc_malloc; + alloc.calloc = tracemalloc_calloc; + alloc.realloc = tracemalloc_realloc; alloc.free = tracemalloc_free; alloc.ctx = &allocators.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); - alloc.malloc = tracemalloc_malloc_gil; - alloc.calloc = tracemalloc_calloc_gil; - alloc.realloc = tracemalloc_realloc_gil; - alloc.free = tracemalloc_free; - alloc.ctx = &allocators.mem; PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); @@ -898,17 +940,62 @@ _PyTraceMalloc_Stop(void) +/* Convert an interned filename to a str object. intern_filenames + (const char* => str object, can be NULL) shares the str objects. */ static PyObject* -frame_to_pyobject(frame_t *frame) +filename_to_pyobject(const char *filename, _Py_hashtable_t *intern_filenames) +{ + PyObject *filename_obj; + if (intern_filenames != NULL) { + filename_obj = _Py_hashtable_get(intern_filenames, filename); + if (filename_obj != NULL) { + return Py_NewRef(filename_obj); + } + } + + if (filename == UNKNOWN_FILENAME) { + filename_obj = Py_NewRef(&_Py_STR(anon_unknown)); + } + else { + filename_obj = PyUnicode_DecodeUTF8(filename, + (Py_ssize_t)strlen(filename), + "surrogatepass"); + if (filename_obj == NULL) { + return NULL; + } + } + + if (intern_filenames != NULL) { + if (_Py_hashtable_set(intern_filenames, filename, filename_obj) < 0) { + Py_DECREF(filename_obj); + PyErr_NoMemory(); + return NULL; + } + /* intern_filenames keeps a new reference to filename_obj */ + Py_INCREF(filename_obj); + } + return filename_obj; +} + + +static PyObject* +frame_to_pyobject(frame_t *frame, _Py_hashtable_t *intern_filenames) { assert(get_reentrant()); + PyObject *filename_obj = filename_to_pyobject(frame->filename, + intern_filenames); + if (filename_obj == NULL) { + return NULL; + } + PyObject *frame_obj = PyTuple_New(2); if (frame_obj == NULL) { + Py_DECREF(filename_obj); return NULL; } - PyTuple_SET_ITEM(frame_obj, 0, Py_NewRef(frame->filename)); + PyTuple_SET_ITEM(frame_obj, 0, filename_obj); PyObject *lineno_obj = PyLong_FromUnsignedLong(frame->lineno); if (lineno_obj == NULL) { @@ -922,7 +1009,8 @@ frame_to_pyobject(frame_t *frame) static PyObject* -traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) +traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table, + _Py_hashtable_t *intern_filenames) { PyObject *frames; if (intern_table != NULL) { @@ -938,7 +1026,8 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) } for (int i=0; i < traceback->nframe; i++) { - PyObject *frame = frame_to_pyobject(&traceback->frames[i]); + PyObject *frame = frame_to_pyobject(&traceback->frames[i], + intern_filenames); if (frame == NULL) { Py_DECREF(frames); return NULL; @@ -961,7 +1050,8 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) static PyObject* trace_to_pyobject(unsigned int domain, const trace_t *trace, - _Py_hashtable_t *intern_tracebacks) + _Py_hashtable_t *intern_tracebacks, + _Py_hashtable_t *intern_filenames) { assert(get_reentrant()); @@ -984,7 +1074,8 @@ trace_to_pyobject(unsigned int domain, const trace_t *trace, } PyTuple_SET_ITEM(trace_obj, 1, obj); - obj = traceback_to_pyobject(trace->traceback, intern_tracebacks); + obj = traceback_to_pyobject(trace->traceback, intern_tracebacks, + intern_filenames); if (obj == NULL) { Py_DECREF(trace_obj); return NULL; @@ -1006,6 +1097,7 @@ typedef struct { _Py_hashtable_t *traces; _Py_hashtable_t *domains; _Py_hashtable_t *tracebacks; + _Py_hashtable_t *filenames; PyObject *list; unsigned int domain; } get_traces_t; @@ -1100,7 +1192,8 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, const trace_t *trace = (const trace_t *)value; PyObject *tuple = trace_to_pyobject(get_traces->domain, trace, - get_traces->tracebacks); + get_traces->tracebacks, + get_traces->filenames); if (tuple == NULL) { return 1; } @@ -1160,11 +1253,41 @@ tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) +/* Dump an interned filename: write printable ASCII characters as-is, + escape the other bytes. The function is signal-safe. */ +static void +_PyMem_DumpFilename(int fd, const char *filename) +{ + const size_t max_length = 500; + size_t length = strlen(filename); + int truncated = 0; + if (length > max_length) { + length = max_length; + truncated = 1; + } + + for (size_t i = 0; i < length; i++) { + unsigned char ch = (unsigned char)filename[i]; + if (' ' <= ch && ch <= 126) { + /* printable ASCII character */ + char c = (char)ch; + (void)_Py_write_noraise(fd, &c, 1); + } + else { + PUTS(fd, "\\x"); + _Py_DumpHexadecimal(fd, ch, 2); + } + } + if (truncated) { + PUTS(fd, "..."); + } +} + static void _PyMem_DumpFrame(int fd, frame_t * frame) { PUTS(fd, " File \""); - _Py_DumpASCII(fd, frame->filename); + _PyMem_DumpFilename(fd, frame->filename); PUTS(fd, "\", line "); _Py_DumpDecimal(fd, frame->lineno); PUTS(fd, "\n"); @@ -1221,7 +1344,6 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, /* tracemalloc is not tracing: do nothing */ return -2; } - PyGILState_STATE gil_state = PyGILState_Ensure(); TABLES_LOCK(); int result; @@ -1234,7 +1356,6 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, } TABLES_UNLOCK(); - PyGILState_Release(gil_state); return result; } @@ -1324,7 +1445,7 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) PyObject *result; if (traceback) { set_reentrant(1); - result = traceback_to_pyobject(traceback, NULL); + result = traceback_to_pyobject(traceback, NULL, NULL); set_reentrant(0); } else { @@ -1365,6 +1486,7 @@ _PyTraceMalloc_GetTraces(void) get_traces.traces = NULL; get_traces.domains = NULL; get_traces.tracebacks = NULL; + get_traces.filenames = NULL; get_traces.list = PyList_New(0); if (get_traces.list == NULL) { goto finally; @@ -1383,6 +1505,15 @@ _PyTraceMalloc_GetTraces(void) goto no_memory; } + /* the filename hash table is used temporarily to share filename + str objects between tracebacks */ + get_traces.filenames = hashtable_new(_Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, tracemalloc_pyobject_decref); + if (get_traces.filenames == NULL) { + goto no_memory; + } + // Copy all traces so tracemalloc_get_traces_fill() doesn't have to disable // temporarily tracemalloc which would impact other threads and so would // miss allocations while get_traces() is called. @@ -1424,6 +1555,9 @@ _PyTraceMalloc_GetTraces(void) if (get_traces.tracebacks != NULL) { _Py_hashtable_destroy(get_traces.tracebacks); } + if (get_traces.filenames != NULL) { + _Py_hashtable_destroy(get_traces.filenames); + } if (get_traces.traces != NULL) { _Py_hashtable_destroy(get_traces.traces); }