Last active
June 5, 2026 04:55
-
-
Save ergoithz/6cf043e3fdedd1b94fcf to your computer and use it in GitHub Desktop.
Generate unique XPATH for BeautifulSoup element
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # -*- coding: utf-8 -*- | |
| import bs4 | |
| def xpath_soup(element): | |
| # type: (typing.Union[bs4.element.Tag, bs4.element.NavigableString]) -> str | |
| """ | |
| Generate xpath from BeautifulSoup4 element. | |
| :param element: BeautifulSoup4 element. | |
| :type element: bs4.element.Tag or bs4.element.NavigableString | |
| :return: xpath as string | |
| :rtype: str | |
| Usage | |
| ----- | |
| >>> import bs4 | |
| >>> html = ( | |
| ... '<html><head><title>title</title></head>' | |
| ... '<body><p>p <i>1</i></p><p>p <i>2</i></p></body></html>' | |
| ... ) | |
| >>> soup = bs4.BeautifulSoup(html, 'html.parser') | |
| >>> xpath_soup(soup.html.body.p.i) | |
| '/html/body/p[1]/i' | |
| >>> import bs4 | |
| >>> xml = ( | |
| ... '<?xml version="1.0" encoding="UTF-8"?>' | |
| ... '<doc xmlns:ns1="http://localhost/ns1"' | |
| ... ' xmlns:ns2="http://localhost/ns2">' | |
| ... '<ns1:elm/><ns2:elm/><ns2:elm/></doc>' | |
| ... ) | |
| >>> soup = bs4.BeautifulSoup(xml, 'lxml-xml') | |
| >>> xpath_soup(soup.doc.find('ns2:elm').next_sibling) | |
| '/doc/ns2:elm[2]' | |
| """ | |
| components = [] | |
| target = element if element.name else element.parent | |
| for node in (target, *target.parents)[-2::-1]: # type: bs4.element.Tag | |
| tag = '%s:%s' % (node.prefix, node.name) if node.prefix else node.name | |
| siblings = node.parent.find_all(tag, recursive=False) | |
| components.append(tag if len(siblings) == 1 else '%s[%d]' % (tag, next( | |
| index | |
| for index, sibling in enumerate(siblings, 1) | |
| if sibling is node | |
| ))) | |
| return '/%s' % '/'.join(components) | |
| if __name__ == '__main__': | |
| import doctest | |
| doctest.testmod(verbose=True, raise_on_error=True) |
Author
Hi @ergoithz
This looks throwing wrong error. If not then can you please guide me the error in below HTML5 semantic .
<footer id="copyright-container"> <div id="copyright-content"> <small class="copyright">Copyright 2021-21</small> <p> All Rights Reserved.</p> </div> </footer>Below Error I am getting
E AssertionError: current tag id: copyright-content (located at xpath: /html/body/footer/div) has wrong usage (xpath algorithm code: https://gist.github.com/ergoithz/6cf043e3fdedd1b94fcf)
I tried to change div to section/article but it does not work
this platform verification was so weird, the accepted solution was;
<footer id="copyright-container"><details id="copyright-content"><summary class="copyright">Copyright 1999-2020.</summary><p> by platform ® All Rights Reserved.</p></details></footer>
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@funway thanks for the heads up, snippet updated with your suggestion.
(You might take a look at your code tho, your sibling search wasn't prefixed).