add tests to confirm erroneous behaviour

* SetHml remains unchanged, see comment on the isParent param of
eachNodeHtml
* Removed unused functions
* Added documentation on eachNodeHtml
* Fixed node parsing cache keys and added test
* Refactored some tests for clarity

.travis.yml

@@ -1,7 +1,6 @@
 language: go
 
 go:
-    - 1.1
     - 1.2.x
     - 1.3.x
     - 1.4.x
@@ -10,6 +9,9 @@ go:
     - 1.7.x
     - 1.8.x
     - 1.9.x
-    - "1.10.x"
+    - 1.10.x
+    - 1.11.x
+    - 1.12.x
+    - 1.13.x
     - tip
 

README.md

@@ -37,6 +37,8 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2020-02-04 (v1.5.1)** : Update module dependencies.
+*    **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
 *    **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
 *    **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
 *    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
@@ -137,8 +139,13 @@ func main() {
 - [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
 - [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
 - [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
-- [asciimoo/colly](https://github.com/asciimoo/colly), a lightning fast and elegant Scraping Framework
+- [gocolly/colly](https://github.com/gocolly/colly), a lightning fast and elegant Scraping Framework
 - [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
+- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
+- [tacusci/berrycms](https://github.com/tacusci/berrycms), a modern simple to use CMS with easy to write plugins
+- [Dataflow kit](https://github.com/slotix/dataflowkit), Web Scraping framework for Gophers.  
+- [Geziyor](https://github.com/geziyor/geziyor), a fast web crawling & scraping framework for Go. Supports JS rendering.
+- [Pagser](https://github.com/foolin/pagser), a simple, easy, extensible, configurable HTML parser to struct based on goquery and struct tags.
 
 ## Support
 

bench_iteration_test.go

@@ -1,6 +1,7 @@
 package goquery
 
 import (
+	"strconv"
 	"testing"
 )
 
@@ -31,7 +32,7 @@ func BenchmarkMap(b *testing.B) {
 	sel := DocW().Find("td")
 	f := func(i int, s *Selection) string {
 		tmp++
-		return string(tmp)
+		return strconv.Itoa(tmp)
 	}
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {

doc/tips.md

@@ -4,13 +4,13 @@
 
 The `go.net/html` package used by `goquery` requires that the html document is UTF-8 encoded. When you know the encoding of the html page is not UTF-8, you can use the `iconv` package to convert it to UTF-8 (there are various implementation of the `iconv` API, see [godoc.org][iconv] for other options):
 
-```
+```bash
 $ go get -u github.com/djimenez/iconv-go
 ```
 
 and then:
 
-```
+```golang
 // Load the URL
 res, err := http.Get(url)
 if err != nil {
@@ -51,7 +51,7 @@ You can find a code example using `otto` [in this gist][exotto]. Thanks to githu
 
 If all you need is a normal `for` loop over all nodes in the current selection, where `Map/Each`-style iteration is not necessary, you can use the following:
 
-```
+```golang
 sel := Doc().Find(".selector")
 for i := range sel.Nodes {
 	single := sel.Eq(i)

go.mod

@@ -0,0 +1,8 @@
+module github.com/PuerkitoBio/goquery
+
+require (
+	github.com/andybalholm/cascadia v1.1.0
+	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
+)
+
+go 1.13

go.sum

@@ -0,0 +1,8 @@
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

manipulation.go

@@ -39,8 +39,14 @@ func (s *Selection) AfterSelection(sel *Selection) *Selection {
 // AfterHtml parses the html and inserts it after the set of matched elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) AfterHtml(html string) *Selection {
-	return s.AfterNodes(parseHtml(html)...)
+func (s *Selection) AfterHtml(htmlStr string) *Selection {
+	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			if node.Parent != nil {
+				node.Parent.InsertBefore(n, node.NextSibling)
+			}
+		}
+	})
 }
 
 // AfterNodes inserts the nodes after each element in the set of matched elements.
@@ -85,8 +91,12 @@ func (s *Selection) AppendSelection(sel *Selection) *Selection {
 }
 
 // AppendHtml parses the html and appends it to the set of matched elements.
-func (s *Selection) AppendHtml(html string) *Selection {
-	return s.AppendNodes(parseHtml(html)...)
+func (s *Selection) AppendHtml(htmlStr string) *Selection {
+	return s.eachNodeHtml(htmlStr, false, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			node.AppendChild(n)
+		}
+	})
 }
 
 // AppendNodes appends the specified nodes to each node in the set of matched elements.
@@ -123,8 +133,14 @@ func (s *Selection) BeforeSelection(sel *Selection) *Selection {
 // BeforeHtml parses the html and inserts it before the set of matched elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) BeforeHtml(html string) *Selection {
-	return s.BeforeNodes(parseHtml(html)...)
+func (s *Selection) BeforeHtml(htmlStr string) *Selection {
+	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			if node.Parent != nil {
+				node.Parent.InsertBefore(n, node)
+			}
+		}
+	})
 }
 
 // BeforeNodes inserts the nodes before each element in the set of matched elements.
@@ -184,8 +200,12 @@ func (s *Selection) PrependSelection(sel *Selection) *Selection {
 }
 
 // PrependHtml parses the html and prepends it to the set of matched elements.
-func (s *Selection) PrependHtml(html string) *Selection {
-	return s.PrependNodes(parseHtml(html)...)
+func (s *Selection) PrependHtml(htmlStr string) *Selection {
+	return s.eachNodeHtml(htmlStr, false, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			node.InsertBefore(n, node.FirstChild)
+		}
+	})
 }
 
 // PrependNodes prepends the specified nodes to each node in the set of
@@ -212,14 +232,19 @@ func (s *Selection) Remove() *Selection {
 	return s
 }
 
-// RemoveFiltered removes the set of matched elements by selector.
-// It returns the Selection of removed nodes.
+// RemoveFiltered removes from the current set of matched elements those that
+// match the selector filter. It returns the Selection of removed nodes.
+//
+// For example if the selection s contains "<h1>", "<h2>" and "<h3>"
+// and s.RemoveFiltered("h2") is called, only the "<h2>" node is removed
+// (and returned), while "<h1>" and "<h3>" are kept in the document.
 func (s *Selection) RemoveFiltered(selector string) *Selection {
 	return s.RemoveMatcher(compileMatcher(selector))
 }
 
-// RemoveMatcher removes the set of matched elements.
-// It returns the Selection of removed nodes.
+// RemoveMatcher removes from the current set of matched elements those that
+// match the Matcher filter. It returns the Selection of removed nodes.
+// See RemoveFiltered for additional information.
 func (s *Selection) RemoveMatcher(m Matcher) *Selection {
 	return s.FilterMatcher(m).Remove()
 }
@@ -256,8 +281,15 @@ func (s *Selection) ReplaceWithSelection(sel *Selection) *Selection {
 // It returns the removed elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) ReplaceWithHtml(html string) *Selection {
-	return s.ReplaceWithNodes(parseHtml(html)...)
+func (s *Selection) ReplaceWithHtml(htmlStr string) *Selection {
+	s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			if node.Parent != nil {
+				node.Parent.InsertBefore(n, node.NextSibling)
+			}
+		}
+	})
+	return s.Remove()
 }
 
 // ReplaceWithNodes replaces each element in the set of matched elements with
@@ -272,8 +304,17 @@ func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
 
 // SetHtml sets the html content of each element in the selection to
 // specified html string.
-func (s *Selection) SetHtml(html string) *Selection {
-	return setHtmlNodes(s, parseHtml(html)...)
+func (s *Selection) SetHtml(htmlStr string) *Selection {
+	for _, context := range s.Nodes {
+		for c := context.FirstChild; c != nil; c = context.FirstChild {
+			context.RemoveChild(c)
+		}
+	}
+	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
+		for _, n := range nodes {
+			node.AppendChild(n)
+		}
+	})
 }
 
 // SetText sets the content of each element in the selection to specified content.
@@ -329,8 +370,23 @@ func (s *Selection) WrapSelection(sel *Selection) *Selection {
 // most child of the given HTML.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapHtml(html string) *Selection {
-	return s.wrapNodes(parseHtml(html)...)
+func (s *Selection) WrapHtml(htmlStr string) *Selection {
+	nodesMap := make(map[html.NodeType][]*html.Node)
+	for _, context := range s.Nodes {
+		var parent *html.Node
+		if context.Parent != nil {
+			parent = context.Parent
+		} else {
+			parent = &html.Node{Type: html.ElementNode}
+		}
+		nodes, found := nodesMap[parent.Type]
+		if !found {
+			nodes = parseHtmlWithContext(htmlStr, parent)
+			nodesMap[parent.Type] = nodes
+		}
+		newSingleSelection(context, s.document).wrapAllNodes(cloneNodes(nodes)...)
+	}
+	return s
 }
 
 // WrapNode wraps each element in the set of matched elements inside the inner-
@@ -382,8 +438,18 @@ func (s *Selection) WrapAllSelection(sel *Selection) *Selection {
 // document.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapAllHtml(html string) *Selection {
-	return s.wrapAllNodes(parseHtml(html)...)
+func (s *Selection) WrapAllHtml(htmlStr string) *Selection {
+	var context *html.Node
+	var nodes []*html.Node
+	if len(s.Nodes) > 0 {
+		context = s.Nodes[0]
+		if context.Parent != nil {
+			nodes = parseHtmlWithContext(htmlStr, context)
+		} else {
+			nodes = parseHtml(htmlStr)
+		}
+	}
+	return s.wrapAllNodes(nodes...)
 }
 
 func (s *Selection) wrapAllNodes(ns ...*html.Node) *Selection {
@@ -452,8 +518,17 @@ func (s *Selection) WrapInnerSelection(sel *Selection) *Selection {
 // cloned before being inserted into the document.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapInnerHtml(html string) *Selection {
-	return s.wrapInnerNodes(parseHtml(html)...)
+func (s *Selection) WrapInnerHtml(htmlStr string) *Selection {
+	nodesMap := make(map[html.NodeType][]*html.Node)
+	for _, context := range s.Nodes {
+		nodes, found := nodesMap[context.Type]
+		if !found {
+			nodes = parseHtmlWithContext(htmlStr, context)
+			nodesMap[context.Type] = nodes
+		}
+		newSingleSelection(context, s.document).wrapInnerNodes(cloneNodes(nodes)...)
+	}
+	return s
 }
 
 // WrapInnerNode wraps an HTML structure, matched by the given selector, around
@@ -493,16 +568,14 @@ func parseHtml(h string) []*html.Node {
 	return nodes
 }
 
-func setHtmlNodes(s *Selection, ns ...*html.Node) *Selection {
-	for _, n := range s.Nodes {
-		for c := n.FirstChild; c != nil; c = n.FirstChild {
-			n.RemoveChild(c)
-		}
-		for _, c := range ns {
-			n.AppendChild(cloneNode(c))
-		}
+func parseHtmlWithContext(h string, context *html.Node) []*html.Node {
+	// Errors are only returned when the io.Reader returns any error besides
+	// EOF, but strings.Reader never will
+	nodes, err := html.ParseFragment(strings.NewReader(h), context)
+	if err != nil {
+		panic("goquery: failed to parse HTML: " + err.Error())
 	}
-	return s
+	return nodes
 }
 
 // Get the first child that is an ElementNode
@@ -572,3 +645,29 @@ func (s *Selection) manipulateNodes(ns []*html.Node, reverse bool,
 
 	return s
 }
+
+// eachNodeHtml parses the given html string and inserts the resulting nodes in the dom with the mergeFn.
+// The parsed nodes are inserted for each element of the selection.
+// isParent can be used to indicate that the elements of the selection should be treated as the parent for the parsed html.
+// A cache is used to avoid parsing the html multiple times should the elements of the selection result in the same context.
+func (s *Selection) eachNodeHtml(htmlStr string, isParent bool, mergeFn func(n *html.Node, nodes []*html.Node)) *Selection {
+	// cache to avoid parsing the html for the same context multiple times
+	nodeCache := make(map[string][]*html.Node)
+	var context *html.Node
+	for _, n := range s.Nodes {
+		if isParent {
+			context = n.Parent
+		} else {
+			context = n
+		}
+		if context != nil {
+			nodes, found := nodeCache[nodeName(context)]
+			if !found {
+				nodes = parseHtmlWithContext(htmlStr, context)
+				nodeCache[nodeName(context)] = nodes
+			}
+			mergeFn(n, cloneNodes(nodes))
+		}
+	}
+	return s
+}

manipulation_test.go

@@ -56,6 +56,25 @@ func TestAfterHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestAfterHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<td>Before1</td>
+					</tr>
+					<tr>
+						<td>Before2</td>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table tr td").AfterHtml("<td>Test</td>")
+	assertLength(t, doc.Find("table tr td").Nodes, 4)
+	printSel(t, doc.Selection)
+}
+
 func TestAppend(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#main").Append("#nf6")
@@ -113,6 +132,27 @@ func TestAppendHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestAppendHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<td>Before1</td>
+					</tr>
+					<tr>
+						<td>Before2</td>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table tr").AppendHtml("<td class='new-node'>new node</td>")
+
+	assertLength(t, doc.Find("table td").Nodes, 4)
+	assertClass(t, doc.Find("table td").Last(), "new-node")
+	printSel(t, doc.Selection)
+}
+
 func TestBefore(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#main").Before("#nf6")
@@ -151,6 +191,27 @@ func TestBeforeHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestBeforeHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<td>Before1</td>
+					</tr>
+					<tr>
+						<td>Before2</td>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table tr td:first-child").BeforeHtml("<td class='new-node'>new node</td>")
+
+	assertLength(t, doc.Find("table td").Nodes, 4)
+	assertClass(t, doc.Find("table td").First(), "new-node")
+	printSel(t, doc.Selection)
+}
+
 func TestEmpty(t *testing.T) {
 	doc := Doc2Clone()
 	s := doc.Find("#main").Empty()
@@ -218,6 +279,27 @@ func TestPrependHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestPrependHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<td>Before1</td>
+					</tr>
+					<tr>
+						<td>Before2</td>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table tr").PrependHtml("<td class='c1'>new node</td><td class='c2'>other new node</td>")
+
+	assertLength(t, doc.Find("table td").Nodes, 6)
+	assertClass(t, doc.Find("table tr td").First(), "c1")
+	printSel(t, doc.Selection)
+}
+
 func TestRemove(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#nf1").Remove()
@@ -278,6 +360,27 @@ func TestReplaceWithHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestReplaceWithHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<th>Before1</th>
+					</tr>
+					<tr>
+						<th>Before2</th>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table th").ReplaceWithHtml("<td>Test</td><td>Replace</td>")
+
+	assertLength(t, doc.Find("table th").Nodes, 0)
+	assertLength(t, doc.Find("table tr td").Nodes, 4)
+	printSel(t, doc.Selection)
+}
+
 func TestSetHtml(t *testing.T) {
 	doc := Doc2Clone()
 	q := doc.Find("#main, #foot")
@@ -313,6 +416,28 @@ func TestSetHtmlEmpty(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestSetHtmlContext(t *testing.T) {
+	doc := loadString(t, `
+		<html>
+			<body>
+				<table>
+					<tr>
+						<th>Before1</th>
+					</tr>
+					<tr>
+						<th>Before2</th>
+					</tr>
+				</table>
+			</body>
+		</html>`)
+	doc.Find("table tr").SetHtml("<td class='c1'>Test</td><td class='c2'>Again</td>")
+
+	assertLength(t, doc.Find("table th").Nodes, 0)
+	assertLength(t, doc.Find("table td").Nodes, 4)
+	assertLength(t, doc.Find("table t2").Nodes, 2)
+	printSel(t, doc.Selection)
+}
+
 func TestSetText(t *testing.T) {
 	doc := Doc2Clone()
 	q := doc.Find("#main, #foot")
@@ -511,3 +636,54 @@ func TestWrapInnerHtml(t *testing.T) {
 
 	printSel(t, doc.Selection)
 }
+
+func TestParsingRespectsVaryingContext(t *testing.T) {
+	docA := loadString(t, `
+	<html>
+		<body>
+			<a class="x"></a>
+		</body>
+	</html>`)
+	docTable := loadString(t, `
+	<html>
+		<body>
+			<table class="x"></table>
+		</body>
+	</html>`)
+	docBoth := loadString(t, `
+	<html>
+		<body>
+			<table class="x"></table>
+			<a class="x"></a>
+		</body>
+	</html>`)
+
+	sA := docA.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
+	sTable := docTable.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
+	sBoth := docBoth.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
+
+	printSel(t, docA.Selection)
+	printSel(t, docTable.Selection)
+	printSel(t, docBoth.Selection)
+
+	oA, _ := sA.Html()
+	oTable, _ := sTable.Html()
+
+	if oA == oTable {
+		t.Errorf("Expected inner html of <a> and <table> to not be equal, but got %s and %s", oA, oTable)
+	}
+
+	oBothTable, _ := sBoth.First().Html()
+	if oBothTable != oTable {
+		t.Errorf("Expected inner html of <table> and <table> in doc containing both tags to be equal, but got %s and %s",
+			oTable,
+			oBothTable)
+	}
+
+	oBothA, _ := sBoth.Last().Html()
+	if oBothA != oA {
+		t.Errorf("Expected inner html of <a> and <a> in doc containing both tags to be equal, but got %s and %s",
+			oA,
+			oBothA)
+	}
+}

type_test.go

@@ -121,6 +121,14 @@ func loadDoc(page string) *Document {
 	return NewDocumentFromNode(node)
 }
 
+func loadString(t *testing.T, doc string) *Document {
+	d, err := NewDocumentFromReader(strings.NewReader(doc))
+	if err != nil {
+		t.Error("Failed to parse test document")
+	}
+	return d
+}
+
 func TestNewDocument(t *testing.T) {
 	if f, e := os.Open("./testdata/page.html"); e != nil {
 		t.Error(e.Error())

utilities.go

@@ -36,12 +36,22 @@ func NodeName(s *Selection) string {
 	if s.Length() == 0 {
 		return ""
 	}
-	switch n := s.Get(0); n.Type {
+	return nodeName(s.Get(0))
+}
+
+// nodeName returns the node name of the given html node.
+// See NodeName for additional details on behaviour.
+func nodeName(node *html.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	switch node.Type {
 	case html.ElementNode, html.DoctypeNode:
-		return n.Data
+		return node.Data
 	default:
-		if n.Type >= 0 && int(n.Type) < len(nodeNames) {
-			return nodeNames[n.Type]
+		if node.Type >= 0 && int(node.Type) < len(nodeNames) {
+			return nodeNames[node.Type]
 		}
 		return ""
 	}