add PathForNode and NodeAtPath functions

.travis.yml

@@ -1,6 +1,7 @@
 language: go
 
 go:
+    - 1.1.x
     - 1.2.x
     - 1.3.x
     - 1.4.x
@@ -8,10 +9,4 @@ go:
     - 1.6.x
     - 1.7.x
     - 1.8.x
-    - 1.9.x
-    - 1.10.x
-    - 1.11.x
-    - 1.12.x
-    - 1.13.x
     - tip
-

README.md

@@ -1,5 +1,6 @@
 # goquery - a little like that j-thing, only in Go
-[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.png)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+
 
 goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
 
@@ -14,7 +15,6 @@ Syntax-wise, it is as close as possible to jQuery, with the same function names
 * [API](#api)
 * [Examples](#examples)
 * [Related Projects](#related-projects)
-* [Support](#support)
 * [License](#license)
 
 ## Installation
@@ -37,12 +37,6 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
 
 **Note that goquery's API is now stable, and will not break.**
 
-*    **2020-02-04 (v1.5.1)** : Update module dependencies.
-*    **2018-11-15 (v1.5.0)** : Go module support (thanks @Zaba505).
-*    **2018-06-07 (v1.4.1)** : Add `NewDocumentFromReader` examples.
-*    **2018-03-24 (v1.4.0)** : Deprecate `NewDocument(url)` and `NewDocumentFromResponse(response)`.
-*    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
-*    **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
 *    **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
 *    **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
 *    **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
@@ -98,24 +92,12 @@ package main
 import (
   "fmt"
   "log"
-  "net/http"
 
   "github.com/PuerkitoBio/goquery"
 )
 
 func ExampleScrape() {
-  // Request the HTML page.
-  res, err := http.Get("http://metalsucks.net")
-  if err != nil {
-    log.Fatal(err)
-  }
-  defer res.Body.Close()
-  if res.StatusCode != 200 {
-    log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
-  }
-
-  // Load the HTML document
-  doc, err := goquery.NewDocumentFromReader(res.Body)
+  doc, err := goquery.NewDocument("http://metalsucks.net")
   if err != nil {
     log.Fatal(err)
   }
@@ -139,29 +121,6 @@ func main() {
 - [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
 - [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
 - [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
-- [gocolly/colly](https://github.com/gocolly/colly), a lightning fast and elegant Scraping Framework
-- [gnulnx/goperf](https://github.com/gnulnx/goperf), a website performance test tool that also fetches static assets.
-- [MontFerret/ferret](https://github.com/MontFerret/ferret), declarative web scraping.
-- [tacusci/berrycms](https://github.com/tacusci/berrycms), a modern simple to use CMS with easy to write plugins
-- [Dataflow kit](https://github.com/slotix/dataflowkit), Web Scraping framework for Gophers.  
-- [Geziyor](https://github.com/geziyor/geziyor), a fast web crawling & scraping framework for Go. Supports JS rendering.
-- [Pagser](https://github.com/foolin/pagser), a simple, easy, extensible, configurable HTML parser to struct based on goquery and struct tags.
-
-## Support
-
-There are a number of ways you can support the project:
-
-* Use it, star it, build something with it, spread the word!
-  - If you do build something open-source or otherwise publicly-visible, let me know so I can add it to the [Related Projects](#related-projects) section!
-* Raise issues to improve the project (note: doc typos and clarifications are issues too!)
-  - Please search existing issues before opening a new one - it may have already been adressed.
-* Pull requests: please discuss new code in an issue first, unless the fix is really trivial.
-  - Make sure new code is tested.
-  - Be mindful of existing code - PRs that break existing code have a high probability of being declined, unless it fixes a serious issue.
-
-If you desperately want to send money my way, I have a BuyMeACoffee.com page:
-
-<a href="https://www.buymeacoffee.com/mna" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
 
 ## License
 

array.go

@@ -4,16 +4,6 @@ import (
 	"golang.org/x/net/html"
 )
 
-const (
-	maxUint = ^uint(0)
-	maxInt  = int(maxUint >> 1)
-
-	// ToEnd is a special index value that can be used as end index in a call
-	// to Slice so that all elements are selected until the end of the Selection.
-	// It is equivalent to passing (*Selection).Length().
-	ToEnd = maxInt
-)
-
 // First reduces the set of matched elements to the first in the set.
 // It returns a new Selection object, and an empty Selection object if the
 // the selection is empty.
@@ -45,23 +35,12 @@ func (s *Selection) Eq(index int) *Selection {
 }
 
 // Slice reduces the set of matched elements to a subset specified by a range
-// of indices. The start index is 0-based and indicates the index of the first
-// element to select. The end index is 0-based and indicates the index at which
-// the elements stop being selected (the end index is not selected).
-//
-// The indices may be negative, in which case they represent an offset from the
-// end of the selection.
-//
-// The special value ToEnd may be specified as end index, in which case all elements
-// until the end are selected. This works both for a positive and negative start
-// index.
+// of indices.
 func (s *Selection) Slice(start, end int) *Selection {
 	if start < 0 {
 		start += len(s.Nodes)
 	}
-	if end == ToEnd {
-		end = len(s.Nodes)
-	} else if end < 0 {
+	if end < 0 {
 		end += len(s.Nodes)
 	}
 	return pushStack(s, s.Nodes[start:end])

array_test.go

@@ -98,17 +98,6 @@ func TestSlice(t *testing.T) {
 	sel := Doc().Find(".pvk-content").Slice(0, 2)
 
 	assertLength(t, sel.Nodes, 2)
-	assertSelectionIs(t, sel, "#pc1", "#pc2")
-}
-
-func TestSliceToEnd(t *testing.T) {
-	sel := Doc().Find(".pvk-content").Slice(1, ToEnd)
-
-	assertLength(t, sel.Nodes, 2)
-	assertSelectionIs(t, sel.Eq(0), "#pc2")
-	if _, ok := sel.Eq(1).Attr("id"); ok {
-		t.Error("Want no attribute ID, got one")
-	}
 }
 
 func TestSliceEmpty(t *testing.T) {
@@ -121,11 +110,6 @@ func TestSliceInvalid(t *testing.T) {
 	Doc().Find("").Slice(0, 2)
 }
 
-func TestSliceInvalidToEnd(t *testing.T) {
-	defer assertPanic(t)
-	Doc().Find("").Slice(2, ToEnd)
-}
-
 func TestSliceOutOfBounds(t *testing.T) {
 	defer assertPanic(t)
 	Doc().Find(".pvk-content").Slice(2, 12)
@@ -151,12 +135,6 @@ func TestNegativeSliceBoth(t *testing.T) {
 	assertSelectionIs(t, sel.Eq(1), "#cf3")
 }
 
-func TestNegativeSliceToEnd(t *testing.T) {
-	sel := Doc().Find(".container-fluid").Slice(-3, ToEnd)
-	assertLength(t, sel.Nodes, 3)
-	assertSelectionIs(t, sel, "#cf2", "#cf3", "#cf4")
-}
-
 func TestNegativeSliceOutOfBounds(t *testing.T) {
 	defer assertPanic(t)
 	Doc().Find(".container-fluid").Slice(-12, -7)

bench_filter_test.go

@@ -231,6 +231,6 @@ func BenchmarkEnd(b *testing.B) {
 		}
 	}
 	if n != 373 {
-		b.Fatalf("want 373, got %d", n)
+		b.Fatalf("wnat 373, got %d", n)
 	}
 }

bench_iteration_test.go

@@ -1,7 +1,6 @@
 package goquery
 
 import (
-	"strconv"
 	"testing"
 )
 
@@ -32,7 +31,7 @@ func BenchmarkMap(b *testing.B) {
 	sel := DocW().Find("td")
 	f := func(i int, s *Selection) string {
 		tmp++
-		return strconv.Itoa(tmp)
+		return string(tmp)
 	}
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {

doc/tips.md

@@ -4,13 +4,13 @@
 
 The `go.net/html` package used by `goquery` requires that the html document is UTF-8 encoded. When you know the encoding of the html page is not UTF-8, you can use the `iconv` package to convert it to UTF-8 (there are various implementation of the `iconv` API, see [godoc.org][iconv] for other options):
 
-```bash
+```
 $ go get -u github.com/djimenez/iconv-go
 ```
 
 and then:
 
-```golang
+```
 // Load the URL
 res, err := http.Get(url)
 if err != nil {
@@ -51,7 +51,7 @@ You can find a code example using `otto` [in this gist][exotto]. Thanks to githu
 
 If all you need is a normal `for` loop over all nodes in the current selection, where `Map/Each`-style iteration is not necessary, you can use the following:
 
-```golang
+```
 sel := Doc().Find(".selector")
 for i := range sel.Nodes {
 	single := sel.Eq(i)

example_test.go

@@ -3,27 +3,14 @@ package goquery_test
 import (
 	"fmt"
 	"log"
-	"net/http"
-	"os"
-	"strings"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
 // This example scrapes the reviews shown on the home page of metalsucks.net.
 func Example() {
-	// Request the HTML page.
-	res, err := http.Get("http://metalsucks.net")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		log.Fatalf("status code error: %d %s", res.StatusCode, res.Status)
-	}
-
 	// Load the HTML document
-	doc, err := goquery.NewDocumentFromReader(res.Body)
+	doc, err := goquery.NewDocument("http://metalsucks.net")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -41,42 +28,3 @@ func Example() {
 
 	// xOutput: voluntarily fail the Example output.
 }
-
-// This example shows how to use NewDocumentFromReader from a file.
-func ExampleNewDocumentFromReader_file() {
-	// create from a file
-	f, err := os.Open("some/file.html")
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer f.Close()
-	doc, err := goquery.NewDocumentFromReader(f)
-	if err != nil {
-		log.Fatal(err)
-	}
-	// use the goquery document...
-	_ = doc.Find("h1")
-}
-
-// This example shows how to use NewDocumentFromReader from a string.
-func ExampleNewDocumentFromReader_string() {
-	// create from a string
-	data := `
-<html>
-	<head>
-		<title>My document</title>
-	</head>
-	<body>
-		<h1>Header</h1>
-	</body>
-</html>`
-
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
-	if err != nil {
-		log.Fatal(err)
-	}
-	header := doc.Find("h1").Text()
-	fmt.Println(header)
-
-	// Output: Header
-}

expand.go

@@ -41,30 +41,6 @@ func (s *Selection) AddNodes(nodes ...*html.Node) *Selection {
 // AndSelf adds the previous set of elements on the stack to the current set.
 // It returns a new Selection object containing the current Selection combined
 // with the previous one.
-// Deprecated: This function has been deprecated and is now an alias for AddBack().
 func (s *Selection) AndSelf() *Selection {
-	return s.AddBack()
-}
-
-// AddBack adds the previous set of elements on the stack to the current set.
-// It returns a new Selection object containing the current Selection combined
-// with the previous one.
-func (s *Selection) AddBack() *Selection {
 	return s.AddSelection(s.prevSel)
 }
-
-// AddBackFiltered reduces the previous set of elements on the stack to those that
-// match the selector string, and adds them to the current set.
-// It returns a new Selection object containing the current Selection combined
-// with the filtered previous one
-func (s *Selection) AddBackFiltered(selector string) *Selection {
-	return s.AddSelection(s.prevSel.Filter(selector))
-}
-
-// AddBackMatcher reduces the previous set of elements on the stack to those that match
-// the mateher, and adds them to the curernt set.
-// It returns a new Selection object containing the current Selection combined
-// with the filtered previous one
-func (s *Selection) AddBackMatcher(m Matcher) *Selection {
-	return s.AddSelection(s.prevSel.FilterMatcher(m))
-}

expand_test.go

@@ -94,25 +94,3 @@ func TestAndSelfRollback(t *testing.T) {
 	sel2 := sel.Find("a").AndSelf().End().End()
 	assertEqual(t, sel, sel2)
 }
-
-func TestAddBack(t *testing.T) {
-	sel := Doc().Find(".span12").Last().AddBack()
-	assertLength(t, sel.Nodes, 2)
-}
-
-func TestAddBackRollback(t *testing.T) {
-	sel := Doc().Find(".pvk-content")
-	sel2 := sel.Find("a").AddBack().End().End()
-	assertEqual(t, sel, sel2)
-}
-
-func TestAddBackFiltered(t *testing.T) {
-	sel := Doc().Find(".span12, .footer").Find("h1").AddBackFiltered(".footer")
-	assertLength(t, sel.Nodes, 2)
-}
-
-func TestAddBackFilteredRollback(t *testing.T) {
-	sel := Doc().Find(".span12, .footer")
-	sel2 := sel.Find("h1").AddBackFiltered(".footer").End().End()
-	assertEqual(t, sel, sel2)
-}

go.mod

@@ -1,8 +0,0 @@
-module github.com/PuerkitoBio/goquery
-
-require (
-	github.com/andybalholm/cascadia v1.1.0
-	golang.org/x/net v0.0.0-20200202094626-16171245cfb2
-)
-
-go 1.13

go.sum

@@ -1,8 +0,0 @@
-github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
-github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

manipulation.go

@@ -39,14 +39,8 @@ func (s *Selection) AfterSelection(sel *Selection) *Selection {
 // AfterHtml parses the html and inserts it after the set of matched elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) AfterHtml(htmlStr string) *Selection {
-	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			if node.Parent != nil {
-				node.Parent.InsertBefore(n, node.NextSibling)
-			}
-		}
-	})
+func (s *Selection) AfterHtml(html string) *Selection {
+	return s.AfterNodes(parseHtml(html)...)
 }
 
 // AfterNodes inserts the nodes after each element in the set of matched elements.
@@ -91,12 +85,8 @@ func (s *Selection) AppendSelection(sel *Selection) *Selection {
 }
 
 // AppendHtml parses the html and appends it to the set of matched elements.
-func (s *Selection) AppendHtml(htmlStr string) *Selection {
-	return s.eachNodeHtml(htmlStr, false, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			node.AppendChild(n)
-		}
-	})
+func (s *Selection) AppendHtml(html string) *Selection {
+	return s.AppendNodes(parseHtml(html)...)
 }
 
 // AppendNodes appends the specified nodes to each node in the set of matched elements.
@@ -133,14 +123,8 @@ func (s *Selection) BeforeSelection(sel *Selection) *Selection {
 // BeforeHtml parses the html and inserts it before the set of matched elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) BeforeHtml(htmlStr string) *Selection {
-	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			if node.Parent != nil {
-				node.Parent.InsertBefore(n, node)
-			}
-		}
-	})
+func (s *Selection) BeforeHtml(html string) *Selection {
+	return s.BeforeNodes(parseHtml(html)...)
 }
 
 // BeforeNodes inserts the nodes before each element in the set of matched elements.
@@ -200,12 +184,8 @@ func (s *Selection) PrependSelection(sel *Selection) *Selection {
 }
 
 // PrependHtml parses the html and prepends it to the set of matched elements.
-func (s *Selection) PrependHtml(htmlStr string) *Selection {
-	return s.eachNodeHtml(htmlStr, false, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			node.InsertBefore(n, node.FirstChild)
-		}
-	})
+func (s *Selection) PrependHtml(html string) *Selection {
+	return s.PrependNodes(parseHtml(html)...)
 }
 
 // PrependNodes prepends the specified nodes to each node in the set of
@@ -232,19 +212,14 @@ func (s *Selection) Remove() *Selection {
 	return s
 }
 
-// RemoveFiltered removes from the current set of matched elements those that
-// match the selector filter. It returns the Selection of removed nodes.
-//
-// For example if the selection s contains "<h1>", "<h2>" and "<h3>"
-// and s.RemoveFiltered("h2") is called, only the "<h2>" node is removed
-// (and returned), while "<h1>" and "<h3>" are kept in the document.
+// RemoveFiltered removes the set of matched elements by selector.
+// It returns the Selection of removed nodes.
 func (s *Selection) RemoveFiltered(selector string) *Selection {
 	return s.RemoveMatcher(compileMatcher(selector))
 }
 
-// RemoveMatcher removes from the current set of matched elements those that
-// match the Matcher filter. It returns the Selection of removed nodes.
-// See RemoveFiltered for additional information.
+// RemoveMatcher removes the set of matched elements.
+// It returns the Selection of removed nodes.
 func (s *Selection) RemoveMatcher(m Matcher) *Selection {
 	return s.FilterMatcher(m).Remove()
 }
@@ -281,15 +256,8 @@ func (s *Selection) ReplaceWithSelection(sel *Selection) *Selection {
 // It returns the removed elements.
 //
 // This follows the same rules as Selection.Append.
-func (s *Selection) ReplaceWithHtml(htmlStr string) *Selection {
-	s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			if node.Parent != nil {
-				node.Parent.InsertBefore(n, node.NextSibling)
-			}
-		}
-	})
-	return s.Remove()
+func (s *Selection) ReplaceWithHtml(html string) *Selection {
+	return s.ReplaceWithNodes(parseHtml(html)...)
 }
 
 // ReplaceWithNodes replaces each element in the set of matched elements with
@@ -302,23 +270,13 @@ func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
 	return s.Remove()
 }
 
-// SetHtml sets the html content of each element in the selection to
-// specified html string.
-func (s *Selection) SetHtml(htmlStr string) *Selection {
-	for _, context := range s.Nodes {
-		for c := context.FirstChild; c != nil; c = context.FirstChild {
-			context.RemoveChild(c)
-		}
-	}
-	return s.eachNodeHtml(htmlStr, true, func(node *html.Node, nodes []*html.Node) {
-		for _, n := range nodes {
-			node.AppendChild(n)
-		}
-	})
+// Set the html content of each element in the selection to specified html string.
+func (s *Selection) SetHtml(html string) *Selection {
+	return setHtmlNodes(s, parseHtml(html)...)
 }
 
-// SetText sets the content of each element in the selection to specified content.
-// The provided text string is escaped.
+// Set the content of each element in the selection to specified content. The
+// provided text string is escaped.
 func (s *Selection) SetText(text string) *Selection {
 	return s.SetHtml(html.EscapeString(text))
 }
@@ -370,23 +328,8 @@ func (s *Selection) WrapSelection(sel *Selection) *Selection {
 // most child of the given HTML.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapHtml(htmlStr string) *Selection {
-	nodesMap := make(map[html.NodeType][]*html.Node)
-	for _, context := range s.Nodes {
-		var parent *html.Node
-		if context.Parent != nil {
-			parent = context.Parent
-		} else {
-			parent = &html.Node{Type: html.ElementNode}
-		}
-		nodes, found := nodesMap[parent.Type]
-		if !found {
-			nodes = parseHtmlWithContext(htmlStr, parent)
-			nodesMap[parent.Type] = nodes
-		}
-		newSingleSelection(context, s.document).wrapAllNodes(cloneNodes(nodes)...)
-	}
-	return s
+func (s *Selection) WrapHtml(html string) *Selection {
+	return s.wrapNodes(parseHtml(html)...)
 }
 
 // WrapNode wraps each element in the set of matched elements inside the inner-
@@ -438,18 +381,8 @@ func (s *Selection) WrapAllSelection(sel *Selection) *Selection {
 // document.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapAllHtml(htmlStr string) *Selection {
-	var context *html.Node
-	var nodes []*html.Node
-	if len(s.Nodes) > 0 {
-		context = s.Nodes[0]
-		if context.Parent != nil {
-			nodes = parseHtmlWithContext(htmlStr, context)
-		} else {
-			nodes = parseHtml(htmlStr)
-		}
-	}
-	return s.wrapAllNodes(nodes...)
+func (s *Selection) WrapAllHtml(html string) *Selection {
+	return s.wrapAllNodes(parseHtml(html)...)
 }
 
 func (s *Selection) wrapAllNodes(ns ...*html.Node) *Selection {
@@ -518,17 +451,8 @@ func (s *Selection) WrapInnerSelection(sel *Selection) *Selection {
 // cloned before being inserted into the document.
 //
 // It returns the original set of elements.
-func (s *Selection) WrapInnerHtml(htmlStr string) *Selection {
-	nodesMap := make(map[html.NodeType][]*html.Node)
-	for _, context := range s.Nodes {
-		nodes, found := nodesMap[context.Type]
-		if !found {
-			nodes = parseHtmlWithContext(htmlStr, context)
-			nodesMap[context.Type] = nodes
-		}
-		newSingleSelection(context, s.document).wrapInnerNodes(cloneNodes(nodes)...)
-	}
-	return s
+func (s *Selection) WrapInnerHtml(html string) *Selection {
+	return s.wrapInnerNodes(parseHtml(html)...)
 }
 
 // WrapInnerNode wraps an HTML structure, matched by the given selector, around
@@ -568,14 +492,16 @@ func parseHtml(h string) []*html.Node {
 	return nodes
 }
 
-func parseHtmlWithContext(h string, context *html.Node) []*html.Node {
-	// Errors are only returned when the io.Reader returns any error besides
-	// EOF, but strings.Reader never will
-	nodes, err := html.ParseFragment(strings.NewReader(h), context)
-	if err != nil {
-		panic("goquery: failed to parse HTML: " + err.Error())
+func setHtmlNodes(s *Selection, ns ...*html.Node) *Selection {
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+		}
+		for _, c := range ns {
+			n.AppendChild(cloneNode(c))
+		}
 	}
-	return nodes
+	return s
 }
 
 // Get the first child that is an ElementNode
@@ -645,29 +571,3 @@ func (s *Selection) manipulateNodes(ns []*html.Node, reverse bool,
 
 	return s
 }
-
-// eachNodeHtml parses the given html string and inserts the resulting nodes in the dom with the mergeFn.
-// The parsed nodes are inserted for each element of the selection.
-// isParent can be used to indicate that the elements of the selection should be treated as the parent for the parsed html.
-// A cache is used to avoid parsing the html multiple times should the elements of the selection result in the same context.
-func (s *Selection) eachNodeHtml(htmlStr string, isParent bool, mergeFn func(n *html.Node, nodes []*html.Node)) *Selection {
-	// cache to avoid parsing the html for the same context multiple times
-	nodeCache := make(map[string][]*html.Node)
-	var context *html.Node
-	for _, n := range s.Nodes {
-		if isParent {
-			context = n.Parent
-		} else {
-			context = n
-		}
-		if context != nil {
-			nodes, found := nodeCache[nodeName(context)]
-			if !found {
-				nodes = parseHtmlWithContext(htmlStr, context)
-				nodeCache[nodeName(context)] = nodes
-			}
-			mergeFn(n, cloneNodes(nodes))
-		}
-	}
-	return s
-}

manipulation_test.go

@@ -56,25 +56,6 @@ func TestAfterHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestAfterHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<td>Before1</td>
-					</tr>
-					<tr>
-						<td>Before2</td>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table tr td").AfterHtml("<td>Test</td>")
-	assertLength(t, doc.Find("table tr td").Nodes, 4)
-	printSel(t, doc.Selection)
-}
-
 func TestAppend(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#main").Append("#nf6")
@@ -132,27 +113,6 @@ func TestAppendHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestAppendHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<td>Before1</td>
-					</tr>
-					<tr>
-						<td>Before2</td>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table tr").AppendHtml("<td class='new-node'>new node</td>")
-
-	assertLength(t, doc.Find("table td").Nodes, 4)
-	assertClass(t, doc.Find("table td").Last(), "new-node")
-	printSel(t, doc.Selection)
-}
-
 func TestBefore(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#main").Before("#nf6")
@@ -191,27 +151,6 @@ func TestBeforeHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestBeforeHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<td>Before1</td>
-					</tr>
-					<tr>
-						<td>Before2</td>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table tr td:first-child").BeforeHtml("<td class='new-node'>new node</td>")
-
-	assertLength(t, doc.Find("table td").Nodes, 4)
-	assertClass(t, doc.Find("table td").First(), "new-node")
-	printSel(t, doc.Selection)
-}
-
 func TestEmpty(t *testing.T) {
 	doc := Doc2Clone()
 	s := doc.Find("#main").Empty()
@@ -279,27 +218,6 @@ func TestPrependHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestPrependHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<td>Before1</td>
-					</tr>
-					<tr>
-						<td>Before2</td>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table tr").PrependHtml("<td class='c1'>new node</td><td class='c2'>other new node</td>")
-
-	assertLength(t, doc.Find("table td").Nodes, 6)
-	assertClass(t, doc.Find("table tr td").First(), "c1")
-	printSel(t, doc.Selection)
-}
-
 func TestRemove(t *testing.T) {
 	doc := Doc2Clone()
 	doc.Find("#nf1").Remove()
@@ -360,27 +278,6 @@ func TestReplaceWithHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestReplaceWithHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<th>Before1</th>
-					</tr>
-					<tr>
-						<th>Before2</th>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table th").ReplaceWithHtml("<td>Test</td><td>Replace</td>")
-
-	assertLength(t, doc.Find("table th").Nodes, 0)
-	assertLength(t, doc.Find("table tr td").Nodes, 4)
-	printSel(t, doc.Selection)
-}
-
 func TestSetHtml(t *testing.T) {
 	doc := Doc2Clone()
 	q := doc.Find("#main, #foot")
@@ -416,28 +313,6 @@ func TestSetHtmlEmpty(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
-func TestSetHtmlContext(t *testing.T) {
-	doc := loadString(t, `
-		<html>
-			<body>
-				<table>
-					<tr>
-						<th>Before1</th>
-					</tr>
-					<tr>
-						<th>Before2</th>
-					</tr>
-				</table>
-			</body>
-		</html>`)
-	doc.Find("table tr").SetHtml("<td class='c1'>Test</td><td class='c2'>Again</td>")
-
-	assertLength(t, doc.Find("table th").Nodes, 0)
-	assertLength(t, doc.Find("table td").Nodes, 4)
-	assertLength(t, doc.Find("table t2").Nodes, 2)
-	printSel(t, doc.Selection)
-}
-
 func TestSetText(t *testing.T) {
 	doc := Doc2Clone()
 	q := doc.Find("#main, #foot")
@@ -636,54 +511,3 @@ func TestWrapInnerHtml(t *testing.T) {
 
 	printSel(t, doc.Selection)
 }
-
-func TestParsingRespectsVaryingContext(t *testing.T) {
-	docA := loadString(t, `
-	<html>
-		<body>
-			<a class="x"></a>
-		</body>
-	</html>`)
-	docTable := loadString(t, `
-	<html>
-		<body>
-			<table class="x"></table>
-		</body>
-	</html>`)
-	docBoth := loadString(t, `
-	<html>
-		<body>
-			<table class="x"></table>
-			<a class="x"></a>
-		</body>
-	</html>`)
-
-	sA := docA.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
-	sTable := docTable.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
-	sBoth := docBoth.Find(".x").AppendHtml("<tr><td>Hello</td></tr>")
-
-	printSel(t, docA.Selection)
-	printSel(t, docTable.Selection)
-	printSel(t, docBoth.Selection)
-
-	oA, _ := sA.Html()
-	oTable, _ := sTable.Html()
-
-	if oA == oTable {
-		t.Errorf("Expected inner html of <a> and <table> to not be equal, but got %s and %s", oA, oTable)
-	}
-
-	oBothTable, _ := sBoth.First().Html()
-	if oBothTable != oTable {
-		t.Errorf("Expected inner html of <table> and <table> in doc containing both tags to be equal, but got %s and %s",
-			oTable,
-			oBothTable)
-	}
-
-	oBothA, _ := sBoth.Last().Html()
-	if oBothA != oA {
-		t.Errorf("Expected inner html of <a> and <a> in doc containing both tags to be equal, but got %s and %s",
-			oA,
-			oBothA)
-	}
-}

misc/git/pre-commit

@@ -18,6 +18,21 @@ gosimple $(go list ./... | grep -v /vendor/)
 echo "<<< gosimple"
 echo
 
+echo ">>> staticcheck"
+staticcheck $(go list ./... | grep -v /vendor/)
+echo "<<< staticcheck"
+echo
+
+echo ">>> unused"
+unused $(go list ./... | grep -v /vendor/)
+echo "<<< unused"
+echo
+
+echo ">>> gas"
+gas $(find . -name "*.go" | grep -v /vendor/ | grep -v '_test.go$')
+echo "<<< gas"
+echo
+
 # Check for gofmt problems and report if any.
 gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$' | grep -v /vendor/)
 [ -z "$gofiles" ] && echo "EXIT $vetres" && exit $vetres

query.go

@@ -5,7 +5,11 @@ import "golang.org/x/net/html"
 // Is checks the current matched set of elements against a selector and
 // returns true if at least one of these elements matches.
 func (s *Selection) Is(selector string) bool {
-	return s.IsMatcher(compileMatcher(selector))
+	if len(s.Nodes) > 0 {
+		return s.IsMatcher(compileMatcher(selector))
+	}
+
+	return false
 }
 
 // IsMatcher checks the current matched set of elements against a matcher and

type.go

@@ -31,10 +31,6 @@ func NewDocumentFromNode(root *html.Node) *Document {
 // NewDocument is a Document constructor that takes a string URL as argument.
 // It loads the specified document, parses it, and stores the root Document
 // node, ready to be manipulated.
-//
-// Deprecated: Use the net/http standard library package to make the request
-// and validate the response before calling goquery.NewDocumentFromReader
-// with the response's body.
 func NewDocument(url string) (*Document, error) {
 	// Load the URL
 	res, e := http.Get(url)
@@ -44,10 +40,10 @@ func NewDocument(url string) (*Document, error) {
 	return NewDocumentFromResponse(res)
 }
 
-// NewDocumentFromReader returns a Document from an io.Reader.
+// NewDocumentFromReader returns a Document from a generic reader.
 // It returns an error as second value if the reader's data cannot be parsed
-// as html. It does not check if the reader is also an io.Closer, the
-// provided reader is never closed by this call. It is the responsibility
+// as html. It does *not* check if the reader is also an io.Closer, so the
+// provided reader is never closed by this call, it is the responsibility
 // of the caller to close it if required.
 func NewDocumentFromReader(r io.Reader) (*Document, error) {
 	root, e := html.Parse(r)
@@ -60,8 +56,6 @@ func NewDocumentFromReader(r io.Reader) (*Document, error) {
 // NewDocumentFromResponse is another Document constructor that takes an http response as argument.
 // It loads the specified response's document, parses it, and stores the root Document
 // node, ready to be manipulated. The response's body is closed on return.
-//
-// Deprecated: Use goquery.NewDocumentFromReader with the response's body.
 func NewDocumentFromResponse(res *http.Response) (*Document, error) {
 	if res == nil {
 		return nil, errors.New("Response is nil")

type_test.go

@@ -121,14 +121,6 @@ func loadDoc(page string) *Document {
 	return NewDocumentFromNode(node)
 }
 
-func loadString(t *testing.T, doc string) *Document {
-	d, err := NewDocumentFromReader(strings.NewReader(doc))
-	if err != nil {
-		t.Error("Failed to parse test document")
-	}
-	return d
-}
-
 func TestNewDocument(t *testing.T) {
 	if f, e := os.Open("./testdata/page.html"); e != nil {
 		t.Error(e.Error())

utilities.go

@@ -18,6 +18,62 @@ var nodeNames = []string{
 	html.CommentNode:  "#comment",
 }
 
+// PathForNode returns a unique path to retrieve the specified node
+// from its document tree. The path is a slice of int indices, starting
+// at the root of the tree.
+func PathForNode(n *html.Node) []int {
+	var indices []int
+	for n := n; n != nil; n = n.Parent {
+		ix := 0
+		for prev := n.PrevSibling; prev != nil; prev = prev.PrevSibling {
+			ix++
+		}
+		indices = append(indices, ix)
+	}
+
+	// reverse the slice of indices
+	for l, r := 0, len(indices)-1; l < r; l, r = l+1, r-1 {
+		indices[l], indices[r] = indices[r], indices[l]
+	}
+	return indices
+}
+
+// NodeAtPath returns the HTML node at the specified path in the
+// document tree of the specified n node. The path is followed from
+// the root of the tree. If no node is found by following the path,
+// nil is returned.
+func NodeAtPath(path []int, n *html.Node) *html.Node {
+	if n == nil {
+		return n
+	}
+
+	// start at root
+	for n.Parent != nil {
+		n = n.Parent
+	}
+	for n.PrevSibling != nil {
+		n = n.PrevSibling
+	}
+
+	for i, ix := range path {
+		if i > 0 {
+			n = n.FirstChild
+			if n == nil {
+				return n
+			}
+		}
+
+		for j := 0; j < ix; j++ {
+			n = n.NextSibling
+			if n == nil {
+				return n
+			}
+		}
+	}
+
+	return n
+}
+
 // NodeName returns the node name of the first element in the selection.
 // It tries to behave in a similar way as the DOM's nodeName property
 // (https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName).
@@ -36,22 +92,12 @@ func NodeName(s *Selection) string {
 	if s.Length() == 0 {
 		return ""
 	}
-	return nodeName(s.Get(0))
-}
-
-// nodeName returns the node name of the given html node.
-// See NodeName for additional details on behaviour.
-func nodeName(node *html.Node) string {
-	if node == nil {
-		return ""
-	}
-
-	switch node.Type {
+	switch n := s.Get(0); n.Type {
 	case html.ElementNode, html.DoctypeNode:
-		return node.Data
+		return n.Data
 	default:
-		if node.Type >= 0 && int(node.Type) < len(nodeNames) {
-			return nodeNames[node.Type]
+		if n.Type >= 0 && int(n.Type) < len(nodeNames) {
+			return nodeNames[n.Type]
 		}
 		return ""
 	}

utilities_test.go

@@ -9,6 +9,91 @@ import (
 	"golang.org/x/net/html"
 )
 
+var invalidPathNodes = []struct {
+	in   string
+	path []int
+}{
+	{"<a>", []int{0, 1, 2}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{0, 0, 1, 2, 0}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2, 10}},
+}
+
+var validPathNodes = []struct {
+	in   string
+	el   string
+	path []int
+}{
+	{"<a>", "a", []int{0, 0, 1, 0}},                                                                                                      // root html body(1) a
+	{"<html><head><meta></head><body></body></html>", "meta", []int{0, 0, 0, 0}},                                                         // root html head meta
+	{"<html><head><meta><title></title></head><body></body></html>", "title", []int{0, 0, 0, 1}},                                         // root html head title
+	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "div", []int{0, 0, 1, 0}},                         // root html body(1) div
+	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "p", []int{0, 0, 1, 0, 0}},                        // root html body(1) div p
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "a", []int{0, 0, 1, 0, 1}},    // root html body(1) div a(1)
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "span", []int{0, 0, 1, 0, 2}}, // root html body(1) div span(2)
+}
+
+func TestPathForNode(t *testing.T) {
+	for i, c := range validPathNodes {
+		doc, err := NewDocumentFromReader(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		var n *html.Node
+		if sel := doc.Find(c.el); sel.Length() > 0 {
+			n = sel.Get(0)
+		}
+
+		got := PathForNode(n)
+		if !reflect.DeepEqual(c.path, got) {
+			h, _ := OuterHtml(doc.Selection)
+			t.Errorf("%d: want %v, got %v (html: %s)", i, c.path, got, h)
+		}
+	}
+
+	// test a nil node
+	if got := PathForNode(nil); got != nil {
+		t.Errorf("want nil for nil node, got %v", got)
+	}
+}
+
+func TestNodeAtPath(t *testing.T) {
+	// valid cases
+	for i, c := range validPathNodes {
+		n, err := html.Parse(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		nn := NodeAtPath(c.path, n)
+		if nn.Data != c.el {
+			t.Errorf("%d: want element %s, got %s (%v)", i, c.el, nn.Data, nn)
+		}
+	}
+
+	// invalid cases
+	for i, c := range invalidPathNodes {
+		n, err := html.Parse(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		if got := NodeAtPath(c.path, n); got != nil {
+			t.Errorf("%d: want nil, got %v", i, got)
+		}
+	}
+
+	// test a nil node
+	if got := NodeAtPath([]int{1, 2, 3}, nil); got != nil {
+		t.Errorf("want nil for nil node, got %v", got)
+	}
+}
+
 var allNodes = `<!doctype html>
 <html>
 	<head>