add PathForNode and NodeAtPath functions

Add goquery marshaling library as a downstream project in Readme

.travis.yml

@@ -1,10 +1,12 @@
 language: go
 
 go:
-    - 1.1
-    - 1.2
-    - 1.3
-    - 1.4
-    - 1.5
-    - 1.6
+    - 1.1.x
+    - 1.2.x
+    - 1.3.x
+    - 1.4.x
+    - 1.5.x
+    - 1.6.x
+    - 1.7.x
+    - 1.8.x
     - tip

README.md

@@ -1,4 +1,6 @@
-# goquery - a little like that j-thing, only in Go [![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.png)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery)
+# goquery - a little like that j-thing, only in Go
+[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.png)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+
 
 goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
 
@@ -6,6 +8,15 @@ Also, because the net/html parser requires UTF-8 encoding, so does goquery: it i
 
 Syntax-wise, it is as close as possible to jQuery, with the same function names when possible, and that warm and fuzzy chainable interface. jQuery being the ultra-popular library that it is, I felt that writing a similar HTML-manipulating library was better to follow its API than to start anew (in the same spirit as Go's `fmt` package), even though some of its methods are less than intuitive (looking at you, [index()][index]...).
 
+## Table of Contents
+
+* [Installation](#installation)
+* [Changelog](#changelog)
+* [API](#api)
+* [Examples](#examples)
+* [Related Projects](#related-projects)
+* [License](#license)
+
 ## Installation
 
 Please note that because of the net/html dependency, goquery requires Go1.1+.
@@ -13,7 +24,7 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
     $ go get github.com/PuerkitoBio/goquery
 
 (optional) To run unit tests:
-    
+
     $ cd $GOPATH/src/github.com/PuerkitoBio/goquery
     $ go test
 
@@ -26,6 +37,8 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
+*    **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
 *    **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
 *    **2016-07-27 (v1.0.0)** : Tag version 1.0.0.
 *    **2016-06-15** : Invalid selector strings internally compile to a `Matcher` implementation that never matches any node (instead of a panic). So for example, `doc.Find("~")` returns an empty `*Selection` object.
@@ -84,7 +97,7 @@ import (
 )
 
 func ExampleScrape() {
-  doc, err := goquery.NewDocument("http://metalsucks.net") 
+  doc, err := goquery.NewDocument("http://metalsucks.net")
   if err != nil {
     log.Fatal(err)
   }
@@ -103,6 +116,12 @@ func main() {
 }
 ```
 
+## Related Projects
+
+- [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
+- [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
+- [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
+
 ## License
 
 The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia's license is [here][caslic].
@@ -110,6 +129,7 @@ The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia'
 [jquery]: http://jquery.com/
 [go]: http://golang.org/
 [cascadia]: https://github.com/andybalholm/cascadia
+[cascadiacli]: https://github.com/suntong/cascadia
 [bsd]: http://opensource.org/licenses/BSD-3-Clause
 [golic]: http://golang.org/LICENSE
 [caslic]: https://github.com/andybalholm/cascadia/blob/master/LICENSE
@@ -120,3 +140,4 @@ The [BSD 3-Clause license][bsd], the same as the [Go language][golic]. Cascadia'
 [wiki]: https://github.com/PuerkitoBio/goquery/wiki/Tips-and-tricks
 [thatguystone]: https://github.com/thatguystone
 [piotr]: https://github.com/piotrkowalczuk
+[goq]: https://github.com/andrewstuart/goq

doc/tips.md

@@ -13,7 +13,7 @@ and then:
 ```
 // Load the URL
 res, err := http.Get(url)
-if e != nil {
+if err != nil {
     // handle error
 }
 defer res.Body.Close()
@@ -65,4 +65,4 @@ Thanks to github user @jmoiron.
 [otto]: https://github.com/robertkrimen/otto
 [exotto]: https://gist.github.com/cryptix/87127f76a94183747b53
 [iconv]: http://godoc.org/?q=iconv
-[text]: http://godoc.org/code.google.com/p/go.text/encoding
+[text]: https://godoc.org/golang.org/x/text/encoding

manipulation.go

@@ -270,6 +270,17 @@ func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
 	return s.Remove()
 }
 
+// Set the html content of each element in the selection to specified html string.
+func (s *Selection) SetHtml(html string) *Selection {
+	return setHtmlNodes(s, parseHtml(html)...)
+}
+
+// Set the content of each element in the selection to specified content. The
+// provided text string is escaped.
+func (s *Selection) SetText(text string) *Selection {
+	return s.SetHtml(html.EscapeString(text))
+}
+
 // Unwrap removes the parents of the set of matched elements, leaving the matched
 // elements (and their siblings, if any) in their place.
 // It returns the original selection.
@@ -481,6 +492,18 @@ func parseHtml(h string) []*html.Node {
 	return nodes
 }
 
+func setHtmlNodes(s *Selection, ns ...*html.Node) *Selection {
+	for _, n := range s.Nodes {
+		for c := n.FirstChild; c != nil; c = n.FirstChild {
+			n.RemoveChild(c)
+		}
+		for _, c := range ns {
+			n.AppendChild(cloneNode(c))
+		}
+	}
+	return s
+}
+
 // Get the first child that is an ElementNode
 func getFirstChildEl(n *html.Node) *html.Node {
 	c := n.FirstChild

manipulation_test.go

@@ -278,6 +278,66 @@ func TestReplaceWithHtml(t *testing.T) {
 	printSel(t, doc.Selection)
 }
 
+func TestSetHtml(t *testing.T) {
+	doc := Doc2Clone()
+	q := doc.Find("#main, #foot")
+	q.SetHtml(`<div id="replace">test</div>`)
+
+	assertLength(t, doc.Find("#replace").Nodes, 2)
+	assertLength(t, doc.Find("#main, #foot").Nodes, 2)
+
+	if q.Text() != "testtest" {
+		t.Errorf("Expected text to be %v, found %v", "testtest", q.Text())
+	}
+
+	printSel(t, doc.Selection)
+}
+
+func TestSetHtmlNoMatch(t *testing.T) {
+	doc := Doc2Clone()
+	q := doc.Find("#notthere")
+	q.SetHtml(`<div id="replace">test</div>`)
+
+	assertLength(t, doc.Find("#replace").Nodes, 0)
+
+	printSel(t, doc.Selection)
+}
+
+func TestSetHtmlEmpty(t *testing.T) {
+	doc := Doc2Clone()
+	q := doc.Find("#main")
+	q.SetHtml(``)
+
+	assertLength(t, doc.Find("#main").Nodes, 1)
+	assertLength(t, doc.Find("#main").Children().Nodes, 0)
+	printSel(t, doc.Selection)
+}
+
+func TestSetText(t *testing.T) {
+	doc := Doc2Clone()
+	q := doc.Find("#main, #foot")
+	repl := "<div id=\"replace\">test</div>"
+	q.SetText(repl)
+
+	assertLength(t, doc.Find("#replace").Nodes, 0)
+	assertLength(t, doc.Find("#main, #foot").Nodes, 2)
+
+	if q.Text() != (repl + repl) {
+		t.Errorf("Expected text to be %v, found %v", (repl + repl), q.Text())
+	}
+
+	h, err := q.Html()
+	if err != nil {
+		t.Errorf("Error: %v", err)
+	}
+	esc := "&lt;div id=&#34;replace&#34;&gt;test&lt;/div&gt;"
+	if h != esc {
+		t.Errorf("Expected html to be %v, found %v", esc, h)
+	}
+
+	printSel(t, doc.Selection)
+}
+
 func TestReplaceWithSelection(t *testing.T) {
 	doc := Doc2Clone()
 	sel := doc.Find("#nf6").ReplaceWithSelection(doc.Find("#nf5"))

property.go

@@ -63,9 +63,22 @@ func (s *Selection) Text() string {
 	var buf bytes.Buffer
 
 	// Slightly optimized vs calling Each: no single selection object created
-	for _, n := range s.Nodes {
-		buf.WriteString(getNodeText(n))
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			// Keep newlines and spaces, like jQuery
+			buf.WriteString(n.Data)
+		}
+		if n.FirstChild != nil {
+			for c := n.FirstChild; c != nil; c = c.NextSibling {
+				f(c)
+			}
+		}
 	}
+	for _, n := range s.Nodes {
+		f(n)
+	}
+
 	return buf.String()
 }
 
@@ -192,22 +205,6 @@ func (s *Selection) ToggleClass(class ...string) *Selection {
 	return s
 }
 
-// Get the specified node's text content.
-func getNodeText(node *html.Node) string {
-	if node.Type == html.TextNode {
-		// Keep newlines and spaces, like jQuery
-		return node.Data
-	} else if node.FirstChild != nil {
-		var buf bytes.Buffer
-		for c := node.FirstChild; c != nil; c = c.NextSibling {
-			buf.WriteString(getNodeText(c))
-		}
-		return buf.String()
-	}
-
-	return ""
-}
-
 func getAttributePtr(attrName string, n *html.Node) *html.Attribute {
 	if n == nil {
 		return nil

utilities.go

@@ -18,6 +18,62 @@ var nodeNames = []string{
 	html.CommentNode:  "#comment",
 }
 
+// PathForNode returns a unique path to retrieve the specified node
+// from its document tree. The path is a slice of int indices, starting
+// at the root of the tree.
+func PathForNode(n *html.Node) []int {
+	var indices []int
+	for n := n; n != nil; n = n.Parent {
+		ix := 0
+		for prev := n.PrevSibling; prev != nil; prev = prev.PrevSibling {
+			ix++
+		}
+		indices = append(indices, ix)
+	}
+
+	// reverse the slice of indices
+	for l, r := 0, len(indices)-1; l < r; l, r = l+1, r-1 {
+		indices[l], indices[r] = indices[r], indices[l]
+	}
+	return indices
+}
+
+// NodeAtPath returns the HTML node at the specified path in the
+// document tree of the specified n node. The path is followed from
+// the root of the tree. If no node is found by following the path,
+// nil is returned.
+func NodeAtPath(path []int, n *html.Node) *html.Node {
+	if n == nil {
+		return n
+	}
+
+	// start at root
+	for n.Parent != nil {
+		n = n.Parent
+	}
+	for n.PrevSibling != nil {
+		n = n.PrevSibling
+	}
+
+	for i, ix := range path {
+		if i > 0 {
+			n = n.FirstChild
+			if n == nil {
+				return n
+			}
+		}
+
+		for j := 0; j < ix; j++ {
+			n = n.NextSibling
+			if n == nil {
+				return n
+			}
+		}
+	}
+
+	return n
+}
+
 // NodeName returns the node name of the first element in the selection.
 // It tries to behave in a similar way as the DOM's nodeName property
 // (https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName).

utilities_test.go

@@ -9,6 +9,91 @@ import (
 	"golang.org/x/net/html"
 )
 
+var invalidPathNodes = []struct {
+	in   string
+	path []int
+}{
+	{"<a>", []int{0, 1, 2}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{0, 0, 1, 2, 0}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2}},
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2, 10}},
+}
+
+var validPathNodes = []struct {
+	in   string
+	el   string
+	path []int
+}{
+	{"<a>", "a", []int{0, 0, 1, 0}},                                                                                                      // root html body(1) a
+	{"<html><head><meta></head><body></body></html>", "meta", []int{0, 0, 0, 0}},                                                         // root html head meta
+	{"<html><head><meta><title></title></head><body></body></html>", "title", []int{0, 0, 0, 1}},                                         // root html head title
+	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "div", []int{0, 0, 1, 0}},                         // root html body(1) div
+	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "p", []int{0, 0, 1, 0, 0}},                        // root html body(1) div p
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "a", []int{0, 0, 1, 0, 1}},    // root html body(1) div a(1)
+	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "span", []int{0, 0, 1, 0, 2}}, // root html body(1) div span(2)
+}
+
+func TestPathForNode(t *testing.T) {
+	for i, c := range validPathNodes {
+		doc, err := NewDocumentFromReader(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		var n *html.Node
+		if sel := doc.Find(c.el); sel.Length() > 0 {
+			n = sel.Get(0)
+		}
+
+		got := PathForNode(n)
+		if !reflect.DeepEqual(c.path, got) {
+			h, _ := OuterHtml(doc.Selection)
+			t.Errorf("%d: want %v, got %v (html: %s)", i, c.path, got, h)
+		}
+	}
+
+	// test a nil node
+	if got := PathForNode(nil); got != nil {
+		t.Errorf("want nil for nil node, got %v", got)
+	}
+}
+
+func TestNodeAtPath(t *testing.T) {
+	// valid cases
+	for i, c := range validPathNodes {
+		n, err := html.Parse(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		nn := NodeAtPath(c.path, n)
+		if nn.Data != c.el {
+			t.Errorf("%d: want element %s, got %s (%v)", i, c.el, nn.Data, nn)
+		}
+	}
+
+	// invalid cases
+	for i, c := range invalidPathNodes {
+		n, err := html.Parse(strings.NewReader(c.in))
+		if err != nil {
+			t.Errorf("%d: failed to parse: %v", i, err)
+			continue
+		}
+
+		if got := NodeAtPath(c.path, n); got != nil {
+			t.Errorf("%d: want nil, got %v", i, got)
+		}
+	}
+
+	// test a nil node
+	if got := NodeAtPath([]int{1, 2, 3}, nil); got != nil {
+		t.Errorf("want nil for nil node, got %v", got)
+	}
+}
+
 var allNodes = `<!doctype html>
 <html>
 	<head>