Merge pull request #234 from PuerkitoBio/wip-slice-end

Add ToEnd constant to use as special end index in Slice, closes #232.

.travis.yml

@@ -9,4 +9,5 @@ go:
     - 1.6.x
     - 1.7.x
     - 1.8.x
+    - 1.9.x
     - tip

README.md

@@ -1,5 +1,5 @@
 # goquery - a little like that j-thing, only in Go
-[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.png)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
+[![build status](https://secure.travis-ci.org/PuerkitoBio/goquery.svg?branch=master)](http://travis-ci.org/PuerkitoBio/goquery) [![GoDoc](https://godoc.org/github.com/PuerkitoBio/goquery?status.png)](http://godoc.org/github.com/PuerkitoBio/goquery) [![Sourcegraph Badge](https://sourcegraph.com/github.com/PuerkitoBio/goquery/-/badge.svg)](https://sourcegraph.com/github.com/PuerkitoBio/goquery?badge)
 
 
 goquery brings a syntax and a set of features similar to [jQuery][] to the [Go language][go]. It is based on Go's [net/html package][html] and the CSS Selector library [cascadia][]. Since the net/html parser returns nodes, and not a full-featured DOM tree, jQuery's stateful manipulation functions (like height(), css(), detach()) have been left off.
@@ -37,6 +37,8 @@ Please note that because of the net/html dependency, goquery requires Go1.1+.
 
 **Note that goquery's API is now stable, and will not break.**
 
+*    **2018-01-28 (v1.3.0)** : Add `ToEnd` constant to `Slice` until the end of the selection (thanks to @davidjwilkins for raising the issue).
+*    **2018-01-11 (v1.2.0)** : Add `AddBack*` and deprecate `AndSelf` (thanks to @davidjwilkins).
 *    **2017-02-12 (v1.1.0)** : Add `SetHtml` and `SetText` (thanks to @glebtv).
 *    **2016-12-29 (v1.0.2)** : Optimize allocations for `Selection.Text` (thanks to @radovskyb).
 *    **2016-08-28 (v1.0.1)** : Optimize performance for large documents.
@@ -121,6 +123,7 @@ func main() {
 - [Goq][goq], an HTML deserialization and scraping library based on goquery and struct tags.
 - [andybalholm/cascadia][cascadia], the CSS selector library used by goquery.
 - [suntong/cascadia][cascadiacli], a command-line interface to the cascadia CSS selector library, useful to test selectors.
+- [asciimoo/colly](https://github.com/asciimoo/colly), a lightning fast and elegant Scraping Framework
 
 ## License
 

array.go

@@ -4,6 +4,16 @@ import (
 	"golang.org/x/net/html"
 )
 
+const (
+	maxUint = ^uint(0)
+	maxInt  = int(maxUint >> 1)
+
+	// ToEnd is a special index value that can be used as end index in a call
+	// to Slice so that all elements are selected until the end of the Selection.
+	// It is equivalent to passing (*Selection).Length().
+	ToEnd = maxInt
+)
+
 // First reduces the set of matched elements to the first in the set.
 // It returns a new Selection object, and an empty Selection object if the
 // the selection is empty.
@@ -35,12 +45,23 @@ func (s *Selection) Eq(index int) *Selection {
 }
 
 // Slice reduces the set of matched elements to a subset specified by a range
-// of indices.
+// of indices. The start index is 0-based and indicates the index of the first
+// element to select. The end index is 0-based and indicates the index at which
+// the elements stop being selected (the end index is not selected).
+//
+// The indices may be negative, in which case they represent an offset from the
+// end of the selection.
+//
+// The special value ToEnd may be specified as end index, in which case all elements
+// until the end are selected. This works both for a positive and negative start
+// index.
 func (s *Selection) Slice(start, end int) *Selection {
 	if start < 0 {
 		start += len(s.Nodes)
 	}
-	if end < 0 {
+	if end == ToEnd {
+		end = len(s.Nodes)
+	} else if end < 0 {
 		end += len(s.Nodes)
 	}
 	return pushStack(s, s.Nodes[start:end])

array_test.go

@@ -98,6 +98,17 @@ func TestSlice(t *testing.T) {
 	sel := Doc().Find(".pvk-content").Slice(0, 2)
 
 	assertLength(t, sel.Nodes, 2)
+	assertSelectionIs(t, sel, "#pc1", "#pc2")
+}
+
+func TestSliceToEnd(t *testing.T) {
+	sel := Doc().Find(".pvk-content").Slice(1, ToEnd)
+
+	assertLength(t, sel.Nodes, 2)
+	assertSelectionIs(t, sel.Eq(0), "#pc2")
+	if _, ok := sel.Eq(1).Attr("id"); ok {
+		t.Error("Want no attribute ID, got one")
+	}
 }
 
 func TestSliceEmpty(t *testing.T) {
@@ -110,6 +121,11 @@ func TestSliceInvalid(t *testing.T) {
 	Doc().Find("").Slice(0, 2)
 }
 
+func TestSliceInvalidToEnd(t *testing.T) {
+	defer assertPanic(t)
+	Doc().Find("").Slice(2, ToEnd)
+}
+
 func TestSliceOutOfBounds(t *testing.T) {
 	defer assertPanic(t)
 	Doc().Find(".pvk-content").Slice(2, 12)
@@ -135,6 +151,12 @@ func TestNegativeSliceBoth(t *testing.T) {
 	assertSelectionIs(t, sel.Eq(1), "#cf3")
 }
 
+func TestNegativeSliceToEnd(t *testing.T) {
+	sel := Doc().Find(".container-fluid").Slice(-3, ToEnd)
+	assertLength(t, sel.Nodes, 3)
+	assertSelectionIs(t, sel, "#cf2", "#cf3", "#cf4")
+}
+
 func TestNegativeSliceOutOfBounds(t *testing.T) {
 	defer assertPanic(t)
 	Doc().Find(".container-fluid").Slice(-12, -7)

bench_filter_test.go

@@ -231,6 +231,6 @@ func BenchmarkEnd(b *testing.B) {
 		}
 	}
 	if n != 373 {
-		b.Fatalf("wnat 373, got %d", n)
+		b.Fatalf("want 373, got %d", n)
 	}
 }

expand.go

@@ -41,6 +41,30 @@ func (s *Selection) AddNodes(nodes ...*html.Node) *Selection {
 // AndSelf adds the previous set of elements on the stack to the current set.
 // It returns a new Selection object containing the current Selection combined
 // with the previous one.
+// Deprecated: This function has been deprecated and is now an alias for AddBack().
 func (s *Selection) AndSelf() *Selection {
+	return s.AddBack()
+}
+
+// AddBack adds the previous set of elements on the stack to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the previous one.
+func (s *Selection) AddBack() *Selection {
 	return s.AddSelection(s.prevSel)
 }
+
+// AddBackFiltered reduces the previous set of elements on the stack to those that
+// match the selector string, and adds them to the current set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackFiltered(selector string) *Selection {
+	return s.AddSelection(s.prevSel.Filter(selector))
+}
+
+// AddBackMatcher reduces the previous set of elements on the stack to those that match
+// the mateher, and adds them to the curernt set.
+// It returns a new Selection object containing the current Selection combined
+// with the filtered previous one
+func (s *Selection) AddBackMatcher(m Matcher) *Selection {
+	return s.AddSelection(s.prevSel.FilterMatcher(m))
+}

expand_test.go

@@ -94,3 +94,25 @@ func TestAndSelfRollback(t *testing.T) {
 	sel2 := sel.Find("a").AndSelf().End().End()
 	assertEqual(t, sel, sel2)
 }
+
+func TestAddBack(t *testing.T) {
+	sel := Doc().Find(".span12").Last().AddBack()
+	assertLength(t, sel.Nodes, 2)
+}
+
+func TestAddBackRollback(t *testing.T) {
+	sel := Doc().Find(".pvk-content")
+	sel2 := sel.Find("a").AddBack().End().End()
+	assertEqual(t, sel, sel2)
+}
+
+func TestAddBackFiltered(t *testing.T) {
+	sel := Doc().Find(".span12, .footer").Find("h1").AddBackFiltered(".footer")
+	assertLength(t, sel.Nodes, 2)
+}
+
+func TestAddBackFilteredRollback(t *testing.T) {
+	sel := Doc().Find(".span12, .footer")
+	sel2 := sel.Find("h1").AddBackFiltered(".footer").End().End()
+	assertEqual(t, sel, sel2)
+}

manipulation.go

@@ -270,13 +270,14 @@ func (s *Selection) ReplaceWithNodes(ns ...*html.Node) *Selection {
 	return s.Remove()
 }
 
-// Set the html content of each element in the selection to specified html string.
+// SetHtml sets the html content of each element in the selection to
+// specified html string.
 func (s *Selection) SetHtml(html string) *Selection {
 	return setHtmlNodes(s, parseHtml(html)...)
 }
 
-// Set the content of each element in the selection to specified content. The
-// provided text string is escaped.
+// SetText sets the content of each element in the selection to specified content.
+// The provided text string is escaped.
 func (s *Selection) SetText(text string) *Selection {
 	return s.SetHtml(html.EscapeString(text))
 }

misc/git/pre-commit

@@ -18,21 +18,6 @@ gosimple $(go list ./... | grep -v /vendor/)
 echo "<<< gosimple"
 echo
 
-echo ">>> staticcheck"
-staticcheck $(go list ./... | grep -v /vendor/)
-echo "<<< staticcheck"
-echo
-
-echo ">>> unused"
-unused $(go list ./... | grep -v /vendor/)
-echo "<<< unused"
-echo
-
-echo ">>> gas"
-gas $(find . -name "*.go" | grep -v /vendor/ | grep -v '_test.go$')
-echo "<<< gas"
-echo
-
 # Check for gofmt problems and report if any.
 gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$' | grep -v /vendor/)
 [ -z "$gofiles" ] && echo "EXIT $vetres" && exit $vetres

query.go

@@ -5,11 +5,7 @@ import "golang.org/x/net/html"
 // Is checks the current matched set of elements against a selector and
 // returns true if at least one of these elements matches.
 func (s *Selection) Is(selector string) bool {
-	if len(s.Nodes) > 0 {
-		return s.IsMatcher(compileMatcher(selector))
-	}
-
-	return false
+	return s.IsMatcher(compileMatcher(selector))
 }
 
 // IsMatcher checks the current matched set of elements against a matcher and

utilities.go

@@ -18,62 +18,6 @@ var nodeNames = []string{
 	html.CommentNode:  "#comment",
 }
 
-// PathForNode returns a unique path to retrieve the specified node
-// from its document tree. The path is a slice of int indices, starting
-// at the root of the tree.
-func PathForNode(n *html.Node) []int {
-	var indices []int
-	for n := n; n != nil; n = n.Parent {
-		ix := 0
-		for prev := n.PrevSibling; prev != nil; prev = prev.PrevSibling {
-			ix++
-		}
-		indices = append(indices, ix)
-	}
-
-	// reverse the slice of indices
-	for l, r := 0, len(indices)-1; l < r; l, r = l+1, r-1 {
-		indices[l], indices[r] = indices[r], indices[l]
-	}
-	return indices
-}
-
-// NodeAtPath returns the HTML node at the specified path in the
-// document tree of the specified n node. The path is followed from
-// the root of the tree. If no node is found by following the path,
-// nil is returned.
-func NodeAtPath(path []int, n *html.Node) *html.Node {
-	if n == nil {
-		return n
-	}
-
-	// start at root
-	for n.Parent != nil {
-		n = n.Parent
-	}
-	for n.PrevSibling != nil {
-		n = n.PrevSibling
-	}
-
-	for i, ix := range path {
-		if i > 0 {
-			n = n.FirstChild
-			if n == nil {
-				return n
-			}
-		}
-
-		for j := 0; j < ix; j++ {
-			n = n.NextSibling
-			if n == nil {
-				return n
-			}
-		}
-	}
-
-	return n
-}
-
 // NodeName returns the node name of the first element in the selection.
 // It tries to behave in a similar way as the DOM's nodeName property
 // (https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeName).

utilities_test.go

@@ -9,91 +9,6 @@ import (
 	"golang.org/x/net/html"
 )
 
-var invalidPathNodes = []struct {
-	in   string
-	path []int
-}{
-	{"<a>", []int{0, 1, 2}},
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{0, 0, 1, 2, 0}},
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1}},
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2}},
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", []int{1, 2, 10}},
-}
-
-var validPathNodes = []struct {
-	in   string
-	el   string
-	path []int
-}{
-	{"<a>", "a", []int{0, 0, 1, 0}},                                                                                                      // root html body(1) a
-	{"<html><head><meta></head><body></body></html>", "meta", []int{0, 0, 0, 0}},                                                         // root html head meta
-	{"<html><head><meta><title></title></head><body></body></html>", "title", []int{0, 0, 0, 1}},                                         // root html head title
-	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "div", []int{0, 0, 1, 0}},                         // root html body(1) div
-	{"<html><head><meta><title></title></head><body><div><p></p></div></body></html>", "p", []int{0, 0, 1, 0, 0}},                        // root html body(1) div p
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "a", []int{0, 0, 1, 0, 1}},    // root html body(1) div a(1)
-	{"<html><head><meta><title></title></head><body><div><p></p><a></a><span></span></div></body></html>", "span", []int{0, 0, 1, 0, 2}}, // root html body(1) div span(2)
-}
-
-func TestPathForNode(t *testing.T) {
-	for i, c := range validPathNodes {
-		doc, err := NewDocumentFromReader(strings.NewReader(c.in))
-		if err != nil {
-			t.Errorf("%d: failed to parse: %v", i, err)
-			continue
-		}
-
-		var n *html.Node
-		if sel := doc.Find(c.el); sel.Length() > 0 {
-			n = sel.Get(0)
-		}
-
-		got := PathForNode(n)
-		if !reflect.DeepEqual(c.path, got) {
-			h, _ := OuterHtml(doc.Selection)
-			t.Errorf("%d: want %v, got %v (html: %s)", i, c.path, got, h)
-		}
-	}
-
-	// test a nil node
-	if got := PathForNode(nil); got != nil {
-		t.Errorf("want nil for nil node, got %v", got)
-	}
-}
-
-func TestNodeAtPath(t *testing.T) {
-	// valid cases
-	for i, c := range validPathNodes {
-		n, err := html.Parse(strings.NewReader(c.in))
-		if err != nil {
-			t.Errorf("%d: failed to parse: %v", i, err)
-			continue
-		}
-
-		nn := NodeAtPath(c.path, n)
-		if nn.Data != c.el {
-			t.Errorf("%d: want element %s, got %s (%v)", i, c.el, nn.Data, nn)
-		}
-	}
-
-	// invalid cases
-	for i, c := range invalidPathNodes {
-		n, err := html.Parse(strings.NewReader(c.in))
-		if err != nil {
-			t.Errorf("%d: failed to parse: %v", i, err)
-			continue
-		}
-
-		if got := NodeAtPath(c.path, n); got != nil {
-			t.Errorf("%d: want nil, got %v", i, got)
-		}
-	}
-
-	// test a nil node
-	if got := NodeAtPath([]int{1, 2, 3}, nil); got != nil {
-		t.Errorf("want nil for nil node, got %v", got)
-	}
-}
-
 var allNodes = `<!doctype html>
 <html>
 	<head>