aboutsummaryrefslogtreecommitdiffstats
path: root/vendor/github.com/golangci/dupl
diff options
context:
space:
mode:
authorDmitry Vyukov <dvyukov@google.com>2020-07-04 11:12:55 +0200
committerDmitry Vyukov <dvyukov@google.com>2020-07-04 15:05:30 +0200
commitc7d7f10bdff703e4a3c0414e8a33d4e45c91eb35 (patch)
tree0dff0ee1f98dbfa3ad8776112053a450d176592b /vendor/github.com/golangci/dupl
parent9573094ce235bd9afe88f5da27a47dd6bcc1e13b (diff)
go.mod: vendor golangci-lint
Diffstat (limited to 'vendor/github.com/golangci/dupl')
-rw-r--r--vendor/github.com/golangci/dupl/.travis.yml5
-rw-r--r--vendor/github.com/golangci/dupl/LICENSE21
-rw-r--r--vendor/github.com/golangci/dupl/README.md63
-rw-r--r--vendor/github.com/golangci/dupl/job/buildtree.go22
-rw-r--r--vendor/github.com/golangci/dupl/job/parse.go36
-rw-r--r--vendor/github.com/golangci/dupl/main.go148
-rw-r--r--vendor/github.com/golangci/dupl/printer/html.go120
-rw-r--r--vendor/github.com/golangci/dupl/printer/plumbing.go50
-rw-r--r--vendor/github.com/golangci/dupl/printer/printer.go11
-rw-r--r--vendor/github.com/golangci/dupl/printer/text.go100
-rw-r--r--vendor/github.com/golangci/dupl/suffixtree/dupl.go98
-rw-r--r--vendor/github.com/golangci/dupl/suffixtree/suffixtree.go216
-rw-r--r--vendor/github.com/golangci/dupl/syntax/golang/golang.go392
-rw-r--r--vendor/github.com/golangci/dupl/syntax/syntax.go175
14 files changed, 1457 insertions, 0 deletions
diff --git a/vendor/github.com/golangci/dupl/.travis.yml b/vendor/github.com/golangci/dupl/.travis.yml
new file mode 100644
index 000000000..33de24c0f
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/.travis.yml
@@ -0,0 +1,5 @@
+language: go
+go:
+ - 1.3
+ - 1.8
+ - 1.9
diff --git a/vendor/github.com/golangci/dupl/LICENSE b/vendor/github.com/golangci/dupl/LICENSE
new file mode 100644
index 000000000..ab317d841
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/LICENSE
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Michal Bohuslávek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/vendor/github.com/golangci/dupl/README.md b/vendor/github.com/golangci/dupl/README.md
new file mode 100644
index 000000000..f34901d7a
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/README.md
@@ -0,0 +1,63 @@
+# dupl [![Build Status](https://travis-ci.org/mibk/dupl.png)](https://travis-ci.org/mibk/dupl)
+
+**dupl** is a tool written in Go for finding code clones. So far it can find clones only
+in the Go source files. The method uses suffix tree for serialized ASTs. It ignores values
+of AST nodes. It just operates with their types (e.g. `if a == 13 {}` and `if x == 100 {}` are
+considered the same provided it exceeds the minimal token sequence size).
+
+Due to the used method dupl can report so called "false positives" on the output. These are
+the ones we do not consider clones (whether they are too small, or the values of the matched
+tokens are completely different).
+
+## Installation
+
+```bash
+go get -u github.com/golangci/dupl
+```
+
+## Usage
+
+```
+Usage of dupl:
+ dupl [flags] [paths]
+
+Paths:
+ If the given path is a file, dupl will use it regardless of
+ the file extension. If it is a directory it will recursively
+ search for *.go files in that directory.
+
+ If no path is given dupl will recursively search for *.go
+ files in the current directory.
+
+Flags:
+ -files
+ read file names from stdin one at each line
+ -html
+ output the results as HTML, including duplicate code fragments
+ -plumbing
+ plumbing (easy-to-parse) output for consumption by scripts or tools
+ -t, -threshold size
+ minimum token sequence size as a clone (default 15)
+ -vendor
+ check files in vendor directory
+ -v, -verbose
+ explain what is being done
+
+Examples:
+ dupl -t 100
+ Search clones in the current directory of size at least
+ 100 tokens.
+ dupl $(find app/ -name '*_test.go')
+ Search for clones in tests in the app directory.
+ find app/ -name '*_test.go' |dupl -files
+ The same as above.
+```
+
+## Example
+
+The reduced output of this command with the following parameters for the [Docker](https://www.docker.com) source code
+looks like [this](http://htmlpreview.github.io/?https://github.com/golangci/dupl/blob/master/_output_example/docker.html).
+
+```bash
+$ dupl -t 200 -html >docker.html
+```
diff --git a/vendor/github.com/golangci/dupl/job/buildtree.go b/vendor/github.com/golangci/dupl/job/buildtree.go
new file mode 100644
index 000000000..e9aad54c0
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/job/buildtree.go
@@ -0,0 +1,22 @@
+package job
+
+import (
+ "github.com/golangci/dupl/suffixtree"
+ "github.com/golangci/dupl/syntax"
+)
+
+func BuildTree(schan chan []*syntax.Node) (t *suffixtree.STree, d *[]*syntax.Node, done chan bool) {
+ t = suffixtree.New()
+ data := make([]*syntax.Node, 0, 100)
+ done = make(chan bool)
+ go func() {
+ for seq := range schan {
+ data = append(data, seq...)
+ for _, node := range seq {
+ t.Update(node)
+ }
+ }
+ done <- true
+ }()
+ return t, &data, done
+}
diff --git a/vendor/github.com/golangci/dupl/job/parse.go b/vendor/github.com/golangci/dupl/job/parse.go
new file mode 100644
index 000000000..eb9d7c625
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/job/parse.go
@@ -0,0 +1,36 @@
+package job
+
+import (
+ "log"
+
+ "github.com/golangci/dupl/syntax"
+ "github.com/golangci/dupl/syntax/golang"
+)
+
+func Parse(fchan chan string) chan []*syntax.Node {
+
+ // parse AST
+ achan := make(chan *syntax.Node)
+ go func() {
+ for file := range fchan {
+ ast, err := golang.Parse(file)
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+ achan <- ast
+ }
+ close(achan)
+ }()
+
+ // serialize
+ schan := make(chan []*syntax.Node)
+ go func() {
+ for ast := range achan {
+ seq := syntax.Serialize(ast)
+ schan <- seq
+ }
+ close(schan)
+ }()
+ return schan
+}
diff --git a/vendor/github.com/golangci/dupl/main.go b/vendor/github.com/golangci/dupl/main.go
new file mode 100644
index 000000000..3030a97ae
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/main.go
@@ -0,0 +1,148 @@
+package dupl
+
+import (
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "sort"
+
+ "github.com/golangci/dupl/job"
+ "github.com/golangci/dupl/printer"
+ "github.com/golangci/dupl/syntax"
+)
+
+const defaultThreshold = 15
+
+var (
+ paths = []string{"."}
+ vendor = flag.Bool("dupl.vendor", false, "")
+ verbose = flag.Bool("dupl.verbose", false, "")
+ files = flag.Bool("dupl.files", false, "")
+
+ html = flag.Bool("dupl.html", false, "")
+ plumbing = flag.Bool("dupl.plumbing", false, "")
+)
+
+const (
+ vendorDirPrefix = "vendor" + string(filepath.Separator)
+ vendorDirInPath = string(filepath.Separator) + vendorDirPrefix
+)
+
+func init() {
+ flag.BoolVar(verbose, "dupl.v", false, "alias for -verbose")
+}
+
+func Run(files []string, threshold int) ([]printer.Issue, error) {
+ fchan := make(chan string, 1024)
+ go func() {
+ for _, f := range files {
+ fchan <- f
+ }
+ close(fchan)
+ }()
+ schan := job.Parse(fchan)
+ t, data, done := job.BuildTree(schan)
+ <-done
+
+ // finish stream
+ t.Update(&syntax.Node{Type: -1})
+
+ mchan := t.FindDuplOver(threshold)
+ duplChan := make(chan syntax.Match)
+ go func() {
+ for m := range mchan {
+ match := syntax.FindSyntaxUnits(*data, m, threshold)
+ if len(match.Frags) > 0 {
+ duplChan <- match
+ }
+ }
+ close(duplChan)
+ }()
+
+ return makeIssues(duplChan)
+}
+
+func makeIssues(duplChan <-chan syntax.Match) ([]printer.Issue, error) {
+ groups := make(map[string][][]*syntax.Node)
+ for dupl := range duplChan {
+ groups[dupl.Hash] = append(groups[dupl.Hash], dupl.Frags...)
+ }
+ keys := make([]string, 0, len(groups))
+ for k := range groups {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ p := printer.NewPlumbing(ioutil.ReadFile)
+
+ var issues []printer.Issue
+ for _, k := range keys {
+ uniq := unique(groups[k])
+ if len(uniq) > 1 {
+ i, err := p.MakeIssues(uniq)
+ if err != nil {
+ return nil, err
+ }
+ issues = append(issues, i...)
+ }
+ }
+
+ return issues, nil
+}
+
+func unique(group [][]*syntax.Node) [][]*syntax.Node {
+ fileMap := make(map[string]map[int]struct{})
+
+ var newGroup [][]*syntax.Node
+ for _, seq := range group {
+ node := seq[0]
+ file, ok := fileMap[node.Filename]
+ if !ok {
+ file = make(map[int]struct{})
+ fileMap[node.Filename] = file
+ }
+ if _, ok := file[node.Pos]; !ok {
+ file[node.Pos] = struct{}{}
+ newGroup = append(newGroup, seq)
+ }
+ }
+ return newGroup
+}
+
+func usage() {
+ fmt.Fprintln(os.Stderr, `Usage: dupl [flags] [paths]
+
+Paths:
+ If the given path is a file, dupl will use it regardless of
+ the file extension. If it is a directory, it will recursively
+ search for *.go files in that directory.
+
+ If no path is given, dupl will recursively search for *.go
+ files in the current directory.
+
+Flags:
+ -files
+ read file names from stdin one at each line
+ -html
+ output the results as HTML, including duplicate code fragments
+ -plumbing
+ plumbing (easy-to-parse) output for consumption by scripts or tools
+ -t, -threshold size
+ minimum token sequence size as a clone (default 15)
+ -vendor
+ check files in vendor directory
+ -v, -verbose
+ explain what is being done
+
+Examples:
+ dupl -t 100
+ Search clones in the current directory of size at least
+ 100 tokens.
+ dupl $(find app/ -name '*_test.go')
+ Search for clones in tests in the app directory.
+ find app/ -name '*_test.go' |dupl -files
+ The same as above.`)
+ os.Exit(2)
+}
diff --git a/vendor/github.com/golangci/dupl/printer/html.go b/vendor/github.com/golangci/dupl/printer/html.go
new file mode 100644
index 000000000..5ad9e25c7
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/printer/html.go
@@ -0,0 +1,120 @@
+package printer
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "regexp"
+ "sort"
+
+ "github.com/golangci/dupl/syntax"
+)
+
+type html struct {
+ iota int
+ w io.Writer
+ ReadFile
+}
+
+func NewHTML(w io.Writer, fread ReadFile) Printer {
+ return &html{w: w, ReadFile: fread}
+}
+
+func (p *html) PrintHeader() error {
+ _, err := fmt.Fprint(p.w, `<!DOCTYPE html>
+<meta charset="utf-8"/>
+<title>Duplicates</title>
+<style>
+ pre {
+ background-color: #FFD;
+ border: 1px solid #E2E2E2;
+ padding: 1ex;
+ }
+</style>
+`)
+ return err
+}
+
+func (p *html) PrintClones(dups [][]*syntax.Node) error {
+ p.iota++
+ fmt.Fprintf(p.w, "<h1>#%d found %d clones</h1>\n", p.iota, len(dups))
+
+ clones := make([]clone, len(dups))
+ for i, dup := range dups {
+ cnt := len(dup)
+ if cnt == 0 {
+ panic("zero length dup")
+ }
+ nstart := dup[0]
+ nend := dup[cnt-1]
+
+ file, err := p.ReadFile(nstart.Filename)
+ if err != nil {
+ return err
+ }
+
+ lineStart, _ := blockLines(file, nstart.Pos, nend.End)
+ cl := clone{filename: nstart.Filename, lineStart: lineStart}
+ start := findLineBeg(file, nstart.Pos)
+ content := append(toWhitespace(file[start:nstart.Pos]), file[nstart.Pos:nend.End]...)
+ cl.fragment = deindent(content)
+ clones[i] = cl
+ }
+
+ sort.Sort(byNameAndLine(clones))
+ for _, cl := range clones {
+ fmt.Fprintf(p.w, "<h2>%s:%d</h2>\n<pre>%s</pre>\n", cl.filename, cl.lineStart, cl.fragment)
+ }
+ return nil
+}
+
+func (*html) PrintFooter() error { return nil }
+
+func findLineBeg(file []byte, index int) int {
+ for i := index; i >= 0; i-- {
+ if file[i] == '\n' {
+ return i + 1
+ }
+ }
+ return 0
+}
+
+func toWhitespace(str []byte) []byte {
+ var out []byte
+ for _, c := range bytes.Runes(str) {
+ if c == '\t' {
+ out = append(out, '\t')
+ } else {
+ out = append(out, ' ')
+ }
+ }
+ return out
+}
+
+func deindent(block []byte) []byte {
+ const maxVal = 99
+ min := maxVal
+ re := regexp.MustCompile(`(^|\n)(\t*)\S`)
+ for _, line := range re.FindAllSubmatch(block, -1) {
+ indent := line[2]
+ if len(indent) < min {
+ min = len(indent)
+ }
+ }
+ if min == 0 || min == maxVal {
+ return block
+ }
+ block = block[min:]
+Loop:
+ for i := 0; i < len(block); i++ {
+ if block[i] == '\n' && i != len(block)-1 {
+ for j := 0; j < min; j++ {
+ if block[i+j+1] != '\t' {
+ continue Loop
+ }
+ }
+ block = append(block[:i+1], block[i+1+min:]...)
+ }
+ }
+ return block
+}
diff --git a/vendor/github.com/golangci/dupl/printer/plumbing.go b/vendor/github.com/golangci/dupl/printer/plumbing.go
new file mode 100644
index 000000000..cf39d01b7
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/printer/plumbing.go
@@ -0,0 +1,50 @@
+package printer
+
+import (
+ "sort"
+
+ "github.com/golangci/dupl/syntax"
+)
+
+type Clone clone
+
+func (c Clone) Filename() string {
+ return c.filename
+}
+
+func (c Clone) LineStart() int {
+ return c.lineStart
+}
+
+func (c Clone) LineEnd() int {
+ return c.lineEnd
+}
+
+type Issue struct {
+ From, To Clone
+}
+
+type Plumbing struct {
+ ReadFile
+}
+
+func NewPlumbing(fread ReadFile) *Plumbing {
+ return &Plumbing{fread}
+}
+
+func (p *Plumbing) MakeIssues(dups [][]*syntax.Node) ([]Issue, error) {
+ clones, err := prepareClonesInfo(p.ReadFile, dups)
+ if err != nil {
+ return nil, err
+ }
+ sort.Sort(byNameAndLine(clones))
+ var issues []Issue
+ for i, cl := range clones {
+ nextCl := clones[(i+1)%len(clones)]
+ issues = append(issues, Issue{
+ From: Clone(cl),
+ To: Clone(nextCl),
+ })
+ }
+ return issues, nil
+}
diff --git a/vendor/github.com/golangci/dupl/printer/printer.go b/vendor/github.com/golangci/dupl/printer/printer.go
new file mode 100644
index 000000000..385217bfc
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/printer/printer.go
@@ -0,0 +1,11 @@
+package printer
+
+import "github.com/golangci/dupl/syntax"
+
+type ReadFile func(filename string) ([]byte, error)
+
+type Printer interface {
+ PrintHeader() error
+ PrintClones(dups [][]*syntax.Node) error
+ PrintFooter() error
+}
diff --git a/vendor/github.com/golangci/dupl/printer/text.go b/vendor/github.com/golangci/dupl/printer/text.go
new file mode 100644
index 000000000..8359fa76f
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/printer/text.go
@@ -0,0 +1,100 @@
+package printer
+
+import (
+ "fmt"
+ "io"
+ "sort"
+
+ "github.com/golangci/dupl/syntax"
+)
+
+type text struct {
+ cnt int
+ w io.Writer
+ ReadFile
+}
+
+func NewText(w io.Writer, fread ReadFile) Printer {
+ return &text{w: w, ReadFile: fread}
+}
+
+func (p *text) PrintHeader() error { return nil }
+
+func (p *text) PrintClones(dups [][]*syntax.Node) error {
+ p.cnt++
+ fmt.Fprintf(p.w, "found %d clones:\n", len(dups))
+ clones, err := prepareClonesInfo(p.ReadFile, dups)
+ if err != nil {
+ return err
+ }
+ sort.Sort(byNameAndLine(clones))
+ for _, cl := range clones {
+ fmt.Fprintf(p.w, " %s:%d,%d\n", cl.filename, cl.lineStart, cl.lineEnd)
+ }
+ return nil
+}
+
+func (p *text) PrintFooter() error {
+ _, err := fmt.Fprintf(p.w, "\nFound total %d clone groups.\n", p.cnt)
+ return err
+}
+
+func prepareClonesInfo(fread ReadFile, dups [][]*syntax.Node) ([]clone, error) {
+ clones := make([]clone, len(dups))
+ for i, dup := range dups {
+ cnt := len(dup)
+ if cnt == 0 {
+ panic("zero length dup")
+ }
+ nstart := dup[0]
+ nend := dup[cnt-1]
+
+ file, err := fread(nstart.Filename)
+ if err != nil {
+ return nil, err
+ }
+
+ cl := clone{filename: nstart.Filename}
+ cl.lineStart, cl.lineEnd = blockLines(file, nstart.Pos, nend.End)
+ clones[i] = cl
+ }
+ return clones, nil
+}
+
+func blockLines(file []byte, from, to int) (int, int) {
+ line := 1
+ lineStart, lineEnd := 0, 0
+ for offset, b := range file {
+ if b == '\n' {
+ line++
+ }
+ if offset == from {
+ lineStart = line
+ }
+ if offset == to-1 {
+ lineEnd = line
+ break
+ }
+ }
+ return lineStart, lineEnd
+}
+
+type clone struct {
+ filename string
+ lineStart int
+ lineEnd int
+ fragment []byte
+}
+
+type byNameAndLine []clone
+
+func (c byNameAndLine) Len() int { return len(c) }
+
+func (c byNameAndLine) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+
+func (c byNameAndLine) Less(i, j int) bool {
+ if c[i].filename == c[j].filename {
+ return c[i].lineStart < c[j].lineStart
+ }
+ return c[i].filename < c[j].filename
+}
diff --git a/vendor/github.com/golangci/dupl/suffixtree/dupl.go b/vendor/github.com/golangci/dupl/suffixtree/dupl.go
new file mode 100644
index 000000000..ab145b4f3
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/suffixtree/dupl.go
@@ -0,0 +1,98 @@
+package suffixtree
+
+import "sort"
+
+type Match struct {
+ Ps []Pos
+ Len Pos
+}
+
+type posList struct {
+ positions []Pos
+}
+
+func newPosList() *posList {
+ return &posList{make([]Pos, 0)}
+}
+
+func (p *posList) append(p2 *posList) {
+ p.positions = append(p.positions, p2.positions...)
+}
+
+func (p *posList) add(pos Pos) {
+ p.positions = append(p.positions, pos)
+}
+
+type contextList struct {
+ lists map[int]*posList
+}
+
+func newContextList() *contextList {
+ return &contextList{make(map[int]*posList)}
+}
+
+func (c *contextList) getAll() []Pos {
+ keys := make([]int, 0, len(c.lists))
+ for k := range c.lists {
+ keys = append(keys, k)
+ }
+ sort.Ints(keys)
+ var ps []Pos
+ for _, k := range keys {
+ ps = append(ps, c.lists[k].positions...)
+ }
+ return ps
+}
+
+func (c *contextList) append(c2 *contextList) {
+ for lc, pl := range c2.lists {
+ if _, ok := c.lists[lc]; ok {
+ c.lists[lc].append(pl)
+ } else {
+ c.lists[lc] = pl
+ }
+ }
+}
+
+// FindDuplOver find pairs of maximal duplicities over a threshold
+// length.
+func (t *STree) FindDuplOver(threshold int) <-chan Match {
+ auxTran := newTran(0, 0, t.root)
+ ch := make(chan Match)
+ go func() {
+ walkTrans(auxTran, 0, threshold, ch)
+ close(ch)
+ }()
+ return ch
+}
+
+func walkTrans(parent *tran, length, threshold int, ch chan<- Match) *contextList {
+ s := parent.state
+
+ cl := newContextList()
+
+ if len(s.trans) == 0 {
+ pl := newPosList()
+ start := parent.end + 1 - Pos(length)
+ pl.add(start)
+ ch := 0
+ if start > 0 {
+ ch = s.tree.data[start-1].Val()
+ }
+ cl.lists[ch] = pl
+ return cl
+ }
+
+ for _, t := range s.trans {
+ ln := length + t.len()
+ cl2 := walkTrans(t, ln, threshold, ch)
+ if ln >= threshold {
+ cl.append(cl2)
+ }
+ }
+ if length >= threshold && len(cl.lists) > 1 {
+ m := Match{cl.getAll(), Pos(length)}
+ ch <- m
+ }
+ return cl
+}
diff --git a/vendor/github.com/golangci/dupl/suffixtree/suffixtree.go b/vendor/github.com/golangci/dupl/suffixtree/suffixtree.go
new file mode 100644
index 000000000..738015025
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/suffixtree/suffixtree.go
@@ -0,0 +1,216 @@
+package suffixtree
+
+import (
+ "bytes"
+ "fmt"
+ "math"
+ "strings"
+)
+
+const infinity = math.MaxInt32
+
+// Pos denotes position in data slice.
+type Pos int32
+
+type Token interface {
+ Val() int
+}
+
+// STree is a struct representing a suffix tree.
+type STree struct {
+ data []Token
+ root *state
+ auxState *state // auxiliary state
+
+ // active point
+ s *state
+ start, end Pos
+}
+
+// New creates new suffix tree.
+func New() *STree {
+ t := new(STree)
+ t.data = make([]Token, 0, 50)
+ t.root = newState(t)
+ t.auxState = newState(t)
+ t.root.linkState = t.auxState
+ t.s = t.root
+ return t
+}
+
+// Update refreshes the suffix tree to by new data.
+func (t *STree) Update(data ...Token) {
+ t.data = append(t.data, data...)
+ for _ = range data {
+ t.update()
+ t.s, t.start = t.canonize(t.s, t.start, t.end)
+ t.end++
+ }
+}
+
+// update transforms suffix tree T(n) to T(n+1).
+func (t *STree) update() {
+ oldr := t.root
+
+ // (s, (start, end)) is the canonical reference pair for the active point
+ s := t.s
+ start, end := t.start, t.end
+ var r *state
+ for {
+ var endPoint bool
+ r, endPoint = t.testAndSplit(s, start, end-1)
+ if endPoint {
+ break
+ }
+ r.fork(end)
+ if oldr != t.root {
+ oldr.linkState = r
+ }
+ oldr = r
+ s, start = t.canonize(s.linkState, start, end-1)
+ }
+ if oldr != t.root {
+ oldr.linkState = r
+ }
+
+ // update active point
+ t.s = s
+ t.start = start
+}
+
+// testAndSplit tests whether a state with canonical ref. pair
+// (s, (start, end)) is the end point, that is, a state that have
+// a c-transition. If not, then state (exs, (start, end)) is made
+// explicit (if not already so).
+func (t *STree) testAndSplit(s *state, start, end Pos) (exs *state, endPoint bool) {
+ c := t.data[t.end]
+ if start <= end {
+ tr := s.findTran(t.data[start])
+ splitPoint := tr.start + end - start + 1
+ if t.data[splitPoint].Val() == c.Val() {
+ return s, true
+ }
+ // make the (s, (start, end)) state explicit
+ newSt := newState(s.tree)
+ newSt.addTran(splitPoint, tr.end, tr.state)
+ tr.end = splitPoint - 1
+ tr.state = newSt
+ return newSt, false
+ }
+ if s == t.auxState || s.findTran(c) != nil {
+ return s, true
+ }
+ return s, false
+}
+
+// canonize returns updated state and start position for ref. pair
+// (s, (start, end)) of state r so the new ref. pair is canonical,
+// that is, referenced from the closest explicit ancestor of r.
+func (t *STree) canonize(s *state, start, end Pos) (*state, Pos) {
+ if s == t.auxState {
+ s, start = t.root, start+1
+ }
+ if start > end {
+ return s, start
+ }
+
+ var tr *tran
+ for {
+ if start <= end {
+ tr = s.findTran(t.data[start])
+ if tr == nil {
+ panic(fmt.Sprintf("there should be some transition for '%d' at %d",
+ t.data[start].Val(), start))
+ }
+ }
+ if tr.end-tr.start > end-start {
+ break
+ }
+ start += tr.end - tr.start + 1
+ s = tr.state
+ }
+ if s == nil {
+ panic("there should always be some suffix link resolution")
+ }
+ return s, start
+}
+
+func (t *STree) At(p Pos) Token {
+ if p < 0 || p >= Pos(len(t.data)) {
+ panic("position out of bounds")
+ }
+ return t.data[p]
+}
+
+func (t *STree) String() string {
+ buf := new(bytes.Buffer)
+ printState(buf, t.root, 0)
+ return buf.String()
+}
+
+func printState(buf *bytes.Buffer, s *state, ident int) {
+ for _, tr := range s.trans {
+ fmt.Fprint(buf, strings.Repeat(" ", ident))
+ fmt.Fprintf(buf, "* (%d, %d)\n", tr.start, tr.ActEnd())
+ printState(buf, tr.state, ident+1)
+ }
+}
+
+// state is an explicit state of the suffix tree.
+type state struct {
+ tree *STree
+ trans []*tran
+ linkState *state
+}
+
+func newState(t *STree) *state {
+ return &state{
+ tree: t,
+ trans: make([]*tran, 0),
+ linkState: nil,
+ }
+}
+
+func (s *state) addTran(start, end Pos, r *state) {
+ s.trans = append(s.trans, newTran(start, end, r))
+}
+
+// fork creates a new branch from the state s.
+func (s *state) fork(i Pos) *state {
+ r := newState(s.tree)
+ s.addTran(i, infinity, r)
+ return r
+}
+
+// findTran finds c-transition.
+func (s *state) findTran(c Token) *tran {
+ for _, tran := range s.trans {
+ if s.tree.data[tran.start].Val() == c.Val() {
+ return tran
+ }
+ }
+ return nil
+}
+
+// tran represents a state's transition.
+type tran struct {
+ start, end Pos
+ state *state
+}
+
+func newTran(start, end Pos, s *state) *tran {
+ return &tran{start, end, s}
+}
+
+func (t *tran) len() int {
+ return int(t.end - t.start + 1)
+}
+
+// ActEnd returns actual end position as consistent with
+// the actual length of the data in the STree.
+func (t *tran) ActEnd() Pos {
+ if t.end == infinity {
+ return Pos(len(t.state.tree.data)) - 1
+ }
+ return t.end
+}
diff --git a/vendor/github.com/golangci/dupl/syntax/golang/golang.go b/vendor/github.com/golangci/dupl/syntax/golang/golang.go
new file mode 100644
index 000000000..a0b1e77e1
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/syntax/golang/golang.go
@@ -0,0 +1,392 @@
+package golang
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+
+ "github.com/golangci/dupl/syntax"
+)
+
+const (
+ BadNode = iota
+ File
+ ArrayType
+ AssignStmt
+ BasicLit
+ BinaryExpr
+ BlockStmt
+ BranchStmt
+ CallExpr
+ CaseClause
+ ChanType
+ CommClause
+ CompositeLit
+ DeclStmt
+ DeferStmt
+ Ellipsis
+ EmptyStmt
+ ExprStmt
+ Field
+ FieldList
+ ForStmt
+ FuncDecl
+ FuncLit
+ FuncType
+ GenDecl
+ GoStmt
+ Ident
+ IfStmt
+ IncDecStmt
+ IndexExpr
+ InterfaceType
+ KeyValueExpr
+ LabeledStmt
+ MapType
+ ParenExpr
+ RangeStmt
+ ReturnStmt
+ SelectStmt
+ SelectorExpr
+ SendStmt
+ SliceExpr
+ StarExpr
+ StructType
+ SwitchStmt
+ TypeAssertExpr
+ TypeSpec
+ TypeSwitchStmt
+ UnaryExpr
+ ValueSpec
+)
+
+// Parse the given file and return uniform syntax tree.
+func Parse(filename string) (*syntax.Node, error) {
+ fset := token.NewFileSet()
+ file, err := parser.ParseFile(fset, filename, nil, 0)
+ if err != nil {
+ return nil, err
+ }
+ t := &transformer{
+ fileset: fset,
+ filename: filename,
+ }
+ return t.trans(file), nil
+}
+
+type transformer struct {
+ fileset *token.FileSet
+ filename string
+}
+
+// trans transforms given golang AST to uniform tree structure.
+func (t *transformer) trans(node ast.Node) (o *syntax.Node) {
+ o = syntax.NewNode()
+ o.Filename = t.filename
+ st, end := node.Pos(), node.End()
+ o.Pos, o.End = t.fileset.File(st).Offset(st), t.fileset.File(end).Offset(end)
+
+ switch n := node.(type) {
+ case *ast.ArrayType:
+ o.Type = ArrayType
+ if n.Len != nil {
+ o.AddChildren(t.trans(n.Len))
+ }
+ o.AddChildren(t.trans(n.Elt))
+
+ case *ast.AssignStmt:
+ o.Type = AssignStmt
+ for _, e := range n.Rhs {
+ o.AddChildren(t.trans(e))
+ }
+
+ for _, e := range n.Lhs {
+ o.AddChildren(t.trans(e))
+ }
+
+ case *ast.BasicLit:
+ o.Type = BasicLit
+
+ case *ast.BinaryExpr:
+ o.Type = BinaryExpr
+ o.AddChildren(t.trans(n.X), t.trans(n.Y))
+
+ case *ast.BlockStmt:
+ o.Type = BlockStmt
+ for _, stmt := range n.List {
+ o.AddChildren(t.trans(stmt))
+ }
+
+ case *ast.BranchStmt:
+ o.Type = BranchStmt
+ if n.Label != nil {
+ o.AddChildren(t.trans(n.Label))
+ }
+
+ case *ast.CallExpr:
+ o.Type = CallExpr
+ o.AddChildren(t.trans(n.Fun))
+ for _, arg := range n.Args {
+ o.AddChildren(t.trans(arg))
+ }
+
+ case *ast.CaseClause:
+ o.Type = CaseClause
+ for _, e := range n.List {
+ o.AddChildren(t.trans(e))
+ }
+ for _, stmt := range n.Body {
+ o.AddChildren(t.trans(stmt))
+ }
+
+ case *ast.ChanType:
+ o.Type = ChanType
+ o.AddChildren(t.trans(n.Value))
+
+ case *ast.CommClause:
+ o.Type = CommClause
+ if n.Comm != nil {
+ o.AddChildren(t.trans(n.Comm))
+ }
+ for _, stmt := range n.Body {
+ o.AddChildren(t.trans(stmt))
+ }
+
+ case *ast.CompositeLit:
+ o.Type = CompositeLit
+ if n.Type != nil {
+ o.AddChildren(t.trans(n.Type))
+ }
+ for _, e := range n.Elts {
+ o.AddChildren(t.trans(e))
+ }
+
+ case *ast.DeclStmt:
+ o.Type = DeclStmt
+ o.AddChildren(t.trans(n.Decl))
+
+ case *ast.DeferStmt:
+ o.Type = DeferStmt
+ o.AddChildren(t.trans(n.Call))
+
+ case *ast.Ellipsis:
+ o.Type = Ellipsis
+ if n.Elt != nil {
+ o.AddChildren(t.trans(n.Elt))
+ }
+
+ case *ast.EmptyStmt:
+ o.Type = EmptyStmt
+
+ case *ast.ExprStmt:
+ o.Type = ExprStmt
+ o.AddChildren(t.trans(n.X))
+
+ case *ast.Field:
+ o.Type = Field
+ for _, name := range n.Names {
+ o.AddChildren(t.trans(name))
+ }
+ o.AddChildren(t.trans(n.Type))
+
+ case *ast.FieldList:
+ o.Type = FieldList
+ for _, field := range n.List {
+ o.AddChildren(t.trans(field))
+ }
+
+ case *ast.File:
+ o.Type = File
+ for _, decl := range n.Decls {
+ if genDecl, ok := decl.(*ast.GenDecl); ok && genDecl.Tok == token.IMPORT {
+ // skip import declarations
+ continue
+ }
+ o.AddChildren(t.trans(decl))
+ }
+
+ case *ast.ForStmt:
+ o.Type = ForStmt
+ if n.Init != nil {
+ o.AddChildren(t.trans(n.Init))
+ }
+ if n.Cond != nil {
+ o.AddChildren(t.trans(n.Cond))
+ }
+ if n.Post != nil {
+ o.AddChildren(t.trans(n.Post))
+ }
+ o.AddChildren(t.trans(n.Body))
+
+ case *ast.FuncDecl:
+ o.Type = FuncDecl
+ if n.Recv != nil {
+ o.AddChildren(t.trans(n.Recv))
+ }
+ o.AddChildren(t.trans(n.Name), t.trans(n.Type))
+ if n.Body != nil {
+ o.AddChildren(t.trans(n.Body))
+ }
+
+ case *ast.FuncLit:
+ o.Type = FuncLit
+ o.AddChildren(t.trans(n.Type), t.trans(n.Body))
+
+ case *ast.FuncType:
+ o.Type = FuncType
+ o.AddChildren(t.trans(n.Params))
+ if n.Results != nil {
+ o.AddChildren(t.trans(n.Results))
+ }
+
+ case *ast.GenDecl:
+ o.Type = GenDecl
+ for _, spec := range n.Specs {
+ o.AddChildren(t.trans(spec))
+ }
+
+ case *ast.GoStmt:
+ o.Type = GoStmt
+ o.AddChildren(t.trans(n.Call))
+
+ case *ast.Ident:
+ o.Type = Ident
+
+ case *ast.IfStmt:
+ o.Type = IfStmt
+ if n.Init != nil {
+ o.AddChildren(t.trans(n.Init))
+ }
+ o.AddChildren(t.trans(n.Cond), t.trans(n.Body))
+ if n.Else != nil {
+ o.AddChildren(t.trans(n.Else))
+ }
+
+ case *ast.IncDecStmt:
+ o.Type = IncDecStmt
+ o.AddChildren(t.trans(n.X))
+
+ case *ast.IndexExpr:
+ o.Type = IndexExpr
+ o.AddChildren(t.trans(n.X), t.trans(n.Index))
+
+ case *ast.InterfaceType:
+ o.Type = InterfaceType
+ o.AddChildren(t.trans(n.Methods))
+
+ case *ast.KeyValueExpr:
+ o.Type = KeyValueExpr
+ o.AddChildren(t.trans(n.Key), t.trans(n.Value))
+
+ case *ast.LabeledStmt:
+ o.Type = LabeledStmt
+ o.AddChildren(t.trans(n.Label), t.trans(n.Stmt))
+
+ case *ast.MapType:
+ o.Type = MapType
+ o.AddChildren(t.trans(n.Key), t.trans(n.Value))
+
+ case *ast.ParenExpr:
+ o.Type = ParenExpr
+ o.AddChildren(t.trans(n.X))
+
+ case *ast.RangeStmt:
+ o.Type = RangeStmt
+ if n.Key != nil {
+ o.AddChildren(t.trans(n.Key))
+ }
+ if n.Value != nil {
+ o.AddChildren(t.trans(n.Value))
+ }
+ o.AddChildren(t.trans(n.X), t.trans(n.Body))
+
+ case *ast.ReturnStmt:
+ o.Type = ReturnStmt
+ for _, e := range n.Results {
+ o.AddChildren(t.trans(e))
+ }
+
+ case *ast.SelectStmt:
+ o.Type = SelectStmt
+ o.AddChildren(t.trans(n.Body))
+
+ case *ast.SelectorExpr:
+ o.Type = SelectorExpr
+ o.AddChildren(t.trans(n.X), t.trans(n.Sel))
+
+ case *ast.SendStmt:
+ o.Type = SendStmt
+ o.AddChildren(t.trans(n.Chan), t.trans(n.Value))
+
+ case *ast.SliceExpr:
+ o.Type = SliceExpr
+ o.AddChildren(t.trans(n.X))
+ if n.Low != nil {
+ o.AddChildren(t.trans(n.Low))
+ }
+ if n.High != nil {
+ o.AddChildren(t.trans(n.High))
+ }
+ if n.Max != nil {
+ o.AddChildren(t.trans(n.Max))
+ }
+
+ case *ast.StarExpr:
+ o.Type = StarExpr
+ o.AddChildren(t.trans(n.X))
+
+ case *ast.StructType:
+ o.Type = StructType
+ o.AddChildren(t.trans(n.Fields))
+
+ case *ast.SwitchStmt:
+ o.Type = SwitchStmt
+ if n.Init != nil {
+ o.AddChildren(t.trans(n.Init))
+ }
+ if n.Tag != nil {
+ o.AddChildren(t.trans(n.Tag))
+ }
+ o.AddChildren(t.trans(n.Body))
+
+ case *ast.TypeAssertExpr:
+ o.Type = TypeAssertExpr
+ o.AddChildren(t.trans(n.X))
+ if n.Type != nil {
+ o.AddChildren(t.trans(n.Type))
+ }
+
+ case *ast.TypeSpec:
+ o.Type = TypeSpec
+ o.AddChildren(t.trans(n.Name), t.trans(n.Type))
+
+ case *ast.TypeSwitchStmt:
+ o.Type = TypeSwitchStmt
+ if n.Init != nil {
+ o.AddChildren(t.trans(n.Init))
+ }
+ o.AddChildren(t.trans(n.Assign), t.trans(n.Body))
+
+ case *ast.UnaryExpr:
+ o.Type = UnaryExpr
+ o.AddChildren(t.trans(n.X))
+
+ case *ast.ValueSpec:
+ o.Type = ValueSpec
+ for _, name := range n.Names {
+ o.AddChildren(t.trans(name))
+ }
+ if n.Type != nil {
+ o.AddChildren(t.trans(n.Type))
+ }
+ for _, val := range n.Values {
+ o.AddChildren(t.trans(val))
+ }
+
+ default:
+ o.Type = BadNode
+
+ }
+
+ return o
+}
diff --git a/vendor/github.com/golangci/dupl/syntax/syntax.go b/vendor/github.com/golangci/dupl/syntax/syntax.go
new file mode 100644
index 000000000..e2c750afd
--- /dev/null
+++ b/vendor/github.com/golangci/dupl/syntax/syntax.go
@@ -0,0 +1,175 @@
+package syntax
+
+import (
+ "crypto/sha1"
+
+ "github.com/golangci/dupl/suffixtree"
+)
+
+type Node struct {
+ Type int
+ Filename string
+ Pos, End int
+ Children []*Node
+ Owns int
+}
+
+func NewNode() *Node {
+ return &Node{}
+}
+
+func (n *Node) AddChildren(children ...*Node) {
+ n.Children = append(n.Children, children...)
+}
+
+func (n *Node) Val() int {
+ return n.Type
+}
+
+type Match struct {
+ Hash string
+ Frags [][]*Node
+}
+
+func Serialize(n *Node) []*Node {
+ stream := make([]*Node, 0, 10)
+ serial(n, &stream)
+ return stream
+}
+
+func serial(n *Node, stream *[]*Node) int {
+ *stream = append(*stream, n)
+ var count int
+ for _, child := range n.Children {
+ count += serial(child, stream)
+ }
+ n.Owns = count
+ return count + 1
+}
+
+// FindSyntaxUnits finds all complete syntax units in the match group and returns them
+// with the corresponding hash.
+func FindSyntaxUnits(data []*Node, m suffixtree.Match, threshold int) Match {
+ if len(m.Ps) == 0 {
+ return Match{}
+ }
+ firstSeq := data[m.Ps[0] : m.Ps[0]+m.Len]
+ indexes := getUnitsIndexes(firstSeq, threshold)
+
+ // TODO: is this really working?
+ indexCnt := len(indexes)
+ if indexCnt > 0 {
+ lasti := indexes[indexCnt-1]
+ firstn := firstSeq[lasti]
+ for i := 1; i < len(m.Ps); i++ {
+ n := data[int(m.Ps[i])+lasti]
+ if firstn.Owns != n.Owns {
+ indexes = indexes[:indexCnt-1]
+ break
+ }
+ }
+ }
+ if len(indexes) == 0 || isCyclic(indexes, firstSeq) || spansMultipleFiles(indexes, firstSeq) {
+ return Match{}
+ }
+
+ match := Match{Frags: make([][]*Node, len(m.Ps))}
+ for i, pos := range m.Ps {
+ match.Frags[i] = make([]*Node, len(indexes))
+ for j, index := range indexes {
+ match.Frags[i][j] = data[int(pos)+index]
+ }
+ }
+
+ lastIndex := indexes[len(indexes)-1]
+ match.Hash = hashSeq(firstSeq[indexes[0] : lastIndex+firstSeq[lastIndex].Owns])
+ return match
+}
+
+func getUnitsIndexes(nodeSeq []*Node, threshold int) []int {
+ var indexes []int
+ var split bool
+ for i := 0; i < len(nodeSeq); {
+ n := nodeSeq[i]
+ switch {
+ case n.Owns >= len(nodeSeq)-i:
+ // not complete syntax unit
+ i++
+ split = true
+ continue
+ case n.Owns+1 < threshold:
+ split = true
+ default:
+ if split {
+ indexes = indexes[:0]
+ split = false
+ }
+ indexes = append(indexes, i)
+ }
+ i += n.Owns + 1
+ }
+ return indexes
+}
+
+// isCyclic finds out whether there is a repetive pattern in the found clone. If positive,
+// it return false to point out that the clone would be redundant.
+func isCyclic(indexes []int, nodes []*Node) bool {
+ cnt := len(indexes)
+ if cnt <= 1 {
+ return false
+ }
+
+ alts := make(map[int]bool)
+ for i := 1; i <= cnt/2; i++ {
+ if cnt%i == 0 {
+ alts[i] = true
+ }
+ }
+
+ for i := 0; i < indexes[cnt/2]; i++ {
+ nstart := nodes[i+indexes[0]]
+ AltLoop:
+ for alt := range alts {
+ for j := alt; j < cnt; j += alt {
+ index := i + indexes[j]
+ if index < len(nodes) {
+ nalt := nodes[index]
+ if nstart.Owns == nalt.Owns && nstart.Type == nalt.Type {
+ continue
+ }
+ } else if i >= indexes[alt] {
+ return true
+ }
+ delete(alts, alt)
+ continue AltLoop
+ }
+ }
+ if len(alts) == 0 {
+ return false
+ }
+ }
+ return true
+}
+
+func spansMultipleFiles(indexes []int, nodes []*Node) bool {
+ if len(indexes) < 2 {
+ return false
+ }
+ f := nodes[indexes[0]].Filename
+ for i := 1; i < len(indexes); i++ {
+ if nodes[indexes[i]].Filename != f {
+ return true
+ }
+ }
+ return false
+}
+
+func hashSeq(nodes []*Node) string {
+ h := sha1.New()
+ bytes := make([]byte, len(nodes))
+ for i, node := range nodes {
+ bytes[i] = byte(node.Type)
+ }
+ h.Write(bytes)
+ return string(h.Sum(nil))
+}