1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
package main
import (
"crypto/sha1"
"encoding/hex"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
)
type Sig [sha1.Size]byte
// PersistentSet is a set of binary blobs with a persistent mirror on disk.
type PersistentSet struct {
dir string
m map[Sig][]byte
a [][]byte
}
func hash(data []byte) Sig {
return Sig(sha1.Sum(data))
}
func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet {
ps := &PersistentSet{
dir: dir,
m: make(map[Sig][]byte),
}
os.MkdirAll(dir, 0770)
filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
log.Fatalf("error during dir walk: %v\n", err)
}
if info.IsDir() {
return nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
log.Fatalf("error during file read: %v\n", err)
return nil
}
sig := hash(data)
if _, ok := ps.m[sig]; ok {
return nil
}
name := info.Name()
if len(data) == 0 {
// This can happen is master runs on machine-under-test,
// and it has crashed midway.
log.Printf("removing empty file %v", name)
os.Remove(path)
return nil
}
const hexLen = 2 * sha1.Size
if len(name) > hexLen+1 && isHexString(name[:hexLen]) && name[hexLen] == '.' {
return nil // description file
}
if len(name) != hexLen || !isHexString(name) {
log.Fatalf("unknown file in persistent dir %v: %v", dir, name)
}
if verify != nil && !verify(data) {
os.Remove(path)
return nil
}
if name != hex.EncodeToString(sig[:]) {
log.Printf("bad hash in persistent dir %v for file %v, expect %v", dir, name, hex.EncodeToString(sig[:]))
if err := ioutil.WriteFile(filepath.Join(ps.dir, hex.EncodeToString(sig[:])), data, 0660); err != nil {
log.Fatalf("failed to write file: %v", err)
}
os.Remove(path)
}
ps.m[sig] = data
ps.a = append(ps.a, data)
return nil
})
return ps
}
func isHexString(s string) bool {
for _, v := range []byte(s) {
if v >= '0' && v <= '9' || v >= 'a' && v <= 'f' {
continue
}
return false
}
return true
}
func (ps *PersistentSet) add(data []byte) bool {
sig := hash(data)
if _, ok := ps.m[sig]; ok {
return false
}
data = append([]byte{}, data...)
ps.m[sig] = data
ps.a = append(ps.a, data)
fname := filepath.Join(ps.dir, hex.EncodeToString(sig[:]))
if err := ioutil.WriteFile(fname, data, 0660); err != nil {
log.Fatalf("failed to write file: %v", err)
}
return true
}
// addDescription creates a complementary to data file on disk.
func (ps *PersistentSet) addDescription(data []byte, desc []byte, typ string) {
sig := hash(data)
fname := filepath.Join(ps.dir, fmt.Sprintf("%v.%v", hex.EncodeToString(sig[:]), typ))
if err := ioutil.WriteFile(fname, desc, 0660); err != nil {
log.Fatalf("failed to write file: %v", err)
}
}
func (ps *PersistentSet) minimize(set map[string]bool) {
ps.a = nil
for sig, data := range ps.m {
s := hex.EncodeToString(sig[:])
if set[s] {
ps.a = append(ps.a, data)
} else {
delete(ps.m, sig)
os.Remove(filepath.Join(ps.dir, s))
}
}
}
|
