aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--syz-cluster/controller/Dockerfile1
-rw-r--r--syz-cluster/controller/deployment.yaml7
-rw-r--r--syz-cluster/dashboard/Dockerfile1
-rw-r--r--syz-cluster/dashboard/deployment.yaml7
-rw-r--r--syz-cluster/db-mgmt/Dockerfile1
-rw-r--r--syz-cluster/overlays/dev/fake-gcs.yaml2
-rw-r--r--syz-cluster/overlays/dev/global-config.yaml3
-rw-r--r--syz-cluster/overlays/dev/kustomization.yaml1
-rw-r--r--syz-cluster/pkg/app/env.go19
-rw-r--r--syz-cluster/pkg/blob/gcs.go86
-rw-r--r--syz-cluster/pkg/blob/storage.go7
-rw-r--r--syz-cluster/series-tracker/deployment.yaml5
12 files changed, 104 insertions, 36 deletions
diff --git a/syz-cluster/controller/Dockerfile b/syz-cluster/controller/Dockerfile
index c6692e290..7b05e16fe 100644
--- a/syz-cluster/controller/Dockerfile
+++ b/syz-cluster/controller/Dockerfile
@@ -6,6 +6,7 @@ WORKDIR /build
COPY go.mod ./
COPY go.sum ./
RUN go mod download
+COPY pkg/gcs/ pkg/gcs/
# Build the tool.
COPY syz-cluster/controller/ syz-cluster/controller/
diff --git a/syz-cluster/controller/deployment.yaml b/syz-cluster/controller/deployment.yaml
index 9b52864c8..05ece9aa3 100644
--- a/syz-cluster/controller/deployment.yaml
+++ b/syz-cluster/controller/deployment.yaml
@@ -23,10 +23,3 @@ spec:
name: global-config
ports:
- containerPort: 8080
- volumeMounts:
- - name: blobs-storage-disk
- mountPath: /blob-storage
- volumes:
- - name: blobs-storage-disk
- persistentVolumeClaim:
- claimName: blob-storage-disk-claim
diff --git a/syz-cluster/dashboard/Dockerfile b/syz-cluster/dashboard/Dockerfile
index c813d856e..53925a53b 100644
--- a/syz-cluster/dashboard/Dockerfile
+++ b/syz-cluster/dashboard/Dockerfile
@@ -6,6 +6,7 @@ WORKDIR /build
COPY go.mod ./
COPY go.sum ./
RUN go mod download
+COPY pkg/gcs/ pkg/gcs/
# Build the tool.
COPY syz-cluster/dashboard/ syz-cluster/dashboard/
diff --git a/syz-cluster/dashboard/deployment.yaml b/syz-cluster/dashboard/deployment.yaml
index e6699f211..a3e56dadf 100644
--- a/syz-cluster/dashboard/deployment.yaml
+++ b/syz-cluster/dashboard/deployment.yaml
@@ -23,10 +23,3 @@ spec:
name: global-config
ports:
- containerPort: 8081
- volumeMounts:
- - name: blobs-storage-disk
- mountPath: /blob-storage
- volumes:
- - name: blobs-storage-disk
- persistentVolumeClaim:
- claimName: blob-storage-disk-claim
diff --git a/syz-cluster/db-mgmt/Dockerfile b/syz-cluster/db-mgmt/Dockerfile
index a999a31d3..96e93bba6 100644
--- a/syz-cluster/db-mgmt/Dockerfile
+++ b/syz-cluster/db-mgmt/Dockerfile
@@ -6,6 +6,7 @@ WORKDIR /build
COPY go.mod ./
COPY go.sum ./
RUN go mod download
+COPY pkg/gcs/ pkg/gcs/
# Build the tool.
COPY syz-cluster/db-mgmt/*.go syz-cluster/db-mgmt/
diff --git a/syz-cluster/overlays/dev/fake-gcs.yaml b/syz-cluster/overlays/dev/fake-gcs.yaml
index 362da197c..e9ebcd4e8 100644
--- a/syz-cluster/overlays/dev/fake-gcs.yaml
+++ b/syz-cluster/overlays/dev/fake-gcs.yaml
@@ -18,7 +18,7 @@ spec:
initContainers:
- name: create-test-bucket
image: busybox
- command: ["sh", "-c", "mkdir -p /data/test-bucket"]
+ command: ["sh", "-c", "mkdir -p /data/test-bucket /data/blobs"]
volumeMounts:
- name: data-volume
mountPath: /data
diff --git a/syz-cluster/overlays/dev/global-config.yaml b/syz-cluster/overlays/dev/global-config.yaml
index 0ee6bb8b5..83293e81e 100644
--- a/syz-cluster/overlays/dev/global-config.yaml
+++ b/syz-cluster/overlays/dev/global-config.yaml
@@ -8,5 +8,6 @@ metadata:
data:
SPANNER_EMULATOR_HOST: "cloud-spanner-emulator:9010"
SPANNER_DATABASE_URI: "projects/my-project/instances/my-instance/databases/db"
- LOCAL_BLOB_STORAGE_PATH: "/blob-storage"
+ STORAGE_EMULATOR_HOST: "http://fake-gcs-server.default.svc.cluster.local:4443"
+ BLOB_STORAGE_GCS_BUCKET: "blobs" # Initialized in fake-gcs.yaml
PARALLEL_WORKERS: "1" # Process only one series at a time.
diff --git a/syz-cluster/overlays/dev/kustomization.yaml b/syz-cluster/overlays/dev/kustomization.yaml
index 815048f4e..0aa368cb6 100644
--- a/syz-cluster/overlays/dev/kustomization.yaml
+++ b/syz-cluster/overlays/dev/kustomization.yaml
@@ -11,7 +11,6 @@ resources:
- workflow-roles.yaml
- fake-gcs.yaml
- workflow-artifacts.yaml
- - blobs-pvc-dev.yaml
patches:
- target:
diff --git a/syz-cluster/pkg/app/env.go b/syz-cluster/pkg/app/env.go
index 53ba9a77f..e54cb30b2 100644
--- a/syz-cluster/pkg/app/env.go
+++ b/syz-cluster/pkg/app/env.go
@@ -25,7 +25,7 @@ func Environment(ctx context.Context) (*AppEnvironment, error) {
if err != nil {
return nil, fmt.Errorf("failed to set up a Spanner client: %w", err)
}
- storage, err := DefaultStorage()
+ storage, err := DefaultStorage(ctx)
if err != nil {
return nil, fmt.Errorf("failed to set up the blob storage: %w", err)
}
@@ -60,18 +60,13 @@ func DefaultSpanner(ctx context.Context) (*spanner.Client, error) {
return spanner.NewClient(ctx, uri.Full)
}
-func DefaultStorage() (blob.Storage, error) {
- // LOCAL_BLOB_STORAGE_PATH is set in the dev environment.
- path := os.Getenv("LOCAL_BLOB_STORAGE_PATH")
- if path == "" {
- // TODO: implement GCS support.
- return nil, fmt.Errorf("empty LOCAL_BLOB_STORAGE_PATH")
+func DefaultStorage(ctx context.Context) (blob.Storage, error) {
+ // BLOB_STORAGE_GCS_BUCKET is the only supported option.
+ bucket := os.Getenv("BLOB_STORAGE_GCS_BUCKET")
+ if bucket == "" {
+ return nil, fmt.Errorf("empty BLOB0_STORAGE_GCS_BUCKET")
}
- err := os.MkdirAll(path, 0666)
- if err != nil {
- return nil, err
- }
- return blob.NewLocalStorage(path), nil
+ return blob.NewGCSClient(ctx, bucket)
}
func DefaultClient() *api.Client {
diff --git a/syz-cluster/pkg/blob/gcs.go b/syz-cluster/pkg/blob/gcs.go
new file mode 100644
index 000000000..1a865c1a2
--- /dev/null
+++ b/syz-cluster/pkg/blob/gcs.go
@@ -0,0 +1,86 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package blob
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "regexp"
+
+ "github.com/google/syzkaller/pkg/gcs"
+ "github.com/google/uuid"
+)
+
+type gcsDriver struct {
+ bucket string
+ client *gcs.Client
+}
+
+func NewGCSClient(ctx context.Context, bucket string) (Storage, error) {
+ client, err := gcs.NewClient(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &gcsDriver{
+ bucket: bucket,
+ client: client,
+ }, nil
+}
+
+func (gcs *gcsDriver) Store(source io.Reader) (string, error) {
+ object := uuid.New().String()
+ err := gcs.writeObject(object, source)
+ if err != nil {
+ return "", err
+ }
+ return gcs.objectURI(object), nil
+}
+
+func (gcs *gcsDriver) Update(uri string, source io.Reader) error {
+ object, err := gcs.objectName(uri)
+ if err != nil {
+ return err
+ }
+ return gcs.writeObject(object, source)
+}
+
+func (gcs *gcsDriver) Read(uri string) (io.ReadCloser, error) {
+ object, err := gcs.objectName(uri)
+ if err != nil {
+ return nil, err
+ }
+ file, err := gcs.client.Read(fmt.Sprintf("%s/%s", gcs.bucket, object))
+ if err != nil {
+ return nil, err
+ }
+ return file.Reader()
+}
+
+var gcsObjectRe = regexp.MustCompile(`^gcs://([\w-]+)/([\w-]+)$`)
+
+func (gcs *gcsDriver) objectName(uri string) (string, error) {
+ match := gcsObjectRe.FindStringSubmatch(uri)
+ if len(match) == 0 {
+ return "", fmt.Errorf("invalid GCS URI")
+ } else if match[1] != gcs.bucket {
+ return "", fmt.Errorf("unexpected GCS bucket")
+ }
+ return match[2], nil
+}
+
+func (gcs *gcsDriver) objectURI(object string) string {
+ return fmt.Sprintf("gcs://%s/%s", gcs.bucket, object)
+}
+
+func (gcs *gcsDriver) writeObject(object string, source io.Reader) error {
+ w, err := gcs.client.FileWriterExt(fmt.Sprintf("%s/%s", gcs.bucket, object), "", "")
+ if err != nil {
+ return err
+ }
+ defer w.Close()
+
+ _, err = io.Copy(w, source)
+ return err
+}
diff --git a/syz-cluster/pkg/blob/storage.go b/syz-cluster/pkg/blob/storage.go
index a38899c33..ba85d68d1 100644
--- a/syz-cluster/pkg/blob/storage.go
+++ b/syz-cluster/pkg/blob/storage.go
@@ -9,7 +9,8 @@ import (
"os"
"path/filepath"
"strings"
- "time"
+
+ "github.com/google/uuid"
)
// Storage is not assumed to be used for partciularly large objects (e.g. GB of size),
@@ -23,6 +24,8 @@ type Storage interface {
var _ Storage = (*LocalStorage)(nil)
+// LocalStorage keeps objets in the specified local directory.
+// It's intended to be used only for unit tests.
type LocalStorage struct {
baseFolder string
}
@@ -34,7 +37,7 @@ func NewLocalStorage(baseFolder string) *LocalStorage {
const localStoragePrefix = "local://"
func (ls *LocalStorage) Store(source io.Reader) (string, error) {
- name := fmt.Sprint(time.Now().UnixNano())
+ name := uuid.New().String()
err := ls.writeFile(name, source)
if err != nil {
return "", err
diff --git a/syz-cluster/series-tracker/deployment.yaml b/syz-cluster/series-tracker/deployment.yaml
index 2a84f0722..b6e067a37 100644
--- a/syz-cluster/series-tracker/deployment.yaml
+++ b/syz-cluster/series-tracker/deployment.yaml
@@ -24,12 +24,7 @@ spec:
volumeMounts:
- name: series-tracker-repo-disk
mountPath: /git-repo
- - name: blobs-storage-disk
- mountPath: /blob-storage
volumes:
- name: series-tracker-repo-disk
persistentVolumeClaim:
claimName: series-tracker-repo-disk-claim
- - name: blobs-storage-disk
- persistentVolumeClaim:
- claimName: blob-storage-disk-claim
generated by ggit (джигит) mrf-deployment (git 2.46.0) at 2026-03-11 20:50:50 +0000