syz-cluster: provide API for uploading artifacts archive

The archive would be a useful source of debugging information.

Provide an HTTP endpoint that accepts a multipart form request with
the archived data.

Provide an *api.Client method to encapsulate the encoding of the data.

Add a test.

7 files changed, 130 insertions, 16 deletions

diff --git a/syz-cluster/pkg/api/client.go b/syz-cluster/pkg/api/client.go
index d0509cfe0..787748e51 100644
--- a/syz-cluster/pkg/api/client.go
+++ b/syz-cluster/pkg/api/client.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"strings"
 	"time"
 )
@@ -75,6 +76,16 @@ func (client Client) UploadTestResult(ctx context.Context, req *TestResult) erro
 	return err
 }
 
+func (client Client) UploadTestArtifacts(ctx context.Context, sessionID, testName string,
+	tarGzContent io.Reader) error {
+	v := url.Values{}
+	v.Add("session", sessionID)
+	v.Add("test", testName)
+	url := client.baseURL + "/tests/upload_artifacts?" + v.Encode()
+	_, err := postMultiPartFile[any](ctx, url, tarGzContent)
+	return err
+}
+
 func (client Client) UploadFinding(ctx context.Context, req *NewFinding) error {
 	_, err := postJSON[NewFinding, any](ctx, client.baseURL+"/findings/upload", req)
 	return err
diff --git a/syz-cluster/pkg/api/http.go b/syz-cluster/pkg/api/http.go
index 420b787a7..4523b5374 100644
--- a/syz-cluster/pkg/api/http.go
+++ b/syz-cluster/pkg/api/http.go
@@ -7,7 +7,9 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
+	"mime/multipart"
 	"net/http"
 )
 
@@ -32,6 +34,28 @@ func postJSON[Req any, Resp any](ctx context.Context, url string, req *Req) (*Re
 	return finishRequest[Resp](httpReq)
 }
 
+func postMultiPartFile[Resp any](ctx context.Context, url string, reader io.Reader) (*Resp, error) {
+	// TODO: this will work well only up to some size of the file. We need a pipe and a goroutine.
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("content", "content")
+	if err != nil {
+		return nil, fmt.Errorf("failed to create a form file part: %w", err)
+	}
+	if _, err := io.Copy(part, reader); err != nil {
+		return nil, err
+	}
+	if err := writer.Close(); err != nil {
+		return nil, err
+	}
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
+	if err != nil {
+		return nil, err
+	}
+	httpReq.Header.Set("Content-Type", writer.FormDataContentType())
+	return finishRequest[Resp](httpReq)
+}
+
 func ReplyJSON[T any](w http.ResponseWriter, resp T) {
 	w.Header().Set("Content-Type", "application/json")
 	err := json.NewEncoder(w).Encode(resp)
diff --git a/syz-cluster/pkg/controller/api.go b/syz-cluster/pkg/controller/api.go
index bcab7813f..6dbdc4889 100644
--- a/syz-cluster/pkg/controller/api.go
+++ b/syz-cluster/pkg/controller/api.go
@@ -43,6 +43,7 @@ func (c APIServer) Mux() *http.ServeMux {
 	mux.HandleFunc("/sessions/upload", c.uploadSession)
 	mux.HandleFunc("/sessions/{session_id}/series", c.getSessionSeries)
 	mux.HandleFunc("/sessions/{session_id}/skip", c.skipSession)
+	mux.HandleFunc("/tests/upload_artifacts", c.uploadTestArtifact)
 	mux.HandleFunc("/tests/upload", c.uploadTest)
 	mux.HandleFunc("/trees", c.getTrees)
 	return mux
@@ -116,6 +117,36 @@ func (c APIServer) uploadTest(w http.ResponseWriter, r *http.Request) {
 	api.ReplyJSON[interface{}](w, nil)
 }
 
+func (c APIServer) uploadTestArtifact(w http.ResponseWriter, r *http.Request) {
+	const maxMemory = 16 * 1000 * 1000 // 16 MB.
+	if err := r.ParseMultipartForm(maxMemory); err != nil {
+		http.Error(w, "could not parse the multipart form", http.StatusBadRequest)
+		return
+	}
+	defer r.MultipartForm.RemoveAll()
+
+	file, _, err := r.FormFile("content")
+	if err != nil {
+		if err == http.ErrMissingFile {
+			http.Error(w, "the 'content' file must be present", http.StatusBadRequest)
+			return
+		}
+		http.Error(w, fmt.Sprintf("failed to query the file: %s", err), http.StatusInternalServerError)
+		return
+	}
+	defer file.Close()
+
+	err = c.testService.SaveArtifacts(r.Context(),
+		r.FormValue("session"),
+		r.FormValue("test"),
+		file)
+	if err != nil {
+		http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
+		return
+	}
+	api.ReplyJSON[interface{}](w, nil)
+}
+
 func (c APIServer) uploadFinding(w http.ResponseWriter, r *http.Request) {
 	req := api.ParseJSON[api.NewFinding](w, r)
 	if req == nil {
diff --git a/syz-cluster/pkg/controller/api_test.go b/syz-cluster/pkg/controller/api_test.go
index 2b14fbe5d..fa8491208 100644
--- a/syz-cluster/pkg/controller/api_test.go
+++ b/syz-cluster/pkg/controller/api_test.go
@@ -4,6 +4,7 @@
 package controller
 
 import (
+	"bytes"
 	"testing"
 	"time"
 
@@ -80,6 +81,24 @@ func TestAPISaveFinding(t *testing.T) {
 	})
 }
 
+func TestAPIUploadTestArtifacts(t *testing.T) {
+	env, ctx := app.TestEnvironment(t)
+	client := TestServer(t, env)
+
+	_, sessionID := UploadTestSeries(t, ctx, client, testSeries)
+	buildResp := UploadTestBuild(t, ctx, client, testBuild)
+	err := client.UploadTestResult(ctx, &api.TestResult{
+		SessionID:   sessionID,
+		BaseBuildID: buildResp.ID,
+		TestName:    "test",
+		Result:      api.TestRunning,
+		Log:         []byte("some log"),
+	})
+	assert.NoError(t, err)
+	err = client.UploadTestArtifacts(ctx, sessionID, "test", bytes.NewReader([]byte("artifacts content")))
+	assert.NoError(t, err)
+}
+
 var testSeries = &api.Series{
 	ExtID:       "ext-id",
 	AuthorEmail: "some@email.com",
diff --git a/syz-cluster/pkg/db/entities.go b/syz-cluster/pkg/db/entities.go
index 43f0a4968..688bb8971 100644
--- a/syz-cluster/pkg/db/entities.go
+++ b/syz-cluster/pkg/db/entities.go
@@ -106,13 +106,14 @@ func (s *Session) SetSkipReason(reason string) {
 }
 
 type SessionTest struct {
-	SessionID      string             `spanner:"SessionID"`
-	BaseBuildID    spanner.NullString `spanner:"BaseBuildID"`
-	PatchedBuildID spanner.NullString `spanner:"PatchedBuildID"`
-	UpdatedAt      time.Time          `spanner:"UpdatedAt"`
-	TestName       string             `spanner:"TestName"`
-	Result         string             `spanner:"Result"`
-	LogURI         string             `spanner:"LogURI"`
+	SessionID           string             `spanner:"SessionID"`
+	BaseBuildID         spanner.NullString `spanner:"BaseBuildID"`
+	PatchedBuildID      spanner.NullString `spanner:"PatchedBuildID"`
+	UpdatedAt           time.Time          `spanner:"UpdatedAt"`
+	TestName            string             `spanner:"TestName"`
+	Result              string             `spanner:"Result"`
+	LogURI              string             `spanner:"LogURI"`
+	ArtifactsArchiveURI string             `spanner:"ArtifactsArchiveURI"`
 }
 
 type Finding struct {
diff --git a/syz-cluster/pkg/db/migrations/1_initialize.up.sql b/syz-cluster/pkg/db/migrations/1_initialize.up.sql
index e7a1b396d..c8938ca53 100644
--- a/syz-cluster/pkg/db/migrations/1_initialize.up.sql
+++ b/syz-cluster/pkg/db/migrations/1_initialize.up.sql
@@ -72,6 +72,7 @@ CREATE TABLE SessionTests (
     BaseBuildID STRING(36),
     PatchedBuildID STRING(36),
     LogURI STRING(256) NOT NULL,
+    ArtifactsArchiveURI STRING(256) NOT NULL,
     CONSTRAINT FK_SessionResults FOREIGN KEY (SessionID) REFERENCES Sessions (ID),
     CONSTRAINT ResultEnum CHECK (Result IN ('passed', 'failed', 'error', 'running')),
     CONSTRAINT FK_BaseBuild FOREIGN KEY (BaseBuildID) REFERENCES Builds (ID),
diff --git a/syz-cluster/pkg/service/sessiontest.go b/syz-cluster/pkg/service/sessiontest.go
index 0e4e56761..a59126860 100644
--- a/syz-cluster/pkg/service/sessiontest.go
+++ b/syz-cluster/pkg/service/sessiontest.go
@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"time"
 
 	"cloud.google.com/go/spanner"
@@ -46,18 +47,44 @@ func (s *SessionTestService) Save(ctx context.Context, req *api.TestResult) erro
 	if req.PatchedBuildID != "" {
 		entity.PatchedBuildID = spanner.NullString{StringVal: req.PatchedBuildID, Valid: true}
 	}
-	if entity.LogURI != "" {
-		err := s.blobStorage.Update(entity.LogURI, bytes.NewReader(req.Log))
+	// TODO: the code does not really handle simultaneous requests.
+	if len(req.Log) > 0 {
+		entity.LogURI, err = s.uploadOrUpdate(ctx, entity.LogURI, bytes.NewReader(req.Log))
 		if err != nil {
-			return fmt.Errorf("failed to update the log: %w", err)
+			return fmt.Errorf("failed to save the log: %w", err)
 		}
-	} else if len(req.Log) > 0 {
-		// TODO: it will leak if we fail to save the entity.
-		uri, err := s.blobStorage.Store(bytes.NewReader(req.Log))
+	}
+	return s.testRepo.InsertOrUpdate(ctx, entity)
+}
+
+// TODO: this function has the same problems as Save().
+func (s *SessionTestService) SaveArtifacts(ctx context.Context, sessionID, testName string, reader io.Reader) error {
+	entity, err := s.testRepo.Get(ctx, sessionID, testName)
+	if err != nil {
+		return fmt.Errorf("failed to query the test: %w", err)
+	} else if entity == nil {
+		return fmt.Errorf("the test has not been submitted yet")
+	}
+	newArchiveURI, err := s.uploadOrUpdate(ctx, entity.ArtifactsArchiveURI, reader)
+	if err != nil {
+		return fmt.Errorf("failed to save the artifacts archive: %w", err)
+	}
+	entity.ArtifactsArchiveURI = newArchiveURI
+	return s.testRepo.InsertOrUpdate(ctx, entity)
+}
+
+func (s *SessionTestService) uploadOrUpdate(ctx context.Context, uri string, reader io.Reader) (string, error) {
+	if uri != "" {
+		err := s.blobStorage.Update(uri, reader)
 		if err != nil {
-			return fmt.Errorf("failed to save the log: %w", err)
+			return "", fmt.Errorf("failed to update: %w", err)
 		}
-		entity.LogURI = uri
+		return uri, nil
+	}
+	// TODO: it will leak if we fail to save the entity.
+	uri, err := s.blobStorage.Store(reader)
+	if err != nil {
+		return "", fmt.Errorf("failed to save: %w", err)
 	}
-	return s.testRepo.InsertOrUpdate(context.Background(), entity)
+	return uri, nil
 }